duel 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/docs/ci.md CHANGED
@@ -12,9 +12,10 @@ example, a series of changes that are each 1.5% slower would each pass.
12
12
  To catch that kind of slow creep, also compare against your last release
13
13
  on a schedule. The slowdowns add up there until they're a regression.
14
14
 
15
- The following GitHub Actions workflow does both, and reports on each
16
- pull request. `duel init` writes it for you, as
17
- `.github/workflows/perf.yml`:
15
+ The following GitHub Actions workflow does both. `duel init` writes it
16
+ for you, as `.github/workflows/perf.yml`, with a second workflow that
17
+ comments on a pull request when a benchmark fails (see
18
+ [Comment on a pull request](#comment-on-a-pull-request)):
18
19
 
19
20
  ```yaml
20
21
  # .github/workflows/perf.yml
@@ -22,90 +23,599 @@ name: Perf
22
23
 
23
24
  on:
24
25
  pull_request:
26
+ merge_group: # (a merge queue's)
25
27
  push:
26
- branches: [main]
28
+ branches: [main] # (and any other branch to guard, a stable one, say)
27
29
  schedule:
28
30
  - cron: "0 4 * * *"
29
31
  workflow_dispatch:
30
32
 
33
+ # A pull request's newer push cancels its run, if that's still going, as
34
+ # only its newest commit's result counts. Every other run (a push's, say)
35
+ # runs to the end, so that each push is compared.
36
+ concurrency:
37
+ group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
38
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
39
+
40
+ # The Duel job runs the branch's code (its benchmarks too), so it only
41
+ # reads. Commenting on a commit is the Comment on the commit job's, which
42
+ # runs none of it, and commenting on a pull request is the Perf comment
43
+ # workflow's.
31
44
  permissions:
32
45
  contents: read
33
- pull-requests: write # to comment on a pull request with the report
46
+ pull-requests: read # for the pull request's labels (see c.accept_label)
34
47
 
35
48
  jobs:
36
49
  duel:
37
50
  name: Duel
38
51
  runs-on: ubuntu-latest
52
+ # Whether to comment (post, change or none), the run's status (passed,
53
+ # failed or neutral), and whether to mark a push passed (true, or
54
+ # nothing), from Duel.
55
+ outputs:
56
+ comment: ${{ steps.compare.outputs.comment }}
57
+ status: ${{ steps.compare.outputs.status }}
58
+ passed: ${{ steps.compare.outputs.passed }}
39
59
  steps:
40
60
  - uses: actions/checkout@v7
41
61
  with:
42
62
  fetch-depth: 0 # the baseline comes from history, tags included
63
+ persist-credentials: false
43
64
  # (And whatever else your tests need set up: packages, services.)
44
65
  - uses: ruby/setup-ruby@v1
45
66
  with:
46
67
  ruby-version: "3.4" # (or none, with a .ruby-version)
47
68
  bundler-cache: true
48
- # Against the main branch for a pull request, what was there before
49
- # for a push, and the last release on a schedule (slow creep). Duel
50
- # adds its report to the job's summary, and writes it to duel.md for
51
- # the comment.
69
+ # The labels of the pull request (or for a push, of the one that
70
+ # merged it), for c.accept_label in .duel.rb, as a JSON array. They're
71
+ # read as the job runs, so after adding the label, run it again.
72
+ - name: Find the pull request's labels
73
+ id: labels
74
+ if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'
75
+ continue-on-error: true # (without them, nothing's accepted)
76
+ env:
77
+ GH_TOKEN: ${{ github.token }}
78
+ PR: ${{ github.event.pull_request.number }}
79
+ QUEUED: ${{ github.event.merge_group.head_ref }}
80
+ run: |
81
+ # (In a merge queue, the last pull request queued, which the queue's
82
+ # branch names: gh-readonly-queue/main/pr-123-<sha>.)
83
+ if [ -n "$QUEUED" ]; then
84
+ PR=${QUEUED##*/pr-}
85
+ PR=${PR%%-*}
86
+ fi
87
+ if [ -n "$PR" ]; then
88
+ labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
89
+ else
90
+ labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
91
+ fi
92
+ echo "labels=$labels" >> "$GITHUB_OUTPUT"
93
+ # Against the branch it targets for a pull request, the queue's base
94
+ # for a merge queue, what was there before for a push (or as
95
+ # c.push_base in .duel.rb says, the last commit Duel passed), and the
96
+ # last release on a schedule (slow creep). Duel adds its report to
97
+ # the job's summary, writes it to duel.md for the comment, and sets
98
+ # the step's outputs to say whether to comment, as c.comment in
99
+ # .duel.rb says, whatever the outcome.
52
100
  - name: Compare
101
+ id: compare
53
102
  env:
54
- DUEL_BASE: ${{ github.event_name == 'schedule' && 'last-release' || github.event_name == 'push' && github.event.before || '' }}
103
+ DUEL_BASE: ${{ github.event_name == 'schedule' && 'last-release' || github.event_name == 'push' && github.event.before || github.event_name == 'merge_group' && github.event.merge_group.base_sha || '' }}
55
104
  # (Run by hand, it compares anyway: with nothing changed, the code
56
105
  # with itself, which checks the setup.)
57
106
  DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}
107
+ DUEL_LABELS: ${{ steps.labels.outputs.labels }}
108
+ run: bundle exec duel --markdown "$RUNNER_TEMP/duel/duel.md"
109
+ # For a pull request, what Duel said, for the Perf comment workflow,
110
+ # which can't see this job's outputs.
111
+ - name: Say whether to comment
112
+ if: ${{ !cancelled() && github.event_name == 'pull_request' }}
113
+ env:
114
+ PR: ${{ github.event.pull_request.number }}
115
+ COMMENT: ${{ steps.compare.outputs.comment }}
116
+ STATUS: ${{ steps.compare.outputs.status }}
58
117
  run: |
59
- # (A branch's first push, or one that rewrote history, has no
60
- # commit before it here to compare against.)
61
- if [ "$GITHUB_EVENT_NAME" = push ] && ! git cat-file -e "$DUEL_BASE^{commit}" 2>/dev/null; then
62
- echo "::notice::No commit before this push to compare against"
63
- exit 0
64
- fi
65
- bundle exec duel --markdown "$RUNNER_TEMP/duel.md"
66
- # The report, as a comment on the pull request: one comment, updated
67
- # on every push. (A pull request from a fork gets no comment, as its
68
- # token can't write one: the job's summary has the report.)
69
- - name: Comment on the pull request
70
- if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
118
+ mkdir -p "$RUNNER_TEMP/duel"
119
+ printf 'pr=%s\ncomment=%s\nstatus=%s\n' "$PR" "$COMMENT" "$STATUS" > "$RUNNER_TEMP/duel/comment.env"
120
+ # The report, for the comments.
121
+ - name: Keep the report
122
+ if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'push') }}
123
+ uses: actions/upload-artifact@v7
124
+ with:
125
+ name: duel-report
126
+ path: ${{ runner.temp }}/duel/
127
+ if-no-files-found: ignore # none when Duel didn't get to compare
128
+ retention-days: 1
129
+
130
+ # The report, as a new comment on a pushed commit, which notifies its
131
+ # author, when Duel says to, as for a pull request (see the Perf comment
132
+ # workflow). It does only where a regression in a push passes, with
133
+ # c.fail_on = :pull_requests in .duel.rb, so the comment says what the
134
+ # check doesn't. Its last comment is the last one on any commit. This
135
+ # job checks nothing out and runs none of the code.
136
+ commit-comment:
137
+ name: Comment on the commit
138
+ needs: duel
139
+ if: ${{ !cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
140
+ runs-on: ubuntu-latest
141
+ permissions:
142
+ contents: write # to comment on the commit with the report
143
+ steps:
144
+ - name: Get the report
145
+ id: report
146
+ continue-on-error: true # there's none when the Duel job failed before comparing
147
+ uses: actions/download-artifact@v8
148
+ with:
149
+ name: duel-report
150
+ path: ${{ runner.temp }}/report
151
+ - name: Comment on the commit
152
+ if: steps.report.outcome == 'success'
71
153
  env:
72
154
  GH_TOKEN: ${{ github.token }}
73
- PR: ${{ github.event.pull_request.number }}
155
+ COMMENT: ${{ needs.duel.outputs.comment }}
156
+ STATUS: ${{ needs.duel.outputs.status }}
74
157
  run: |
75
- [ -f "$RUNNER_TEMP/duel.md" ] || exit 0
76
- marker="<!-- duel -->"
77
- { echo "$marker"; cat "$RUNNER_TEMP/duel.md"; } > "$RUNNER_TEMP/comment.md"
78
- ids=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
79
- --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id")
80
- if [ -n "$ids" ]; then
81
- gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/${ids%%$'\n'*}" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
82
- else
83
- gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
158
+ marker="<!-- duel"
159
+ last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/comments" \
160
+ --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\"))" | tail -n 1)
161
+ if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
162
+ { echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
163
+ gh api --method POST "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
84
164
  fi
165
+
166
+ # With c.push_base = :last_passed in .duel.rb, a push is compared with
167
+ # the last commit Duel passed, which a branch marks (each branch pushed
168
+ # has its own), so slowdowns too small for any one push to fail add up
169
+ # until they fail one. This job moves the branch to a push that passed
170
+ # (or makes it). It checks nothing out and runs none of the code. To
171
+ # accept a regression, move the branch yourself (for main, git push
172
+ # origin main:duel/passed/main).
173
+ passed:
174
+ name: Mark the commit passed
175
+ needs: duel
176
+ if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
177
+ runs-on: ubuntu-latest
178
+ permissions:
179
+ contents: write # to move the branch
180
+ steps:
181
+ - name: Move the branch to the commit
182
+ env:
183
+ GH_TOKEN: ${{ github.token }}
184
+ BRANCH: duel/passed/${{ github.ref_name }}
185
+ run: |
186
+ gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" -F force=true > /dev/null 2>&1 ||
187
+ gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null
85
188
  ```
86
189
 
190
+ A newer push to a pull request cancels the pull request's run, if it's
191
+ still going, because only the newest commit's result counts. Every other
192
+ run, such as a push's to `main`, runs to the end, so each push is
193
+ compared.
194
+
87
195
  The workflow compares the following:
88
196
 
89
- - A pull request, against the main branch.
90
- - A push to `main`, against the commit before the push. If that commit
91
- isn't in the history, such as after a force push or for a branch's
92
- first push, the workflow skips the comparison with a notice.
197
+ - A pull request, against the branch that it targets: usually `main`,
198
+ but a stable branch, or another pull request's branch in a stack of
199
+ them, too.
200
+ - A merge queue's group, against the queue's base, the commit that the
201
+ group merges into. With a merge queue, a required Duel check must run
202
+ on `merge_group` too, or the queue waits for it forever. A regression
203
+ fails a merge queue's run as it fails a pull request's, and the run
204
+ reads the labels of the last pull request queued, for
205
+ [`c.accept_label`](#accept-a-regression-with-a-label).
206
+ - A push to `main`, against the commit before the push, or with
207
+ [`c.push_base = :last_passed`](#compare-a-push-with-the-last-commit-that-passed),
208
+ the last commit that Duel passed. If that commit isn't in the history,
209
+ such as after a force push or for a branch's first push, Duel reports
210
+ that there's nothing to compare against, and passes.
93
211
  - On a schedule, the last release. Until you tag your first release,
94
212
  Duel reports that there's nothing to compare against, and passes.
95
213
  - When you run the workflow manually, your code against itself. This
96
214
  checks that the benchmarks run on the CI machines.
97
215
 
216
+ Duel compares a change that touches your code (`lib/` or `ext/`, or what
217
+ `c.watch` names), your benchmarks, `.duel.rb`, or, with
218
+ `c.ruby = :version_file`, your version file. Each version runs your
219
+ branch's benchmarks and configuration, so a change to only those compares
220
+ the same code with itself, which checks that they run and pass. A change
221
+ to only the workflow has nothing to compare: to check it, run the
222
+ workflow manually.
223
+
98
224
  ## Comment on a pull request
99
225
 
100
226
  The workflow writes Duel's report
101
227
  ([Report on a pull request](../README.md#report-on-a-pull-request))
102
- to a file, and then comments on the pull request with it. It updates
103
- the same comment on every push, rather than adding one each time. It
104
- finds its comment by the hidden marker on the comment's first line, so
105
- give each workflow that comments a marker of its own. Commenting takes
106
- the `pull-requests: write` permission. A pull request from a fork gets
107
- no comment, because its token can't write one, but the job's summary
108
- still has the report.
228
+ to a file, and comments on the pull request with it when
229
+ [`c.comment`](configuration.md#choose-when-to-comment-on-a-pull-request)
230
+ says to. By default, it comments when a benchmark fails, or when Duel
231
+ couldn't compare. Whatever `c.comment` says, a regression fails the
232
+ check, because `duel` exits with a nonzero status, and the job's
233
+ summary has the report.
234
+
235
+ The workflow never edits or deletes a comment, so the pull request
236
+ keeps every report that it posted. Each comment starts with a hidden
237
+ marker that records the run's status: `<!-- duel status=passed -->`,
238
+ `status=failed` (a run that couldn't compare too), or `status=neutral`
239
+ (nothing to compare). When a run's outcome wouldn't get a comment of
240
+ its own, the workflow still posts one if it commented on the pull
241
+ request before, and its last comment's status differs from this run's.
242
+ With the default, after a comment that a benchmark failed, the first
243
+ push that passes gets one comment saying so, and the pushes after it
244
+ get none. So the last comment is never stale. A comment from Duel 0.1.0,
245
+ whose marker has no status, counts as a comment with another status.
246
+
247
+ The workflow finds its comments by the marker's start. For a project
248
+ in a subdirectory of its repository, the marker names the directory,
249
+ such as `<!-- duel gems/parser status=failed -->`, so each of a
250
+ monorepo's workflows has comments of its own.
251
+
252
+ Duel decides, because it knows the outcome and reads `.duel.rb`. On
253
+ GitHub Actions, the Compare step's `duel` sets two step outputs, which
254
+ the Duel job passes on as its own: `comment`, and `status` (`passed`,
255
+ `failed`, or `neutral`). `comment` is one of the following:
256
+
257
+ | `comment` | What the Perf comment workflow does |
258
+ |---|---|
259
+ | `post` | Posts a new comment. |
260
+ | `change` | Posts a new comment only if the pull request has one from the workflow, and the last one's status isn't this run's. |
261
+ | `none` | Nothing. |
262
+
263
+ The Duel job runs the pull request's code, its benchmarks included, so
264
+ it only reads (`contents: read`). It keeps the report as an artifact
265
+ for a day, with a file, `comment.env`, that says what Duel decided:
266
+ the pull request's number, `comment`, and `status`. A second workflow,
267
+ `.github/workflows/perf-comment.yml`, posts the report. It runs when
268
+ the Perf workflow completes (`workflow_run`), with your repository's
269
+ own copy of the workflow and a token that can write, so a pull request
270
+ from a fork gets a comment too. It checks nothing out and runs none of
271
+ the pull request's code. The pull request's code ran in the Duel job,
272
+ and could have written anything into the artifact, so the workflow
273
+ trusts only what Duel could have said there: a number that names the
274
+ pull request whose head the run compared, and a `comment` and a
275
+ `status` that Duel uses. The report's Markdown is the comment's body,
276
+ which can only say something on the pull request that the pull request
277
+ could say itself. When the Duel job fails before it compares, there's
278
+ no report, and no comment. GitHub runs a `workflow_run` workflow only
279
+ from your default branch, so the pull request that adds the workflow
280
+ gets no comment.
281
+
282
+ ```yaml
283
+ # .github/workflows/perf-comment.yml
284
+ name: Perf comment
285
+
286
+ # Duel's report, as a new comment on the pull request, when Duel says to:
287
+ # always (post), or only when its last comment's status differs from the
288
+ # run's (change), so the latest comment is never stale. It never edits or
289
+ # deletes a comment. It runs once the Perf workflow has, with this
290
+ # repository's own workflow and a token that can write, so a pull request
291
+ # from a fork gets a comment too. It checks nothing out and runs none of
292
+ # the pull request's code, and of what the Duel job kept (where that code
293
+ # ran), it trusts only what that job could have said: the pull request's
294
+ # number, if it names the pull request the run compared, and the comment
295
+ # and the status, if they're ones Duel says. (The workflow must be on the
296
+ # default branch before it runs at all.)
297
+ on:
298
+ workflow_run:
299
+ workflows: ["Perf"]
300
+ types: [completed]
301
+
302
+ permissions: {}
303
+
304
+ jobs:
305
+ comment:
306
+ name: Comment
307
+ if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }}
308
+ runs-on: ubuntu-latest
309
+ permissions:
310
+ actions: read # to get the report from the Perf run
311
+ pull-requests: write # to comment on the pull request with the report
312
+ steps:
313
+ - name: Get the report
314
+ id: report
315
+ continue-on-error: true # there's none when the Duel job failed before comparing
316
+ uses: actions/download-artifact@v8
317
+ with:
318
+ name: duel-report
319
+ path: ${{ runner.temp }}/report
320
+ run-id: ${{ github.event.workflow_run.id }}
321
+ github-token: ${{ github.token }}
322
+ - name: Comment on the pull request
323
+ if: steps.report.outcome == 'success'
324
+ env:
325
+ GH_TOKEN: ${{ github.token }}
326
+ HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
327
+ run: |
328
+ said() { grep -xE "$1=($2)" "$RUNNER_TEMP/report/comment.env" | tail -n 1 | cut -d= -f2; }
329
+ PR=$(said pr '[0-9]+')
330
+ COMMENT=$(said comment 'post|change')
331
+ STATUS=$(said status 'passed|failed|neutral')
332
+ [ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
333
+ [ "$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq .head.sha)" = "$HEAD_SHA" ] || exit 0
334
+ marker="<!-- duel"
335
+ # The first line of the last comment Duel posted here: its marker,
336
+ # with its status (none in Duel 0.1.0's, which counts as another).
337
+ last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
338
+ --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
339
+ if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
340
+ { echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
341
+ gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
342
+ fi
343
+ ```
344
+
345
+ ## Comment on a pushed commit
346
+
347
+ With [`c.fail_on = :pull_requests`](configuration.md#fail-only-a-pull-request),
348
+ a regression in a push to `main` passes, and the workflow's Comment on
349
+ the commit job comments on the pushed commit with the report instead.
350
+ GitHub notifies the commit's author. It comments as `c.comment` says,
351
+ the way the Comment job does on a pull request, but its last comment is
352
+ the workflow's last one on any commit. So with the default, each push
353
+ that fails gets a comment, the first push after them that passes gets
354
+ one, and the passing pushes after it get none.
355
+
356
+ By default, a push gets no comment: Duel sets the `comment` output to
357
+ `none`, and a regression fails the push's check instead. The Comment on
358
+ the commit job checks nothing out either, and it alone gets the
359
+ `contents: write` permission that a commit comment takes.
360
+
361
+ ## Accept a regression with a label
362
+
363
+ With [`c.accept_label`](configuration.md#accept-a-regression-with-a-label),
364
+ a pull request with that label passes a regression. The Duel job's Find
365
+ the pull request's labels step reads the labels of the pull request,
366
+ or for a push, of the pull request that merged it, and passes them to
367
+ `duel` as a JSON array, in `DUEL_LABELS`. That takes the
368
+ `pull-requests: read` permission. Without the labels, such as when
369
+ GitHub's API fails, nothing is accepted.
370
+
371
+ ## Compare a push with the last commit that passed
372
+
373
+ By default, the workflow compares each push with the commit before it.
374
+ A series of pushes that are each a little slower each pass, and a push
375
+ that the workflow doesn't compare, such as one whose run was cancelled,
376
+ is never compared. With
377
+ [`c.push_base = :last_passed`](configuration.md#compare-a-push-with-the-last-commit-that-passed),
378
+ the workflow compares each push with the last commit that Duel passed,
379
+ so the slowdowns add up until they fail a push.
380
+
381
+ A branch, `duel/passed/main` for `main`, marks that commit. Each
382
+ branch that the workflow guards has its own (see
383
+ [Guard other branches](#guard-other-branches)). When a push passes, or has
384
+ nothing to compare, Duel sets the `passed` output to `true`, and the
385
+ Mark the commit passed job moves the branch to the push, or creates the
386
+ branch. That job checks nothing out either, and it gets the
387
+ `contents: write` permission that moving a branch takes. Until the
388
+ branch exists, and when the push's history doesn't have its commit,
389
+ Duel compares the push with the commit before it. For a project in a
390
+ subdirectory of its repository, the branch names the directory, such as
391
+ `duel/passed-gems-parser/main`.
392
+
393
+ After a regression, the branch stays where it is, so every push fails
394
+ (or with [`c.fail_on = :pull_requests`](#comment-on-a-pushed-commit),
395
+ is reported) until one is no slower. To accept a regression, move the
396
+ branch yourself:
397
+
398
+ ```sh
399
+ git push origin main:duel/passed/main
400
+ ```
401
+
402
+ If you set `c.push_base` back to `:before`, delete the branches too
403
+ (`git push origin --delete duel/passed/main`), because the step that
404
+ finds the baseline's Ruby uses one whenever it exists.
405
+
406
+ ## Guard other branches
407
+
408
+ The workflow compares the pushes to `main`. To guard another branch,
409
+ such as a stable branch that gets backports, add it to the workflow's
410
+ `push` branches:
411
+
412
+ ```yaml
413
+ push:
414
+ branches: [main, 2-x-stable]
415
+ ```
416
+
417
+ Duel compares a push to each branch with the commit before it, or with
418
+ `c.push_base = :last_passed`, with the last commit that Duel passed on
419
+ that branch, which a branch of its own marks: `duel/passed/2-x-stable`.
420
+ A pull request into the branch is compared with the branch, whatever
421
+ the workflow guards.
422
+
423
+ ## Run Duel on GitLab CI
424
+
425
+ `duel init --gitlab` writes GitLab CI jobs, in
426
+ `.gitlab/duel.gitlab-ci.yml`, instead of the GitHub Actions workflows.
427
+ Include them from your `.gitlab-ci.yml`:
428
+
429
+ ```yaml
430
+ include:
431
+ - local: .gitlab/duel.gitlab-ci.yml
432
+ ```
433
+
434
+ The Duel job compares a merge request with the branch that it targets,
435
+ a push to your default branch with the commit before it (or with
436
+ [`c.push_base = :last_passed`](#compare-a-push-with-the-last-commit-that-passed),
437
+ the last commit that Duel passed), and on a schedule, your last
438
+ release. It runs on the `ruby` image of the version that your
439
+ `.ruby-version` names, or else of your Ruby. On GitLab, a branch that
440
+ upgrades Ruby can't compare each version on its own Ruby, because the
441
+ image has only one. Duel reads GitLab's variables: a merge request's
442
+ target, and its labels, for
443
+ [`c.accept_label`](configuration.md#accept-a-regression-with-a-label).
444
+ A push's labels aren't available.
445
+
446
+ With [`c.fail_on = :pull_requests`](configuration.md#fail-only-a-pull-request),
447
+ a regression outside a merge request makes `duel` exit with status 3,
448
+ and the job lets that fail (`allow_failure: exit_codes: [3]`), so the
449
+ pipeline shows a warning, not a failure.
450
+
451
+ The Duel job writes what it decided to a dotenv report (`DUEL_OUTPUT`):
452
+ `DUEL_COMMENT`, `DUEL_STATUS`, and `DUEL_PASSED`. The jobs after it act
453
+ on them. They check nothing out and run none of the code. GitLab's own
454
+ job token can't comment, so they need project access tokens, in masked
455
+ CI/CD variables. Without them, the jobs don't run, and the Duel job's
456
+ log and its `duel.md` artifact have the report.
457
+
458
+ - `DUEL_COMMENT_TOKEN`, with the Reporter role and the `api` scope, and
459
+ not protected, so a merge request's pipeline has it, to comment on a
460
+ merge request. Every job in that pipeline has it, the Duel job too,
461
+ which runs the merge request's code, so give it no more than the
462
+ Reporter role.
463
+ - `DUEL_TOKEN`, with the Developer role and the `api` scope, and
464
+ protected, so only your protected branches' pipelines have it, to
465
+ comment on a pushed commit and to mark a push passed.
466
+
467
+ The comment on a pushed commit looks only at the comments on the commit
468
+ before it for its last status. A merge request from a fork runs its
469
+ pipeline in the fork, without your variables, so it gets no comment.
470
+
471
+ ```yaml
472
+ # .gitlab/duel.gitlab-ci.yml
473
+ # Duel on GitLab CI: https://github.com/sferik/duel-ruby/blob/main/docs/ci.md#run-duel-on-gitlab-ci
474
+ #
475
+ # Include it from .gitlab-ci.yml:
476
+ #
477
+ # include:
478
+ # - local: .gitlab/duel.gitlab-ci.yml
479
+ #
480
+ # It compares a merge request with the branch it targets, a push to the
481
+ # default branch with the commit before it (or as c.push_base in .duel.rb
482
+ # says, the last commit Duel passed), and on a schedule, the last release.
483
+ # The Duel job's log, and its duel.md artifact, have the report. Comments
484
+ # need project access tokens, in masked CI/CD variables:
485
+ #
486
+ # - DUEL_COMMENT_TOKEN (the Reporter role, with the api scope, and not
487
+ # protected), to comment on a merge request. Every job of a merge
488
+ # request's pipeline has it, the Duel job too, which runs the merge
489
+ # request's code, so the token can do no more than comment.
490
+ # - DUEL_TOKEN (the Developer role, with the api scope, and protected, so
491
+ # only the default branch's pipelines have it), to comment on a pushed
492
+ # commit (see c.fail_on), and to mark a push passed (see c.push_base).
493
+
494
+ duel:
495
+ image: ruby:3.4
496
+ stage: test
497
+ variables:
498
+ GIT_DEPTH: 0 # the baseline comes from history, tags included
499
+ DUEL_OUTPUT: duel.env # whether to comment, and mark a push passed
500
+ rules:
501
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
502
+ - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
503
+ variables:
504
+ DUEL_BASE: $CI_COMMIT_BEFORE_SHA
505
+ - if: $CI_PIPELINE_SOURCE == "schedule"
506
+ variables:
507
+ DUEL_BASE: last-release
508
+ # (Run by hand, it compares anyway: with nothing changed, the code
509
+ # with itself, which checks the setup.)
510
+ - if: $CI_PIPELINE_SOURCE == "web"
511
+ variables:
512
+ DUEL_FORCE: "1"
513
+ # A regression outside a merge request, where only a merge request
514
+ # fails (c.fail_on in .duel.rb), exits 3: a warning.
515
+ allow_failure:
516
+ exit_codes: [3]
517
+ before_script:
518
+ # The branch a merge request targets, the last commits Duel passed,
519
+ # and the releases.
520
+ - git fetch --quiet --tags origin "+refs/heads/duel/*:refs/remotes/origin/duel/*"
521
+ - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]; then git fetch --quiet origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"; fi
522
+ - bundle install
523
+ script:
524
+ - bundle exec duel --markdown duel.md
525
+ artifacts:
526
+ when: always
527
+ paths:
528
+ - duel.md
529
+ reports:
530
+ dotenv: duel.env
531
+ expire_in: 1 day
532
+
533
+ # The report, as a new comment on the merge request, when Duel says to:
534
+ # always (post), or only when its last comment's status differs from this
535
+ # run's (change), so the latest comment is never stale. It never edits or
536
+ # deletes a comment.
537
+ duel:comment:
538
+ image: alpine:3
539
+ stage: .post
540
+ needs: [duel]
541
+ rules:
542
+ - if: $DUEL_COMMENT_TOKEN && $CI_PIPELINE_SOURCE == "merge_request_event"
543
+ when: always
544
+ variables:
545
+ GIT_STRATEGY: none # it runs none of the code
546
+ script:
547
+ - apk add --no-cache --quiet curl jq
548
+ - |
549
+ case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
550
+ report=duel.md
551
+ [ -f "$report" ] || exit 0
552
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
553
+ token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
554
+ marker="<!-- duel"
555
+ me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
556
+ # The first line of the last comment Duel posted here: its marker,
557
+ # with its status.
558
+ last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
559
+ jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
560
+ if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
561
+ { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
562
+ curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
563
+ fi
564
+
565
+ # The report, as a new comment on a pushed commit, when Duel says to, as
566
+ # for a merge request: only where a regression in a push passes (see
567
+ # c.fail_on), so the comment says what the check doesn't. Its last comment
568
+ # is the commit before's.
569
+ duel:commit-comment:
570
+ image: alpine:3
571
+ stage: .post
572
+ needs: [duel]
573
+ rules:
574
+ - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
575
+ when: always
576
+ variables:
577
+ GIT_STRATEGY: none # it runs none of the code
578
+ script:
579
+ - apk add --no-cache --quiet curl jq
580
+ - |
581
+ case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
582
+ report=duel.md
583
+ [ -f "$report" ] || exit 0
584
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
585
+ token="PRIVATE-TOKEN: $DUEL_TOKEN"
586
+ marker="<!-- duel"
587
+ me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
588
+ last=$(curl -sf -H "$token" "$api/repository/commits/$CI_COMMIT_BEFORE_SHA/comments?per_page=100" |
589
+ jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .note | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | tail -n 1)
590
+ if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
591
+ { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{note: .}' |
592
+ curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/repository/commits/$CI_COMMIT_SHA/comments" > /dev/null
593
+ fi
594
+
595
+ # With c.push_base = :last_passed in .duel.rb, a push is compared with the
596
+ # last commit Duel passed, which a branch marks (duel/passed/main for
597
+ # main), so slowdowns too small for any one push to fail add up until they
598
+ # fail one. This job moves the branch to a push that passed (or makes it).
599
+ # To accept a regression, move the branch yourself (for main, git push
600
+ # origin main:duel/passed/main).
601
+ duel:passed:
602
+ image: alpine:3
603
+ stage: .post
604
+ needs: [duel]
605
+ rules:
606
+ - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
607
+ variables:
608
+ GIT_STRATEGY: none # it runs none of the code
609
+ script:
610
+ - apk add --no-cache --quiet curl
611
+ - |
612
+ [ "$DUEL_PASSED" = true ] || exit 0
613
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
614
+ token="PRIVATE-TOKEN: $DUEL_TOKEN"
615
+ branch="duel/passed/$CI_COMMIT_BRANCH"
616
+ curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
617
+ curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches"
618
+ ```
109
619
 
110
620
  ## Run each version on its own Ruby on CI
111
621
 
@@ -113,29 +623,77 @@ With [`c.ruby = :version_file`](configuration.md#run-each-version-on-its-own-rub
113
623
  each version runs on the Ruby that its own version file names. On CI, a
114
624
  setup step such as `ruby/setup-ruby` installs the Ruby that
115
625
  runs `duel` where no version manager looks. That's enough while both
116
- versions ask for that Ruby. A pull request that upgrades Ruby needs the
626
+ versions ask for that Ruby. A branch that upgrades Ruby needs the
117
627
  baseline's Ruby installed too, where Duel finds it. For example, for a
118
628
  project with a `.ruby-version`, in the
119
- [GitHub Actions workflow](#catch-slow-creep), install the main branch's
120
- Ruby with a second `ruby/setup-ruby` step, and link it into
121
- `~/.rubies`, where Duel looks for Rubies the way that chruby does.
122
- `duel init` adds these steps for a project that has a `.ruby-version`:
629
+ [GitHub Actions workflow](#catch-slow-creep), find the baseline that the
630
+ Compare step compares with, for each event: where `HEAD` left the main
631
+ branch for a pull request and a manual run, the commit before a push,
632
+ and the last release on a schedule. When its `.ruby-version` names
633
+ another Ruby than yours, install that Ruby with a second
634
+ `ruby/setup-ruby` step, with the bundle for it (`bundler-cache: true`),
635
+ and link it into `~/.rubies`, where Duel looks for Rubies the way that
636
+ chruby does. The baseline's workers load the same bundle as the
637
+ candidate's, and gems installed for one Ruby version don't load on
638
+ another, so without that bundle they can't load the code to compare. Without a baseline, such as for a
639
+ branch's first push, or before your first release, there's no Ruby to
640
+ install. The step finds the main branch and the last release the way
641
+ Duel does by default: if `.duel.rb` sets `c.main_branch` or
642
+ `c.release_tags`, set the step's `MAIN_BRANCH` or `RELEASE_TAGS` to
643
+ match. `duel init` adds these steps for a project that has a
644
+ `.ruby-version`:
123
645
 
124
646
  ```yaml
125
647
  # (Before the step that sets up the candidate's Ruby.)
648
+ # Each version runs on the Ruby its own .ruby-version names (see
649
+ # .duel.rb). When the baseline's names another Ruby than this code's,
650
+ # that one is installed too, and linked where Duel finds it (as
651
+ # chruby would), before the project's own Ruby, which runs Duel. The
652
+ # baseline is the Compare step's: where HEAD left the branch a pull
653
+ # request targets (or run by hand, the main branch), a merge queue's
654
+ # base, the commit before a push (or the last commit Duel passed, PASSED, if there is
655
+ # one: see the Mark the commit passed job), or on a schedule the last
656
+ # release, the newest tag before HEAD that RELEASE_TAGS matches. (If
657
+ # .duel.rb sets c.main_branch or c.release_tags, set MAIN_BRANCH or
658
+ # RELEASE_TAGS to match.)
126
659
  - name: Find the baseline's Ruby
127
660
  id: baseline
128
- if: github.event_name == 'pull_request'
129
- run: echo "ruby=$(git show "origin/$GITHUB_BASE_REF:.ruby-version")" >> "$GITHUB_OUTPUT"
661
+ env:
662
+ BEFORE: ${{ github.event.before }}
663
+ QUEUE_BASE: ${{ github.event.merge_group.base_sha }}
664
+ PASSED: origin/duel/passed/${{ github.ref_name }}
665
+ MAIN_BRANCH: origin/${{ github.base_ref || github.event.repository.default_branch }}
666
+ RELEASE_TAGS: "v*"
667
+ run: |
668
+ case "$GITHUB_EVENT_NAME" in
669
+ push)
670
+ base=$BEFORE
671
+ if git merge-base --is-ancestor "$PASSED" HEAD 2>/dev/null; then base=$PASSED; fi ;;
672
+ merge_group) base=$QUEUE_BASE ;;
673
+ schedule) base=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || base= ;;
674
+ *) base=$(git merge-base HEAD "$MAIN_BRANCH" 2>/dev/null) || base= ;;
675
+ esac
676
+ ruby_of() { git show "$1:./.ruby-version" 2>/dev/null | awk 'NF { print $1; exit }'; }
677
+ ruby=
678
+ # (None without a baseline: a branch's first push, or no release yet.)
679
+ if [ -n "$base" ] && git cat-file -e "$base^{commit}" 2>/dev/null; then
680
+ ruby=$(ruby_of "$base")
681
+ [ "$ruby" = "$(ruby_of HEAD)" ] && ruby=
682
+ fi
683
+ echo "ruby=$ruby" >> "$GITHUB_OUTPUT"
130
684
  - uses: ruby/setup-ruby@v1
131
685
  id: baseline-ruby
132
- if: github.event_name == 'pull_request'
686
+ if: steps.baseline.outputs.ruby != ''
133
687
  with:
134
688
  ruby-version: ${{ steps.baseline.outputs.ruby }}
689
+ # The bundle for this Ruby too, which the baseline's workers load.
690
+ # (Gems installed for one Ruby version don't load on another.)
691
+ bundler-cache: true
135
692
  - name: Link the baseline's Ruby where Duel finds it
136
- if: github.event_name == 'pull_request'
693
+ if: steps.baseline.outputs.ruby != ''
694
+ env:
695
+ PREFIX: ${{ steps.baseline-ruby.outputs.ruby-prefix }}
137
696
  run: |
138
- prefix="${{ steps.baseline-ruby.outputs.ruby-prefix }}"
139
697
  mkdir -p ~/.rubies
140
- ln -sfn "$prefix" ~/.rubies/ruby-"$("$prefix/bin/ruby" -e 'print RUBY_VERSION')"
698
+ ln -sfn "$PREFIX" ~/.rubies/ruby-"$("$PREFIX/bin/ruby" -e 'print RUBY_VERSION')"
141
699
  ```