duel 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/docs/ci.md CHANGED
@@ -12,9 +12,10 @@ example, a series of changes that are each 1.5% slower would each pass.
12
12
  To catch that kind of slow creep, also compare against your last release
13
13
  on a schedule. The slowdowns add up there until they're a regression.
14
14
 
15
- The following GitHub Actions workflow does both, and comments on a pull
16
- request when a benchmark fails. `duel init` writes it for you, as
17
- `.github/workflows/perf.yml`:
15
+ The following GitHub Actions workflow does both. `duel init` writes it
16
+ for you, as `.github/workflows/perf.yml`, with a second workflow that
17
+ comments on a pull request when a benchmark fails (see
18
+ [Comment on a pull request](#comment-on-a-pull-request)):
18
19
 
19
20
  ```yaml
20
21
  # .github/workflows/perf.yml
@@ -22,26 +23,39 @@ name: Perf
22
23
 
23
24
  on:
24
25
  pull_request:
26
+ merge_group: # (a merge queue's)
25
27
  push:
26
- branches: [main]
28
+ branches: [main] # (and any other branch to guard, a stable one, say)
27
29
  schedule:
28
30
  - cron: "0 4 * * *"
29
31
  workflow_dispatch:
30
32
 
33
+ # A pull request's newer push cancels its run, if that's still going, as
34
+ # only its newest commit's result counts. Every other run (a push's, say)
35
+ # runs to the end, so that each push is compared.
36
+ concurrency:
37
+ group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
38
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
39
+
31
40
  # The Duel job runs the branch's code (its benchmarks too), so it only
32
- # reads. Commenting is the Comment job's, which runs none of it.
41
+ # reads. Commenting on a commit is the Comment on the commit job's, which
42
+ # runs none of it, and commenting on a pull request is the Perf comment
43
+ # workflow's.
33
44
  permissions:
34
45
  contents: read
46
+ pull-requests: read # for the pull request's labels (see c.accept_label)
35
47
 
36
48
  jobs:
37
49
  duel:
38
50
  name: Duel
39
51
  runs-on: ubuntu-latest
40
- # Whether the Comment job comments (post, change or none), and the
41
- # run's status (passed, failed or neutral), from Duel.
52
+ # Whether to comment (post, change or none), the run's status (passed,
53
+ # failed or neutral), and whether to mark a push passed (true, or
54
+ # nothing), from Duel.
42
55
  outputs:
43
56
  comment: ${{ steps.compare.outputs.comment }}
44
57
  status: ${{ steps.compare.outputs.status }}
58
+ passed: ${{ steps.compare.outputs.passed }}
45
59
  steps:
46
60
  - uses: actions/checkout@v7
47
61
  with:
@@ -52,50 +66,80 @@ jobs:
52
66
  with:
53
67
  ruby-version: "3.4" # (or none, with a .ruby-version)
54
68
  bundler-cache: true
55
- # Against the main branch for a pull request, what was there before
56
- # for a push, and the last release on a schedule (slow creep). Duel
57
- # adds its report to the job's summary, writes it to duel.md for the
58
- # comment, and sets the step's outputs to say whether to comment, as
59
- # c.comment in .duel.rb says, whatever the outcome.
69
+ # The labels of the pull request (or for a push, of the one that
70
+ # merged it), for c.accept_label in .duel.rb, as a JSON array. They're
71
+ # read as the job runs, so after adding the label, run it again.
72
+ - name: Find the pull request's labels
73
+ id: labels
74
+ if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'
75
+ continue-on-error: true # (without them, nothing's accepted)
76
+ env:
77
+ GH_TOKEN: ${{ github.token }}
78
+ PR: ${{ github.event.pull_request.number }}
79
+ QUEUED: ${{ github.event.merge_group.head_ref }}
80
+ run: |
81
+ # (In a merge queue, the last pull request queued, which the queue's
82
+ # branch names: gh-readonly-queue/main/pr-123-<sha>.)
83
+ if [ -n "$QUEUED" ]; then
84
+ PR=${QUEUED##*/pr-}
85
+ PR=${PR%%-*}
86
+ fi
87
+ if [ -n "$PR" ]; then
88
+ labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
89
+ else
90
+ labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
91
+ fi
92
+ echo "labels=$labels" >> "$GITHUB_OUTPUT"
93
+ # Against the branch it targets for a pull request, the queue's base
94
+ # for a merge queue, what was there before for a push (or as
95
+ # c.push_base in .duel.rb says, the last commit Duel passed), and the
96
+ # last release on a schedule (slow creep). Duel adds its report to
97
+ # the job's summary, writes it to duel.md for the comment, and sets
98
+ # the step's outputs to say whether to comment, as c.comment in
99
+ # .duel.rb says, whatever the outcome.
60
100
  - name: Compare
61
101
  id: compare
62
102
  env:
63
- DUEL_BASE: ${{ github.event_name == 'schedule' && 'last-release' || github.event_name == 'push' && github.event.before || '' }}
103
+ DUEL_BASE: ${{ github.event_name == 'schedule' && 'last-release' || github.event_name == 'push' && github.event.before || github.event_name == 'merge_group' && github.event.merge_group.base_sha || '' }}
64
104
  # (Run by hand, it compares anyway: with nothing changed, the code
65
105
  # with itself, which checks the setup.)
66
106
  DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}
107
+ DUEL_LABELS: ${{ steps.labels.outputs.labels }}
108
+ run: bundle exec duel --markdown "$RUNNER_TEMP/duel/duel.md"
109
+ # For a pull request, what Duel said, for the Perf comment workflow,
110
+ # which can't see this job's outputs.
111
+ - name: Say whether to comment
112
+ if: ${{ !cancelled() && github.event_name == 'pull_request' }}
113
+ env:
114
+ PR: ${{ github.event.pull_request.number }}
115
+ COMMENT: ${{ steps.compare.outputs.comment }}
116
+ STATUS: ${{ steps.compare.outputs.status }}
67
117
  run: |
68
- # (A branch's first push, or one that rewrote history, has no
69
- # commit before it here to compare against.)
70
- if [ "$GITHUB_EVENT_NAME" = push ] && ! git cat-file -e "$DUEL_BASE^{commit}" 2>/dev/null; then
71
- echo "::notice::No commit before this push to compare against"
72
- exit 0
73
- fi
74
- bundle exec duel --markdown "$RUNNER_TEMP/duel.md"
75
- # The report, for the Comment job.
118
+ mkdir -p "$RUNNER_TEMP/duel"
119
+ printf 'pr=%s\ncomment=%s\nstatus=%s\n' "$PR" "$COMMENT" "$STATUS" > "$RUNNER_TEMP/duel/comment.env"
120
+ # The report, for the comments.
76
121
  - name: Keep the report
77
- if: ${{ !cancelled() && github.event_name == 'pull_request' }}
122
+ if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'push') }}
78
123
  uses: actions/upload-artifact@v7
79
124
  with:
80
125
  name: duel-report
81
- path: ${{ runner.temp }}/duel.md
126
+ path: ${{ runner.temp }}/duel/
82
127
  if-no-files-found: ignore # none when Duel didn't get to compare
83
128
  retention-days: 1
84
129
 
85
- # The report, as a new comment on the pull request, when Duel says to:
86
- # always (post), or only when its last comment's status differs from
87
- # this run's (change), so the latest comment is never stale. It never
88
- # edits or deletes a comment. This job checks nothing out and runs none
89
- # of the branch's code, so it alone can write. (A pull request from a
90
- # fork gets no comment, as its token can't write one: the Duel job's
91
- # summary has the report.)
92
- comment:
93
- name: Comment
130
+ # The report, as a new comment on a pushed commit, which notifies its
131
+ # author, when Duel says to, as for a pull request (see the Perf comment
132
+ # workflow). It does only where a regression in a push passes, with
133
+ # c.fail_on = :pull_requests in .duel.rb, so the comment says what the
134
+ # check doesn't. Its last comment is the last one on any commit. This
135
+ # job checks nothing out and runs none of the code.
136
+ commit-comment:
137
+ name: Comment on the commit
94
138
  needs: duel
95
- if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
139
+ if: ${{ !cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
96
140
  runs-on: ubuntu-latest
97
141
  permissions:
98
- pull-requests: write # to comment on the pull request with the report
142
+ contents: write # to comment on the commit with the report
99
143
  steps:
100
144
  - name: Get the report
101
145
  id: report
@@ -104,31 +148,146 @@ jobs:
104
148
  with:
105
149
  name: duel-report
106
150
  path: ${{ runner.temp }}/report
107
- - name: Comment on the pull request
151
+ - name: Comment on the commit
108
152
  if: steps.report.outcome == 'success'
109
153
  env:
110
154
  GH_TOKEN: ${{ github.token }}
111
- PR: ${{ github.event.pull_request.number }}
112
155
  COMMENT: ${{ needs.duel.outputs.comment }}
113
156
  STATUS: ${{ needs.duel.outputs.status }}
114
157
  run: |
115
158
  marker="<!-- duel"
116
- # The first line of the last comment Duel posted here: its marker,
117
- # with its status (none in Duel 0.1.0's, which counts as another).
118
- last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
119
- --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
159
+ last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/comments" \
160
+ --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\"))" | tail -n 1)
120
161
  if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
121
162
  { echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
122
- gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
163
+ gh api --method POST "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
123
164
  fi
165
+
166
+ # With c.push_base = :last_passed in .duel.rb, a push is compared with
167
+ # the last commit Duel passed, which a branch marks (each branch pushed
168
+ # has its own), so slowdowns too small for any one push to fail add up
169
+ # until they fail one. This job moves the branch to a push that passed
170
+ # (or makes it), but never back: where a newer push's run moved it
171
+ # first, it stays. It checks nothing out and runs none of the code. To
172
+ # accept a regression, move the branch yourself (for main, git push
173
+ # origin main:duel/passed/main).
174
+ passed:
175
+ name: Mark the commit passed
176
+ needs: duel
177
+ if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
178
+ runs-on: ubuntu-latest
179
+ permissions:
180
+ contents: write # to move the branch
181
+ steps:
182
+ - name: Move the branch to the commit
183
+ env:
184
+ GH_TOKEN: ${{ github.token }}
185
+ BRANCH: duel/passed/${{ github.ref_name }}
186
+ run: |
187
+ # (Git can't make a branch where another's name is a folder of its
188
+ # name: a branch named duel blocks duel/passed/main. If one does,
189
+ # this names it.)
190
+ blocked() {
191
+ prefix=$BRANCH
192
+ while [ "${prefix%/*}" != "$prefix" ]; do
193
+ prefix=${prefix%/*}
194
+ if gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$prefix" > /dev/null 2>&1; then
195
+ echo "::error::The branch $prefix keeps Duel from making $BRANCH, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
196
+ fi
197
+ done
198
+ exit 1
199
+ }
200
+ # (Forward: GitHub moves it only where it's behind the push.)
201
+ gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null 2>&1 && exit 0
202
+ at=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$BRANCH" --jq .object.sha 2>/dev/null) || at=
203
+ if [ -z "$at" ]; then
204
+ gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null || blocked
205
+ exit 0
206
+ fi
207
+ compared=$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$at" --jq .status)
208
+ case "$compared" in
209
+ # (A newer push's run moved it, or this one's did, run again.)
210
+ ahead | identical) ;;
211
+ # (History rewritten, a force push's, say: moved anyway.)
212
+ diverged | behind) gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" -F force=true > /dev/null ;;
213
+ *) echo "::error::Couldn't compare $BRANCH with this push: $compared" && exit 1 ;;
214
+ esac
215
+
216
+ # Duel's badge, for the README: green (perf guarded) or red (perf
217
+ # regressed), as the last push to the default branch that Duel compared
218
+ # left it. It's red where the Duel job failed or found a regression,
219
+ # even one that passed (with c.fail_on = :pull_requests in .duel.rb). A
220
+ # push with nothing to compare leaves it as it was, and so does a run
221
+ # for an older push than the badge's. This job writes the badge, alone,
222
+ # to a branch of its own, which the README shows:
223
+ #
224
+ # [![Duel](https://raw.githubusercontent.com/OWNER/REPO/duel/badge/badge.svg)](https://github.com/sferik/duel-ruby)
225
+ #
226
+ # It checks nothing out and runs none of the code.
227
+ badge:
228
+ name: Update the badge
229
+ needs: duel
230
+ if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
231
+ runs-on: ubuntu-latest
232
+ permissions:
233
+ contents: write # to move the badge's branch
234
+ steps:
235
+ - name: Update the badge
236
+ env:
237
+ GH_TOKEN: ${{ github.token }}
238
+ BRANCH: duel/badge
239
+ RESULT: ${{ needs.duel.result }}
240
+ STATUS: ${{ needs.duel.outputs.status }}
241
+ BADGES: https://raw.githubusercontent.com/sferik/duel-ruby/main/assets
242
+ run: |
243
+ # (Git can't make a branch where another's name is a folder of its
244
+ # name: a branch named duel blocks duel/passed/main. If one does,
245
+ # this names it.)
246
+ blocked() {
247
+ prefix=$BRANCH
248
+ while [ "${prefix%/*}" != "$prefix" ]; do
249
+ prefix=${prefix%/*}
250
+ if gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$prefix" > /dev/null 2>&1; then
251
+ echo "::error::The branch $prefix keeps Duel from making $BRANCH, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
252
+ fi
253
+ done
254
+ exit 1
255
+ }
256
+ # (Where the badge is a newer push's, that push's run has the say.
257
+ # Its commit names the push.)
258
+ last=$(gh api "repos/$GITHUB_REPOSITORY/commits/$BRANCH" --jq .commit.message 2>/dev/null | sed -n 's/^Duel.s badge, for \([0-9a-f]*\)$/\1/p') || true
259
+ if [ -n "$last" ] && [ "$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$last" --jq .status 2>/dev/null)" = ahead ]; then exit 0; fi
260
+ badge="badge.svg"
261
+ if [ "$RESULT" = failure ] || [ "$STATUS" = failed ]; then badge="badge-regressed.svg"; fi
262
+ curl -fsSL "$BADGES/$badge" -o "$RUNNER_TEMP/badge.svg"
263
+ tree=$(jq -n --rawfile svg "$RUNNER_TEMP/badge.svg" '{tree: [{path: "badge.svg", mode: "100644", type: "blob", content: $svg}]}' |
264
+ gh api --method POST "repos/$GITHUB_REPOSITORY/git/trees" --input - --jq .sha)
265
+ commit=$(gh api --method POST "repos/$GITHUB_REPOSITORY/git/commits" -f message="Duel's badge, for $GITHUB_SHA" -f tree="$tree" --jq .sha)
266
+ gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$commit" -F force=true > /dev/null 2>&1 ||
267
+ gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$commit" > /dev/null || blocked
124
268
  ```
125
269
 
270
+ A newer push to a pull request cancels the pull request's run, if it's
271
+ still going, because only the newest commit's result counts. Every other
272
+ run, such as a push's to `main`, runs to the end, so each push is
273
+ compared.
274
+
126
275
  The workflow compares the following:
127
276
 
128
- - A pull request, against the main branch.
129
- - A push to `main`, against the commit before the push. If that commit
130
- isn't in the history, such as after a force push or for a branch's
131
- first push, the workflow skips the comparison with a notice.
277
+ - A pull request, against the branch that it targets: usually `main`,
278
+ but a stable branch, or another pull request's branch in a stack of
279
+ them, too.
280
+ - A merge queue's group, against the queue's base, the commit that the
281
+ group merges into. With a merge queue, a required Duel check must run
282
+ on `merge_group` too, or the queue waits for it forever. A regression
283
+ fails a merge queue's run as it fails a pull request's, and the run
284
+ reads the labels of the last pull request queued, for
285
+ [`c.accept_label`](#accept-a-regression-with-a-label).
286
+ - A push to `main`, against the commit before the push, or with
287
+ [`c.push_base = :last_passed`](#compare-a-push-with-the-last-commit-that-passed),
288
+ the last commit that Duel passed. If that commit isn't in the history,
289
+ such as after a force push or for a branch's first push, Duel reports
290
+ that there's nothing to compare against, and passes.
132
291
  - On a schedule, the last release. Until you tag your first release,
133
292
  Duel reports that there's nothing to compare against, and passes.
134
293
  - When you run the workflow manually, your code against itself. This
@@ -175,20 +334,423 @@ GitHub Actions, the Compare step's `duel` sets two step outputs, which
175
334
  the Duel job passes on as its own: `comment`, and `status` (`passed`,
176
335
  `failed`, or `neutral`). `comment` is one of the following:
177
336
 
178
- | `comment` | What the Comment job does |
337
+ | `comment` | What the Perf comment workflow does |
179
338
  |---|---|
180
339
  | `post` | Posts a new comment. |
181
340
  | `change` | Posts a new comment only if the pull request has one from the workflow, and the last one's status isn't this run's. |
182
- | `none` | Nothing: the Comment job doesn't run. |
341
+ | `none` | Nothing. |
183
342
 
184
343
  The Duel job runs the pull request's code, its benchmarks included, so
185
- it only reads (`contents: read`), and it keeps the report as an
186
- artifact for a day. A job of its own, Comment, posts the report. It
187
- checks nothing out and runs none of the pull request's code, so it
188
- alone gets the `pull-requests: write` permission that commenting takes.
189
- When the Duel job fails before it compares, there's no report, and no
190
- comment. A pull request from a fork gets no comment either, because its
191
- token can't write one, but the Duel job's summary still has the report.
344
+ it only reads (`contents: read`). It keeps the report as an artifact
345
+ for a day, with a file, `comment.env`, that says what Duel decided:
346
+ the pull request's number, `comment`, and `status`. A second workflow,
347
+ `.github/workflows/perf-comment.yml`, posts the report. It runs when
348
+ the Perf workflow completes (`workflow_run`), with your repository's
349
+ own copy of the workflow and a token that can write, so a pull request
350
+ from a fork gets a comment too. It checks nothing out and runs none of
351
+ the pull request's code. The pull request's code ran in the Duel job,
352
+ and could have written anything into the artifact, so the workflow
353
+ trusts only what Duel could have said there: a number that names the
354
+ pull request whose head the run compared, and a `comment` and a
355
+ `status` that Duel uses. The report's Markdown is the comment's body,
356
+ which can only say something on the pull request that the pull request
357
+ could say itself. When the Duel job fails before it compares, there's
358
+ no report, and no comment. GitHub runs a `workflow_run` workflow only
359
+ from your default branch, so the pull request that adds the workflow
360
+ gets no comment.
361
+
362
+ ```yaml
363
+ # .github/workflows/perf-comment.yml
364
+ name: Perf comment
365
+
366
+ # Duel's report, as a new comment on the pull request, when Duel says to:
367
+ # always (post), or only when its last comment's status differs from the
368
+ # run's (change), so the latest comment is never stale. It never edits or
369
+ # deletes a comment. It runs once the Perf workflow has, with this
370
+ # repository's own workflow and a token that can write, so a pull request
371
+ # from a fork gets a comment too. It checks nothing out and runs none of
372
+ # the pull request's code, and of what the Duel job kept (where that code
373
+ # ran), it trusts only what that job could have said: the pull request's
374
+ # number, if it names the pull request the run compared, and the comment
375
+ # and the status, if they're ones Duel says. (The workflow must be on the
376
+ # default branch before it runs at all.)
377
+ on:
378
+ workflow_run:
379
+ workflows: ["Perf"]
380
+ types: [completed]
381
+
382
+ permissions: {}
383
+
384
+ jobs:
385
+ comment:
386
+ name: Comment
387
+ if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }}
388
+ runs-on: ubuntu-latest
389
+ permissions:
390
+ actions: read # to get the report from the Perf run
391
+ pull-requests: write # to comment on the pull request with the report
392
+ steps:
393
+ - name: Get the report
394
+ id: report
395
+ continue-on-error: true # there's none when the Duel job failed before comparing
396
+ uses: actions/download-artifact@v8
397
+ with:
398
+ name: duel-report
399
+ path: ${{ runner.temp }}/report
400
+ run-id: ${{ github.event.workflow_run.id }}
401
+ github-token: ${{ github.token }}
402
+ - name: Comment on the pull request
403
+ if: steps.report.outcome == 'success'
404
+ env:
405
+ GH_TOKEN: ${{ github.token }}
406
+ HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
407
+ run: |
408
+ said() { grep -xE "$1=($2)" "$RUNNER_TEMP/report/comment.env" | tail -n 1 | cut -d= -f2; }
409
+ PR=$(said pr '[0-9]+')
410
+ COMMENT=$(said comment 'post|change')
411
+ STATUS=$(said status 'passed|failed|neutral')
412
+ [ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
413
+ [ "$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq .head.sha)" = "$HEAD_SHA" ] || exit 0
414
+ marker="<!-- duel"
415
+ # The first line of the last comment Duel posted here: its marker,
416
+ # with its status (none in Duel 0.1.0's, which counts as another).
417
+ last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
418
+ --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
419
+ if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
420
+ { echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
421
+ gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
422
+ fi
423
+ ```
424
+
425
+ ## Comment on a pushed commit
426
+
427
+ With [`c.fail_on = :pull_requests`](configuration.md#fail-only-a-pull-request),
428
+ a regression in a push to `main` passes, and the workflow's Comment on
429
+ the commit job comments on the pushed commit with the report instead.
430
+ GitHub notifies the commit's author. It comments as `c.comment` says,
431
+ the way the Comment job does on a pull request, but its last comment is
432
+ the workflow's last one on any commit. So with the default, each push
433
+ that fails gets a comment, the first push after them that passes gets
434
+ one, and the passing pushes after it get none.
435
+
436
+ By default, a push gets no comment: Duel sets the `comment` output to
437
+ `none`, and a regression fails the push's check instead. The Comment on
438
+ the commit job checks nothing out either, and it alone gets the
439
+ `contents: write` permission that a commit comment takes.
440
+
441
+ ## Accept a regression with a label
442
+
443
+ With [`c.accept_label`](configuration.md#accept-a-regression-with-a-label),
444
+ a pull request with that label passes a regression. The Duel job's Find
445
+ the pull request's labels step reads the labels of the pull request,
446
+ or for a push, of the pull request that merged it, and passes them to
447
+ `duel` as a JSON array, in `DUEL_LABELS`. That takes the
448
+ `pull-requests: read` permission. Without the labels, such as when
449
+ GitHub's API fails, nothing is accepted.
450
+
451
+ ## Compare a push with the last commit that passed
452
+
453
+ By default, the workflow compares each push with the commit before it.
454
+ A series of pushes that are each a little slower each pass, and a push
455
+ that the workflow doesn't compare, such as one whose run was cancelled,
456
+ is never compared. With
457
+ [`c.push_base = :last_passed`](configuration.md#compare-a-push-with-the-last-commit-that-passed),
458
+ the workflow compares each push with the last commit that Duel passed,
459
+ so the slowdowns add up until they fail a push.
460
+
461
+ A branch, `duel/passed/main` for `main`, marks that commit. Each
462
+ branch that the workflow guards has its own (see
463
+ [Guard other branches](#guard-other-branches)). When a push passes, or has
464
+ nothing to compare, Duel sets the `passed` output to `true`, and the
465
+ Mark the commit passed job moves the branch to the push, or creates the
466
+ branch. It never moves the branch back: when two pushes run at once and
467
+ the older one finishes last, the branch stays at the newer push. After
468
+ a force push rewrites the branch's history, it moves the branch anyway.
469
+ That job checks nothing out either, and it gets the `contents: write`
470
+ permission that moving a branch takes. Until the branch exists, and
471
+ when the push's history doesn't have its commit, Duel compares the push
472
+ with the commit before it. For a project in a subdirectory of its
473
+ repository, the branch names the directory, such as
474
+ `duel/passed-gems-parser/main`.
475
+
476
+ Git can't have a branch named `duel` and a branch named
477
+ `duel/passed/main` at once, because a branch's name is a path, and
478
+ `duel` can't be both a file and a folder. If your repository has a
479
+ branch named `duel`, say the one that added Duel, the job can't make
480
+ its branch, and its error says so. Delete that branch
481
+ (`git push origin --delete duel`), or rename it.
482
+
483
+ After a regression, the branch stays where it is, so every push fails
484
+ (or with [`c.fail_on = :pull_requests`](#comment-on-a-pushed-commit),
485
+ is reported) until one is no slower. To accept a regression, move the
486
+ branch yourself:
487
+
488
+ ```sh
489
+ git push origin main:duel/passed/main
490
+ ```
491
+
492
+ If you set `c.push_base` back to `:before`, delete the branches too
493
+ (`git push origin --delete duel/passed/main`), because the step that
494
+ finds the baseline's Ruby uses one whenever it exists.
495
+
496
+ ## Show whether the last push passed
497
+
498
+ The Update the badge job keeps a badge that turns red when Duel fails.
499
+ After each push to your default branch, it writes `badge.svg` to a
500
+ branch of its own, `duel/badge`, so your README can show it:
501
+
502
+ ```markdown
503
+ [![Duel](https://raw.githubusercontent.com/OWNER/REPO/duel/badge/badge.svg)](https://github.com/sferik/duel-ruby)
504
+ ```
505
+
506
+ The badge says "perf guarded", in green, when Duel passed the push. It
507
+ says "perf regressed", in red, when the Duel job failed or found a
508
+ regression, even one that passed, with
509
+ [`c.fail_on = :pull_requests`](#comment-on-a-pushed-commit), or that a
510
+ label accepted. A push with nothing to compare leaves the badge as it
511
+ was, and so does a run that finishes after a newer push's, so the
512
+ badge is always the newest push's that Duel compared. The branch holds
513
+ one commit, with the badge alone, whose message names that push. The
514
+ job checks nothing out, and it gets the `contents: write` permission
515
+ that moving a branch takes. For a project in a subdirectory of its
516
+ repository, the branch names the directory, such as
517
+ `duel/badge-gems-parser`.
518
+
519
+ Until the first push to your default branch after you add the
520
+ workflow, the branch doesn't exist, and the badge doesn't show.
521
+
522
+ ## Guard other branches
523
+
524
+ The workflow compares the pushes to `main`. To guard another branch,
525
+ such as a stable branch that gets backports, add it to the workflow's
526
+ `push` branches:
527
+
528
+ ```yaml
529
+ push:
530
+ branches: [main, 2-x-stable]
531
+ ```
532
+
533
+ Duel compares a push to each branch with the commit before it, or with
534
+ `c.push_base = :last_passed`, with the last commit that Duel passed on
535
+ that branch, which a branch of its own marks: `duel/passed/2-x-stable`.
536
+ A pull request into the branch is compared with the branch, whatever
537
+ the workflow guards.
538
+
539
+ ## Run Duel on GitLab CI
540
+
541
+ `duel init --gitlab` writes GitLab CI jobs, in
542
+ `.gitlab/duel.gitlab-ci.yml`, instead of the GitHub Actions workflows.
543
+ Include them from your `.gitlab-ci.yml`:
544
+
545
+ ```yaml
546
+ include:
547
+ - local: .gitlab/duel.gitlab-ci.yml
548
+ ```
549
+
550
+ The Duel job compares a merge request with the branch that it targets,
551
+ a push to your default branch with the commit before it (or with
552
+ [`c.push_base = :last_passed`](#compare-a-push-with-the-last-commit-that-passed),
553
+ the last commit that Duel passed), and on a schedule, your last
554
+ release. It runs on the `ruby` image of the version that your
555
+ `.ruby-version` names, or else of your Ruby. On GitLab, a branch that
556
+ upgrades Ruby can't compare each version on its own Ruby, because the
557
+ image has only one. Duel reads GitLab's variables: a merge request's
558
+ target, and its labels, for
559
+ [`c.accept_label`](configuration.md#accept-a-regression-with-a-label).
560
+ A push's labels aren't available.
561
+
562
+ With [`c.fail_on = :pull_requests`](configuration.md#fail-only-a-pull-request),
563
+ a regression outside a merge request makes `duel` exit with status 3,
564
+ and the job lets that fail (`allow_failure: exit_codes: [3]`), so the
565
+ pipeline shows a warning, not a failure.
566
+
567
+ The Duel job writes what it decided to a dotenv report (`DUEL_OUTPUT`):
568
+ `DUEL_COMMENT`, `DUEL_STATUS`, and `DUEL_PASSED`. The jobs after it act
569
+ on them. They check nothing out and run none of the code. GitLab's own
570
+ job token can't comment, so they need project access tokens, in masked
571
+ CI/CD variables. Without them, the jobs don't run, and the Duel job's
572
+ log and its `duel.md` artifact have the report.
573
+
574
+ - `DUEL_COMMENT_TOKEN`, with the Reporter role and the `api` scope, and
575
+ not protected, so a merge request's pipeline has it, to comment on a
576
+ merge request. Every job in that pipeline has it, the Duel job too,
577
+ which runs the merge request's code, so give it no more than the
578
+ Reporter role.
579
+ - `DUEL_TOKEN`, with the Developer role and the `api` scope, and
580
+ protected, so only your protected branches' pipelines have it, to
581
+ comment on a pushed commit and to mark a push passed.
582
+
583
+ The comment on a pushed commit looks only at the comments on the commit
584
+ before it for its last status. A merge request from a fork runs its
585
+ pipeline in the fork, without your variables, so it gets no comment.
586
+
587
+ ```yaml
588
+ # .gitlab/duel.gitlab-ci.yml
589
+ # Duel on GitLab CI: https://github.com/sferik/duel-ruby/blob/main/docs/ci.md#run-duel-on-gitlab-ci
590
+ #
591
+ # Include it from .gitlab-ci.yml:
592
+ #
593
+ # include:
594
+ # - local: .gitlab/duel.gitlab-ci.yml
595
+ #
596
+ # It compares a merge request with the branch it targets, a push to the
597
+ # default branch with the commit before it (or as c.push_base in .duel.rb
598
+ # says, the last commit Duel passed), and on a schedule, the last release.
599
+ # The Duel job's log, and its duel.md artifact, have the report. Comments
600
+ # need project access tokens, in masked CI/CD variables:
601
+ #
602
+ # - DUEL_COMMENT_TOKEN (the Reporter role, with the api scope, and not
603
+ # protected), to comment on a merge request. Every job of a merge
604
+ # request's pipeline has it, the Duel job too, which runs the merge
605
+ # request's code, so the token can do no more than comment.
606
+ # - DUEL_TOKEN (the Developer role, with the api scope, and protected, so
607
+ # only the default branch's pipelines have it), to comment on a pushed
608
+ # commit (see c.fail_on), and to mark a push passed (see c.push_base).
609
+
610
+ duel:
611
+ image: ruby:3.4
612
+ stage: test
613
+ variables:
614
+ GIT_DEPTH: 0 # the baseline comes from history, tags included
615
+ DUEL_OUTPUT: duel.env # whether to comment, and mark a push passed
616
+ rules:
617
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
618
+ - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
619
+ variables:
620
+ DUEL_BASE: $CI_COMMIT_BEFORE_SHA
621
+ - if: $CI_PIPELINE_SOURCE == "schedule"
622
+ variables:
623
+ DUEL_BASE: last-release
624
+ # (Run by hand, it compares anyway: with nothing changed, the code
625
+ # with itself, which checks the setup.)
626
+ - if: $CI_PIPELINE_SOURCE == "web"
627
+ variables:
628
+ DUEL_FORCE: "1"
629
+ # A regression outside a merge request, where only a merge request
630
+ # fails (c.fail_on in .duel.rb), exits 3: a warning.
631
+ allow_failure:
632
+ exit_codes: [3]
633
+ before_script:
634
+ # The branch a merge request targets, the last commits Duel passed,
635
+ # and the releases.
636
+ - git fetch --quiet --tags origin "+refs/heads/duel/*:refs/remotes/origin/duel/*"
637
+ - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]; then git fetch --quiet origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"; fi
638
+ - bundle install
639
+ script:
640
+ - bundle exec duel --markdown duel.md
641
+ artifacts:
642
+ when: always
643
+ paths:
644
+ - duel.md
645
+ reports:
646
+ dotenv: duel.env
647
+ expire_in: 1 day
648
+
649
+ # The report, as a new comment on the merge request, when Duel says to:
650
+ # always (post), or only when its last comment's status differs from this
651
+ # run's (change), so the latest comment is never stale. It never edits or
652
+ # deletes a comment.
653
+ duel:comment:
654
+ image: alpine:3
655
+ stage: .post
656
+ needs: [duel]
657
+ rules:
658
+ - if: $DUEL_COMMENT_TOKEN && $CI_PIPELINE_SOURCE == "merge_request_event"
659
+ when: always
660
+ variables:
661
+ GIT_STRATEGY: none # it runs none of the code
662
+ script:
663
+ - apk add --no-cache --quiet curl jq
664
+ - |
665
+ case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
666
+ report=duel.md
667
+ [ -f "$report" ] || exit 0
668
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
669
+ token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
670
+ marker="<!-- duel"
671
+ me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
672
+ # The first line of the last comment Duel posted here: its marker,
673
+ # with its status.
674
+ last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
675
+ jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
676
+ if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
677
+ { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
678
+ curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
679
+ fi
680
+
681
+ # The report, as a new comment on a pushed commit, when Duel says to, as
682
+ # for a merge request: only where a regression in a push passes (see
683
+ # c.fail_on), so the comment says what the check doesn't. Its last comment
684
+ # is the commit before's.
685
+ duel:commit-comment:
686
+ image: alpine:3
687
+ stage: .post
688
+ needs: [duel]
689
+ rules:
690
+ - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
691
+ when: always
692
+ variables:
693
+ GIT_STRATEGY: none # it runs none of the code
694
+ script:
695
+ - apk add --no-cache --quiet curl jq
696
+ - |
697
+ case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
698
+ report=duel.md
699
+ [ -f "$report" ] || exit 0
700
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
701
+ token="PRIVATE-TOKEN: $DUEL_TOKEN"
702
+ marker="<!-- duel"
703
+ me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
704
+ last=$(curl -sf -H "$token" "$api/repository/commits/$CI_COMMIT_BEFORE_SHA/comments?per_page=100" |
705
+ jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .note | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | tail -n 1)
706
+ if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
707
+ { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{note: .}' |
708
+ curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/repository/commits/$CI_COMMIT_SHA/comments" > /dev/null
709
+ fi
710
+
711
+ # With c.push_base = :last_passed in .duel.rb, a push is compared with the
712
+ # last commit Duel passed, which a branch marks (duel/passed/main for
713
+ # main), so slowdowns too small for any one push to fail add up until they
714
+ # fail one. This job moves the branch to a push that passed (or makes it),
715
+ # but never back: where a newer push's job moved it first, it stays. To
716
+ # accept a regression, move the branch yourself (for main, git push
717
+ # origin main:duel/passed/main).
718
+ duel:passed:
719
+ image: alpine:3
720
+ stage: .post
721
+ needs: [duel]
722
+ rules:
723
+ - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
724
+ variables:
725
+ GIT_STRATEGY: none # it runs none of the code
726
+ script:
727
+ - apk add --no-cache --quiet curl jq
728
+ - |
729
+ [ "$DUEL_PASSED" = true ] || exit 0
730
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
731
+ token="PRIVATE-TOKEN: $DUEL_TOKEN"
732
+ branch="duel/passed/$CI_COMMIT_BRANCH"
733
+ at=$(curl -sf -H "$token" "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')" | jq -r '.commit.id // empty') || at=
734
+ if [ -n "$at" ]; then
735
+ base=$(curl -sf -H "$token" "$api/repository/merge_base?refs%5B%5D=$CI_COMMIT_SHA&refs%5B%5D=$at" | jq -r '.id // empty') || base=
736
+ [ -n "$base" ] || { echo "Couldn't compare $branch with this push" && exit 1; }
737
+ # (A newer push's job moved it, or this one's did, run again.)
738
+ [ "$base" = "$CI_COMMIT_SHA" ] && exit 0
739
+ fi
740
+ curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
741
+ curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches" && exit 0
742
+ # (Git can't make a branch where another's name is a folder of its
743
+ # name: a branch named duel blocks duel/passed/main. If one does,
744
+ # this names it.)
745
+ prefix=$branch
746
+ while [ "${prefix%/*}" != "$prefix" ]; do
747
+ prefix=${prefix%/*}
748
+ if curl -sf -o /dev/null -H "$token" "$api/repository/branches/$(echo "$prefix" | sed 's|/|%2F|g')"; then
749
+ echo "The branch $prefix keeps Duel from making $branch, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
750
+ fi
751
+ done
752
+ exit 1
753
+ ```
192
754
 
193
755
  ## Run each version on its own Ruby on CI
194
756
 
@@ -222,20 +784,27 @@ match. `duel init` adds these steps for a project that has a
222
784
  # .duel.rb). When the baseline's names another Ruby than this code's,
223
785
  # that one is installed too, and linked where Duel finds it (as
224
786
  # chruby would), before the project's own Ruby, which runs Duel. The
225
- # baseline is the Compare step's: where HEAD left the main branch
226
- # (for a pull request, and run by hand), the commit before a push, or
227
- # on a schedule the last release, the newest tag before HEAD that
228
- # RELEASE_TAGS matches. (If .duel.rb sets c.main_branch or
229
- # c.release_tags, set MAIN_BRANCH or RELEASE_TAGS to match.)
787
+ # baseline is the Compare step's: where HEAD left the branch a pull
788
+ # request targets (or run by hand, the main branch), a merge queue's
789
+ # base, the commit before a push (or the last commit Duel passed, PASSED, if there is
790
+ # one: see the Mark the commit passed job), or on a schedule the last
791
+ # release, the newest tag before HEAD that RELEASE_TAGS matches. (If
792
+ # .duel.rb sets c.main_branch or c.release_tags, set MAIN_BRANCH or
793
+ # RELEASE_TAGS to match.)
230
794
  - name: Find the baseline's Ruby
231
795
  id: baseline
232
796
  env:
233
797
  BEFORE: ${{ github.event.before }}
234
- MAIN_BRANCH: origin/${{ github.event.repository.default_branch }}
798
+ QUEUE_BASE: ${{ github.event.merge_group.base_sha }}
799
+ PASSED: origin/duel/passed/${{ github.ref_name }}
800
+ MAIN_BRANCH: origin/${{ github.base_ref || github.event.repository.default_branch }}
235
801
  RELEASE_TAGS: "v*"
236
802
  run: |
237
803
  case "$GITHUB_EVENT_NAME" in
238
- push) base=$BEFORE ;;
804
+ push)
805
+ base=$BEFORE
806
+ if git merge-base --is-ancestor "$PASSED" HEAD 2>/dev/null; then base=$PASSED; fi ;;
807
+ merge_group) base=$QUEUE_BASE ;;
239
808
  schedule) base=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || base= ;;
240
809
  *) base=$(git merge-base HEAD "$MAIN_BRANCH" 2>/dev/null) || base= ;;
241
810
  esac