duel 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/duel/stats.rb CHANGED
@@ -43,6 +43,82 @@ module Duel
43
43
  [sorted.fetch(lower - 1), sorted.fetch(upper - 1)]
44
44
  end
45
45
 
46
+ # The 1-based rank r whose r-th smallest and r-th largest cross
47
+ # ratio (see #cross_ratios) bound a +confidence+ interval for the
48
+ # ratio between two versions, from +m+ samples of one and +n+ of the
49
+ # other, or nil when they're too few for one (fewer than 5 each, at
50
+ # 99%). It's distribution-free, as long as each version's samples
51
+ # are independent and alike: the true ratio lies below the r-th
52
+ # smallest with probability P(U <= r - 1), for the Mann-Whitney U
53
+ # statistic (see #u_lower_tail), whatever their distribution. So a
54
+ # sample can be a worker process's median, and then the interval
55
+ # accounts for how a process's luck (where things landed in its
56
+ # memory) differs from another's, as an interval over pairs from a
57
+ # few processes can't.
58
+ def shift_rank(m, n, confidence)
59
+ tail = (1 - confidence) / 2
60
+ r = u_lower_tail(m, n).index { |p| p > tail }
61
+ # (None only where the tail is a half or more, at no confidence.)
62
+ r unless r.nil? || r.zero?
63
+ end
64
+
65
+ # P(U <= u) for u from 0 to m * n / 2, where U is the Mann-Whitney
66
+ # statistic of samples of +m+ and +n+ from the same distribution:
67
+ # the pairs (one of each) in which the first's is the larger. Each
68
+ # P(U = u) is found from samples of one fewer (the largest of all is
69
+ # one of the first's, beating all +n+, as often as m / (m + n), or
70
+ # one of the second's), in probabilities, which stay in range where
71
+ # the counts of orderings would grow huge.
72
+ def u_lower_tail(m, n)
73
+ cap = m * n / 2
74
+ sum = 0.0
75
+ u_probabilities(m, n, cap, {}).map { |p| sum += p }
76
+ end
77
+
78
+ # P(U = u) for u from 0 to +cap+, for samples of +i+ and +j+ (see
79
+ # #u_lower_tail), each found once, in +found+.
80
+ def u_probabilities(i, j, cap, found)
81
+ found[[i, j]] ||= if i.zero? || j.zero?
82
+ Array.new(cap + 1) { |u| u.zero? ? 1.0 : 0.0 } # (no samples of one: U is 0)
83
+ else
84
+ first = u_probabilities(i - 1, j, cap, found)
85
+ second = u_probabilities(i, j - 1, cap, found)
86
+ Array.new(cap + 1) do |u|
87
+ from_first = (u >= j) ? first.fetch(u - j) * i : 0.0
88
+ (from_first + (second.fetch(u) * j)) / (i + j)
89
+ end
90
+ end
91
+ end
92
+
93
+ # [low, high] bounds of the +confidence+ interval for the ratio of
94
+ # +candidates+ (a version's samples) to +bases+ (the other's), or nil
95
+ # (see #shift_rank).
96
+ def shift_interval(bases, candidates, confidence)
97
+ r = shift_rank(bases.size, candidates.size, confidence)
98
+ return unless r
99
+
100
+ ratios = cross_ratios(bases, candidates)
101
+ [ratios.fetch(r - 1), ratios.fetch(-r)]
102
+ end
103
+
104
+ # The ratio of +candidates+ to +bases+ the interval is for: the
105
+ # median of the cross ratios (the Hodges-Lehmann estimate).
106
+ def shift(bases, candidates) = median(cross_ratios(bases, candidates))
107
+
108
+ # [low, high] bounds of the +confidence+ interval for how much more
109
+ # +candidates+ (a version's samples) are than +bases+ (the other's),
110
+ # as #shift_interval's are for how many times as much, or nil.
111
+ def difference_interval(bases, candidates, confidence)
112
+ r = shift_rank(bases.size, candidates.size, confidence)
113
+ return unless r
114
+
115
+ differences = candidates.flat_map { |c| bases.map { |b| c - b } }.sort
116
+ [differences.fetch(r - 1), differences.fetch(-r)]
117
+ end
118
+
119
+ # Each of +candidates+ over each of +bases+, sorted.
120
+ def cross_ratios(bases, candidates) = candidates.flat_map { |c| bases.map { |b| c.fdiv(b) } }.sort
121
+
46
122
  def median(samples)
47
123
  sorted = samples.sort
48
124
  mid = sorted.size / 2
data/lib/duel/summary.rb CHANGED
@@ -168,7 +168,7 @@ module Duel
168
168
  if timed then format("%s %+.1f%%", metric, (timed.fetch("median") - 1) * 100)
169
169
  elsif counted
170
170
  n = counted.fetch("n")
171
- format("%s %.4g → %.4g", metric, counted.fetch("baseline").fdiv(n), counted.fetch("candidate").fdiv(n))
171
+ "#{metric} #{Runner.number(counted.fetch("baseline").fdiv(n))} → #{Runner.number(counted.fetch("candidate").fdiv(n))}"
172
172
  else
173
173
  # (Peak memory, which grew too much.)
174
174
  memory = result.memory #: Hash[String, Integer]
@@ -247,7 +247,7 @@ module Duel
247
247
  # that grew too much.
248
248
  def counts(result)
249
249
  counted = Summary.changed_counts(result).map do |metric, (base, candidate)|
250
- format("%s %.4g → %.4g", metric, base, candidate)
250
+ "#{metric} #{Runner.number(base)} → #{Runner.number(candidate)}"
251
251
  end
252
252
  if result.more_memory?
253
253
  memory = result.memory #: Hash[String, Integer]
@@ -134,6 +134,9 @@
134
134
  image: alpine:3
135
135
  stage: .post
136
136
  needs: [<%= gitlab_job %>]
137
+ # One at a time, as each deletes the branch and makes it again: two at
138
+ # once could both delete it, and the second to make it would fail.
139
+ resource_group: <%= ["duel", working_directory, "passed"].compact.join("/") %>
137
140
  rules:
138
141
  - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
139
142
  variables:
@@ -40,14 +40,18 @@ Duel.configure do |c|
40
40
 
41
41
  # Which runs a regression fails: :every_run, or :pull_requests, for a
42
42
  # project where people push to main directly (on GitHub Actions, another
43
- # run warns and passes, and a pushed commit gets the comment instead).
43
+ # run warns and passes, and a pushed commit gets the comment instead,
44
+ # from a workflow job written only where this is set: see docs/ci.md).
44
45
  # c.fail_on = :every_run
45
46
 
46
- # A label that accepts a regression in a pull request (nil for none).
47
- # c.accept_label = "duel: accept"
47
+ # A label that accepts a regression in a pull request, for a change
48
+ # that's slower on purpose (nil for none).
49
+ c.accept_label = "duel: accept"
48
50
 
49
51
  # What the workflow compares a push with: :before (the commit before it),
50
52
  # or :last_passed (the last commit Duel passed, which a branch,
51
53
  # duel/passed/main, marks), so small slowdowns add up until they fail.
54
+ # (A workflow job moves the branch, written only where this is set: see
55
+ # docs/ci.md.)
52
56
  # c.push_base = :before
53
57
  end
@@ -9,12 +9,28 @@ name: Perf comment<%= " (#{working_directory})" if working_directory %>
9
9
  # the pull request's code, and of what the Duel job kept (where that code
10
10
  # ran), it trusts only what that job could have said: the pull request's
11
11
  # number, if it names the pull request the run compared, and the comment
12
- # and the status, if they're ones Duel says. (The workflow must be on the
13
- # default branch before it runs at all.)
12
+ # and the status, if they're ones Duel says. A fork's report, which the
13
+ # fork's code wrote, is shown as text.
14
+ <% if accept_label -%>
15
+ #
16
+ # And when the label that accepts a regression (c.accept_label in
17
+ # .duel.rb) is added to a pull request, or taken off, it runs the Duel
18
+ # job again, which reads the labels as it runs: a run that failed, once
19
+ # the label's added, or one that passed, once it's taken off.
20
+ <% end -%>
21
+ #
22
+ # (The workflow must be on the default branch before it runs at all.)
14
23
  on:
15
24
  workflow_run:
16
25
  workflows: ["Perf<%= " (#{working_directory})" if working_directory %>"]
17
26
  types: [completed]
27
+ <% if accept_label -%>
28
+ # (This repository's own workflow, as for the comment, with a token
29
+ # that can run a workflow again: it checks nothing out, and runs none
30
+ # of the pull request's code.)
31
+ pull_request_target:
32
+ types: [labeled, unlabeled]
33
+ <% end -%>
18
34
 
19
35
  permissions: {}
20
36
 
@@ -47,13 +63,61 @@ jobs:
47
63
  COMMENT=$(said comment 'post|change')
48
64
  STATUS=$(said status 'passed|failed|neutral')
49
65
  [ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
50
- [ "$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq .head.sha)" = "$HEAD_SHA" ] || exit 0
66
+ # (The pull request's head commit, and the repository it's from.)
67
+ head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq '"\(.head.sha) \(.head.repo.full_name)"')
68
+ [ "${head%% *}" = "$HEAD_SHA" ] || exit 0
69
+ # The report, as Markdown for a branch of this repository (whose
70
+ # author can push here anyway), but for a fork's, as text: its run
71
+ # ran the fork's code, which could have written anything in it. In
72
+ # a code block, fenced with more backticks than any run of them in
73
+ # the report, it can't mention anyone, link anywhere or show an
74
+ # image. It's cut short where a comment couldn't hold it, and loses
75
+ # any NUL bytes, which GitHub wouldn't take, and grep reads it as
76
+ # text whatever's in it, so it finds every run of backticks.
77
+ report() {
78
+ if [ "${head#* }" = "$GITHUB_REPOSITORY" ]; then cat "$RUNNER_TEMP/report/duel.md" && return; fi
79
+ text="$RUNNER_TEMP/report.txt"
80
+ LC_ALL=C tr -d '\000' < "$RUNNER_TEMP/report/duel.md" | head -c 60000 > "$text"
81
+ fence=$(LC_ALL=C grep -ao '``*' "$text" | awk '{ if (length > n) n = length } END { n = (n < 3) ? 3 : n + 1; while (n-- > 0) printf "`" }')
82
+ printf 'From a fork, so shown as text:\n\n%s\n' "$fence"
83
+ cat "$text"
84
+ printf '\n%s\n' "$fence"
85
+ }
51
86
  marker="<!-- <%= marker %>"
52
87
  # The first line of the last comment Duel posted here: its marker,
53
88
  # with its status (none in Duel 0.1.0's, which counts as another).
54
89
  last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
55
90
  --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
56
91
  if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
57
- { echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
92
+ { echo "$marker status=$STATUS -->"; report; } > "$RUNNER_TEMP/comment.md"
58
93
  gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
59
94
  fi
95
+ <% if accept_label -%>
96
+
97
+ again:
98
+ name: Run Duel again
99
+ if: >-
100
+ github.event_name == 'pull_request_target' && github.event.label.name == '<%= accept_label.gsub("'", "''") %>'
101
+ runs-on: ubuntu-latest
102
+ permissions:
103
+ actions: write # to run the Perf workflow's jobs again
104
+ steps:
105
+ - name: Run Duel again
106
+ env:
107
+ GH_TOKEN: ${{ github.token }}
108
+ GH_REPO: ${{ github.repository }}
109
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
110
+ ACTION: ${{ github.event.action }}
111
+ run: |
112
+ # The newest Perf run of the pull request's head commit, once it's
113
+ # done (one that's running may have read the labels already).
114
+ id=$(gh run list --workflow "Perf<%= " (#{working_directory})" if working_directory %>" --event pull_request --commit "$HEAD_SHA" --limit 1 \
115
+ --json databaseId --jq '.[0].databaseId // empty')
116
+ [ -n "$id" ] || exit 0
117
+ gh run watch "$id" --interval 30 > /dev/null || true
118
+ conclusion=$(gh run view "$id" --json conclusion --jq .conclusion)
119
+ case "$ACTION:$conclusion" in
120
+ labeled:failure) gh run rerun "$id" --failed ;;
121
+ unlabeled:success) gh run rerun "$id" ;;
122
+ esac
123
+ <% end -%>
@@ -47,6 +47,51 @@ jobs:
47
47
  with:
48
48
  fetch-depth: 0 # the baseline comes from history, tags included
49
49
  persist-credentials: false
50
+ # The labels of the pull request (or for a push, of the one that
51
+ # merged it), for c.accept_label in .duel.rb, as a JSON array. They're
52
+ # read as the job runs, and adding the label (or taking it off) runs
53
+ # the job again: see the Perf comment workflow.
54
+ <% if accept_label -%>
55
+ # On a schedule, the last commit since the last release that merged a
56
+ # pull request with the label (ACCEPT_LABEL, as .duel.rb names it):
57
+ # what that slowed down was accepted, so it's compared with instead.
58
+ <% end -%>
59
+ - name: Find the pull request's labels
60
+ id: labels
61
+ if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %> || github.event_name == 'schedule'<% end %>
62
+ continue-on-error: true # (without them, nothing's accepted)
63
+ env:
64
+ GH_TOKEN: ${{ github.token }}
65
+ PR: ${{ github.event.pull_request.number }}
66
+ QUEUED: ${{ github.event.merge_group.head_ref }}
67
+ <% if accept_label -%>
68
+ ACCEPT_LABEL: '<%= accept_label.gsub("'", "''") %>'
69
+ RELEASE_TAGS: "<%= release_tags %>"
70
+ <% end -%>
71
+ run: |
72
+ <% if accept_label -%>
73
+ if [ "$GITHUB_EVENT_NAME" = schedule ]; then
74
+ tag=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || exit 0
75
+ since=$(TZ=UTC git log -1 --date=format-local:%Y-%m-%dT%H:%M:%SZ --format=%cd "$tag")
76
+ merged=$(gh pr list --repo "$GITHUB_REPOSITORY" --state merged --label "$ACCEPT_LABEL" --search "merged:>=$since" --limit 1000 \
77
+ --json mergeCommit --jq '.[].mergeCommit.oid')
78
+ accepted=$(git rev-list "$tag..HEAD" | grep -Fx -m 1 -f <(echo "$merged")) || accepted=
79
+ echo "accepted=$accepted" >> "$GITHUB_OUTPUT"
80
+ exit 0
81
+ fi
82
+ <% end -%>
83
+ # (In a merge queue, the last pull request queued, which the queue's
84
+ # branch names: gh-readonly-queue/main/pr-123-<sha>.)
85
+ if [ -n "$QUEUED" ]; then
86
+ PR=${QUEUED##*/pr-}
87
+ PR=${PR%%-*}
88
+ fi
89
+ if [ -n "$PR" ]; then
90
+ labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
91
+ else
92
+ labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
93
+ fi
94
+ echo "labels=$labels" >> "$GITHUB_OUTPUT"
50
95
  # (And whatever else your tests need set up: packages, services.)
51
96
  <% if version_file -%>
52
97
  # Each version runs on the Ruby its own .ruby-version names (see
@@ -57,7 +102,8 @@ jobs:
57
102
  # request targets (or run by hand, the main branch), a merge queue's
58
103
  # base, the commit before a push (or the last commit Duel passed, PASSED, if there is
59
104
  # one: see the Mark the commit passed job), or on a schedule the last
60
- # release, the newest tag before HEAD that RELEASE_TAGS matches. (If
105
+ # release, the newest tag before HEAD that RELEASE_TAGS matches<% if accept_label %>
106
+ # (or ACCEPTED, the last change accepted since then)<% end %>. (If
61
107
  # .duel.rb sets c.main_branch or c.release_tags, set MAIN_BRANCH or
62
108
  # RELEASE_TAGS to match.)
63
109
  - name: Find the baseline's Ruby
@@ -68,13 +114,16 @@ jobs:
68
114
  PASSED: origin/<%= passed_branch %>/${{ github.ref_name }}
69
115
  MAIN_BRANCH: origin/${{ github.base_ref || github.event.repository.default_branch }}
70
116
  RELEASE_TAGS: "<%= release_tags %>"
117
+ <% if accept_label -%>
118
+ ACCEPTED: ${{ steps.labels.outputs.accepted }}
119
+ <% end -%>
71
120
  run: |
72
121
  case "$GITHUB_EVENT_NAME" in
73
122
  push)
74
123
  base=$BEFORE
75
124
  if git merge-base --is-ancestor "$PASSED" HEAD 2>/dev/null; then base=$PASSED; fi ;;
76
125
  merge_group) base=$QUEUE_BASE ;;
77
- schedule) base=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || base= ;;
126
+ schedule) base=${ACCEPTED:-$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null)} || base= ;;
78
127
  *) base=$(git merge-base HEAD "$MAIN_BRANCH" 2>/dev/null) || base= ;;
79
128
  esac
80
129
  ruby_of() { git show "$1:./.ruby-version" 2>/dev/null | awk 'NF { print $1; exit }'; }
@@ -113,30 +162,6 @@ jobs:
113
162
  ruby-version: "<%= ruby_version %>"
114
163
  <% end -%>
115
164
  bundler-cache: true
116
- # The labels of the pull request (or for a push, of the one that
117
- # merged it), for c.accept_label in .duel.rb, as a JSON array. They're
118
- # read as the job runs, so after adding the label, run it again.
119
- - name: Find the pull request's labels
120
- id: labels
121
- if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'
122
- continue-on-error: true # (without them, nothing's accepted)
123
- env:
124
- GH_TOKEN: ${{ github.token }}
125
- PR: ${{ github.event.pull_request.number }}
126
- QUEUED: ${{ github.event.merge_group.head_ref }}
127
- run: |
128
- # (In a merge queue, the last pull request queued, which the queue's
129
- # branch names: gh-readonly-queue/main/pr-123-<sha>.)
130
- if [ -n "$QUEUED" ]; then
131
- PR=${QUEUED##*/pr-}
132
- PR=${PR%%-*}
133
- fi
134
- if [ -n "$PR" ]; then
135
- labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
136
- else
137
- labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
138
- fi
139
- echo "labels=$labels" >> "$GITHUB_OUTPUT"
140
165
  # Against the branch it targets for a pull request, the queue's base
141
166
  # for a merge queue, what was there before for a push (or as
142
167
  # c.push_base in .duel.rb says, the last commit Duel passed), and the
@@ -152,6 +177,9 @@ jobs:
152
177
  # with itself, which checks the setup.)
153
178
  DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}
154
179
  DUEL_LABELS: ${{ steps.labels.outputs.labels }}
180
+ <% if accept_label -%>
181
+ DUEL_ACCEPTED: ${{ steps.labels.outputs.accepted }}
182
+ <% end -%>
155
183
  run: bundle exec duel --markdown "$RUNNER_TEMP/duel/duel.md"
156
184
  # For a pull request, what Duel said, for the Perf comment workflow,
157
185
  # which can't see this job's outputs.
@@ -173,6 +201,7 @@ jobs:
173
201
  path: ${{ runner.temp }}/duel/
174
202
  if-no-files-found: ignore # none when Duel didn't get to compare
175
203
  retention-days: 1
204
+ <% if comment_on_pushes? -%>
176
205
 
177
206
  # The report, as a new comment on a pushed commit, which notifies its
178
207
  # author, when Duel says to, as for a pull request (see the Perf comment
@@ -209,6 +238,8 @@ jobs:
209
238
  { echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
210
239
  gh api --method POST "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
211
240
  fi
241
+ <% end -%>
242
+ <% if mark_passed? -%>
212
243
 
213
244
  # With c.push_base = :last_passed in .duel.rb, a push is compared with
214
245
  # the last commit Duel passed, which a branch marks (each branch pushed
@@ -221,6 +252,13 @@ jobs:
221
252
  passed:
222
253
  name: Mark the commit passed
223
254
  needs: duel
255
+ # One at a time for each branch, so that two runs can't both find no
256
+ # branch and race to make it (a push's run doesn't wait for the one
257
+ # before it). One that's running is never cancelled, and one that's
258
+ # waiting gives way to a newer push's, which moves it further anyway.
259
+ concurrency:
260
+ group: ${{ github.workflow }}-passed-${{ github.ref_name }}
261
+ cancel-in-progress: false
224
262
  if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
225
263
  runs-on: ubuntu-latest
226
264
  permissions:
@@ -244,13 +282,16 @@ jobs:
244
282
  done
245
283
  exit 1
246
284
  }
247
- # (Forward: GitHub moves it only where it's behind the push.)
248
- gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null 2>&1 && exit 0
249
- at=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$BRANCH" --jq .object.sha 2>/dev/null) || at=
250
- if [ -z "$at" ]; then
251
- gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null || blocked
252
- exit 0
253
- fi
285
+ # (Forward: GitHub moves it only where it's behind the push. Where
286
+ # there's no branch, and making it fails, it looks once more, as
287
+ # something else may have made it: a push of it, say.)
288
+ for _ in 1 2; do
289
+ gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null 2>&1 && exit 0
290
+ at=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$BRANCH" --jq .object.sha 2>/dev/null) || at=
291
+ [ -n "$at" ] && break
292
+ gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null && exit 0
293
+ done
294
+ [ -n "$at" ] || blocked
254
295
  compared=$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$at" --jq .status 2>&1) || case "$compared" in
255
296
  # (GitHub can't compare commits with no history in common.)
256
297
  *"No common ancestor"*) compared=unrelated ;;
@@ -262,6 +303,8 @@ jobs:
262
303
  diverged | behind | unrelated) gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" -F force=true > /dev/null ;;
263
304
  *) echo "::error::Couldn't compare $BRANCH with this push: $compared" && exit 1 ;;
264
305
  esac
306
+ <% end -%>
307
+ <% if badge? -%>
265
308
 
266
309
  # Duel's badge, for the README: green (perf guarded) or red (perf
267
310
  # regressed), as the last push to the default branch that Duel compared
@@ -273,10 +316,18 @@ jobs:
273
316
  #
274
317
  # [![Duel](https://raw.githubusercontent.com/OWNER/REPO/<%= badge_branch %>/badge.svg)](https://github.com/sferik/duel-ruby)
275
318
  #
276
- # It checks nothing out and runs none of the code.
319
+ # It checks nothing out, runs none of the code, and fetches nothing.
277
320
  badge:
278
321
  name: Update the badge
279
322
  needs: duel
323
+ # One at a time, so that one run's check of which push the badge
324
+ # shows can't interleave with another run's write (a push's run
325
+ # doesn't wait for the one before it). A badge job that's running is
326
+ # never cancelled, and one that's waiting gives way to a newer push's,
327
+ # whose result the badge should show anyway.
328
+ concurrency:
329
+ group: ${{ github.workflow }}-badge
330
+ cancel-in-progress: false
280
331
  if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
281
332
  runs-on: ubuntu-latest
282
333
  permissions:
@@ -288,7 +339,6 @@ jobs:
288
339
  BRANCH: <%= badge_branch %>
289
340
  RESULT: ${{ needs.duel.result }}
290
341
  STATUS: ${{ needs.duel.outputs.status }}
291
- BADGES: https://raw.githubusercontent.com/sferik/duel-ruby/main/assets
292
342
  run: |
293
343
  # (Git can't make a branch where another's name is a folder of its
294
344
  # name: a branch named duel blocks duel/passed/main. If one does,
@@ -307,11 +357,59 @@ jobs:
307
357
  # Its commit names the push.)
308
358
  last=$(gh api "repos/$GITHUB_REPOSITORY/commits/$BRANCH" --jq .commit.message 2>/dev/null | sed -n 's/^Duel.s badge, for \([0-9a-f]*\)$/\1/p') || true
309
359
  if [ -n "$last" ] && [ "$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$last" --jq .status 2>/dev/null)" = ahead ]; then exit 0; fi
310
- badge="badge.svg"
311
- if [ "$RESULT" = failure ] || [ "$STATUS" = failed ]; then badge="badge-regressed.svg"; fi
312
- curl -fsSL "$BADGES/$badge" -o "$RUNNER_TEMP/badge.svg"
360
+ # (Its image, written here rather than fetched from anywhere: green,
361
+ # or red where the Duel job failed or found a regression.)
362
+ state="perf guarded" width=133 top="#34D058" bottom="#28A745" right="76.939C78.629 0 80 1.343 80"
363
+ if [ "$RESULT" = failure ] || [ "$STATUS" = failed ]; then
364
+ state="perf regressed" width=142 top="#EA4A5A" bottom="#D73A49" right="85.939C87.629 0 89 1.343 89"
365
+ fi
366
+ cat > "$RUNNER_TEMP/badge.svg" <<SVG
367
+ <svg xmlns="http://www.w3.org/2000/svg" width="$width" height="20">
368
+ <title>Duel - $state</title>
369
+ <defs>
370
+ <linearGradient id="duel-fill" x1="50%" y1="0%" x2="50%" y2="100%">
371
+ <stop stop-color="#444D56" offset="0%"></stop>
372
+ <stop stop-color="#24292E" offset="100%"></stop>
373
+ </linearGradient>
374
+ <linearGradient id="state-fill" x1="50%" y1="0%" x2="50%" y2="100%">
375
+ <stop stop-color="$top" offset="0%"></stop>
376
+ <stop stop-color="$bottom" offset="100%"></stop>
377
+ </linearGradient>
378
+ </defs>
379
+ <g fill="none" fill-rule="evenodd">
380
+ <g font-family="'DejaVu Sans',Verdana,Geneva,sans-serif" font-size="11">
381
+ <path d="M0,3 C0,1.3431 1.3552,0 3.02702703,0 L53,0 L53,20 L3.02702703,20 C1.3552,20 0,18.6569 0,17 L0,3 Z" fill="url(#duel-fill)" fill-rule="nonzero"></path>
382
+ <text fill="#010101" fill-opacity=".3">
383
+ <tspan x="22.1981982" y="15" aria-hidden="true">Duel</tspan>
384
+ </text>
385
+ <text fill="#FFFFFF">
386
+ <tspan x="22.1981982" y="14">Duel</tspan>
387
+ </text>
388
+ </g>
389
+ <g transform="translate(53)" font-family="'DejaVu Sans',Verdana,Geneva,sans-serif" font-size="11">
390
+ <path d="M0 0h$right 3v14c0 1.657-1.37 3-3.061 3H0V0z" fill="url(#state-fill)" fill-rule="nonzero"></path>
391
+ <text fill="#010101" fill-opacity=".3" aria-hidden="true">
392
+ <tspan x="4" y="15">$state</tspan>
393
+ </text>
394
+ <text fill="#FFFFFF">
395
+ <tspan x="4" y="14">$state</tspan>
396
+ </text>
397
+ </g>
398
+ <g fill="#959DA5">
399
+ <path d="M4.6 3.6L6.9 4.5L14.3 11.9L12.9 13.3L5.5 5.9Z M17.4 3.6L15.1 4.5L7.7 11.9L9.1 13.3L16.5 5.9Z"></path>
400
+ <circle cx="16.6" cy="15.6" r="1.1"></circle>
401
+ <circle cx="5.4" cy="15.6" r="1.1"></circle>
402
+ </g>
403
+ <g stroke="#959DA5" stroke-linecap="round">
404
+ <path d="M12.1 14.5L15.5 11.1 M9.9 14.5L6.5 11.1" stroke-width="1.6"></path>
405
+ <path d="M13.8 12.8L16 15 M8.2 12.8L6 15" stroke-width="1.8"></path>
406
+ </g>
407
+ </g>
408
+ </svg>
409
+ SVG
313
410
  tree=$(jq -n --rawfile svg "$RUNNER_TEMP/badge.svg" '{tree: [{path: "badge.svg", mode: "100644", type: "blob", content: $svg}]}' |
314
411
  gh api --method POST "repos/$GITHUB_REPOSITORY/git/trees" --input - --jq .sha)
315
412
  commit=$(gh api --method POST "repos/$GITHUB_REPOSITORY/git/commits" -f message="Duel's badge, for $GITHUB_SHA" -f tree="$tree" --jq .sha)
316
413
  gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$commit" -F force=true > /dev/null 2>&1 ||
317
414
  gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$commit" > /dev/null || blocked
415
+ <% end -%>
data/lib/duel/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Duel
4
- VERSION = "0.1.4"
4
+ VERSION = "0.1.6"
5
5
  end
data/lib/duel/worker.rb CHANGED
@@ -18,10 +18,11 @@ module Duel
18
18
  # closed.)
19
19
  # @api private
20
20
  class Worker
21
- # The built-in metrics, outermost first: allocations inside
22
- # instructions, whose counter allocates as it's read (a sample's own
23
- # allocations are none but the benchmark's), and CPU time innermost.
24
- BUILT_IN = [Metrics::Instructions, Metrics::Allocations, Metrics::CPU].freeze
21
+ # The built-in metrics, outermost first: allocations (CRuby's objects,
22
+ # or JRuby's bytes) inside instructions, whose counter allocates as
23
+ # it's read (a sample's own allocations are none but the benchmark's),
24
+ # and CPU time innermost.
25
+ BUILT_IN = [Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
25
26
 
26
27
  def self.main(version_root, candidate_root, control, config_path = nil)
27
28
  config = config_path ? Duel.load_config(config_path) : (Duel.config = Config.new(candidate_root))
data/sig/duel/across.rbs CHANGED
@@ -5,17 +5,39 @@ module Duel
5
5
  # A timed metric's verdict across the board ("confirmed": slower in a
6
6
  # second measurement too, see Runner#across).
7
7
  type verdict = { "median" => Float, "interval" => Array[Float]?, "limit" => Float, "benchmarks" => Integer,
8
- "verdict" => String, ?"confirmed" => bool }
8
+ ?"processes" => Integer, "verdict" => String, ?"confirmed" => bool }
9
+
10
+ # The fewest benchmarks across the board is judged on by process.
11
+ MIN_BENCHMARKS: Integer
9
12
 
10
13
  @results: Array[Result]
11
14
  @confidence: Float
15
+ @by_process: bool
12
16
 
13
17
  # The across-the-board limit for a metric whose own limit is +limit+.
14
18
  def self.limit: (Float limit) -> Float
15
19
 
16
- def initialize: (Array[Result] results, confidence: Float) -> void
20
+ def initialize: (Array[Result] results, confidence: Float, ?by_process: bool) -> void
17
21
 
18
22
  # Every timed metric any benchmark was judged on, by name.
19
23
  def verdicts: () -> Hash[String, verdict]
24
+
25
+ private
26
+
27
+ # The verdicts of a run that judged benchmarks over workers.
28
+ def by_process_or_benchmark: (Hash[String, Array[Float]] medians, Hash[String, Float] limits) -> Hash[String, verdict]
29
+
30
+ # Each benchmark's workers' medians, of those judged by process, by
31
+ # metric.
32
+ def workers: () -> Hash[String, Array[Result::processes]]
33
+
34
+ # The verdict over every benchmark's median ratio.
35
+ def by_benchmark: (Array[Float] list, Float limit) -> verdict
36
+
37
+ # The verdict over the workers' scores.
38
+ def by_process: (Array[Result::processes] processes, Float limit) -> verdict
39
+
40
+ # Each worker's score, by version.
41
+ def scores: (Array[Result::processes] processes) -> Hash[String, Array[Float]]
20
42
  end
21
43
  end
@@ -9,6 +9,7 @@ module Duel
9
9
  @config: Config
10
10
  @settings: Settings
11
11
  @ref: String?
12
+ @release: String?
12
13
  @prefix: String?
13
14
 
14
15
  def initialize: (Config config, Settings settings) -> void
@@ -17,6 +18,17 @@ module Duel
17
18
  # a push with no commit before it here.
18
19
  def ref: () -> String?
19
20
 
21
+ # The newest release tag before HEAD, or nil.
22
+ def release: () -> String?
23
+
24
+ # The last release, or the commit since then that merged the last
25
+ # pull request accepted with Config#accept_label (Settings#accepted),
26
+ # or nil with no release.
27
+ def last_release: () -> String?
28
+
29
+ # Whether +commit+ is +of+ or an ancestor of it.
30
+ def ancestor?: (String commit, String of) -> bool
31
+
20
32
  # The branch that marks the last commit Duel passed in a push to the
21
33
  # branch pushed.
22
34
  def passed_branch: () -> String
data/sig/duel/cli.rbs CHANGED
@@ -153,6 +153,10 @@ module Duel
153
153
  # The candidate, in Markdown.
154
154
  def described_candidate: (Config config, Settings settings) -> String
155
155
 
156
+ # What chose a baseline of the last release: the release, or a change
157
+ # since it accepted with the label.
158
+ def last_release: (Config config, Baseline baseline) -> String
159
+
156
160
  # The baseline, for JSON: the ref that named it, its commit, and its
157
161
  # tree.
158
162
  def baseline_json: (Config config, Settings settings, Baseline baseline, String root) -> Hash[String, String?]