duel 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/duel/stats.rb CHANGED
@@ -43,6 +43,82 @@ module Duel
43
43
  [sorted.fetch(lower - 1), sorted.fetch(upper - 1)]
44
44
  end
45
45
 
46
+ # The 1-based rank r whose r-th smallest and r-th largest cross
47
+ # ratio (see #cross_ratios) bound a +confidence+ interval for the
48
+ # ratio between two versions, from +m+ samples of one and +n+ of the
49
+ # other, or nil when they're too few for one (fewer than 5 each, at
50
+ # 99%). It's distribution-free, as long as each version's samples
51
+ # are independent and alike: the true ratio lies below the r-th
52
+ # smallest with probability P(U <= r - 1), for the Mann-Whitney U
53
+ # statistic (see #u_lower_tail), whatever their distribution. So a
54
+ # sample can be a worker process's median, and then the interval
55
+ # accounts for how a process's luck (where things landed in its
56
+ # memory) differs from another's, as an interval over pairs from a
57
+ # few processes can't.
58
+ def shift_rank(m, n, confidence)
59
+ tail = (1 - confidence) / 2
60
+ r = u_lower_tail(m, n).index { |p| p > tail }
61
+ # (None only where the tail is a half or more, at no confidence.)
62
+ r unless r.nil? || r.zero?
63
+ end
64
+
65
+ # P(U <= u) for u from 0 to m * n / 2, where U is the Mann-Whitney
66
+ # statistic of samples of +m+ and +n+ from the same distribution:
67
+ # the pairs (one of each) in which the first's is the larger. Each
68
+ # P(U = u) is found from samples of one fewer (the largest of all is
69
+ # one of the first's, beating all +n+, as often as m / (m + n), or
70
+ # one of the second's), in probabilities, which stay in range where
71
+ # the counts of orderings would grow huge.
72
+ def u_lower_tail(m, n)
73
+ cap = m * n / 2
74
+ sum = 0.0
75
+ u_probabilities(m, n, cap, {}).map { |p| sum += p }
76
+ end
77
+
78
+ # P(U = u) for u from 0 to +cap+, for samples of +i+ and +j+ (see
79
+ # #u_lower_tail), each found once, in +found+.
80
+ def u_probabilities(i, j, cap, found)
81
+ found[[i, j]] ||= if i.zero? || j.zero?
82
+ Array.new(cap + 1) { |u| u.zero? ? 1.0 : 0.0 } # (no samples of one: U is 0)
83
+ else
84
+ first = u_probabilities(i - 1, j, cap, found)
85
+ second = u_probabilities(i, j - 1, cap, found)
86
+ Array.new(cap + 1) do |u|
87
+ from_first = (u >= j) ? first.fetch(u - j) * i : 0.0
88
+ (from_first + (second.fetch(u) * j)) / (i + j)
89
+ end
90
+ end
91
+ end
92
+
93
+ # [low, high] bounds of the +confidence+ interval for the ratio of
94
+ # +candidates+ (a version's samples) to +bases+ (the other's), or nil
95
+ # (see #shift_rank).
96
+ def shift_interval(bases, candidates, confidence)
97
+ r = shift_rank(bases.size, candidates.size, confidence)
98
+ return unless r
99
+
100
+ ratios = cross_ratios(bases, candidates)
101
+ [ratios.fetch(r - 1), ratios.fetch(-r)]
102
+ end
103
+
104
+ # The ratio of +candidates+ to +bases+ the interval is for: the
105
+ # median of the cross ratios (the Hodges-Lehmann estimate).
106
+ def shift(bases, candidates) = median(cross_ratios(bases, candidates))
107
+
108
+ # [low, high] bounds of the +confidence+ interval for how much more
109
+ # +candidates+ (a version's samples) are than +bases+ (the other's),
110
+ # as #shift_interval's are for how many times as much, or nil.
111
+ def difference_interval(bases, candidates, confidence)
112
+ r = shift_rank(bases.size, candidates.size, confidence)
113
+ return unless r
114
+
115
+ differences = candidates.flat_map { |c| bases.map { |b| c - b } }.sort
116
+ [differences.fetch(r - 1), differences.fetch(-r)]
117
+ end
118
+
119
+ # Each of +candidates+ over each of +bases+, sorted.
120
+ def cross_ratios(bases, candidates) = candidates.flat_map { |c| bases.map { |b| c.fdiv(b) } }.sort
121
+
46
122
  def median(samples)
47
123
  sorted = samples.sort
48
124
  mid = sorted.size / 2
data/lib/duel/summary.rb CHANGED
@@ -168,7 +168,7 @@ module Duel
168
168
  if timed then format("%s %+.1f%%", metric, (timed.fetch("median") - 1) * 100)
169
169
  elsif counted
170
170
  n = counted.fetch("n")
171
- format("%s %.4g → %.4g", metric, counted.fetch("baseline").fdiv(n), counted.fetch("candidate").fdiv(n))
171
+ "#{metric} #{Runner.number(counted.fetch("baseline").fdiv(n))} → #{Runner.number(counted.fetch("candidate").fdiv(n))}"
172
172
  else
173
173
  # (Peak memory, which grew too much.)
174
174
  memory = result.memory #: Hash[String, Integer]
@@ -247,7 +247,7 @@ module Duel
247
247
  # that grew too much.
248
248
  def counts(result)
249
249
  counted = Summary.changed_counts(result).map do |metric, (base, candidate)|
250
- format("%s %.4g → %.4g", metric, base, candidate)
250
+ "#{metric} #{Runner.number(base)} → #{Runner.number(candidate)}"
251
251
  end
252
252
  if result.more_memory?
253
253
  memory = result.memory #: Hash[String, Integer]
@@ -134,6 +134,9 @@
134
134
  image: alpine:3
135
135
  stage: .post
136
136
  needs: [<%= gitlab_job %>]
137
+ # One at a time, as each deletes the branch and makes it again: two at
138
+ # once could both delete it, and the second to make it would fail.
139
+ resource_group: <%= ["duel", working_directory, "passed"].compact.join("/") %>
137
140
  rules:
138
141
  - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
139
142
  variables:
@@ -10,12 +10,27 @@ name: Perf comment<%= " (#{working_directory})" if working_directory %>
10
10
  # ran), it trusts only what that job could have said: the pull request's
11
11
  # number, if it names the pull request the run compared, and the comment
12
12
  # and the status, if they're ones Duel says. A fork's report, which the
13
- # fork's code wrote, is shown as text. (The workflow must be on the
14
- # default branch before it runs at all.)
13
+ # fork's code wrote, is shown as text.
14
+ <% if accept_label -%>
15
+ #
16
+ # And when the label that accepts a regression (c.accept_label in
17
+ # .duel.rb) is added to a pull request, or taken off, it runs the Duel
18
+ # job again, which reads the labels as it runs: a run that failed, once
19
+ # the label's added, or one that passed, once it's taken off.
20
+ <% end -%>
21
+ #
22
+ # (The workflow must be on the default branch before it runs at all.)
15
23
  on:
16
24
  workflow_run:
17
25
  workflows: ["Perf<%= " (#{working_directory})" if working_directory %>"]
18
26
  types: [completed]
27
+ <% if accept_label -%>
28
+ # (This repository's own workflow, as for the comment, with a token
29
+ # that can run a workflow again: it checks nothing out, and runs none
30
+ # of the pull request's code.)
31
+ pull_request_target:
32
+ types: [labeled, unlabeled]
33
+ <% end -%>
19
34
 
20
35
  permissions: {}
21
36
 
@@ -56,12 +71,16 @@ jobs:
56
71
  # ran the fork's code, which could have written anything in it. In
57
72
  # a code block, fenced with more backticks than any run of them in
58
73
  # the report, it can't mention anyone, link anywhere or show an
59
- # image. (And it's cut short where a comment couldn't hold it.)
74
+ # image. It's cut short where a comment couldn't hold it, and loses
75
+ # any NUL bytes, which GitHub wouldn't take, and grep reads it as
76
+ # text whatever's in it, so it finds every run of backticks.
60
77
  report() {
61
78
  if [ "${head#* }" = "$GITHUB_REPOSITORY" ]; then cat "$RUNNER_TEMP/report/duel.md" && return; fi
62
- fence=$(grep -o '``*' "$RUNNER_TEMP/report/duel.md" | awk '{ if (length > n) n = length } END { n = (n < 3) ? 3 : n + 1; while (n-- > 0) printf "`" }')
79
+ text="$RUNNER_TEMP/report.txt"
80
+ LC_ALL=C tr -d '\000' < "$RUNNER_TEMP/report/duel.md" | head -c 60000 > "$text"
81
+ fence=$(LC_ALL=C grep -ao '``*' "$text" | awk '{ if (length > n) n = length } END { n = (n < 3) ? 3 : n + 1; while (n-- > 0) printf "`" }')
63
82
  printf 'From a fork, so shown as text:\n\n%s\n' "$fence"
64
- head -c 60000 "$RUNNER_TEMP/report/duel.md"
83
+ cat "$text"
65
84
  printf '\n%s\n' "$fence"
66
85
  }
67
86
  marker="<!-- <%= marker %>"
@@ -73,3 +92,32 @@ jobs:
73
92
  { echo "$marker status=$STATUS -->"; report; } > "$RUNNER_TEMP/comment.md"
74
93
  gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
75
94
  fi
95
+ <% if accept_label -%>
96
+
97
+ again:
98
+ name: Run Duel again
99
+ if: >-
100
+ github.event_name == 'pull_request_target' && github.event.label.name == '<%= accept_label.gsub("'", "''") %>'
101
+ runs-on: ubuntu-latest
102
+ permissions:
103
+ actions: write # to run the Perf workflow's jobs again
104
+ steps:
105
+ - name: Run Duel again
106
+ env:
107
+ GH_TOKEN: ${{ github.token }}
108
+ GH_REPO: ${{ github.repository }}
109
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
110
+ ACTION: ${{ github.event.action }}
111
+ run: |
112
+ # The newest Perf run of the pull request's head commit, once it's
113
+ # done (one that's running may have read the labels already).
114
+ id=$(gh run list --workflow "Perf<%= " (#{working_directory})" if working_directory %>" --event pull_request --commit "$HEAD_SHA" --limit 1 \
115
+ --json databaseId --jq '.[0].databaseId // empty')
116
+ [ -n "$id" ] || exit 0
117
+ gh run watch "$id" --interval 30 > /dev/null || true
118
+ conclusion=$(gh run view "$id" --json conclusion --jq .conclusion)
119
+ case "$ACTION:$conclusion" in
120
+ labeled:failure) gh run rerun "$id" --failed ;;
121
+ unlabeled:success) gh run rerun "$id" ;;
122
+ esac
123
+ <% end -%>
@@ -47,6 +47,51 @@ jobs:
47
47
  with:
48
48
  fetch-depth: 0 # the baseline comes from history, tags included
49
49
  persist-credentials: false
50
+ # The labels of the pull request (or for a push, of the one that
51
+ # merged it), for c.accept_label in .duel.rb, as a JSON array. They're
52
+ # read as the job runs, and adding the label (or taking it off) runs
53
+ # the job again: see the Perf comment workflow.
54
+ <% if accept_label -%>
55
+ # On a schedule, the last commit since the last release that merged a
56
+ # pull request with the label (ACCEPT_LABEL, as .duel.rb names it):
57
+ # what that slowed down was accepted, so it's compared with instead.
58
+ <% end -%>
59
+ - name: Find the pull request's labels
60
+ id: labels
61
+ if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %> || github.event_name == 'schedule'<% end %>
62
+ continue-on-error: true # (without them, nothing's accepted)
63
+ env:
64
+ GH_TOKEN: ${{ github.token }}
65
+ PR: ${{ github.event.pull_request.number }}
66
+ QUEUED: ${{ github.event.merge_group.head_ref }}
67
+ <% if accept_label -%>
68
+ ACCEPT_LABEL: '<%= accept_label.gsub("'", "''") %>'
69
+ RELEASE_TAGS: "<%= release_tags %>"
70
+ <% end -%>
71
+ run: |
72
+ <% if accept_label -%>
73
+ if [ "$GITHUB_EVENT_NAME" = schedule ]; then
74
+ tag=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || exit 0
75
+ since=$(TZ=UTC git log -1 --date=format-local:%Y-%m-%dT%H:%M:%SZ --format=%cd "$tag")
76
+ merged=$(gh pr list --repo "$GITHUB_REPOSITORY" --state merged --label "$ACCEPT_LABEL" --search "merged:>=$since" --limit 1000 \
77
+ --json mergeCommit --jq '.[].mergeCommit.oid')
78
+ accepted=$(git rev-list "$tag..HEAD" | grep -Fx -m 1 -f <(echo "$merged")) || accepted=
79
+ echo "accepted=$accepted" >> "$GITHUB_OUTPUT"
80
+ exit 0
81
+ fi
82
+ <% end -%>
83
+ # (In a merge queue, the last pull request queued, which the queue's
84
+ # branch names: gh-readonly-queue/main/pr-123-<sha>.)
85
+ if [ -n "$QUEUED" ]; then
86
+ PR=${QUEUED##*/pr-}
87
+ PR=${PR%%-*}
88
+ fi
89
+ if [ -n "$PR" ]; then
90
+ labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
91
+ else
92
+ labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
93
+ fi
94
+ echo "labels=$labels" >> "$GITHUB_OUTPUT"
50
95
  # (And whatever else your tests need set up: packages, services.)
51
96
  <% if version_file -%>
52
97
  # Each version runs on the Ruby its own .ruby-version names (see
@@ -57,7 +102,8 @@ jobs:
57
102
  # request targets (or run by hand, the main branch), a merge queue's
58
103
  # base, the commit before a push (or the last commit Duel passed, PASSED, if there is
59
104
  # one: see the Mark the commit passed job), or on a schedule the last
60
- # release, the newest tag before HEAD that RELEASE_TAGS matches. (If
105
+ # release, the newest tag before HEAD that RELEASE_TAGS matches<% if accept_label %>
106
+ # (or ACCEPTED, the last change accepted since then)<% end %>. (If
61
107
  # .duel.rb sets c.main_branch or c.release_tags, set MAIN_BRANCH or
62
108
  # RELEASE_TAGS to match.)
63
109
  - name: Find the baseline's Ruby
@@ -68,13 +114,16 @@ jobs:
68
114
  PASSED: origin/<%= passed_branch %>/${{ github.ref_name }}
69
115
  MAIN_BRANCH: origin/${{ github.base_ref || github.event.repository.default_branch }}
70
116
  RELEASE_TAGS: "<%= release_tags %>"
117
+ <% if accept_label -%>
118
+ ACCEPTED: ${{ steps.labels.outputs.accepted }}
119
+ <% end -%>
71
120
  run: |
72
121
  case "$GITHUB_EVENT_NAME" in
73
122
  push)
74
123
  base=$BEFORE
75
124
  if git merge-base --is-ancestor "$PASSED" HEAD 2>/dev/null; then base=$PASSED; fi ;;
76
125
  merge_group) base=$QUEUE_BASE ;;
77
- schedule) base=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || base= ;;
126
+ schedule) base=${ACCEPTED:-$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null)} || base= ;;
78
127
  *) base=$(git merge-base HEAD "$MAIN_BRANCH" 2>/dev/null) || base= ;;
79
128
  esac
80
129
  ruby_of() { git show "$1:./.ruby-version" 2>/dev/null | awk 'NF { print $1; exit }'; }
@@ -113,30 +162,6 @@ jobs:
113
162
  ruby-version: "<%= ruby_version %>"
114
163
  <% end -%>
115
164
  bundler-cache: true
116
- # The labels of the pull request (or for a push, of the one that
117
- # merged it), for c.accept_label in .duel.rb, as a JSON array. They're
118
- # read as the job runs, so after adding the label, run it again.
119
- - name: Find the pull request's labels
120
- id: labels
121
- if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'
122
- continue-on-error: true # (without them, nothing's accepted)
123
- env:
124
- GH_TOKEN: ${{ github.token }}
125
- PR: ${{ github.event.pull_request.number }}
126
- QUEUED: ${{ github.event.merge_group.head_ref }}
127
- run: |
128
- # (In a merge queue, the last pull request queued, which the queue's
129
- # branch names: gh-readonly-queue/main/pr-123-<sha>.)
130
- if [ -n "$QUEUED" ]; then
131
- PR=${QUEUED##*/pr-}
132
- PR=${PR%%-*}
133
- fi
134
- if [ -n "$PR" ]; then
135
- labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
136
- else
137
- labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
138
- fi
139
- echo "labels=$labels" >> "$GITHUB_OUTPUT"
140
165
  # Against the branch it targets for a pull request, the queue's base
141
166
  # for a merge queue, what was there before for a push (or as
142
167
  # c.push_base in .duel.rb says, the last commit Duel passed), and the
@@ -152,6 +177,9 @@ jobs:
152
177
  # with itself, which checks the setup.)
153
178
  DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}
154
179
  DUEL_LABELS: ${{ steps.labels.outputs.labels }}
180
+ <% if accept_label -%>
181
+ DUEL_ACCEPTED: ${{ steps.labels.outputs.accepted }}
182
+ <% end -%>
155
183
  run: bundle exec duel --markdown "$RUNNER_TEMP/duel/duel.md"
156
184
  # For a pull request, what Duel said, for the Perf comment workflow,
157
185
  # which can't see this job's outputs.
@@ -224,6 +252,13 @@ jobs:
224
252
  passed:
225
253
  name: Mark the commit passed
226
254
  needs: duel
255
+ # One at a time for each branch, so that two runs can't both find no
256
+ # branch and race to make it (a push's run doesn't wait for the one
257
+ # before it). One that's running is never cancelled, and one that's
258
+ # waiting gives way to a newer push's, which moves it further anyway.
259
+ concurrency:
260
+ group: ${{ github.workflow }}-passed-${{ github.ref_name }}
261
+ cancel-in-progress: false
227
262
  if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
228
263
  runs-on: ubuntu-latest
229
264
  permissions:
@@ -247,13 +282,16 @@ jobs:
247
282
  done
248
283
  exit 1
249
284
  }
250
- # (Forward: GitHub moves it only where it's behind the push.)
251
- gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null 2>&1 && exit 0
252
- at=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$BRANCH" --jq .object.sha 2>/dev/null) || at=
253
- if [ -z "$at" ]; then
254
- gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null || blocked
255
- exit 0
256
- fi
285
+ # (Forward: GitHub moves it only where it's behind the push. Where
286
+ # there's no branch, and making it fails, it looks once more, as
287
+ # something else may have made it: a push of it, say.)
288
+ for _ in 1 2; do
289
+ gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null 2>&1 && exit 0
290
+ at=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$BRANCH" --jq .object.sha 2>/dev/null) || at=
291
+ [ -n "$at" ] && break
292
+ gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null && exit 0
293
+ done
294
+ [ -n "$at" ] || blocked
257
295
  compared=$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$at" --jq .status 2>&1) || case "$compared" in
258
296
  # (GitHub can't compare commits with no history in common.)
259
297
  *"No common ancestor"*) compared=unrelated ;;
@@ -268,13 +306,15 @@ jobs:
268
306
  <% end -%>
269
307
  <% if badge? -%>
270
308
 
271
- # Duel's badge, for the README: green (perf guarded) or red (perf
272
- # regressed), as the last push to the default branch that Duel compared
273
- # left it. It's red where the Duel job failed or found a regression,
274
- # even one that passed (with c.fail_on = :pull_requests in .duel.rb). A
275
- # push with nothing to compare leaves it as it was, and so does a run
276
- # for an older push than the badge's. This job writes the badge, alone,
277
- # to a branch of its own, which the README shows:
309
+ # Duel's badge, for the README: green (perf guarded), red (perf
310
+ # regressed) or grey (not measured), as the last push to the default
311
+ # branch that Duel compared left it. It's red where Duel failed the push
312
+ # or found a regression, even one that passed (with
313
+ # c.fail_on = :pull_requests in .duel.rb), and grey where the Duel job
314
+ # failed before Duel could judge the push (its runner ran out of memory,
315
+ # say). A push with nothing to compare leaves it as it was, and so does
316
+ # a run for an older push than the badge's. This job writes the badge,
317
+ # alone, to a branch of its own, which the README shows:
278
318
  #
279
319
  # [![Duel](https://raw.githubusercontent.com/OWNER/REPO/<%= badge_branch %>/badge.svg)](https://github.com/sferik/duel-ruby)
280
320
  #
@@ -320,10 +360,13 @@ jobs:
320
360
  last=$(gh api "repos/$GITHUB_REPOSITORY/commits/$BRANCH" --jq .commit.message 2>/dev/null | sed -n 's/^Duel.s badge, for \([0-9a-f]*\)$/\1/p') || true
321
361
  if [ -n "$last" ] && [ "$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$last" --jq .status 2>/dev/null)" = ahead ]; then exit 0; fi
322
362
  # (Its image, written here rather than fetched from anywhere: green,
323
- # or red where the Duel job failed or found a regression.)
363
+ # red where Duel failed the push or found a regression, or grey
364
+ # where the Duel job failed before Duel said either.)
324
365
  state="perf guarded" width=133 top="#34D058" bottom="#28A745" right="76.939C78.629 0 80 1.343 80"
325
- if [ "$RESULT" = failure ] || [ "$STATUS" = failed ]; then
366
+ if [ "$STATUS" = failed ]; then
326
367
  state="perf regressed" width=142 top="#EA4A5A" bottom="#D73A49" right="85.939C87.629 0 89 1.343 89"
368
+ elif [ "$RESULT" = failure ] && [ "$STATUS" != passed ]; then
369
+ state="not measured" width=138 top="#959DA5" bottom="#6A737D" right="81.939C83.629 0 85 1.343 85"
327
370
  fi
328
371
  cat > "$RUNNER_TEMP/badge.svg" <<SVG
329
372
  <svg xmlns="http://www.w3.org/2000/svg" width="$width" height="20">
data/lib/duel/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Duel
4
- VERSION = "0.1.5"
4
+ VERSION = "0.1.7"
5
5
  end
data/lib/duel/worker.rb CHANGED
@@ -18,10 +18,11 @@ module Duel
18
18
  # closed.)
19
19
  # @api private
20
20
  class Worker
21
- # The built-in metrics, outermost first: allocations inside
22
- # instructions, whose counter allocates as it's read (a sample's own
23
- # allocations are none but the benchmark's), and CPU time innermost.
24
- BUILT_IN = [Metrics::Instructions, Metrics::Allocations, Metrics::CPU].freeze
21
+ # The built-in metrics, outermost first: allocations (CRuby's objects,
22
+ # or JRuby's bytes) inside instructions, whose counter allocates as
23
+ # it's read (a sample's own allocations are none but the benchmark's),
24
+ # and CPU time innermost.
25
+ BUILT_IN = [Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
25
26
 
26
27
  def self.main(version_root, candidate_root, control, config_path = nil)
27
28
  config = config_path ? Duel.load_config(config_path) : (Duel.config = Config.new(candidate_root))
data/sig/duel/across.rbs CHANGED
@@ -5,17 +5,39 @@ module Duel
5
5
  # A timed metric's verdict across the board ("confirmed": slower in a
6
6
  # second measurement too, see Runner#across).
7
7
  type verdict = { "median" => Float, "interval" => Array[Float]?, "limit" => Float, "benchmarks" => Integer,
8
- "verdict" => String, ?"confirmed" => bool }
8
+ ?"processes" => Integer, "verdict" => String, ?"confirmed" => bool }
9
+
10
+ # The fewest benchmarks across the board is judged on by process.
11
+ MIN_BENCHMARKS: Integer
9
12
 
10
13
  @results: Array[Result]
11
14
  @confidence: Float
15
+ @by_process: bool
12
16
 
13
17
  # The across-the-board limit for a metric whose own limit is +limit+.
14
18
  def self.limit: (Float limit) -> Float
15
19
 
16
- def initialize: (Array[Result] results, confidence: Float) -> void
20
+ def initialize: (Array[Result] results, confidence: Float, ?by_process: bool) -> void
17
21
 
18
22
  # Every timed metric any benchmark was judged on, by name.
19
23
  def verdicts: () -> Hash[String, verdict]
24
+
25
+ private
26
+
27
+ # The verdicts of a run that judged benchmarks over workers.
28
+ def by_process_or_benchmark: (Hash[String, Array[Float]] medians, Hash[String, Float] limits) -> Hash[String, verdict]
29
+
30
+ # Each benchmark's workers' medians, of those judged by process, by
31
+ # metric.
32
+ def workers: () -> Hash[String, Array[Result::processes]]
33
+
34
+ # The verdict over every benchmark's median ratio.
35
+ def by_benchmark: (Array[Float] list, Float limit) -> verdict
36
+
37
+ # The verdict over the workers' scores.
38
+ def by_process: (Array[Result::processes] processes, Float limit) -> verdict
39
+
40
+ # Each worker's score, by version.
41
+ def scores: (Array[Result::processes] processes) -> Hash[String, Array[Float]]
20
42
  end
21
43
  end
@@ -9,6 +9,7 @@ module Duel
9
9
  @config: Config
10
10
  @settings: Settings
11
11
  @ref: String?
12
+ @release: String?
12
13
  @prefix: String?
13
14
 
14
15
  def initialize: (Config config, Settings settings) -> void
@@ -17,6 +18,17 @@ module Duel
17
18
  # a push with no commit before it here.
18
19
  def ref: () -> String?
19
20
 
21
+ # The newest release tag before HEAD, or nil.
22
+ def release: () -> String?
23
+
24
+ # The last release, or the commit since then that merged the last
25
+ # pull request accepted with Config#accept_label (Settings#accepted),
26
+ # or nil with no release.
27
+ def last_release: () -> String?
28
+
29
+ # Whether +commit+ is +of+ or an ancestor of it.
30
+ def ancestor?: (String commit, String of) -> bool
31
+
20
32
  # The branch that marks the last commit Duel passed in a push to the
21
33
  # branch pushed.
22
34
  def passed_branch: () -> String
data/sig/duel/cli.rbs CHANGED
@@ -153,6 +153,10 @@ module Duel
153
153
  # The candidate, in Markdown.
154
154
  def described_candidate: (Config config, Settings settings) -> String
155
155
 
156
+ # What chose a baseline of the last release: the release, or a change
157
+ # since it accepted with the label.
158
+ def last_release: (Config config, Baseline baseline) -> String
159
+
156
160
  # The baseline, for JSON: the ref that named it, its commit, and its
157
161
  # tree.
158
162
  def baseline_json: (Config config, Settings settings, Baseline baseline, String root) -> Hash[String, String?]
data/sig/duel/init.rbs CHANGED
@@ -75,6 +75,12 @@ module Duel
75
75
 
76
76
  def load_settings: () -> Config?
77
77
 
78
+ # The settings of the .duel.rb it writes.
79
+ def written_settings: () -> Config
80
+
81
+ # The label that accepts a regression, for the workflows, or nil.
82
+ def accept_label: () -> String?
83
+
78
84
  # Whether the workflow comments on a pushed commit.
79
85
  def comment_on_pushes?: () -> bool
80
86
 
@@ -0,0 +1,30 @@
1
+ module Duel
2
+ # The memory a run's workers can take: what the machine has available,
3
+ # or what its cgroup's limit leaves.
4
+ # @api private
5
+ module Memory
6
+ # The kinds of pages macOS's vm_stat counts as available.
7
+ MAC_PAGES: Array[String]
8
+
9
+ # Where Linux's proc and sys are.
10
+ ROOT: String
11
+
12
+ # The bytes available, or nil where Duel can't tell.
13
+ def self.available: (?root: String, ?vm_stat: ^() -> String?) -> Integer?
14
+
15
+ # MemAvailable, at most what the cgroup leaves, or nil.
16
+ def self.linux: (String root) -> Integer?
17
+
18
+ # What a cgroup (v2) at +dir+ leaves of its limit, or nil.
19
+ def self.cgroup: (String dir) -> Integer?
20
+
21
+ # The bytes in vm_stat's available pages, or nil.
22
+ def self.mac: (String? output) -> Integer?
23
+
24
+ # A file's contents, or nil.
25
+ def self.read: (String path) -> String?
26
+
27
+ # What a command prints, or nil.
28
+ def self.command: (*String args) -> String?
29
+ end
30
+ end
data/sig/duel/metric.rbs CHANGED
@@ -307,8 +307,16 @@ module Duel
307
307
  def self.process: () -> Integer
308
308
  end
309
309
 
310
+ # Bytes allocated, by the benchmark thread or by every thread (JRuby).
311
+ module Allocated
312
+ def self.thread: () -> Integer
313
+
314
+ def self.process: () -> Integer
315
+ end
316
+
310
317
  # The built-in metrics: their measures are named ("cpu",
311
- # "instructions", "allocations"), as Config#measure takes them.
318
+ # "instructions", "allocations", "allocated_bytes"), as Config#measure
319
+ # takes them.
312
320
  # @api private
313
321
  module Metrics
314
322
  # CPU time, in nanoseconds.
@@ -347,5 +355,15 @@ module Duel
347
355
 
348
356
  def stop_sample: () -> Integer
349
357
  end
358
+
359
+ # Bytes allocated (JRuby).
360
+ class AllocatedBytes < Metric
361
+ @scope: :thread | :process
362
+ @started: Integer
363
+
364
+ def start_sample: (Sample sample) -> void
365
+
366
+ def stop_sample: () -> Integer
367
+ end
350
368
  end
351
369
  end
data/sig/duel/rubies.rbs CHANGED
@@ -17,6 +17,19 @@ module Duel
17
17
  # The Ruby that +choice+ gives the version +name+d at +root+.
18
18
  def self.find: (Config::ruby choice, String root, String name) -> String
19
19
 
20
+ # The JVM's options for a JRuby worker.
21
+ JRUBY_WORKER: Array[String]
22
+
23
+ # A JVM option that chooses its garbage collector.
24
+ GARBAGE_COLLECTOR: Regexp
25
+
26
+ # The engine of +ruby+ (RUBY_ENGINE).
27
+ def self.engine: (String ruby) -> String
28
+
29
+ # What a worker on +engine+ starts with: +env+, and on JRuby, the
30
+ # JVM's options for a worker.
31
+ def self.worker_environment: (Hash[String, String?] env, String engine, ?String? inherited) -> Hash[String, String?]
32
+
20
33
  # Whether +ruby+ is the runner's own.
21
34
  def self.runner?: (String ruby) -> bool
22
35