duel 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -1
- data/README.md +5 -5
- data/docs/ci.md +120 -25
- data/docs/configuration.md +8 -4
- data/docs/how-it-works.md +109 -35
- data/docs/json.md +16 -10
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +18 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +19 -3
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +337 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/perf-comment.yml.erb +53 -5
- data/lib/duel/templates/perf.yml.erb +71 -33
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +6 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +112 -6
- data/sig/duel/stats.rbs +25 -0
- metadata +1 -1
data/lib/duel/stats.rb
CHANGED
|
@@ -43,6 +43,82 @@ module Duel
|
|
|
43
43
|
[sorted.fetch(lower - 1), sorted.fetch(upper - 1)]
|
|
44
44
|
end
|
|
45
45
|
|
|
46
|
+
# The 1-based rank r whose r-th smallest and r-th largest cross
|
|
47
|
+
# ratio (see #cross_ratios) bound a +confidence+ interval for the
|
|
48
|
+
# ratio between two versions, from +m+ samples of one and +n+ of the
|
|
49
|
+
# other, or nil when they're too few for one (fewer than 5 each, at
|
|
50
|
+
# 99%). It's distribution-free, as long as each version's samples
|
|
51
|
+
# are independent and alike: the true ratio lies below the r-th
|
|
52
|
+
# smallest with probability P(U <= r - 1), for the Mann-Whitney U
|
|
53
|
+
# statistic (see #u_lower_tail), whatever their distribution. So a
|
|
54
|
+
# sample can be a worker process's median, and then the interval
|
|
55
|
+
# accounts for how a process's luck (where things landed in its
|
|
56
|
+
# memory) differs from another's, as an interval over pairs from a
|
|
57
|
+
# few processes can't.
|
|
58
|
+
def shift_rank(m, n, confidence)
|
|
59
|
+
tail = (1 - confidence) / 2
|
|
60
|
+
r = u_lower_tail(m, n).index { |p| p > tail }
|
|
61
|
+
# (None only where the tail is a half or more, at no confidence.)
|
|
62
|
+
r unless r.nil? || r.zero?
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# P(U <= u) for u from 0 to m * n / 2, where U is the Mann-Whitney
|
|
66
|
+
# statistic of samples of +m+ and +n+ from the same distribution:
|
|
67
|
+
# the pairs (one of each) in which the first's is the larger. Each
|
|
68
|
+
# P(U = u) is found from samples of one fewer (the largest of all is
|
|
69
|
+
# one of the first's, beating all +n+, as often as m / (m + n), or
|
|
70
|
+
# one of the second's), in probabilities, which stay in range where
|
|
71
|
+
# the counts of orderings would grow huge.
|
|
72
|
+
def u_lower_tail(m, n)
|
|
73
|
+
cap = m * n / 2
|
|
74
|
+
sum = 0.0
|
|
75
|
+
u_probabilities(m, n, cap, {}).map { |p| sum += p }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# P(U = u) for u from 0 to +cap+, for samples of +i+ and +j+ (see
|
|
79
|
+
# #u_lower_tail), each found once, in +found+.
|
|
80
|
+
def u_probabilities(i, j, cap, found)
|
|
81
|
+
found[[i, j]] ||= if i.zero? || j.zero?
|
|
82
|
+
Array.new(cap + 1) { |u| u.zero? ? 1.0 : 0.0 } # (no samples of one: U is 0)
|
|
83
|
+
else
|
|
84
|
+
first = u_probabilities(i - 1, j, cap, found)
|
|
85
|
+
second = u_probabilities(i, j - 1, cap, found)
|
|
86
|
+
Array.new(cap + 1) do |u|
|
|
87
|
+
from_first = (u >= j) ? first.fetch(u - j) * i : 0.0
|
|
88
|
+
(from_first + (second.fetch(u) * j)) / (i + j)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# [low, high] bounds of the +confidence+ interval for the ratio of
|
|
94
|
+
# +candidates+ (a version's samples) to +bases+ (the other's), or nil
|
|
95
|
+
# (see #shift_rank).
|
|
96
|
+
def shift_interval(bases, candidates, confidence)
|
|
97
|
+
r = shift_rank(bases.size, candidates.size, confidence)
|
|
98
|
+
return unless r
|
|
99
|
+
|
|
100
|
+
ratios = cross_ratios(bases, candidates)
|
|
101
|
+
[ratios.fetch(r - 1), ratios.fetch(-r)]
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The ratio of +candidates+ to +bases+ the interval is for: the
|
|
105
|
+
# median of the cross ratios (the Hodges-Lehmann estimate).
|
|
106
|
+
def shift(bases, candidates) = median(cross_ratios(bases, candidates))
|
|
107
|
+
|
|
108
|
+
# [low, high] bounds of the +confidence+ interval for how much more
|
|
109
|
+
# +candidates+ (a version's samples) are than +bases+ (the other's),
|
|
110
|
+
# as #shift_interval's are for how many times as much, or nil.
|
|
111
|
+
def difference_interval(bases, candidates, confidence)
|
|
112
|
+
r = shift_rank(bases.size, candidates.size, confidence)
|
|
113
|
+
return unless r
|
|
114
|
+
|
|
115
|
+
differences = candidates.flat_map { |c| bases.map { |b| c - b } }.sort
|
|
116
|
+
[differences.fetch(r - 1), differences.fetch(-r)]
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Each of +candidates+ over each of +bases+, sorted.
|
|
120
|
+
def cross_ratios(bases, candidates) = candidates.flat_map { |c| bases.map { |b| c.fdiv(b) } }.sort
|
|
121
|
+
|
|
46
122
|
def median(samples)
|
|
47
123
|
sorted = samples.sort
|
|
48
124
|
mid = sorted.size / 2
|
data/lib/duel/summary.rb
CHANGED
|
@@ -168,7 +168,7 @@ module Duel
|
|
|
168
168
|
if timed then format("%s %+.1f%%", metric, (timed.fetch("median") - 1) * 100)
|
|
169
169
|
elsif counted
|
|
170
170
|
n = counted.fetch("n")
|
|
171
|
-
|
|
171
|
+
"#{metric} #{Runner.number(counted.fetch("baseline").fdiv(n))} → #{Runner.number(counted.fetch("candidate").fdiv(n))}"
|
|
172
172
|
else
|
|
173
173
|
# (Peak memory, which grew too much.)
|
|
174
174
|
memory = result.memory #: Hash[String, Integer]
|
|
@@ -247,7 +247,7 @@ module Duel
|
|
|
247
247
|
# that grew too much.
|
|
248
248
|
def counts(result)
|
|
249
249
|
counted = Summary.changed_counts(result).map do |metric, (base, candidate)|
|
|
250
|
-
|
|
250
|
+
"#{metric} #{Runner.number(base)} → #{Runner.number(candidate)}"
|
|
251
251
|
end
|
|
252
252
|
if result.more_memory?
|
|
253
253
|
memory = result.memory #: Hash[String, Integer]
|
|
@@ -134,6 +134,9 @@
|
|
|
134
134
|
image: alpine:3
|
|
135
135
|
stage: .post
|
|
136
136
|
needs: [<%= gitlab_job %>]
|
|
137
|
+
# One at a time, as each deletes the branch and makes it again: two at
|
|
138
|
+
# once could both delete it, and the second to make it would fail.
|
|
139
|
+
resource_group: <%= ["duel", working_directory, "passed"].compact.join("/") %>
|
|
137
140
|
rules:
|
|
138
141
|
- if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
139
142
|
variables:
|
|
@@ -10,12 +10,27 @@ name: Perf comment<%= " (#{working_directory})" if working_directory %>
|
|
|
10
10
|
# ran), it trusts only what that job could have said: the pull request's
|
|
11
11
|
# number, if it names the pull request the run compared, and the comment
|
|
12
12
|
# and the status, if they're ones Duel says. A fork's report, which the
|
|
13
|
-
# fork's code wrote, is shown as text.
|
|
14
|
-
|
|
13
|
+
# fork's code wrote, is shown as text.
|
|
14
|
+
<% if accept_label -%>
|
|
15
|
+
#
|
|
16
|
+
# And when the label that accepts a regression (c.accept_label in
|
|
17
|
+
# .duel.rb) is added to a pull request, or taken off, it runs the Duel
|
|
18
|
+
# job again, which reads the labels as it runs: a run that failed, once
|
|
19
|
+
# the label's added, or one that passed, once it's taken off.
|
|
20
|
+
<% end -%>
|
|
21
|
+
#
|
|
22
|
+
# (The workflow must be on the default branch before it runs at all.)
|
|
15
23
|
on:
|
|
16
24
|
workflow_run:
|
|
17
25
|
workflows: ["Perf<%= " (#{working_directory})" if working_directory %>"]
|
|
18
26
|
types: [completed]
|
|
27
|
+
<% if accept_label -%>
|
|
28
|
+
# (This repository's own workflow, as for the comment, with a token
|
|
29
|
+
# that can run a workflow again: it checks nothing out, and runs none
|
|
30
|
+
# of the pull request's code.)
|
|
31
|
+
pull_request_target:
|
|
32
|
+
types: [labeled, unlabeled]
|
|
33
|
+
<% end -%>
|
|
19
34
|
|
|
20
35
|
permissions: {}
|
|
21
36
|
|
|
@@ -56,12 +71,16 @@ jobs:
|
|
|
56
71
|
# ran the fork's code, which could have written anything in it. In
|
|
57
72
|
# a code block, fenced with more backticks than any run of them in
|
|
58
73
|
# the report, it can't mention anyone, link anywhere or show an
|
|
59
|
-
# image.
|
|
74
|
+
# image. It's cut short where a comment couldn't hold it, and loses
|
|
75
|
+
# any NUL bytes, which GitHub wouldn't take, and grep reads it as
|
|
76
|
+
# text whatever's in it, so it finds every run of backticks.
|
|
60
77
|
report() {
|
|
61
78
|
if [ "${head#* }" = "$GITHUB_REPOSITORY" ]; then cat "$RUNNER_TEMP/report/duel.md" && return; fi
|
|
62
|
-
|
|
79
|
+
text="$RUNNER_TEMP/report.txt"
|
|
80
|
+
LC_ALL=C tr -d '\000' < "$RUNNER_TEMP/report/duel.md" | head -c 60000 > "$text"
|
|
81
|
+
fence=$(LC_ALL=C grep -ao '``*' "$text" | awk '{ if (length > n) n = length } END { n = (n < 3) ? 3 : n + 1; while (n-- > 0) printf "`" }')
|
|
63
82
|
printf 'From a fork, so shown as text:\n\n%s\n' "$fence"
|
|
64
|
-
|
|
83
|
+
cat "$text"
|
|
65
84
|
printf '\n%s\n' "$fence"
|
|
66
85
|
}
|
|
67
86
|
marker="<!-- <%= marker %>"
|
|
@@ -73,3 +92,32 @@ jobs:
|
|
|
73
92
|
{ echo "$marker status=$STATUS -->"; report; } > "$RUNNER_TEMP/comment.md"
|
|
74
93
|
gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
|
|
75
94
|
fi
|
|
95
|
+
<% if accept_label -%>
|
|
96
|
+
|
|
97
|
+
again:
|
|
98
|
+
name: Run Duel again
|
|
99
|
+
if: >-
|
|
100
|
+
github.event_name == 'pull_request_target' && github.event.label.name == '<%= accept_label.gsub("'", "''") %>'
|
|
101
|
+
runs-on: ubuntu-latest
|
|
102
|
+
permissions:
|
|
103
|
+
actions: write # to run the Perf workflow's jobs again
|
|
104
|
+
steps:
|
|
105
|
+
- name: Run Duel again
|
|
106
|
+
env:
|
|
107
|
+
GH_TOKEN: ${{ github.token }}
|
|
108
|
+
GH_REPO: ${{ github.repository }}
|
|
109
|
+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
110
|
+
ACTION: ${{ github.event.action }}
|
|
111
|
+
run: |
|
|
112
|
+
# The newest Perf run of the pull request's head commit, once it's
|
|
113
|
+
# done (one that's running may have read the labels already).
|
|
114
|
+
id=$(gh run list --workflow "Perf<%= " (#{working_directory})" if working_directory %>" --event pull_request --commit "$HEAD_SHA" --limit 1 \
|
|
115
|
+
--json databaseId --jq '.[0].databaseId // empty')
|
|
116
|
+
[ -n "$id" ] || exit 0
|
|
117
|
+
gh run watch "$id" --interval 30 > /dev/null || true
|
|
118
|
+
conclusion=$(gh run view "$id" --json conclusion --jq .conclusion)
|
|
119
|
+
case "$ACTION:$conclusion" in
|
|
120
|
+
labeled:failure) gh run rerun "$id" --failed ;;
|
|
121
|
+
unlabeled:success) gh run rerun "$id" ;;
|
|
122
|
+
esac
|
|
123
|
+
<% end -%>
|
|
@@ -47,6 +47,51 @@ jobs:
|
|
|
47
47
|
with:
|
|
48
48
|
fetch-depth: 0 # the baseline comes from history, tags included
|
|
49
49
|
persist-credentials: false
|
|
50
|
+
# The labels of the pull request (or for a push, of the one that
|
|
51
|
+
# merged it), for c.accept_label in .duel.rb, as a JSON array. They're
|
|
52
|
+
# read as the job runs, and adding the label (or taking it off) runs
|
|
53
|
+
# the job again: see the Perf comment workflow.
|
|
54
|
+
<% if accept_label -%>
|
|
55
|
+
# On a schedule, the last commit since the last release that merged a
|
|
56
|
+
# pull request with the label (ACCEPT_LABEL, as .duel.rb names it):
|
|
57
|
+
# what that slowed down was accepted, so it's compared with instead.
|
|
58
|
+
<% end -%>
|
|
59
|
+
- name: Find the pull request's labels
|
|
60
|
+
id: labels
|
|
61
|
+
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %> || github.event_name == 'schedule'<% end %>
|
|
62
|
+
continue-on-error: true # (without them, nothing's accepted)
|
|
63
|
+
env:
|
|
64
|
+
GH_TOKEN: ${{ github.token }}
|
|
65
|
+
PR: ${{ github.event.pull_request.number }}
|
|
66
|
+
QUEUED: ${{ github.event.merge_group.head_ref }}
|
|
67
|
+
<% if accept_label -%>
|
|
68
|
+
ACCEPT_LABEL: '<%= accept_label.gsub("'", "''") %>'
|
|
69
|
+
RELEASE_TAGS: "<%= release_tags %>"
|
|
70
|
+
<% end -%>
|
|
71
|
+
run: |
|
|
72
|
+
<% if accept_label -%>
|
|
73
|
+
if [ "$GITHUB_EVENT_NAME" = schedule ]; then
|
|
74
|
+
tag=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || exit 0
|
|
75
|
+
since=$(TZ=UTC git log -1 --date=format-local:%Y-%m-%dT%H:%M:%SZ --format=%cd "$tag")
|
|
76
|
+
merged=$(gh pr list --repo "$GITHUB_REPOSITORY" --state merged --label "$ACCEPT_LABEL" --search "merged:>=$since" --limit 1000 \
|
|
77
|
+
--json mergeCommit --jq '.[].mergeCommit.oid')
|
|
78
|
+
accepted=$(git rev-list "$tag..HEAD" | grep -Fx -m 1 -f <(echo "$merged")) || accepted=
|
|
79
|
+
echo "accepted=$accepted" >> "$GITHUB_OUTPUT"
|
|
80
|
+
exit 0
|
|
81
|
+
fi
|
|
82
|
+
<% end -%>
|
|
83
|
+
# (In a merge queue, the last pull request queued, which the queue's
|
|
84
|
+
# branch names: gh-readonly-queue/main/pr-123-<sha>.)
|
|
85
|
+
if [ -n "$QUEUED" ]; then
|
|
86
|
+
PR=${QUEUED##*/pr-}
|
|
87
|
+
PR=${PR%%-*}
|
|
88
|
+
fi
|
|
89
|
+
if [ -n "$PR" ]; then
|
|
90
|
+
labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
|
|
91
|
+
else
|
|
92
|
+
labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
|
|
93
|
+
fi
|
|
94
|
+
echo "labels=$labels" >> "$GITHUB_OUTPUT"
|
|
50
95
|
# (And whatever else your tests need set up: packages, services.)
|
|
51
96
|
<% if version_file -%>
|
|
52
97
|
# Each version runs on the Ruby its own .ruby-version names (see
|
|
@@ -57,7 +102,8 @@ jobs:
|
|
|
57
102
|
# request targets (or run by hand, the main branch), a merge queue's
|
|
58
103
|
# base, the commit before a push (or the last commit Duel passed, PASSED, if there is
|
|
59
104
|
# one: see the Mark the commit passed job), or on a schedule the last
|
|
60
|
-
# release, the newest tag before HEAD that RELEASE_TAGS matches
|
|
105
|
+
# release, the newest tag before HEAD that RELEASE_TAGS matches<% if accept_label %>
|
|
106
|
+
# (or ACCEPTED, the last change accepted since then)<% end %>. (If
|
|
61
107
|
# .duel.rb sets c.main_branch or c.release_tags, set MAIN_BRANCH or
|
|
62
108
|
# RELEASE_TAGS to match.)
|
|
63
109
|
- name: Find the baseline's Ruby
|
|
@@ -68,13 +114,16 @@ jobs:
|
|
|
68
114
|
PASSED: origin/<%= passed_branch %>/${{ github.ref_name }}
|
|
69
115
|
MAIN_BRANCH: origin/${{ github.base_ref || github.event.repository.default_branch }}
|
|
70
116
|
RELEASE_TAGS: "<%= release_tags %>"
|
|
117
|
+
<% if accept_label -%>
|
|
118
|
+
ACCEPTED: ${{ steps.labels.outputs.accepted }}
|
|
119
|
+
<% end -%>
|
|
71
120
|
run: |
|
|
72
121
|
case "$GITHUB_EVENT_NAME" in
|
|
73
122
|
push)
|
|
74
123
|
base=$BEFORE
|
|
75
124
|
if git merge-base --is-ancestor "$PASSED" HEAD 2>/dev/null; then base=$PASSED; fi ;;
|
|
76
125
|
merge_group) base=$QUEUE_BASE ;;
|
|
77
|
-
schedule) base=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || base= ;;
|
|
126
|
+
schedule) base=${ACCEPTED:-$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null)} || base= ;;
|
|
78
127
|
*) base=$(git merge-base HEAD "$MAIN_BRANCH" 2>/dev/null) || base= ;;
|
|
79
128
|
esac
|
|
80
129
|
ruby_of() { git show "$1:./.ruby-version" 2>/dev/null | awk 'NF { print $1; exit }'; }
|
|
@@ -113,30 +162,6 @@ jobs:
|
|
|
113
162
|
ruby-version: "<%= ruby_version %>"
|
|
114
163
|
<% end -%>
|
|
115
164
|
bundler-cache: true
|
|
116
|
-
# The labels of the pull request (or for a push, of the one that
|
|
117
|
-
# merged it), for c.accept_label in .duel.rb, as a JSON array. They're
|
|
118
|
-
# read as the job runs, so after adding the label, run it again.
|
|
119
|
-
- name: Find the pull request's labels
|
|
120
|
-
id: labels
|
|
121
|
-
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'
|
|
122
|
-
continue-on-error: true # (without them, nothing's accepted)
|
|
123
|
-
env:
|
|
124
|
-
GH_TOKEN: ${{ github.token }}
|
|
125
|
-
PR: ${{ github.event.pull_request.number }}
|
|
126
|
-
QUEUED: ${{ github.event.merge_group.head_ref }}
|
|
127
|
-
run: |
|
|
128
|
-
# (In a merge queue, the last pull request queued, which the queue's
|
|
129
|
-
# branch names: gh-readonly-queue/main/pr-123-<sha>.)
|
|
130
|
-
if [ -n "$QUEUED" ]; then
|
|
131
|
-
PR=${QUEUED##*/pr-}
|
|
132
|
-
PR=${PR%%-*}
|
|
133
|
-
fi
|
|
134
|
-
if [ -n "$PR" ]; then
|
|
135
|
-
labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
|
|
136
|
-
else
|
|
137
|
-
labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
|
|
138
|
-
fi
|
|
139
|
-
echo "labels=$labels" >> "$GITHUB_OUTPUT"
|
|
140
165
|
# Against the branch it targets for a pull request, the queue's base
|
|
141
166
|
# for a merge queue, what was there before for a push (or as
|
|
142
167
|
# c.push_base in .duel.rb says, the last commit Duel passed), and the
|
|
@@ -152,6 +177,9 @@ jobs:
|
|
|
152
177
|
# with itself, which checks the setup.)
|
|
153
178
|
DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}
|
|
154
179
|
DUEL_LABELS: ${{ steps.labels.outputs.labels }}
|
|
180
|
+
<% if accept_label -%>
|
|
181
|
+
DUEL_ACCEPTED: ${{ steps.labels.outputs.accepted }}
|
|
182
|
+
<% end -%>
|
|
155
183
|
run: bundle exec duel --markdown "$RUNNER_TEMP/duel/duel.md"
|
|
156
184
|
# For a pull request, what Duel said, for the Perf comment workflow,
|
|
157
185
|
# which can't see this job's outputs.
|
|
@@ -224,6 +252,13 @@ jobs:
|
|
|
224
252
|
passed:
|
|
225
253
|
name: Mark the commit passed
|
|
226
254
|
needs: duel
|
|
255
|
+
# One at a time for each branch, so that two runs can't both find no
|
|
256
|
+
# branch and race to make it (a push's run doesn't wait for the one
|
|
257
|
+
# before it). One that's running is never cancelled, and one that's
|
|
258
|
+
# waiting gives way to a newer push's, which moves it further anyway.
|
|
259
|
+
concurrency:
|
|
260
|
+
group: ${{ github.workflow }}-passed-${{ github.ref_name }}
|
|
261
|
+
cancel-in-progress: false
|
|
227
262
|
if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
|
|
228
263
|
runs-on: ubuntu-latest
|
|
229
264
|
permissions:
|
|
@@ -247,13 +282,16 @@ jobs:
|
|
|
247
282
|
done
|
|
248
283
|
exit 1
|
|
249
284
|
}
|
|
250
|
-
# (Forward: GitHub moves it only where it's behind the push.
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
gh api --method
|
|
255
|
-
|
|
256
|
-
|
|
285
|
+
# (Forward: GitHub moves it only where it's behind the push. Where
|
|
286
|
+
# there's no branch, and making it fails, it looks once more, as
|
|
287
|
+
# something else may have made it: a push of it, say.)
|
|
288
|
+
for _ in 1 2; do
|
|
289
|
+
gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null 2>&1 && exit 0
|
|
290
|
+
at=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$BRANCH" --jq .object.sha 2>/dev/null) || at=
|
|
291
|
+
[ -n "$at" ] && break
|
|
292
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null && exit 0
|
|
293
|
+
done
|
|
294
|
+
[ -n "$at" ] || blocked
|
|
257
295
|
compared=$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$at" --jq .status 2>&1) || case "$compared" in
|
|
258
296
|
# (GitHub can't compare commits with no history in common.)
|
|
259
297
|
*"No common ancestor"*) compared=unrelated ;;
|
data/lib/duel/version.rb
CHANGED
data/lib/duel/worker.rb
CHANGED
|
@@ -18,10 +18,11 @@ module Duel
|
|
|
18
18
|
# closed.)
|
|
19
19
|
# @api private
|
|
20
20
|
class Worker
|
|
21
|
-
# The built-in metrics, outermost first: allocations
|
|
22
|
-
# instructions, whose counter allocates as
|
|
23
|
-
# allocations are none but the benchmark's),
|
|
24
|
-
|
|
21
|
+
# The built-in metrics, outermost first: allocations (CRuby's objects,
|
|
22
|
+
# or JRuby's bytes) inside instructions, whose counter allocates as
|
|
23
|
+
# it's read (a sample's own allocations are none but the benchmark's),
|
|
24
|
+
# and CPU time innermost.
|
|
25
|
+
BUILT_IN = [Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
|
|
25
26
|
|
|
26
27
|
def self.main(version_root, candidate_root, control, config_path = nil)
|
|
27
28
|
config = config_path ? Duel.load_config(config_path) : (Duel.config = Config.new(candidate_root))
|
data/sig/duel/across.rbs
CHANGED
|
@@ -5,17 +5,39 @@ module Duel
|
|
|
5
5
|
# A timed metric's verdict across the board ("confirmed": slower in a
|
|
6
6
|
# second measurement too, see Runner#across).
|
|
7
7
|
type verdict = { "median" => Float, "interval" => Array[Float]?, "limit" => Float, "benchmarks" => Integer,
|
|
8
|
-
"verdict" => String, ?"confirmed" => bool }
|
|
8
|
+
?"processes" => Integer, "verdict" => String, ?"confirmed" => bool }
|
|
9
|
+
|
|
10
|
+
# The fewest benchmarks across the board is judged on by process.
|
|
11
|
+
MIN_BENCHMARKS: Integer
|
|
9
12
|
|
|
10
13
|
@results: Array[Result]
|
|
11
14
|
@confidence: Float
|
|
15
|
+
@by_process: bool
|
|
12
16
|
|
|
13
17
|
# The across-the-board limit for a metric whose own limit is +limit+.
|
|
14
18
|
def self.limit: (Float limit) -> Float
|
|
15
19
|
|
|
16
|
-
def initialize: (Array[Result] results, confidence: Float) -> void
|
|
20
|
+
def initialize: (Array[Result] results, confidence: Float, ?by_process: bool) -> void
|
|
17
21
|
|
|
18
22
|
# Every timed metric any benchmark was judged on, by name.
|
|
19
23
|
def verdicts: () -> Hash[String, verdict]
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
# The verdicts of a run that judged benchmarks over workers.
|
|
28
|
+
def by_process_or_benchmark: (Hash[String, Array[Float]] medians, Hash[String, Float] limits) -> Hash[String, verdict]
|
|
29
|
+
|
|
30
|
+
# Each benchmark's workers' medians, of those judged by process, by
|
|
31
|
+
# metric.
|
|
32
|
+
def workers: () -> Hash[String, Array[Result::processes]]
|
|
33
|
+
|
|
34
|
+
# The verdict over every benchmark's median ratio.
|
|
35
|
+
def by_benchmark: (Array[Float] list, Float limit) -> verdict
|
|
36
|
+
|
|
37
|
+
# The verdict over the workers' scores.
|
|
38
|
+
def by_process: (Array[Result::processes] processes, Float limit) -> verdict
|
|
39
|
+
|
|
40
|
+
# Each worker's score, by version.
|
|
41
|
+
def scores: (Array[Result::processes] processes) -> Hash[String, Array[Float]]
|
|
20
42
|
end
|
|
21
43
|
end
|
data/sig/duel/baseline.rbs
CHANGED
|
@@ -9,6 +9,7 @@ module Duel
|
|
|
9
9
|
@config: Config
|
|
10
10
|
@settings: Settings
|
|
11
11
|
@ref: String?
|
|
12
|
+
@release: String?
|
|
12
13
|
@prefix: String?
|
|
13
14
|
|
|
14
15
|
def initialize: (Config config, Settings settings) -> void
|
|
@@ -17,6 +18,17 @@ module Duel
|
|
|
17
18
|
# a push with no commit before it here.
|
|
18
19
|
def ref: () -> String?
|
|
19
20
|
|
|
21
|
+
# The newest release tag before HEAD, or nil.
|
|
22
|
+
def release: () -> String?
|
|
23
|
+
|
|
24
|
+
# The last release, or the commit since then that merged the last
|
|
25
|
+
# pull request accepted with Config#accept_label (Settings#accepted),
|
|
26
|
+
# or nil with no release.
|
|
27
|
+
def last_release: () -> String?
|
|
28
|
+
|
|
29
|
+
# Whether +commit+ is +of+ or an ancestor of it.
|
|
30
|
+
def ancestor?: (String commit, String of) -> bool
|
|
31
|
+
|
|
20
32
|
# The branch that marks the last commit Duel passed in a push to the
|
|
21
33
|
# branch pushed.
|
|
22
34
|
def passed_branch: () -> String
|
data/sig/duel/cli.rbs
CHANGED
|
@@ -153,6 +153,10 @@ module Duel
|
|
|
153
153
|
# The candidate, in Markdown.
|
|
154
154
|
def described_candidate: (Config config, Settings settings) -> String
|
|
155
155
|
|
|
156
|
+
# What chose a baseline of the last release: the release, or a change
|
|
157
|
+
# since it accepted with the label.
|
|
158
|
+
def last_release: (Config config, Baseline baseline) -> String
|
|
159
|
+
|
|
156
160
|
# The baseline, for JSON: the ref that named it, its commit, and its
|
|
157
161
|
# tree.
|
|
158
162
|
def baseline_json: (Config config, Settings settings, Baseline baseline, String root) -> Hash[String, String?]
|
data/sig/duel/init.rbs
CHANGED
|
@@ -75,6 +75,12 @@ module Duel
|
|
|
75
75
|
|
|
76
76
|
def load_settings: () -> Config?
|
|
77
77
|
|
|
78
|
+
# The settings of the .duel.rb it writes.
|
|
79
|
+
def written_settings: () -> Config
|
|
80
|
+
|
|
81
|
+
# The label that accepts a regression, for the workflows, or nil.
|
|
82
|
+
def accept_label: () -> String?
|
|
83
|
+
|
|
78
84
|
# Whether the workflow comments on a pushed commit.
|
|
79
85
|
def comment_on_pushes?: () -> bool
|
|
80
86
|
|
data/sig/duel/metric.rbs
CHANGED
|
@@ -307,8 +307,16 @@ module Duel
|
|
|
307
307
|
def self.process: () -> Integer
|
|
308
308
|
end
|
|
309
309
|
|
|
310
|
+
# Bytes allocated, by the benchmark thread or by every thread (JRuby).
|
|
311
|
+
module Allocated
|
|
312
|
+
def self.thread: () -> Integer
|
|
313
|
+
|
|
314
|
+
def self.process: () -> Integer
|
|
315
|
+
end
|
|
316
|
+
|
|
310
317
|
# The built-in metrics: their measures are named ("cpu",
|
|
311
|
-
# "instructions", "allocations"), as Config#measure
|
|
318
|
+
# "instructions", "allocations", "allocated_bytes"), as Config#measure
|
|
319
|
+
# takes them.
|
|
312
320
|
# @api private
|
|
313
321
|
module Metrics
|
|
314
322
|
# CPU time, in nanoseconds.
|
|
@@ -347,5 +355,15 @@ module Duel
|
|
|
347
355
|
|
|
348
356
|
def stop_sample: () -> Integer
|
|
349
357
|
end
|
|
358
|
+
|
|
359
|
+
# Bytes allocated (JRuby).
|
|
360
|
+
class AllocatedBytes < Metric
|
|
361
|
+
@scope: :thread | :process
|
|
362
|
+
@started: Integer
|
|
363
|
+
|
|
364
|
+
def start_sample: (Sample sample) -> void
|
|
365
|
+
|
|
366
|
+
def stop_sample: () -> Integer
|
|
367
|
+
end
|
|
350
368
|
end
|
|
351
369
|
end
|
data/sig/duel/rubies.rbs
CHANGED
|
@@ -17,6 +17,19 @@ module Duel
|
|
|
17
17
|
# The Ruby that +choice+ gives the version +name+d at +root+.
|
|
18
18
|
def self.find: (Config::ruby choice, String root, String name) -> String
|
|
19
19
|
|
|
20
|
+
# The JVM's options for a JRuby worker.
|
|
21
|
+
JRUBY_WORKER: Array[String]
|
|
22
|
+
|
|
23
|
+
# A JVM option that chooses its garbage collector.
|
|
24
|
+
GARBAGE_COLLECTOR: Regexp
|
|
25
|
+
|
|
26
|
+
# The engine of +ruby+ (RUBY_ENGINE).
|
|
27
|
+
def self.engine: (String ruby) -> String
|
|
28
|
+
|
|
29
|
+
# What a worker on +engine+ starts with: +env+, and on JRuby, the
|
|
30
|
+
# JVM's options for a worker.
|
|
31
|
+
def self.worker_environment: (Hash[String, String?] env, String engine, ?String? inherited) -> Hash[String, String?]
|
|
32
|
+
|
|
20
33
|
# Whether +ruby+ is the runner's own.
|
|
21
34
|
def self.runner?: (String ruby) -> bool
|
|
22
35
|
|