duel 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +68 -31
- data/CHANGELOG.md +158 -1
- data/README.md +16 -13
- data/docs/README.md +1 -1
- data/docs/benchmarks.md +32 -3
- data/docs/ci.md +365 -167
- data/docs/configuration.md +11 -4
- data/docs/how-it-works.md +109 -35
- data/docs/json.md +16 -10
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +18 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +55 -1
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rake_task.rb +10 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +337 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/duel.rb.erb +7 -3
- data/lib/duel/templates/perf-comment.yml.erb +68 -4
- data/lib/duel/templates/perf.yml.erb +136 -38
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +23 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rake_task.rbs +3 -0
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +113 -7
- data/sig/duel/stats.rbs +25 -0
- metadata +1 -1
data/lib/duel/stats.rb
CHANGED
|
@@ -43,6 +43,82 @@ module Duel
|
|
|
43
43
|
[sorted.fetch(lower - 1), sorted.fetch(upper - 1)]
|
|
44
44
|
end
|
|
45
45
|
|
|
46
|
+
# The 1-based rank r whose r-th smallest and r-th largest cross
|
|
47
|
+
# ratio (see #cross_ratios) bound a +confidence+ interval for the
|
|
48
|
+
# ratio between two versions, from +m+ samples of one and +n+ of the
|
|
49
|
+
# other, or nil when they're too few for one (fewer than 5 each, at
|
|
50
|
+
# 99%). It's distribution-free, as long as each version's samples
|
|
51
|
+
# are independent and alike: the true ratio lies below the r-th
|
|
52
|
+
# smallest with probability P(U <= r - 1), for the Mann-Whitney U
|
|
53
|
+
# statistic (see #u_lower_tail), whatever their distribution. So a
|
|
54
|
+
# sample can be a worker process's median, and then the interval
|
|
55
|
+
# accounts for how a process's luck (where things landed in its
|
|
56
|
+
# memory) differs from another's, as an interval over pairs from a
|
|
57
|
+
# few processes can't.
|
|
58
|
+
def shift_rank(m, n, confidence)
|
|
59
|
+
tail = (1 - confidence) / 2
|
|
60
|
+
r = u_lower_tail(m, n).index { |p| p > tail }
|
|
61
|
+
# (None only where the tail is a half or more, at no confidence.)
|
|
62
|
+
r unless r.nil? || r.zero?
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# P(U <= u) for u from 0 to m * n / 2, where U is the Mann-Whitney
|
|
66
|
+
# statistic of samples of +m+ and +n+ from the same distribution:
|
|
67
|
+
# the pairs (one of each) in which the first's is the larger. Each
|
|
68
|
+
# P(U = u) is found from samples of one fewer (the largest of all is
|
|
69
|
+
# one of the first's, beating all +n+, as often as m / (m + n), or
|
|
70
|
+
# one of the second's), in probabilities, which stay in range where
|
|
71
|
+
# the counts of orderings would grow huge.
|
|
72
|
+
def u_lower_tail(m, n)
|
|
73
|
+
cap = m * n / 2
|
|
74
|
+
sum = 0.0
|
|
75
|
+
u_probabilities(m, n, cap, {}).map { |p| sum += p }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# P(U = u) for u from 0 to +cap+, for samples of +i+ and +j+ (see
|
|
79
|
+
# #u_lower_tail), each found once, in +found+.
|
|
80
|
+
def u_probabilities(i, j, cap, found)
|
|
81
|
+
found[[i, j]] ||= if i.zero? || j.zero?
|
|
82
|
+
Array.new(cap + 1) { |u| u.zero? ? 1.0 : 0.0 } # (no samples of one: U is 0)
|
|
83
|
+
else
|
|
84
|
+
first = u_probabilities(i - 1, j, cap, found)
|
|
85
|
+
second = u_probabilities(i, j - 1, cap, found)
|
|
86
|
+
Array.new(cap + 1) do |u|
|
|
87
|
+
from_first = (u >= j) ? first.fetch(u - j) * i : 0.0
|
|
88
|
+
(from_first + (second.fetch(u) * j)) / (i + j)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# [low, high] bounds of the +confidence+ interval for the ratio of
|
|
94
|
+
# +candidates+ (a version's samples) to +bases+ (the other's), or nil
|
|
95
|
+
# (see #shift_rank).
|
|
96
|
+
def shift_interval(bases, candidates, confidence)
|
|
97
|
+
r = shift_rank(bases.size, candidates.size, confidence)
|
|
98
|
+
return unless r
|
|
99
|
+
|
|
100
|
+
ratios = cross_ratios(bases, candidates)
|
|
101
|
+
[ratios.fetch(r - 1), ratios.fetch(-r)]
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The ratio of +candidates+ to +bases+ the interval is for: the
|
|
105
|
+
# median of the cross ratios (the Hodges-Lehmann estimate).
|
|
106
|
+
def shift(bases, candidates) = median(cross_ratios(bases, candidates))
|
|
107
|
+
|
|
108
|
+
# [low, high] bounds of the +confidence+ interval for how much more
|
|
109
|
+
# +candidates+ (a version's samples) are than +bases+ (the other's),
|
|
110
|
+
# as #shift_interval's are for how many times as much, or nil.
|
|
111
|
+
def difference_interval(bases, candidates, confidence)
|
|
112
|
+
r = shift_rank(bases.size, candidates.size, confidence)
|
|
113
|
+
return unless r
|
|
114
|
+
|
|
115
|
+
differences = candidates.flat_map { |c| bases.map { |b| c - b } }.sort
|
|
116
|
+
[differences.fetch(r - 1), differences.fetch(-r)]
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Each of +candidates+ over each of +bases+, sorted.
|
|
120
|
+
def cross_ratios(bases, candidates) = candidates.flat_map { |c| bases.map { |b| c.fdiv(b) } }.sort
|
|
121
|
+
|
|
46
122
|
def median(samples)
|
|
47
123
|
sorted = samples.sort
|
|
48
124
|
mid = sorted.size / 2
|
data/lib/duel/summary.rb
CHANGED
|
@@ -168,7 +168,7 @@ module Duel
|
|
|
168
168
|
if timed then format("%s %+.1f%%", metric, (timed.fetch("median") - 1) * 100)
|
|
169
169
|
elsif counted
|
|
170
170
|
n = counted.fetch("n")
|
|
171
|
-
|
|
171
|
+
"#{metric} #{Runner.number(counted.fetch("baseline").fdiv(n))} → #{Runner.number(counted.fetch("candidate").fdiv(n))}"
|
|
172
172
|
else
|
|
173
173
|
# (Peak memory, which grew too much.)
|
|
174
174
|
memory = result.memory #: Hash[String, Integer]
|
|
@@ -247,7 +247,7 @@ module Duel
|
|
|
247
247
|
# that grew too much.
|
|
248
248
|
def counts(result)
|
|
249
249
|
counted = Summary.changed_counts(result).map do |metric, (base, candidate)|
|
|
250
|
-
|
|
250
|
+
"#{metric} #{Runner.number(base)} → #{Runner.number(candidate)}"
|
|
251
251
|
end
|
|
252
252
|
if result.more_memory?
|
|
253
253
|
memory = result.memory #: Hash[String, Integer]
|
|
@@ -134,6 +134,9 @@
|
|
|
134
134
|
image: alpine:3
|
|
135
135
|
stage: .post
|
|
136
136
|
needs: [<%= gitlab_job %>]
|
|
137
|
+
# One at a time, as each deletes the branch and makes it again: two at
|
|
138
|
+
# once could both delete it, and the second to make it would fail.
|
|
139
|
+
resource_group: <%= ["duel", working_directory, "passed"].compact.join("/") %>
|
|
137
140
|
rules:
|
|
138
141
|
- if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
139
142
|
variables:
|
|
@@ -40,14 +40,18 @@ Duel.configure do |c|
|
|
|
40
40
|
|
|
41
41
|
# Which runs a regression fails: :every_run, or :pull_requests, for a
|
|
42
42
|
# project where people push to main directly (on GitHub Actions, another
|
|
43
|
-
# run warns and passes, and a pushed commit gets the comment instead
|
|
43
|
+
# run warns and passes, and a pushed commit gets the comment instead,
|
|
44
|
+
# from a workflow job written only where this is set: see docs/ci.md).
|
|
44
45
|
# c.fail_on = :every_run
|
|
45
46
|
|
|
46
|
-
# A label that accepts a regression in a pull request
|
|
47
|
-
#
|
|
47
|
+
# A label that accepts a regression in a pull request, for a change
|
|
48
|
+
# that's slower on purpose (nil for none).
|
|
49
|
+
c.accept_label = "duel: accept"
|
|
48
50
|
|
|
49
51
|
# What the workflow compares a push with: :before (the commit before it),
|
|
50
52
|
# or :last_passed (the last commit Duel passed, which a branch,
|
|
51
53
|
# duel/passed/main, marks), so small slowdowns add up until they fail.
|
|
54
|
+
# (A workflow job moves the branch, written only where this is set: see
|
|
55
|
+
# docs/ci.md.)
|
|
52
56
|
# c.push_base = :before
|
|
53
57
|
end
|
|
@@ -9,12 +9,28 @@ name: Perf comment<%= " (#{working_directory})" if working_directory %>
|
|
|
9
9
|
# the pull request's code, and of what the Duel job kept (where that code
|
|
10
10
|
# ran), it trusts only what that job could have said: the pull request's
|
|
11
11
|
# number, if it names the pull request the run compared, and the comment
|
|
12
|
-
# and the status, if they're ones Duel says.
|
|
13
|
-
#
|
|
12
|
+
# and the status, if they're ones Duel says. A fork's report, which the
|
|
13
|
+
# fork's code wrote, is shown as text.
|
|
14
|
+
<% if accept_label -%>
|
|
15
|
+
#
|
|
16
|
+
# And when the label that accepts a regression (c.accept_label in
|
|
17
|
+
# .duel.rb) is added to a pull request, or taken off, it runs the Duel
|
|
18
|
+
# job again, which reads the labels as it runs: a run that failed, once
|
|
19
|
+
# the label's added, or one that passed, once it's taken off.
|
|
20
|
+
<% end -%>
|
|
21
|
+
#
|
|
22
|
+
# (The workflow must be on the default branch before it runs at all.)
|
|
14
23
|
on:
|
|
15
24
|
workflow_run:
|
|
16
25
|
workflows: ["Perf<%= " (#{working_directory})" if working_directory %>"]
|
|
17
26
|
types: [completed]
|
|
27
|
+
<% if accept_label -%>
|
|
28
|
+
# (This repository's own workflow, as for the comment, with a token
|
|
29
|
+
# that can run a workflow again: it checks nothing out, and runs none
|
|
30
|
+
# of the pull request's code.)
|
|
31
|
+
pull_request_target:
|
|
32
|
+
types: [labeled, unlabeled]
|
|
33
|
+
<% end -%>
|
|
18
34
|
|
|
19
35
|
permissions: {}
|
|
20
36
|
|
|
@@ -47,13 +63,61 @@ jobs:
|
|
|
47
63
|
COMMENT=$(said comment 'post|change')
|
|
48
64
|
STATUS=$(said status 'passed|failed|neutral')
|
|
49
65
|
[ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
|
|
50
|
-
|
|
66
|
+
# (The pull request's head commit, and the repository it's from.)
|
|
67
|
+
head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq '"\(.head.sha) \(.head.repo.full_name)"')
|
|
68
|
+
[ "${head%% *}" = "$HEAD_SHA" ] || exit 0
|
|
69
|
+
# The report, as Markdown for a branch of this repository (whose
|
|
70
|
+
# author can push here anyway), but for a fork's, as text: its run
|
|
71
|
+
# ran the fork's code, which could have written anything in it. In
|
|
72
|
+
# a code block, fenced with more backticks than any run of them in
|
|
73
|
+
# the report, it can't mention anyone, link anywhere or show an
|
|
74
|
+
# image. It's cut short where a comment couldn't hold it, and loses
|
|
75
|
+
# any NUL bytes, which GitHub wouldn't take, and grep reads it as
|
|
76
|
+
# text whatever's in it, so it finds every run of backticks.
|
|
77
|
+
report() {
|
|
78
|
+
if [ "${head#* }" = "$GITHUB_REPOSITORY" ]; then cat "$RUNNER_TEMP/report/duel.md" && return; fi
|
|
79
|
+
text="$RUNNER_TEMP/report.txt"
|
|
80
|
+
LC_ALL=C tr -d '\000' < "$RUNNER_TEMP/report/duel.md" | head -c 60000 > "$text"
|
|
81
|
+
fence=$(LC_ALL=C grep -ao '``*' "$text" | awk '{ if (length > n) n = length } END { n = (n < 3) ? 3 : n + 1; while (n-- > 0) printf "`" }')
|
|
82
|
+
printf 'From a fork, so shown as text:\n\n%s\n' "$fence"
|
|
83
|
+
cat "$text"
|
|
84
|
+
printf '\n%s\n' "$fence"
|
|
85
|
+
}
|
|
51
86
|
marker="<!-- <%= marker %>"
|
|
52
87
|
# The first line of the last comment Duel posted here: its marker,
|
|
53
88
|
# with its status (none in Duel 0.1.0's, which counts as another).
|
|
54
89
|
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
|
|
55
90
|
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
|
|
56
91
|
if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
|
|
57
|
-
{ echo "$marker status=$STATUS -->";
|
|
92
|
+
{ echo "$marker status=$STATUS -->"; report; } > "$RUNNER_TEMP/comment.md"
|
|
58
93
|
gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
|
|
59
94
|
fi
|
|
95
|
+
<% if accept_label -%>
|
|
96
|
+
|
|
97
|
+
again:
|
|
98
|
+
name: Run Duel again
|
|
99
|
+
if: >-
|
|
100
|
+
github.event_name == 'pull_request_target' && github.event.label.name == '<%= accept_label.gsub("'", "''") %>'
|
|
101
|
+
runs-on: ubuntu-latest
|
|
102
|
+
permissions:
|
|
103
|
+
actions: write # to run the Perf workflow's jobs again
|
|
104
|
+
steps:
|
|
105
|
+
- name: Run Duel again
|
|
106
|
+
env:
|
|
107
|
+
GH_TOKEN: ${{ github.token }}
|
|
108
|
+
GH_REPO: ${{ github.repository }}
|
|
109
|
+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
110
|
+
ACTION: ${{ github.event.action }}
|
|
111
|
+
run: |
|
|
112
|
+
# The newest Perf run of the pull request's head commit, once it's
|
|
113
|
+
# done (one that's running may have read the labels already).
|
|
114
|
+
id=$(gh run list --workflow "Perf<%= " (#{working_directory})" if working_directory %>" --event pull_request --commit "$HEAD_SHA" --limit 1 \
|
|
115
|
+
--json databaseId --jq '.[0].databaseId // empty')
|
|
116
|
+
[ -n "$id" ] || exit 0
|
|
117
|
+
gh run watch "$id" --interval 30 > /dev/null || true
|
|
118
|
+
conclusion=$(gh run view "$id" --json conclusion --jq .conclusion)
|
|
119
|
+
case "$ACTION:$conclusion" in
|
|
120
|
+
labeled:failure) gh run rerun "$id" --failed ;;
|
|
121
|
+
unlabeled:success) gh run rerun "$id" ;;
|
|
122
|
+
esac
|
|
123
|
+
<% end -%>
|
|
@@ -47,6 +47,51 @@ jobs:
|
|
|
47
47
|
with:
|
|
48
48
|
fetch-depth: 0 # the baseline comes from history, tags included
|
|
49
49
|
persist-credentials: false
|
|
50
|
+
# The labels of the pull request (or for a push, of the one that
|
|
51
|
+
# merged it), for c.accept_label in .duel.rb, as a JSON array. They're
|
|
52
|
+
# read as the job runs, and adding the label (or taking it off) runs
|
|
53
|
+
# the job again: see the Perf comment workflow.
|
|
54
|
+
<% if accept_label -%>
|
|
55
|
+
# On a schedule, the last commit since the last release that merged a
|
|
56
|
+
# pull request with the label (ACCEPT_LABEL, as .duel.rb names it):
|
|
57
|
+
# what that slowed down was accepted, so it's compared with instead.
|
|
58
|
+
<% end -%>
|
|
59
|
+
- name: Find the pull request's labels
|
|
60
|
+
id: labels
|
|
61
|
+
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %> || github.event_name == 'schedule'<% end %>
|
|
62
|
+
continue-on-error: true # (without them, nothing's accepted)
|
|
63
|
+
env:
|
|
64
|
+
GH_TOKEN: ${{ github.token }}
|
|
65
|
+
PR: ${{ github.event.pull_request.number }}
|
|
66
|
+
QUEUED: ${{ github.event.merge_group.head_ref }}
|
|
67
|
+
<% if accept_label -%>
|
|
68
|
+
ACCEPT_LABEL: '<%= accept_label.gsub("'", "''") %>'
|
|
69
|
+
RELEASE_TAGS: "<%= release_tags %>"
|
|
70
|
+
<% end -%>
|
|
71
|
+
run: |
|
|
72
|
+
<% if accept_label -%>
|
|
73
|
+
if [ "$GITHUB_EVENT_NAME" = schedule ]; then
|
|
74
|
+
tag=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || exit 0
|
|
75
|
+
since=$(TZ=UTC git log -1 --date=format-local:%Y-%m-%dT%H:%M:%SZ --format=%cd "$tag")
|
|
76
|
+
merged=$(gh pr list --repo "$GITHUB_REPOSITORY" --state merged --label "$ACCEPT_LABEL" --search "merged:>=$since" --limit 1000 \
|
|
77
|
+
--json mergeCommit --jq '.[].mergeCommit.oid')
|
|
78
|
+
accepted=$(git rev-list "$tag..HEAD" | grep -Fx -m 1 -f <(echo "$merged")) || accepted=
|
|
79
|
+
echo "accepted=$accepted" >> "$GITHUB_OUTPUT"
|
|
80
|
+
exit 0
|
|
81
|
+
fi
|
|
82
|
+
<% end -%>
|
|
83
|
+
# (In a merge queue, the last pull request queued, which the queue's
|
|
84
|
+
# branch names: gh-readonly-queue/main/pr-123-<sha>.)
|
|
85
|
+
if [ -n "$QUEUED" ]; then
|
|
86
|
+
PR=${QUEUED##*/pr-}
|
|
87
|
+
PR=${PR%%-*}
|
|
88
|
+
fi
|
|
89
|
+
if [ -n "$PR" ]; then
|
|
90
|
+
labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
|
|
91
|
+
else
|
|
92
|
+
labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
|
|
93
|
+
fi
|
|
94
|
+
echo "labels=$labels" >> "$GITHUB_OUTPUT"
|
|
50
95
|
# (And whatever else your tests need set up: packages, services.)
|
|
51
96
|
<% if version_file -%>
|
|
52
97
|
# Each version runs on the Ruby its own .ruby-version names (see
|
|
@@ -57,7 +102,8 @@ jobs:
|
|
|
57
102
|
# request targets (or run by hand, the main branch), a merge queue's
|
|
58
103
|
# base, the commit before a push (or the last commit Duel passed, PASSED, if there is
|
|
59
104
|
# one: see the Mark the commit passed job), or on a schedule the last
|
|
60
|
-
# release, the newest tag before HEAD that RELEASE_TAGS matches
|
|
105
|
+
# release, the newest tag before HEAD that RELEASE_TAGS matches<% if accept_label %>
|
|
106
|
+
# (or ACCEPTED, the last change accepted since then)<% end %>. (If
|
|
61
107
|
# .duel.rb sets c.main_branch or c.release_tags, set MAIN_BRANCH or
|
|
62
108
|
# RELEASE_TAGS to match.)
|
|
63
109
|
- name: Find the baseline's Ruby
|
|
@@ -68,13 +114,16 @@ jobs:
|
|
|
68
114
|
PASSED: origin/<%= passed_branch %>/${{ github.ref_name }}
|
|
69
115
|
MAIN_BRANCH: origin/${{ github.base_ref || github.event.repository.default_branch }}
|
|
70
116
|
RELEASE_TAGS: "<%= release_tags %>"
|
|
117
|
+
<% if accept_label -%>
|
|
118
|
+
ACCEPTED: ${{ steps.labels.outputs.accepted }}
|
|
119
|
+
<% end -%>
|
|
71
120
|
run: |
|
|
72
121
|
case "$GITHUB_EVENT_NAME" in
|
|
73
122
|
push)
|
|
74
123
|
base=$BEFORE
|
|
75
124
|
if git merge-base --is-ancestor "$PASSED" HEAD 2>/dev/null; then base=$PASSED; fi ;;
|
|
76
125
|
merge_group) base=$QUEUE_BASE ;;
|
|
77
|
-
schedule) base=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || base= ;;
|
|
126
|
+
schedule) base=${ACCEPTED:-$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null)} || base= ;;
|
|
78
127
|
*) base=$(git merge-base HEAD "$MAIN_BRANCH" 2>/dev/null) || base= ;;
|
|
79
128
|
esac
|
|
80
129
|
ruby_of() { git show "$1:./.ruby-version" 2>/dev/null | awk 'NF { print $1; exit }'; }
|
|
@@ -113,30 +162,6 @@ jobs:
|
|
|
113
162
|
ruby-version: "<%= ruby_version %>"
|
|
114
163
|
<% end -%>
|
|
115
164
|
bundler-cache: true
|
|
116
|
-
# The labels of the pull request (or for a push, of the one that
|
|
117
|
-
# merged it), for c.accept_label in .duel.rb, as a JSON array. They're
|
|
118
|
-
# read as the job runs, so after adding the label, run it again.
|
|
119
|
-
- name: Find the pull request's labels
|
|
120
|
-
id: labels
|
|
121
|
-
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'
|
|
122
|
-
continue-on-error: true # (without them, nothing's accepted)
|
|
123
|
-
env:
|
|
124
|
-
GH_TOKEN: ${{ github.token }}
|
|
125
|
-
PR: ${{ github.event.pull_request.number }}
|
|
126
|
-
QUEUED: ${{ github.event.merge_group.head_ref }}
|
|
127
|
-
run: |
|
|
128
|
-
# (In a merge queue, the last pull request queued, which the queue's
|
|
129
|
-
# branch names: gh-readonly-queue/main/pr-123-<sha>.)
|
|
130
|
-
if [ -n "$QUEUED" ]; then
|
|
131
|
-
PR=${QUEUED##*/pr-}
|
|
132
|
-
PR=${PR%%-*}
|
|
133
|
-
fi
|
|
134
|
-
if [ -n "$PR" ]; then
|
|
135
|
-
labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
|
|
136
|
-
else
|
|
137
|
-
labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
|
|
138
|
-
fi
|
|
139
|
-
echo "labels=$labels" >> "$GITHUB_OUTPUT"
|
|
140
165
|
# Against the branch it targets for a pull request, the queue's base
|
|
141
166
|
# for a merge queue, what was there before for a push (or as
|
|
142
167
|
# c.push_base in .duel.rb says, the last commit Duel passed), and the
|
|
@@ -152,6 +177,9 @@ jobs:
|
|
|
152
177
|
# with itself, which checks the setup.)
|
|
153
178
|
DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}
|
|
154
179
|
DUEL_LABELS: ${{ steps.labels.outputs.labels }}
|
|
180
|
+
<% if accept_label -%>
|
|
181
|
+
DUEL_ACCEPTED: ${{ steps.labels.outputs.accepted }}
|
|
182
|
+
<% end -%>
|
|
155
183
|
run: bundle exec duel --markdown "$RUNNER_TEMP/duel/duel.md"
|
|
156
184
|
# For a pull request, what Duel said, for the Perf comment workflow,
|
|
157
185
|
# which can't see this job's outputs.
|
|
@@ -173,6 +201,7 @@ jobs:
|
|
|
173
201
|
path: ${{ runner.temp }}/duel/
|
|
174
202
|
if-no-files-found: ignore # none when Duel didn't get to compare
|
|
175
203
|
retention-days: 1
|
|
204
|
+
<% if comment_on_pushes? -%>
|
|
176
205
|
|
|
177
206
|
# The report, as a new comment on a pushed commit, which notifies its
|
|
178
207
|
# author, when Duel says to, as for a pull request (see the Perf comment
|
|
@@ -209,6 +238,8 @@ jobs:
|
|
|
209
238
|
{ echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
|
|
210
239
|
gh api --method POST "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
|
|
211
240
|
fi
|
|
241
|
+
<% end -%>
|
|
242
|
+
<% if mark_passed? -%>
|
|
212
243
|
|
|
213
244
|
# With c.push_base = :last_passed in .duel.rb, a push is compared with
|
|
214
245
|
# the last commit Duel passed, which a branch marks (each branch pushed
|
|
@@ -221,6 +252,13 @@ jobs:
|
|
|
221
252
|
passed:
|
|
222
253
|
name: Mark the commit passed
|
|
223
254
|
needs: duel
|
|
255
|
+
# One at a time for each branch, so that two runs can't both find no
|
|
256
|
+
# branch and race to make it (a push's run doesn't wait for the one
|
|
257
|
+
# before it). One that's running is never cancelled, and one that's
|
|
258
|
+
# waiting gives way to a newer push's, which moves it further anyway.
|
|
259
|
+
concurrency:
|
|
260
|
+
group: ${{ github.workflow }}-passed-${{ github.ref_name }}
|
|
261
|
+
cancel-in-progress: false
|
|
224
262
|
if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
|
|
225
263
|
runs-on: ubuntu-latest
|
|
226
264
|
permissions:
|
|
@@ -244,13 +282,16 @@ jobs:
|
|
|
244
282
|
done
|
|
245
283
|
exit 1
|
|
246
284
|
}
|
|
247
|
-
# (Forward: GitHub moves it only where it's behind the push.
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
gh api --method
|
|
252
|
-
|
|
253
|
-
|
|
285
|
+
# (Forward: GitHub moves it only where it's behind the push. Where
|
|
286
|
+
# there's no branch, and making it fails, it looks once more, as
|
|
287
|
+
# something else may have made it: a push of it, say.)
|
|
288
|
+
for _ in 1 2; do
|
|
289
|
+
gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null 2>&1 && exit 0
|
|
290
|
+
at=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$BRANCH" --jq .object.sha 2>/dev/null) || at=
|
|
291
|
+
[ -n "$at" ] && break
|
|
292
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null && exit 0
|
|
293
|
+
done
|
|
294
|
+
[ -n "$at" ] || blocked
|
|
254
295
|
compared=$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$at" --jq .status 2>&1) || case "$compared" in
|
|
255
296
|
# (GitHub can't compare commits with no history in common.)
|
|
256
297
|
*"No common ancestor"*) compared=unrelated ;;
|
|
@@ -262,6 +303,8 @@ jobs:
|
|
|
262
303
|
diverged | behind | unrelated) gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" -F force=true > /dev/null ;;
|
|
263
304
|
*) echo "::error::Couldn't compare $BRANCH with this push: $compared" && exit 1 ;;
|
|
264
305
|
esac
|
|
306
|
+
<% end -%>
|
|
307
|
+
<% if badge? -%>
|
|
265
308
|
|
|
266
309
|
# Duel's badge, for the README: green (perf guarded) or red (perf
|
|
267
310
|
# regressed), as the last push to the default branch that Duel compared
|
|
@@ -273,10 +316,18 @@ jobs:
|
|
|
273
316
|
#
|
|
274
317
|
# [](https://github.com/sferik/duel-ruby)
|
|
275
318
|
#
|
|
276
|
-
# It checks nothing out
|
|
319
|
+
# It checks nothing out, runs none of the code, and fetches nothing.
|
|
277
320
|
badge:
|
|
278
321
|
name: Update the badge
|
|
279
322
|
needs: duel
|
|
323
|
+
# One at a time, so that one run's check of which push the badge
|
|
324
|
+
# shows can't interleave with another run's write (a push's run
|
|
325
|
+
# doesn't wait for the one before it). A badge job that's running is
|
|
326
|
+
# never cancelled, and one that's waiting gives way to a newer push's,
|
|
327
|
+
# whose result the badge should show anyway.
|
|
328
|
+
concurrency:
|
|
329
|
+
group: ${{ github.workflow }}-badge
|
|
330
|
+
cancel-in-progress: false
|
|
280
331
|
if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
|
|
281
332
|
runs-on: ubuntu-latest
|
|
282
333
|
permissions:
|
|
@@ -288,7 +339,6 @@ jobs:
|
|
|
288
339
|
BRANCH: <%= badge_branch %>
|
|
289
340
|
RESULT: ${{ needs.duel.result }}
|
|
290
341
|
STATUS: ${{ needs.duel.outputs.status }}
|
|
291
|
-
BADGES: https://raw.githubusercontent.com/sferik/duel-ruby/main/assets
|
|
292
342
|
run: |
|
|
293
343
|
# (Git can't make a branch where another's name is a folder of its
|
|
294
344
|
# name: a branch named duel blocks duel/passed/main. If one does,
|
|
@@ -307,11 +357,59 @@ jobs:
|
|
|
307
357
|
# Its commit names the push.)
|
|
308
358
|
last=$(gh api "repos/$GITHUB_REPOSITORY/commits/$BRANCH" --jq .commit.message 2>/dev/null | sed -n 's/^Duel.s badge, for \([0-9a-f]*\)$/\1/p') || true
|
|
309
359
|
if [ -n "$last" ] && [ "$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$last" --jq .status 2>/dev/null)" = ahead ]; then exit 0; fi
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
360
|
+
# (Its image, written here rather than fetched from anywhere: green,
|
|
361
|
+
# or red where the Duel job failed or found a regression.)
|
|
362
|
+
state="perf guarded" width=133 top="#34D058" bottom="#28A745" right="76.939C78.629 0 80 1.343 80"
|
|
363
|
+
if [ "$RESULT" = failure ] || [ "$STATUS" = failed ]; then
|
|
364
|
+
state="perf regressed" width=142 top="#EA4A5A" bottom="#D73A49" right="85.939C87.629 0 89 1.343 89"
|
|
365
|
+
fi
|
|
366
|
+
cat > "$RUNNER_TEMP/badge.svg" <<SVG
|
|
367
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="$width" height="20">
|
|
368
|
+
<title>Duel - $state</title>
|
|
369
|
+
<defs>
|
|
370
|
+
<linearGradient id="duel-fill" x1="50%" y1="0%" x2="50%" y2="100%">
|
|
371
|
+
<stop stop-color="#444D56" offset="0%"></stop>
|
|
372
|
+
<stop stop-color="#24292E" offset="100%"></stop>
|
|
373
|
+
</linearGradient>
|
|
374
|
+
<linearGradient id="state-fill" x1="50%" y1="0%" x2="50%" y2="100%">
|
|
375
|
+
<stop stop-color="$top" offset="0%"></stop>
|
|
376
|
+
<stop stop-color="$bottom" offset="100%"></stop>
|
|
377
|
+
</linearGradient>
|
|
378
|
+
</defs>
|
|
379
|
+
<g fill="none" fill-rule="evenodd">
|
|
380
|
+
<g font-family="'DejaVu Sans',Verdana,Geneva,sans-serif" font-size="11">
|
|
381
|
+
<path d="M0,3 C0,1.3431 1.3552,0 3.02702703,0 L53,0 L53,20 L3.02702703,20 C1.3552,20 0,18.6569 0,17 L0,3 Z" fill="url(#duel-fill)" fill-rule="nonzero"></path>
|
|
382
|
+
<text fill="#010101" fill-opacity=".3">
|
|
383
|
+
<tspan x="22.1981982" y="15" aria-hidden="true">Duel</tspan>
|
|
384
|
+
</text>
|
|
385
|
+
<text fill="#FFFFFF">
|
|
386
|
+
<tspan x="22.1981982" y="14">Duel</tspan>
|
|
387
|
+
</text>
|
|
388
|
+
</g>
|
|
389
|
+
<g transform="translate(53)" font-family="'DejaVu Sans',Verdana,Geneva,sans-serif" font-size="11">
|
|
390
|
+
<path d="M0 0h$right 3v14c0 1.657-1.37 3-3.061 3H0V0z" fill="url(#state-fill)" fill-rule="nonzero"></path>
|
|
391
|
+
<text fill="#010101" fill-opacity=".3" aria-hidden="true">
|
|
392
|
+
<tspan x="4" y="15">$state</tspan>
|
|
393
|
+
</text>
|
|
394
|
+
<text fill="#FFFFFF">
|
|
395
|
+
<tspan x="4" y="14">$state</tspan>
|
|
396
|
+
</text>
|
|
397
|
+
</g>
|
|
398
|
+
<g fill="#959DA5">
|
|
399
|
+
<path d="M4.6 3.6L6.9 4.5L14.3 11.9L12.9 13.3L5.5 5.9Z M17.4 3.6L15.1 4.5L7.7 11.9L9.1 13.3L16.5 5.9Z"></path>
|
|
400
|
+
<circle cx="16.6" cy="15.6" r="1.1"></circle>
|
|
401
|
+
<circle cx="5.4" cy="15.6" r="1.1"></circle>
|
|
402
|
+
</g>
|
|
403
|
+
<g stroke="#959DA5" stroke-linecap="round">
|
|
404
|
+
<path d="M12.1 14.5L15.5 11.1 M9.9 14.5L6.5 11.1" stroke-width="1.6"></path>
|
|
405
|
+
<path d="M13.8 12.8L16 15 M8.2 12.8L6 15" stroke-width="1.8"></path>
|
|
406
|
+
</g>
|
|
407
|
+
</g>
|
|
408
|
+
</svg>
|
|
409
|
+
SVG
|
|
313
410
|
tree=$(jq -n --rawfile svg "$RUNNER_TEMP/badge.svg" '{tree: [{path: "badge.svg", mode: "100644", type: "blob", content: $svg}]}' |
|
|
314
411
|
gh api --method POST "repos/$GITHUB_REPOSITORY/git/trees" --input - --jq .sha)
|
|
315
412
|
commit=$(gh api --method POST "repos/$GITHUB_REPOSITORY/git/commits" -f message="Duel's badge, for $GITHUB_SHA" -f tree="$tree" --jq .sha)
|
|
316
413
|
gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$commit" -F force=true > /dev/null 2>&1 ||
|
|
317
414
|
gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$commit" > /dev/null || blocked
|
|
415
|
+
<% end -%>
|
data/lib/duel/version.rb
CHANGED
data/lib/duel/worker.rb
CHANGED
|
@@ -18,10 +18,11 @@ module Duel
|
|
|
18
18
|
# closed.)
|
|
19
19
|
# @api private
|
|
20
20
|
class Worker
|
|
21
|
-
# The built-in metrics, outermost first: allocations
|
|
22
|
-
# instructions, whose counter allocates as
|
|
23
|
-
# allocations are none but the benchmark's),
|
|
24
|
-
|
|
21
|
+
# The built-in metrics, outermost first: allocations (CRuby's objects,
|
|
22
|
+
# or JRuby's bytes) inside instructions, whose counter allocates as
|
|
23
|
+
# it's read (a sample's own allocations are none but the benchmark's),
|
|
24
|
+
# and CPU time innermost.
|
|
25
|
+
BUILT_IN = [Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
|
|
25
26
|
|
|
26
27
|
def self.main(version_root, candidate_root, control, config_path = nil)
|
|
27
28
|
config = config_path ? Duel.load_config(config_path) : (Duel.config = Config.new(candidate_root))
|
data/sig/duel/across.rbs
CHANGED
|
@@ -5,17 +5,39 @@ module Duel
|
|
|
5
5
|
# A timed metric's verdict across the board ("confirmed": slower in a
|
|
6
6
|
# second measurement too, see Runner#across).
|
|
7
7
|
type verdict = { "median" => Float, "interval" => Array[Float]?, "limit" => Float, "benchmarks" => Integer,
|
|
8
|
-
"verdict" => String, ?"confirmed" => bool }
|
|
8
|
+
?"processes" => Integer, "verdict" => String, ?"confirmed" => bool }
|
|
9
|
+
|
|
10
|
+
# The fewest benchmarks across the board is judged on by process.
|
|
11
|
+
MIN_BENCHMARKS: Integer
|
|
9
12
|
|
|
10
13
|
@results: Array[Result]
|
|
11
14
|
@confidence: Float
|
|
15
|
+
@by_process: bool
|
|
12
16
|
|
|
13
17
|
# The across-the-board limit for a metric whose own limit is +limit+.
|
|
14
18
|
def self.limit: (Float limit) -> Float
|
|
15
19
|
|
|
16
|
-
def initialize: (Array[Result] results, confidence: Float) -> void
|
|
20
|
+
def initialize: (Array[Result] results, confidence: Float, ?by_process: bool) -> void
|
|
17
21
|
|
|
18
22
|
# Every timed metric any benchmark was judged on, by name.
|
|
19
23
|
def verdicts: () -> Hash[String, verdict]
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
# The verdicts of a run that judged benchmarks over workers.
|
|
28
|
+
def by_process_or_benchmark: (Hash[String, Array[Float]] medians, Hash[String, Float] limits) -> Hash[String, verdict]
|
|
29
|
+
|
|
30
|
+
# Each benchmark's workers' medians, of those judged by process, by
|
|
31
|
+
# metric.
|
|
32
|
+
def workers: () -> Hash[String, Array[Result::processes]]
|
|
33
|
+
|
|
34
|
+
# The verdict over every benchmark's median ratio.
|
|
35
|
+
def by_benchmark: (Array[Float] list, Float limit) -> verdict
|
|
36
|
+
|
|
37
|
+
# The verdict over the workers' scores.
|
|
38
|
+
def by_process: (Array[Result::processes] processes, Float limit) -> verdict
|
|
39
|
+
|
|
40
|
+
# Each worker's score, by version.
|
|
41
|
+
def scores: (Array[Result::processes] processes) -> Hash[String, Array[Float]]
|
|
20
42
|
end
|
|
21
43
|
end
|
data/sig/duel/baseline.rbs
CHANGED
|
@@ -9,6 +9,7 @@ module Duel
|
|
|
9
9
|
@config: Config
|
|
10
10
|
@settings: Settings
|
|
11
11
|
@ref: String?
|
|
12
|
+
@release: String?
|
|
12
13
|
@prefix: String?
|
|
13
14
|
|
|
14
15
|
def initialize: (Config config, Settings settings) -> void
|
|
@@ -17,6 +18,17 @@ module Duel
|
|
|
17
18
|
# a push with no commit before it here.
|
|
18
19
|
def ref: () -> String?
|
|
19
20
|
|
|
21
|
+
# The newest release tag before HEAD, or nil.
|
|
22
|
+
def release: () -> String?
|
|
23
|
+
|
|
24
|
+
# The last release, or the commit since then that merged the last
|
|
25
|
+
# pull request accepted with Config#accept_label (Settings#accepted),
|
|
26
|
+
# or nil with no release.
|
|
27
|
+
def last_release: () -> String?
|
|
28
|
+
|
|
29
|
+
# Whether +commit+ is +of+ or an ancestor of it.
|
|
30
|
+
def ancestor?: (String commit, String of) -> bool
|
|
31
|
+
|
|
20
32
|
# The branch that marks the last commit Duel passed in a push to the
|
|
21
33
|
# branch pushed.
|
|
22
34
|
def passed_branch: () -> String
|
data/sig/duel/cli.rbs
CHANGED
|
@@ -153,6 +153,10 @@ module Duel
|
|
|
153
153
|
# The candidate, in Markdown.
|
|
154
154
|
def described_candidate: (Config config, Settings settings) -> String
|
|
155
155
|
|
|
156
|
+
# What chose a baseline of the last release: the release, or a change
|
|
157
|
+
# since it accepted with the label.
|
|
158
|
+
def last_release: (Config config, Baseline baseline) -> String
|
|
159
|
+
|
|
156
160
|
# The baseline, for JSON: the ref that named it, its commit, and its
|
|
157
161
|
# tree.
|
|
158
162
|
def baseline_json: (Config config, Settings settings, Baseline baseline, String root) -> Hash[String, String?]
|