duel 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +203 -1
- data/README.md +41 -1
- data/docs/ci.md +170 -76
- data/docs/configuration.md +38 -2
- data/docs/how-it-works.md +18 -9
- data/docs/plugins.md +23 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +96 -28
- data/lib/duel/comment.rb +296 -0
- data/lib/duel/config.rb +124 -4
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +255 -20
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +33 -1044
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +21 -15
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +61 -44
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +35 -3
- data/sig/duel/cli.rbs +19 -0
- data/sig/duel/comment.rbs +97 -0
- data/sig/duel/config.rbs +57 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +95 -1
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +6 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +6 -3
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -105
|
@@ -1,16 +1,46 @@
|
|
|
1
|
+
<% if reusable? -%>
|
|
2
|
+
name: Duel comment
|
|
3
|
+
|
|
4
|
+
# Duel's Perf comment workflow, reusable: a project's own Perf comment
|
|
5
|
+
# workflow (as `duel init --reusable` writes it) calls it, triggered as
|
|
6
|
+
# that workflow is (once the Perf workflow has run, and where there's a
|
|
7
|
+
# label that accepts a regression, when it's added or taken off), and
|
|
8
|
+
# passes on what .duel.rb sets that it needs as its inputs. Its jobs run
|
|
9
|
+
# as the Perf comment workflow's that `duel init` writes do, and the
|
|
10
|
+
# caller grants what they need (actions: read, or write to run the Duel
|
|
11
|
+
# job again, and pull-requests: write), which each job narrows. See
|
|
12
|
+
# Duel's docs/ci.md.
|
|
13
|
+
on:
|
|
14
|
+
workflow_call:
|
|
15
|
+
inputs:
|
|
16
|
+
accept-label:
|
|
17
|
+
description: The label that accepts a regression (c.accept_label), or none
|
|
18
|
+
type: string
|
|
19
|
+
default: ""
|
|
20
|
+
perf-workflow:
|
|
21
|
+
description: The name of the Perf workflow, whose runs it comments on
|
|
22
|
+
type: string
|
|
23
|
+
default: Perf
|
|
24
|
+
marker:
|
|
25
|
+
description: What marks Duel's comments
|
|
26
|
+
type: string
|
|
27
|
+
default: duel
|
|
28
|
+
|
|
29
|
+
<% else -%>
|
|
1
30
|
name: Perf comment<%= " (#{working_directory})" if working_directory %>
|
|
2
31
|
|
|
3
32
|
# Duel's report, as a new comment on the pull request, when Duel says to:
|
|
4
33
|
# always (post), or only when its last comment's status differs from the
|
|
5
34
|
# run's (change), so the latest comment is never stale. It never edits or
|
|
6
|
-
# deletes a comment
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
35
|
+
# deletes a comment, but hides Duel's earlier ones as outdated, so the
|
|
36
|
+
# newest is the one in view. It runs once the Perf workflow has, with
|
|
37
|
+
# this repository's own workflow and a token that can write, so a pull
|
|
38
|
+
# request from a fork gets a comment too. It checks nothing out and runs
|
|
39
|
+
# none of the pull request's code, only `duel comment` (see Duel's
|
|
40
|
+
# docs/ci.md), which trusts only what the Duel job could have said in
|
|
41
|
+
# what it kept: the pull request's number, if it names the pull request
|
|
42
|
+
# the run compared, and the comment and the status, if they're ones Duel
|
|
43
|
+
# says. Its report, a fork's too, is the comment, as Markdown.
|
|
14
44
|
<% if accept_label -%>
|
|
15
45
|
#
|
|
16
46
|
# And when the label that accepts a regression (c.accept_label in
|
|
@@ -32,6 +62,7 @@ on:
|
|
|
32
62
|
types: [labeled, unlabeled]
|
|
33
63
|
<% end -%>
|
|
34
64
|
|
|
65
|
+
<% end -%>
|
|
35
66
|
permissions: {}
|
|
36
67
|
|
|
37
68
|
jobs:
|
|
@@ -48,56 +79,39 @@ jobs:
|
|
|
48
79
|
continue-on-error: true # there's none when the Duel job failed before comparing
|
|
49
80
|
uses: actions/download-artifact@v8
|
|
50
81
|
with:
|
|
82
|
+
<% if reusable? -%>
|
|
83
|
+
# (Every call's, where the Perf workflow called Duel's more than
|
|
84
|
+
# once, each in a directory of its own, which make one comment.)
|
|
85
|
+
pattern: duel-report*
|
|
86
|
+
<% else -%>
|
|
51
87
|
name: duel-report
|
|
88
|
+
<% end -%>
|
|
52
89
|
path: ${{ runner.temp }}/report
|
|
53
90
|
run-id: ${{ github.event.workflow_run.id }}
|
|
54
91
|
github-token: ${{ github.token }}
|
|
92
|
+
- uses: ruby/setup-ruby@v1
|
|
93
|
+
if: steps.report.outcome == 'success'
|
|
94
|
+
with:
|
|
95
|
+
ruby-version: ruby
|
|
96
|
+
# Duel's own code, from RubyGems (none of the pull request's), so a
|
|
97
|
+
# fix to how it comments comes with a release of Duel.
|
|
55
98
|
- name: Comment on the pull request
|
|
56
99
|
if: steps.report.outcome == 'success'
|
|
57
100
|
env:
|
|
58
101
|
GH_TOKEN: ${{ github.token }}
|
|
59
102
|
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
103
|
+
<% if reusable? -%>
|
|
104
|
+
MARKER: ${{ inputs.marker }}
|
|
105
|
+
<% end -%>
|
|
60
106
|
run: |
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
COMMENT=$(said comment 'post|change')
|
|
64
|
-
STATUS=$(said status 'passed|failed|neutral')
|
|
65
|
-
[ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
|
|
66
|
-
# (The pull request's head commit, and the repository it's from.)
|
|
67
|
-
head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq '"\(.head.sha) \(.head.repo.full_name)"')
|
|
68
|
-
[ "${head%% *}" = "$HEAD_SHA" ] || exit 0
|
|
69
|
-
# The report, as Markdown for a branch of this repository (whose
|
|
70
|
-
# author can push here anyway), but for a fork's, as text: its run
|
|
71
|
-
# ran the fork's code, which could have written anything in it. In
|
|
72
|
-
# a code block, fenced with more backticks than any run of them in
|
|
73
|
-
# the report, it can't mention anyone, link anywhere or show an
|
|
74
|
-
# image. It's cut short where a comment couldn't hold it, and loses
|
|
75
|
-
# any NUL bytes, which GitHub wouldn't take, and grep reads it as
|
|
76
|
-
# text whatever's in it, so it finds every run of backticks.
|
|
77
|
-
report() {
|
|
78
|
-
if [ "${head#* }" = "$GITHUB_REPOSITORY" ]; then cat "$RUNNER_TEMP/report/duel.md" && return; fi
|
|
79
|
-
text="$RUNNER_TEMP/report.txt"
|
|
80
|
-
LC_ALL=C tr -d '\000' < "$RUNNER_TEMP/report/duel.md" | head -c 60000 > "$text"
|
|
81
|
-
fence=$(LC_ALL=C grep -ao '``*' "$text" | awk '{ if (length > n) n = length } END { n = (n < 3) ? 3 : n + 1; while (n-- > 0) printf "`" }')
|
|
82
|
-
printf 'From a fork, so shown as text:\n\n%s\n' "$fence"
|
|
83
|
-
cat "$text"
|
|
84
|
-
printf '\n%s\n' "$fence"
|
|
85
|
-
}
|
|
86
|
-
marker="<!-- <%= marker %>"
|
|
87
|
-
# The first line of the last comment Duel posted here: its marker,
|
|
88
|
-
# with its status (none in Duel 0.1.0's, which counts as another).
|
|
89
|
-
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
|
|
90
|
-
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
|
|
91
|
-
if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
|
|
92
|
-
{ echo "$marker status=$STATUS -->"; report; } > "$RUNNER_TEMP/comment.md"
|
|
93
|
-
gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
|
|
94
|
-
fi
|
|
107
|
+
gem install duel --version '~> <%= Duel::Comment::SINCE %>' --no-document
|
|
108
|
+
duel comment --report "$RUNNER_TEMP/report" --head "$HEAD_SHA" --marker <%= reusable? ? '"$MARKER"' : "'#{marker.gsub("'") { "'\\''" }}'" %>
|
|
95
109
|
<% if accept_label -%>
|
|
96
110
|
|
|
97
111
|
again:
|
|
98
112
|
name: Run Duel again
|
|
99
113
|
if: >-
|
|
100
|
-
github.event_name == 'pull_request_target' && github.event.label.name == '
|
|
114
|
+
github.event_name == 'pull_request_target' && <%= reusable? ? "inputs.accept-label != '' && github.event.label.name == inputs.accept-label" : "github.event.label.name == '#{accept_label.gsub("'", "''")}'" %>
|
|
101
115
|
runs-on: ubuntu-latest
|
|
102
116
|
permissions:
|
|
103
117
|
actions: write # to run the Perf workflow's jobs again
|
|
@@ -108,10 +122,13 @@ jobs:
|
|
|
108
122
|
GH_REPO: ${{ github.repository }}
|
|
109
123
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
110
124
|
ACTION: ${{ github.event.action }}
|
|
125
|
+
<% if reusable? -%>
|
|
126
|
+
PERF_WORKFLOW: ${{ inputs.perf-workflow }}
|
|
127
|
+
<% end -%>
|
|
111
128
|
run: |
|
|
112
129
|
# The newest Perf run of the pull request's head commit, once it's
|
|
113
130
|
# done (one that's running may have read the labels already).
|
|
114
|
-
id=$(gh run list --workflow "
|
|
131
|
+
id=$(gh run list --workflow "<%= reusable? ? "$PERF_WORKFLOW" : "Perf#{" (#{working_directory})" if working_directory}" %>" --event pull_request --commit "$HEAD_SHA" --limit 1 \
|
|
115
132
|
--json databaseId --jq '.[0].databaseId // empty')
|
|
116
133
|
[ -n "$id" ] || exit 0
|
|
117
134
|
gh run watch "$id" --interval 30 > /dev/null || true
|
|
@@ -1,3 +1,101 @@
|
|
|
1
|
+
<% if reusable? -%>
|
|
2
|
+
name: Duel
|
|
3
|
+
|
|
4
|
+
# Duel's Perf workflow, reusable: a project's own Perf workflow (as
|
|
5
|
+
# `duel init --reusable` writes it) calls it, with its triggers and its
|
|
6
|
+
# concurrency, and passes on what .duel.rb sets that it needs as its
|
|
7
|
+
# inputs. Its jobs run as the Perf workflow's that `duel init` writes
|
|
8
|
+
# do, and the caller grants what they need (contents: write to mark a
|
|
9
|
+
# push passed, comment on it, or keep the badge, and pull-requests: read
|
|
10
|
+
# for the labels), which each job narrows. See Duel's docs/ci.md.
|
|
11
|
+
on:
|
|
12
|
+
workflow_call:
|
|
13
|
+
inputs:
|
|
14
|
+
ruby-version:
|
|
15
|
+
description: The Ruby the Duel job runs (default, as .ruby-version names it)
|
|
16
|
+
type: string
|
|
17
|
+
default: default
|
|
18
|
+
version-file:
|
|
19
|
+
description: Whether each version runs on the Ruby its .ruby-version names (c.ruby = :version_file)
|
|
20
|
+
type: boolean
|
|
21
|
+
default: false
|
|
22
|
+
yjit:
|
|
23
|
+
description: Whether the benchmarks run with YJIT on (RUBY_YJIT_ENABLE=1)
|
|
24
|
+
type: boolean
|
|
25
|
+
default: false
|
|
26
|
+
runs-on:
|
|
27
|
+
description: The runner the Duel job runs on (a larger one, say, for steadier timings), Linux or macOS
|
|
28
|
+
type: string
|
|
29
|
+
default: ubuntu-latest
|
|
30
|
+
timeout-minutes:
|
|
31
|
+
description: The minutes the Duel job may take before it's cancelled
|
|
32
|
+
type: number
|
|
33
|
+
default: 60
|
|
34
|
+
working-directory:
|
|
35
|
+
description: The project's directory, from the repository's root
|
|
36
|
+
type: string
|
|
37
|
+
default: .
|
|
38
|
+
gemfile:
|
|
39
|
+
description: The Gemfile of the bundle Duel runs in (BUNDLE_GEMFILE), from the project's directory, or none for the project's own
|
|
40
|
+
type: string
|
|
41
|
+
default: ""
|
|
42
|
+
submodules:
|
|
43
|
+
description: Whether to check out the repository's submodules too (true, or recursive), as actions/checkout takes it
|
|
44
|
+
type: string
|
|
45
|
+
default: "false"
|
|
46
|
+
setup:
|
|
47
|
+
description: Shell commands that set up what else the tests need (packages, services), before Ruby's
|
|
48
|
+
type: string
|
|
49
|
+
default: ""
|
|
50
|
+
report:
|
|
51
|
+
description: A name for this call's report (yjit, say), where the workflow calls this one more than once. The Perf comment workflow comments with every call's, each under its name
|
|
52
|
+
type: string
|
|
53
|
+
default: ""
|
|
54
|
+
accept-label:
|
|
55
|
+
description: The label that accepts a regression (c.accept_label), or none
|
|
56
|
+
type: string
|
|
57
|
+
default: ""
|
|
58
|
+
release-tags:
|
|
59
|
+
description: The glob release tags match (c.release_tags)
|
|
60
|
+
type: string
|
|
61
|
+
default: v*
|
|
62
|
+
comment-on-pushes:
|
|
63
|
+
description: Whether to comment on a pushed commit (c.fail_on = :pull_requests)
|
|
64
|
+
type: boolean
|
|
65
|
+
default: false
|
|
66
|
+
mark-passed:
|
|
67
|
+
description: Whether to mark a push passed (c.push_base = :last_passed)
|
|
68
|
+
type: boolean
|
|
69
|
+
default: false
|
|
70
|
+
badge:
|
|
71
|
+
description: Whether to keep a badge for the README
|
|
72
|
+
type: boolean
|
|
73
|
+
default: false
|
|
74
|
+
passed-branch:
|
|
75
|
+
description: What names the branches that mark the last commit Duel passed
|
|
76
|
+
type: string
|
|
77
|
+
default: duel/passed
|
|
78
|
+
badge-branch:
|
|
79
|
+
description: The branch that holds the badge
|
|
80
|
+
type: string
|
|
81
|
+
default: duel/badge
|
|
82
|
+
marker:
|
|
83
|
+
description: What marks Duel's comments
|
|
84
|
+
type: string
|
|
85
|
+
default: duel
|
|
86
|
+
# What Duel said, for the caller's own jobs (a notice, say).
|
|
87
|
+
outputs:
|
|
88
|
+
comment:
|
|
89
|
+
description: Whether Duel said to comment (post or change), or nothing
|
|
90
|
+
value: ${{ jobs.duel.outputs.comment }}
|
|
91
|
+
status:
|
|
92
|
+
description: The run's status (passed, failed or neutral), or nothing where Duel didn't get to compare
|
|
93
|
+
value: ${{ jobs.duel.outputs.status }}
|
|
94
|
+
passed:
|
|
95
|
+
description: Whether to mark a push passed (true), or nothing
|
|
96
|
+
value: ${{ jobs.duel.outputs.passed }}
|
|
97
|
+
|
|
98
|
+
<% else -%>
|
|
1
99
|
name: Perf<%= " (#{working_directory})" if working_directory %>
|
|
2
100
|
|
|
3
101
|
on:
|
|
@@ -16,6 +114,7 @@ concurrency:
|
|
|
16
114
|
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
|
|
17
115
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
18
116
|
|
|
117
|
+
<% end -%>
|
|
19
118
|
# The Duel job runs the branch's code (its benchmarks too), so it only
|
|
20
119
|
# reads. Commenting on a commit is the Comment on the commit job's, which
|
|
21
120
|
# runs none of it, and commenting on a pull request is the Perf comment
|
|
@@ -26,8 +125,14 @@ permissions:
|
|
|
26
125
|
|
|
27
126
|
jobs:
|
|
28
127
|
duel:
|
|
128
|
+
<% if reusable? -%>
|
|
129
|
+
name: Duel${{ inputs.report != '' && format(' ({0})', inputs.report) || '' }}
|
|
130
|
+
runs-on: ${{ inputs.runs-on }}
|
|
131
|
+
timeout-minutes: ${{ inputs.timeout-minutes }}
|
|
132
|
+
<% else -%>
|
|
29
133
|
name: Duel
|
|
30
134
|
runs-on: ubuntu-latest
|
|
135
|
+
<% end -%>
|
|
31
136
|
# Whether to comment (post, change or none), the run's status (passed,
|
|
32
137
|
# failed or neutral), and whether to mark a push passed (true, or
|
|
33
138
|
# nothing), from Duel.
|
|
@@ -47,6 +152,9 @@ jobs:
|
|
|
47
152
|
with:
|
|
48
153
|
fetch-depth: 0 # the baseline comes from history, tags included
|
|
49
154
|
persist-credentials: false
|
|
155
|
+
<% if reusable? -%>
|
|
156
|
+
submodules: ${{ inputs.submodules }}
|
|
157
|
+
<% end -%>
|
|
50
158
|
# The labels of the pull request (or for a push, of the one that
|
|
51
159
|
# merged it), for c.accept_label in .duel.rb, as a JSON array. They're
|
|
52
160
|
# read as the job runs, and adding the label (or taking it off) runs
|
|
@@ -58,7 +166,7 @@ jobs:
|
|
|
58
166
|
<% end -%>
|
|
59
167
|
- name: Find the pull request's labels
|
|
60
168
|
id: labels
|
|
61
|
-
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label
|
|
169
|
+
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'<% if accept_label %><%= reusable? ? " || (github.event_name == 'schedule' && inputs.accept-label != '')" : " || github.event_name == 'schedule'" %><% end %>
|
|
62
170
|
continue-on-error: true # (without them, nothing's accepted)
|
|
63
171
|
env:
|
|
64
172
|
GH_TOKEN: ${{ github.token }}
|
|
@@ -92,7 +200,29 @@ jobs:
|
|
|
92
200
|
labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
|
|
93
201
|
fi
|
|
94
202
|
echo "labels=$labels" >> "$GITHUB_OUTPUT"
|
|
203
|
+
<% if reusable? -%>
|
|
204
|
+
# What else the tests need set up (packages, services), as the
|
|
205
|
+
# caller says.
|
|
206
|
+
- name: Set up
|
|
207
|
+
if: inputs.setup != ''
|
|
208
|
+
env:
|
|
209
|
+
SETUP: ${{ inputs.setup }}
|
|
210
|
+
run: eval "$SETUP"
|
|
211
|
+
# The bundle Duel and both versions' workers run in, where the caller
|
|
212
|
+
# names one of its own (a Gemfile for Duel alone, say): an absolute
|
|
213
|
+
# path, as the baseline's checkout is elsewhere.
|
|
214
|
+
- name: Use the caller's Gemfile
|
|
215
|
+
if: inputs.gemfile != ''
|
|
216
|
+
env:
|
|
217
|
+
GEMFILE: ${{ inputs.gemfile }}
|
|
218
|
+
run: echo "BUNDLE_GEMFILE=$PWD/$GEMFILE" >> "$GITHUB_ENV"
|
|
219
|
+
# YJIT, where the caller says to, for every worker of both versions.
|
|
220
|
+
- name: Turn on YJIT
|
|
221
|
+
if: inputs.yjit
|
|
222
|
+
run: echo "RUBY_YJIT_ENABLE=1" >> "$GITHUB_ENV"
|
|
223
|
+
<% else -%>
|
|
95
224
|
# (And whatever else your tests need set up: packages, services.)
|
|
225
|
+
<% end -%>
|
|
96
226
|
<% if version_file -%>
|
|
97
227
|
# Each version runs on the Ruby its own .ruby-version names (see
|
|
98
228
|
# .duel.rb). When the baseline's names another Ruby than this code's,
|
|
@@ -108,6 +238,9 @@ jobs:
|
|
|
108
238
|
# RELEASE_TAGS to match.)
|
|
109
239
|
- name: Find the baseline's Ruby
|
|
110
240
|
id: baseline
|
|
241
|
+
<% if reusable? -%>
|
|
242
|
+
if: inputs.version-file
|
|
243
|
+
<% end -%>
|
|
111
244
|
env:
|
|
112
245
|
BEFORE: ${{ github.event.before }}
|
|
113
246
|
QUEUE_BASE: ${{ github.event.merge_group.base_sha }}
|
|
@@ -197,7 +330,7 @@ jobs:
|
|
|
197
330
|
if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'push') }}
|
|
198
331
|
uses: actions/upload-artifact@v7
|
|
199
332
|
with:
|
|
200
|
-
name: duel-report
|
|
333
|
+
name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
|
|
201
334
|
path: ${{ runner.temp }}/duel/
|
|
202
335
|
if-no-files-found: ignore # none when Duel didn't get to compare
|
|
203
336
|
retention-days: 1
|
|
@@ -212,7 +345,7 @@ jobs:
|
|
|
212
345
|
commit-comment:
|
|
213
346
|
name: Comment on the commit
|
|
214
347
|
needs: duel
|
|
215
|
-
if: ${{
|
|
348
|
+
if: ${{ <%= "inputs.comment-on-pushes && " if reusable? %>!cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
|
|
216
349
|
runs-on: ubuntu-latest
|
|
217
350
|
permissions:
|
|
218
351
|
contents: write # to comment on the commit with the report
|
|
@@ -222,7 +355,7 @@ jobs:
|
|
|
222
355
|
continue-on-error: true # there's none when the Duel job failed before comparing
|
|
223
356
|
uses: actions/download-artifact@v8
|
|
224
357
|
with:
|
|
225
|
-
name: duel-report
|
|
358
|
+
name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
|
|
226
359
|
path: ${{ runner.temp }}/report
|
|
227
360
|
- name: Comment on the commit
|
|
228
361
|
if: steps.report.outcome == 'success'
|
|
@@ -230,8 +363,11 @@ jobs:
|
|
|
230
363
|
GH_TOKEN: ${{ github.token }}
|
|
231
364
|
COMMENT: ${{ needs.duel.outputs.comment }}
|
|
232
365
|
STATUS: ${{ needs.duel.outputs.status }}
|
|
366
|
+
<% if reusable? -%>
|
|
367
|
+
MARKER: ${{ inputs.marker }}
|
|
368
|
+
<% end -%>
|
|
233
369
|
run: |
|
|
234
|
-
marker="<!-- <%= marker %>"
|
|
370
|
+
marker="<!-- <%= reusable? ? "$MARKER" : marker %>"
|
|
235
371
|
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/comments" \
|
|
236
372
|
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\"))" | tail -n 1)
|
|
237
373
|
if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
|
|
@@ -259,7 +395,7 @@ jobs:
|
|
|
259
395
|
concurrency:
|
|
260
396
|
group: ${{ github.workflow }}-passed-${{ github.ref_name }}
|
|
261
397
|
cancel-in-progress: false
|
|
262
|
-
if: ${{
|
|
398
|
+
if: ${{ <%= "inputs.mark-passed && " if reusable? %>!cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
|
|
263
399
|
runs-on: ubuntu-latest
|
|
264
400
|
permissions:
|
|
265
401
|
contents: write # to move the branch
|
|
@@ -330,7 +466,7 @@ jobs:
|
|
|
330
466
|
concurrency:
|
|
331
467
|
group: ${{ github.workflow }}-badge
|
|
332
468
|
cancel-in-progress: false
|
|
333
|
-
if: ${{
|
|
469
|
+
if: ${{ <%= "inputs.badge && " if reusable? %>!cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
|
|
334
470
|
runs-on: ubuntu-latest
|
|
335
471
|
permissions:
|
|
336
472
|
contents: write # to move the badge's branch
|
data/lib/duel/testing.rb
CHANGED
|
@@ -217,6 +217,9 @@ module Duel
|
|
|
217
217
|
run = Settings.defaults
|
|
218
218
|
run.config_path = config_path if File.exist?(config_path)
|
|
219
219
|
run.processes = 2
|
|
220
|
+
# (.duel.rb's run settings, c.limit say, but where one's given, and
|
|
221
|
+
# its processes: a test's are few.)
|
|
222
|
+
config.run_settings.except(:processes).each { |name, value| run[name] = value }
|
|
220
223
|
settings.each { |setting, value| run[setting] = value }
|
|
221
224
|
out = StringIO.new
|
|
222
225
|
runner = Runner.new(config, run, base, out: out)
|
data/lib/duel/version.rb
CHANGED
data/lib/duel/worker.rb
CHANGED
|
@@ -5,6 +5,7 @@ require "socket"
|
|
|
5
5
|
require_relative "../duel"
|
|
6
6
|
require_relative "context"
|
|
7
7
|
require_relative "message"
|
|
8
|
+
require_relative "control"
|
|
8
9
|
require_relative "profile"
|
|
9
10
|
|
|
10
11
|
module Duel
|
|
@@ -12,7 +13,7 @@ module Duel
|
|
|
12
13
|
# benchmarks, and answers the runner's requests. Started (see
|
|
13
14
|
# WorkerProcess) with Worker.main(VERSION_ROOT, CANDIDATE_ROOT, CONTROL
|
|
14
15
|
# [, CONFIG]), it gives its main object the DSL, loads the benchmarks,
|
|
15
|
-
# connects to the runner
|
|
16
|
+
# connects to the runner at CONTROL (see Control), sends what it has
|
|
16
17
|
# (see #hello), and answers each request with a reply (see Message).
|
|
17
18
|
# (Not over stdin: a JRuby started by JRuby sees its stdin pipe
|
|
18
19
|
# closed.)
|
|
@@ -21,8 +22,16 @@ module Duel
|
|
|
21
22
|
# The built-in metrics, outermost first: allocations (CRuby's objects,
|
|
22
23
|
# or JRuby's bytes) inside instructions, whose counter allocates as
|
|
23
24
|
# it's read (a sample's own allocations are none but the benchmark's),
|
|
24
|
-
# and CPU time innermost.
|
|
25
|
-
|
|
25
|
+
# and CPU time innermost. But on JRuby on Windows, JRuby's bytes are
|
|
26
|
+
# innermost, inside CPU time, whose readings of Windows's cycles
|
|
27
|
+
# (through Fiddle: see CycleClock) allocate there.
|
|
28
|
+
# simplecov:disable JRuby on Windows's (its test run covers it).
|
|
29
|
+
BUILT_IN = if Platform::JRUBY_ON_WINDOWS
|
|
30
|
+
[Metrics::Instructions, Metrics::Allocations, Metrics::CPU, Metrics::AllocatedBytes].freeze
|
|
31
|
+
else
|
|
32
|
+
[Metrics::Instructions, Metrics::Allocations, Metrics::AllocatedBytes, Metrics::CPU].freeze
|
|
33
|
+
end
|
|
34
|
+
# simplecov:enable
|
|
26
35
|
|
|
27
36
|
def self.main(version_root, candidate_root, control, config_path = nil)
|
|
28
37
|
config = config_path ? Duel.load_config(config_path) : (Duel.config = Config.new(candidate_root))
|
|
@@ -40,7 +49,7 @@ module Duel
|
|
|
40
49
|
GC.start
|
|
41
50
|
worker = new(config.metrics, config.measures, hooks: config.hooks, before: before)
|
|
42
51
|
begin
|
|
43
|
-
socket =
|
|
52
|
+
socket = Control.connect(control)
|
|
44
53
|
Message.write(socket, worker.hello)
|
|
45
54
|
while (request = Message.read(socket))
|
|
46
55
|
Message.write(socket, worker.handle(request))
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../duel"
|
|
4
|
+
require_relative "control"
|
|
5
|
+
require_relative "message"
|
|
6
|
+
require_relative "process_sample"
|
|
7
|
+
require_relative "rubies"
|
|
8
|
+
require_relative "spawn"
|
|
9
|
+
|
|
10
|
+
module Duel
|
|
11
|
+
# A worker process (Duel::Worker) running one version of the code.
|
|
12
|
+
# @api private
|
|
13
|
+
class WorkerProcess
|
|
14
|
+
LIB = File.expand_path("..", __dir__)
|
|
15
|
+
# Seconds between looks at a worker that's loading.
|
|
16
|
+
POLL = 0.01
|
|
17
|
+
# Seconds a worker may take to end, once its connection is closed,
|
|
18
|
+
# before it's killed.
|
|
19
|
+
STOP_WAIT = 10
|
|
20
|
+
# The gems that are Duel's: Duel, and its plugins (duel-redis, say).
|
|
21
|
+
DUEL_GEM = /\Aduel(-|\z)/
|
|
22
|
+
|
|
23
|
+
attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :yjit
|
|
24
|
+
|
|
25
|
+
# The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
|
|
26
|
+
# loaded (see Worker.extensions), those it loaded twice (see
|
|
27
|
+
# Worker.twice), and the directories gems are installed in (see
|
|
28
|
+
# Worker.gem_dirs).
|
|
29
|
+
attr_reader :description, :extensions, :twice, :gem_dirs
|
|
30
|
+
|
|
31
|
+
# Where a worker on +ruby+ loads Duel from, ahead of anything else
|
|
32
|
+
# (a version's own bundle, which may not have Duel, or an older one):
|
|
33
|
+
# the runner's Duel, and every Duel gem (+specs+) the runner loaded
|
|
34
|
+
# (its plugins), so both versions are measured with the same. On the
|
|
35
|
+
# runner's own Ruby, Duel's dependencies (Fiddle) too: on another,
|
|
36
|
+
# their builds wouldn't load.
|
|
37
|
+
def self.load_path(ruby, specs: Gem.loaded_specs)
|
|
38
|
+
duel = specs.select { |name, _| name.match?(DUEL_GEM) }.values
|
|
39
|
+
none = [] #: Array[Gem::BasicSpecification]
|
|
40
|
+
dependencies = Rubies.runner?(ruby) ? specs["duel"]&.runtime_dependencies.to_a.filter_map { |d| specs[d.name] } : none
|
|
41
|
+
[LIB, *(duel + dependencies).flat_map(&:full_require_paths)].uniq
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# +timeout+: the seconds it may take to load, or to answer a request,
|
|
45
|
+
# before it's taken for hung (and killed). It runs on +ruby+ (whose
|
|
46
|
+
# engine is +engine+), with +env+, and on JRuby, the JVM's options for
|
|
47
|
+
# a worker (see Rubies.worker_environment), not given to the processes
|
|
48
|
+
# it starts for a process benchmark (see #run): their boot is measured
|
|
49
|
+
# as it is. It connects over a Unix socket, unless +unix+ is false
|
|
50
|
+
# (see Control).
|
|
51
|
+
def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:, unix: Control::UNIX)
|
|
52
|
+
@label = label
|
|
53
|
+
@timeout = timeout
|
|
54
|
+
@root = root
|
|
55
|
+
@dir = dir
|
|
56
|
+
@env = env
|
|
57
|
+
@ruby = ruby
|
|
58
|
+
@engine = engine
|
|
59
|
+
server, control = Control.listen(File.join(dir, "#{label}.sock"), unix: unix)
|
|
60
|
+
# (A nil config_path is left out. And the worker gets no other
|
|
61
|
+
# worker's connection: TruffleRuby's sockets aren't close-on-exec,
|
|
62
|
+
# and one left open in another worker would keep it from ending.)
|
|
63
|
+
worker_env = Rubies.worker_environment(env, engine)
|
|
64
|
+
@child = Spawn::STARTER.started(worker_env, [*Rubies.launch(ruby, engine, worker_env),
|
|
65
|
+
*WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker", "-e", "Duel::Worker.main(*ARGV)", root,
|
|
66
|
+
candidate_root, control, *[config_path].compact], close_others: true)
|
|
67
|
+
@socket = accept(server, control)
|
|
68
|
+
hello = read
|
|
69
|
+
@benchmarks = hello.fetch("benchmarks").to_h { |b| [b.fetch("name"), b] }
|
|
70
|
+
@metrics = hello.fetch("metrics")
|
|
71
|
+
@memory = hello.fetch("memory")
|
|
72
|
+
@profiler = hello.fetch("profiler")
|
|
73
|
+
@yjit = hello.fetch("yjit")
|
|
74
|
+
@description = hello.fetch("ruby")
|
|
75
|
+
@extensions = hello.fetch("extensions")
|
|
76
|
+
@twice = hello.fetch("twice")
|
|
77
|
+
@gem_dirs = hello.fetch("gem_dirs")
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Sends a request, and returns the reply, or raises Failed with its
|
|
81
|
+
# error.
|
|
82
|
+
def request(op, name, **args)
|
|
83
|
+
# (By name: +args+ come by keyword, as Symbols.)
|
|
84
|
+
Message.write(@socket, {"op" => op, "name" => name, **args.transform_keys(&:to_s)})
|
|
85
|
+
reply = read
|
|
86
|
+
raise Failed, "#{@label}: #{reply.fetch("error")}\n #{reply.fetch("backtrace").join("\n ")}" if reply.key?("error")
|
|
87
|
+
|
|
88
|
+
reply
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def sample(name, n, runs: 1, count: false) = request("sample", name, n: n, runs: runs, count: count)
|
|
92
|
+
|
|
93
|
+
# A run of a process benchmark (see DSL#duel_process): +process+, a
|
|
94
|
+
# fresh process, started as this worker was (on its Ruby, in its
|
|
95
|
+
# version's tree, with its environment), and measured (see
|
|
96
|
+
# ProcessSample#run).
|
|
97
|
+
def run(process)
|
|
98
|
+
# (Code, without a command.)
|
|
99
|
+
code = process.fetch("ruby") #: String
|
|
100
|
+
command = process.fetch("command") || [*(@launch ||= Rubies.launch(@ruby, @engine, @env)), "-I", File.join(@root, "lib"), "-e", code]
|
|
101
|
+
runs = @runs ||= ProcessSample.new(@label, env: @env, chdir: @root, log: File.join(@dir, "#{@label}.log"), timeout: @timeout)
|
|
102
|
+
runs.run(command)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Closing the connection ends the worker. One still running +wait+
|
|
106
|
+
# seconds later is killed.
|
|
107
|
+
def stop(wait: STOP_WAIT)
|
|
108
|
+
@runs&.stop
|
|
109
|
+
@socket.close
|
|
110
|
+
@child.stop(wait)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
private
|
|
114
|
+
|
|
115
|
+
# The worker's connection, once it has loaded (or Failed if it died,
|
|
116
|
+
# or took too long, first).
|
|
117
|
+
def accept(server, path)
|
|
118
|
+
deadline = monotonic + @timeout
|
|
119
|
+
until server.wait_readable(POLL)
|
|
120
|
+
raise Failed, "#{@label} worker exited while loading" unless @child.alive?
|
|
121
|
+
hung("load") if monotonic > deadline
|
|
122
|
+
end
|
|
123
|
+
server.accept
|
|
124
|
+
ensure
|
|
125
|
+
server.close
|
|
126
|
+
Control.remove(path)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def read
|
|
130
|
+
hung("answer") unless @socket.wait_readable(@timeout)
|
|
131
|
+
Message.read(@socket) or raise Failed, "#{@label} worker exited"
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Kills a worker that didn't +what+ within the timeout, and fails.
|
|
135
|
+
def hung(what)
|
|
136
|
+
@child.kill
|
|
137
|
+
raise Failed, "#{@label} worker didn't #{what} within #{@timeout}s (killed: a benchmark that hangs?)"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
141
|
+
end
|
|
142
|
+
end
|