duel 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +203 -1
- data/README.md +41 -1
- data/docs/ci.md +170 -76
- data/docs/configuration.md +38 -2
- data/docs/how-it-works.md +18 -9
- data/docs/plugins.md +23 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +96 -28
- data/lib/duel/comment.rb +296 -0
- data/lib/duel/config.rb +124 -4
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +255 -20
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +33 -1044
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +21 -15
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +61 -44
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +35 -3
- data/sig/duel/cli.rbs +19 -0
- data/sig/duel/comment.rbs +97 -0
- data/sig/duel/config.rbs +57 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +95 -1
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +6 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +6 -3
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -105
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3914db6930c721c09f8b53b0b0e8b9f15e04fa2106d7a385a2f773d36545e2f1
|
|
4
|
+
data.tar.gz: ab23ea38c226c83ba1da5145caee9548f48a22ffd76fb225e3ad05f5b0da5f27
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e2e23a16ee7d7d893ce503667b17f1fc677cbd91cda676401cb514aec820cf6bcea97294af7c3f9aceadc2b34e249b402ec751641bcc3dad4496aed58f1f764d
|
|
7
|
+
data.tar.gz: ae3d97e628cce46263ef4a255c937ff7f53e26c2582ec999bed4ed030881e3c6b3dc69fad0dfe5bb59616821ed774ec9444a182712f03fecd4e7e8a9544f868e
|
data/AGENT_PROMPT.md
CHANGED
|
@@ -201,8 +201,8 @@ at the API.
|
|
|
201
201
|
- `.github/workflows/perf-comment.yml` comments on a pull request
|
|
202
202
|
with Duel's report once the first has run (`c.comment` in
|
|
203
203
|
`.duel.rb` chooses when, and Duel also adds the report to the
|
|
204
|
-
job's summary). It runs none of the pull request's code
|
|
205
|
-
|
|
204
|
+
job's summary). It runs none of the pull request's code. Keep it as
|
|
205
|
+
it is.
|
|
206
206
|
- `duel init` writes the jobs for pushes (a comment on a pushed
|
|
207
207
|
commit, and a `duel/passed/` branch that marks the last commit
|
|
208
208
|
Duel passed) only where `.duel.rb` sets `c.fail_on =
|
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,206 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.2.0] - 2026-11-16
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- Duel's workflows are reusable: `.github/workflows/duel.yml` and
|
|
15
|
+
`duel-comment.yml`, in Duel's repository, are the Perf and Perf
|
|
16
|
+
comment workflows that `duel init` writes, with each setting of
|
|
17
|
+
`.duel.rb` they need as an input, plus a `setup` input, for what else a
|
|
18
|
+
project's tests need, a `gemfile` input, for a bundle of Duel's own
|
|
19
|
+
that keeps it out of the project's other bundles, a `yjit` input, to
|
|
20
|
+
measure with YJIT on, and `runs-on`, `timeout-minutes` and
|
|
21
|
+
`submodules` inputs for the Duel job. A workflow can call Duel's more
|
|
22
|
+
than once (with YJIT and without, say), each call with a `report`
|
|
23
|
+
input of its own, and the Perf comment workflow comments with every
|
|
24
|
+
call's report, as one comment (`duel comment` merges the reports it's
|
|
25
|
+
given as `duel-report*` directories). Duel's Perf workflow says what
|
|
26
|
+
Duel said as its outputs. `duel init --reusable` writes short workflows
|
|
27
|
+
that call them, at the release's tag, passing on what `.duel.rb` sets,
|
|
28
|
+
and `--check --reusable` compares those. `rake workflows` renders the
|
|
29
|
+
reusable workflows from the templates, and a test keeps them the
|
|
30
|
+
same. See docs/ci.md.
|
|
31
|
+
- The start of Windows support (CRuby). CPU time there is the cycles
|
|
32
|
+
the thread (or the process) was charged, in nanoseconds at a rate
|
|
33
|
+
the run measures once for every worker, as Windows's own CPU times
|
|
34
|
+
move in ticks of 15.6 ms, more than a sample takes. A process
|
|
35
|
+
benchmark's CPU time and peak memory are its process's own (not what
|
|
36
|
+
it starts, as on Unix), from a handle on it, as Windows keeps no
|
|
37
|
+
account of a child's for its parent. Resident memory comes from the
|
|
38
|
+
process's working set, the memory available from
|
|
39
|
+
`GlobalMemoryStatusEx`, a process benchmark's process that hangs is
|
|
40
|
+
killed with what it started by `taskkill /T`, and a resource's
|
|
41
|
+
processes are killed as it stops, with what they started, as Windows
|
|
42
|
+
can send them no other signal (an `at_stop` block can ask one to stop
|
|
43
|
+
first). A run's lock
|
|
44
|
+
works there too, though Windows can't read a held lock, so another
|
|
45
|
+
run's isn't named. There are no instructions there. The tests run on
|
|
46
|
+
macOS and Windows in CI too.
|
|
47
|
+
- `.duel.rb` sets a run's settings too, as their options do, for a run
|
|
48
|
+
in CI, where the workflow runs `duel` with no options: `c.limit`,
|
|
49
|
+
`c.confidence`, `c.processes`, `c.timeout`, `c.time_limit` and
|
|
50
|
+
`c.warm`, each nil for Duel's own default, and an option given over
|
|
51
|
+
it. The confidence of every interval (0.99) has an option of its own,
|
|
52
|
+
`--confidence`. An option given a value out of its range (a
|
|
53
|
+
confidence of 1.5, or a limit below 1) fails as an invalid argument.
|
|
54
|
+
- JRuby 10 on Windows. Its `jruby.exe` starts the JVM in a process of
|
|
55
|
+
its own, so Duel starts each worker's JVM, and a process benchmark's
|
|
56
|
+
of `ruby:` code, itself, with the command `jruby -Xcommand` prints:
|
|
57
|
+
what's measured, waited for and killed is the JVM, not the launcher.
|
|
58
|
+
A worker talks to the runner over a port of 127.0.0.1's where there
|
|
59
|
+
are no Unix sockets (JRuby has none there). Where Process.spawn
|
|
60
|
+
redirects nothing and doesn't wait (JRuby's there), workers, a
|
|
61
|
+
resource's processes, and a process benchmark's Ruby process and its
|
|
62
|
+
runs start through Open3, their output copied to their logs, and
|
|
63
|
+
every command Duel runs through Open3 gets ENV (JRuby's gives the
|
|
64
|
+
JVM's own environment, and ignores `unsetenv_others`), each variable by
|
|
65
|
+
ENV's name for it (JRuby's PATH given doesn't replace ENV's Path), its
|
|
66
|
+
program found by the PATH and PATHEXT (JRuby's runs only an .exe by
|
|
67
|
+
name, so `gem` and `bundle`, which are .bat files, didn't run), and
|
|
68
|
+
its arguments quoted as Windows takes them (Java took out the quotes
|
|
69
|
+
in one, so `ruby: %(require "x")` ran `require x`). CPU time there is the cycles Windows charged,
|
|
70
|
+
as on CRuby, not the JVM's CPU time, in Windows's ticks of 15.6 ms
|
|
71
|
+
(and allocated bytes are read inside it, as reading the cycles, through
|
|
72
|
+
Fiddle, allocates). A
|
|
73
|
+
run's lock held by another run is told from the Java exception JRuby
|
|
74
|
+
raises for one, and a process that's ended from its EPERM. The tests
|
|
75
|
+
run on JRuby 10.0 and 10.1 on Windows in CI.
|
|
76
|
+
- `w.port`, in a prepare block: a port of 127.0.0.1's that's free, for
|
|
77
|
+
a server to listen on where it can't on a Unix socket (the plugins'
|
|
78
|
+
servers, on Windows). The Redis, memcached, MySQL and PostgreSQL
|
|
79
|
+
plugins use it, in place of a method each of their own. And
|
|
80
|
+
`Duel.unix_sockets?`, for a plugin's default: whether its server can
|
|
81
|
+
listen on a Unix socket that a worker can reach (not on Windows, nor
|
|
82
|
+
where Ruby has none), or should listen on that port.
|
|
83
|
+
- `Duel.windows?`, for a plugin that does something differently on
|
|
84
|
+
Windows. The PostgreSQL plugin uses it for its `pg_ctl:` default.
|
|
85
|
+
- `Duel.load_config` is public: it loads a `.duel.rb` and returns its
|
|
86
|
+
configuration, for a tool that reads a project's settings, as the
|
|
87
|
+
Rails plugin's `duel:install` generator does.
|
|
88
|
+
|
|
89
|
+
### Changed
|
|
90
|
+
|
|
91
|
+
- On macOS, instructions are the benchmark thread's
|
|
92
|
+
(`thread_selfcounts`), as they are on Linux, so they're no longer
|
|
93
|
+
counted for benchmarks with threads of their own. Where the thread's
|
|
94
|
+
can't be counted (in a virtual machine, say), they're the whole
|
|
95
|
+
process's, as before.
|
|
96
|
+
- The Perf comment workflow and GitLab CI's comment job that `duel
|
|
97
|
+
init` writes install Duel 0.2 (`~> 0.2.0`), whose `duel comment`
|
|
98
|
+
merges the reports of a workflow that called Duel's reusable one more
|
|
99
|
+
than once. `duel init --check` says to update an older one.
|
|
100
|
+
|
|
101
|
+
### Fixed
|
|
102
|
+
|
|
103
|
+
- `c.measure`'s `calibrate: false`, or `kind: :ratio`, for a measure of
|
|
104
|
+
running totals (`config.metric`'s, as a plugin's Redis commands or
|
|
105
|
+
queries are) is what decides whether reading it is calibrated. It was
|
|
106
|
+
what the metric declared, so what reading it cost was taken away
|
|
107
|
+
still.
|
|
108
|
+
- A command a prepare block runs (`w.run`) that a signal Ruby has no
|
|
109
|
+
name for killed fails saying the signal's number, as a process
|
|
110
|
+
benchmark's does. It said "SIG" with nothing after it, and on JRuby,
|
|
111
|
+
raised an ArgumentError instead.
|
|
112
|
+
- A resource's program given its arguments, in a path with a space in
|
|
113
|
+
it (`C:/Program Files/PostgreSQL/17/bin/initdb`, say), has its log
|
|
114
|
+
named for the program (`initdb.log`), not the path's first word
|
|
115
|
+
(`Program.log`).
|
|
116
|
+
- A process benchmark's Ruby process takes what it runs with (the
|
|
117
|
+
environment, the directory, the log) as its input's first line, not
|
|
118
|
+
as an argument: on Windows, an environment could be longer than a
|
|
119
|
+
command may be, and JRuby's spawn there split the argument at its
|
|
120
|
+
spaces.
|
|
121
|
+
- On a Mac that counts no instructions (a virtual machine, say),
|
|
122
|
+
resident memory, and so peak memory, is measured all the same: it
|
|
123
|
+
was left out with the instructions.
|
|
124
|
+
- `duel init` writes its files with LF line endings on Windows too,
|
|
125
|
+
and `duel init --check` takes a workflow whose lines end with CRLF
|
|
126
|
+
(as git can check it out on Windows) for the same.
|
|
127
|
+
- A process benchmark's process that doesn't end once its input is
|
|
128
|
+
closed is killed ten seconds later, with what it started, rather
|
|
129
|
+
than waited for without end.
|
|
130
|
+
- A process benchmark's run that doesn't end within the timeout is
|
|
131
|
+
killed, where its reply could be waited for without end once the pipe
|
|
132
|
+
said it was ready (as JRuby's on Windows says before it is).
|
|
133
|
+
- A benchmark's counts are taken once it's timed, not right after its
|
|
134
|
+
warm-up: on JRuby, the bytes it allocates fall as the JIT compiles it,
|
|
135
|
+
at different times in different workers, so a short warm-up (`--warm`
|
|
136
|
+
or `c.warm`) could find one version's workers allocating a small
|
|
137
|
+
object or two more than the other's, and fail it
|
|
138
|
+
(https://github.com/sferik/duel-ruby/issues/6).
|
|
139
|
+
- A memory sample's peak can't count memory outside an iteration: a
|
|
140
|
+
reading could land between a pause's changes, on JRuby, whose threads
|
|
141
|
+
run at once.
|
|
142
|
+
- `duel init` writes LF line endings from templates checked out with
|
|
143
|
+
CRLF, and a failure shows a log's lines without a Windows program's CR.
|
|
144
|
+
- On Windows, a resource's log is added to, as it is elsewhere: a
|
|
145
|
+
command's output overwrote the start of it, as a child given a file
|
|
146
|
+
opened to append writes from where the file's at.
|
|
147
|
+
- A process benchmark's command that can't start (there's no such
|
|
148
|
+
program, say) fails saying why ("couldn't start: No such file or
|
|
149
|
+
directory - ..."), not "the process running ... exited", and the next
|
|
150
|
+
run goes on in the same Ruby process. It was that process that
|
|
151
|
+
ended. (JRuby runs a program it can't find through the shell, which
|
|
152
|
+
says why, and exits with 127.)
|
|
153
|
+
|
|
154
|
+
## [0.1.8] - 2026-11-12
|
|
155
|
+
|
|
156
|
+
### Added
|
|
157
|
+
|
|
158
|
+
- `duel comment`, which the Perf comment workflow now runs instead of
|
|
159
|
+
a shell script: it installs Duel from RubyGems (the newest release of
|
|
160
|
+
the same minor version) and comments with it, so a fix to how it
|
|
161
|
+
comments comes with a release of Duel, without a change to the
|
|
162
|
+
workflow. `duel init --check` shows how a project's workflows differ
|
|
163
|
+
from what the installed Duel writes.
|
|
164
|
+
- Once the Perf comment workflow posts a comment, it hides Duel's
|
|
165
|
+
earlier ones on the pull request as outdated (those not hidden
|
|
166
|
+
already), which GitHub collapses, so the newest is the one in view.
|
|
167
|
+
- `duel init` offers to create the label that `c.accept_label` names,
|
|
168
|
+
with `gh`, where it can (at a terminal, in a GitHub repository that
|
|
169
|
+
hasn't the label). Where it doesn't, its next steps say to.
|
|
170
|
+
- A plugin can say how much memory what it starts for each worker
|
|
171
|
+
takes at most (`def self.worker_memory`), and Duel counts at least
|
|
172
|
+
that much for each worker as it fits them in the memory available. A
|
|
173
|
+
server takes more as it's used than as it starts, when Duel measures
|
|
174
|
+
it.
|
|
175
|
+
- `duel comment --gitlab`, which the GitLab CI jobs' comment job now
|
|
176
|
+
runs instead of a shell script, as the GitHub workflow runs `duel
|
|
177
|
+
comment`: it trusts only what Duel said, comments only if the merge
|
|
178
|
+
request's head is still the run's, and comes from RubyGems, so a fix
|
|
179
|
+
comes with a release. In a merged results pipeline, the report names
|
|
180
|
+
the merge request's own commit, as on GitHub, and `duel init --gitlab`
|
|
181
|
+
offers to create the accept label with `glab`.
|
|
182
|
+
|
|
183
|
+
### Changed
|
|
184
|
+
|
|
185
|
+
- On a pull request's run on GitHub Actions, the report names the pull
|
|
186
|
+
request's own commit, "merged with `main`", instead of the commit
|
|
187
|
+
that merges it, which GitHub makes for the run and isn't in the pull
|
|
188
|
+
request.
|
|
189
|
+
- A run whose failures `c.accept_label` accepted says so in its
|
|
190
|
+
heading, under 🟡 instead of 🔻: "1 benchmark failed, accepted".
|
|
191
|
+
- The Perf comment workflow posts a report from a fork's pull request
|
|
192
|
+
as Markdown, like any other, instead of as text in a code block.
|
|
193
|
+
- A regression that `c.accept_label` accepts is commented on as a run
|
|
194
|
+
that passed would be. With the default `c.comment = :failures`, Duel
|
|
195
|
+
no longer posts the report again when the label is added, since the
|
|
196
|
+
last comment already showed it failing.
|
|
197
|
+
|
|
198
|
+
### Fixed
|
|
199
|
+
|
|
200
|
+
- On TruffleRuby, Duel fits the workers in the memory available as
|
|
201
|
+
they'll be once warm, not as they start: its workers grow some 2.9
|
|
202
|
+
times as its JIT compiles, so where 8 of each version seemed to fit,
|
|
203
|
+
they took 24 GB, more than a 16 GB CI runner has, and the runner
|
|
204
|
+
stopped answering.
|
|
205
|
+
|
|
206
|
+
To pick up the Perf comment workflow's changes, run `duel init --check`
|
|
207
|
+
(or see duel-rails's generator's template), and copy them into
|
|
208
|
+
`.github/workflows/perf-comment.yml`.
|
|
209
|
+
|
|
10
210
|
## [0.1.7] - 2026-11-10
|
|
11
211
|
|
|
12
212
|
### Fixed
|
|
@@ -706,7 +906,9 @@ needs Ruby 3.4 or later (or JRuby 10, or TruffleRuby 40).
|
|
|
706
906
|
Anything marked `@api private`, in lib/ and sig/, may change in any
|
|
707
907
|
release.
|
|
708
908
|
|
|
709
|
-
[Unreleased]: https://github.com/sferik/duel-ruby/compare/v0.
|
|
909
|
+
[Unreleased]: https://github.com/sferik/duel-ruby/compare/v0.2.0...HEAD
|
|
910
|
+
[0.2.0]: https://github.com/sferik/duel-ruby/compare/v0.1.8...v0.2.0
|
|
911
|
+
[0.1.8]: https://github.com/sferik/duel-ruby/compare/v0.1.7...v0.1.8
|
|
710
912
|
[0.1.7]: https://github.com/sferik/duel-ruby/compare/v0.1.6...v0.1.7
|
|
711
913
|
[0.1.6]: https://github.com/sferik/duel-ruby/compare/v0.1.5...v0.1.6
|
|
712
914
|
[0.1.5]: https://github.com/sferik/duel-ruby/compare/v0.1.4...v0.1.5
|
data/README.md
CHANGED
|
@@ -69,6 +69,38 @@ out are Duel's.
|
|
|
69
69
|
It adds StackProf to your `Gemfile`, for profiles of regressions, so
|
|
70
70
|
run `bundle install` again.
|
|
71
71
|
|
|
72
|
+
Duel needs Ruby 3.4 or later. It runs on Linux, macOS and Windows, on
|
|
73
|
+
CRuby and JRuby 10 (and on Linux, TruffleRuby).
|
|
74
|
+
|
|
75
|
+
### Windows
|
|
76
|
+
|
|
77
|
+
Duel runs on Windows with CRuby and JRuby 10. Its plugins run there on
|
|
78
|
+
CRuby, and start their servers (MySQL, PostgreSQL, Redis, memcached) on
|
|
79
|
+
a port of 127.0.0.1's rather than on a Unix socket. A few things work
|
|
80
|
+
differently:
|
|
81
|
+
|
|
82
|
+
- **CPU time** is the cycles Windows charged the thread or the process,
|
|
83
|
+
as its own CPU times move in ticks of 15.6 ms, longer than a sample
|
|
84
|
+
takes. There are no instruction counts, so CPU time judges alone.
|
|
85
|
+
- **A process benchmark** measures its own process's CPU time and peak
|
|
86
|
+
memory, but not that of the processes it starts, as Windows keeps no
|
|
87
|
+
account of a child's for its parent.
|
|
88
|
+
- **Stopping:** Windows can send a process no signal but to kill it,
|
|
89
|
+
so a resource's processes are killed as it stops. An `at_stop` block
|
|
90
|
+
can ask one to stop first.
|
|
91
|
+
- **JRuby** starts its JVM in a process of its own there (`jruby.exe`
|
|
92
|
+
starts `javaw.exe`), so Duel starts each worker's JVM itself, and a
|
|
93
|
+
process benchmark's of `ruby:` code, with the command `jruby -Xcommand`
|
|
94
|
+
prints: what it measures and stops is the JVM. (A process benchmark of
|
|
95
|
+
a command of its own that starts JRuby is measured as that command's
|
|
96
|
+
process, not the JVM it starts.) JRuby has no Unix sockets there, so a
|
|
97
|
+
worker talks to the runner over a port of 127.0.0.1's, and its spawn
|
|
98
|
+
redirects no output, so Duel starts processes through Open3, copying
|
|
99
|
+
their output to their logs.
|
|
100
|
+
|
|
101
|
+
The workflow that `duel init` writes runs on Linux, and Duel's reusable
|
|
102
|
+
one on Linux or macOS.
|
|
103
|
+
|
|
72
104
|
### Set up a project
|
|
73
105
|
|
|
74
106
|
`duel init` sets Duel up in a gem or an app, and writes the following:
|
|
@@ -86,6 +118,9 @@ out are Duel's.
|
|
|
86
118
|
use `--dry-run`. For a badge in your README that turns red when Duel
|
|
87
119
|
fails a push, use `--badge` (see
|
|
88
120
|
[Show that a project is guarded](#show-that-a-project-is-guarded)).
|
|
121
|
+
To write short workflows that call Duel's reusable ones instead of
|
|
122
|
+
copies of them, use `--reusable` (see
|
|
123
|
+
[Call Duel's reusable workflows](docs/ci.md#call-duels-reusable-workflows)).
|
|
89
124
|
When it's done, `duel init` lists the next steps.
|
|
90
125
|
|
|
91
126
|
For a Rails app, use
|
|
@@ -312,7 +347,12 @@ close to the limit to call doesn't fail the run, and is in the details.
|
|
|
312
347
|
A benchmark that the baseline can't run (a new one, say) is skipped, and
|
|
313
348
|
the heading counts it. A run that has nothing to compare, or no release
|
|
314
349
|
to compare against, says so under ⬜, and one that couldn't compare
|
|
315
|
-
says why under 🔻.
|
|
350
|
+
says why under 🔻. One whose failures the pull request's label accepted
|
|
351
|
+
(see [`c.accept_label`](docs/configuration.md#accept-a-regression-with-a-label))
|
|
352
|
+
says so under 🟡. On a pull request, GitHub Actions checks out a commit
|
|
353
|
+
that merges it with the branch it targets, which isn't in the pull
|
|
354
|
+
request, so the report names the pull request's own commit, "merged
|
|
355
|
+
with `main`".
|
|
316
356
|
|
|
317
357
|
The workflow that `duel init` writes posts this report as a comment on
|
|
318
358
|
the pull request when a benchmark fails, or when Duel couldn't compare.
|
data/docs/ci.md
CHANGED
|
@@ -193,7 +193,9 @@ check, because `duel` exits with a nonzero status, and the job's
|
|
|
193
193
|
summary has the report.
|
|
194
194
|
|
|
195
195
|
The workflow never edits or deletes a comment, so the pull request
|
|
196
|
-
keeps every report that it posted.
|
|
196
|
+
keeps every report that it posted. Once it posts one, it hides its
|
|
197
|
+
earlier ones as outdated, which GitHub collapses, so the newest is the
|
|
198
|
+
one in view. Each comment starts with a hidden
|
|
197
199
|
marker that records the run's status: `<!-- duel status=passed -->`,
|
|
198
200
|
`status=failed` (a run that couldn't compare too), or `status=neutral`
|
|
199
201
|
(nothing to compare). When a run's outcome wouldn't get a comment of
|
|
@@ -228,16 +230,14 @@ the pull request's number, `comment`, and `status`. A second workflow,
|
|
|
228
230
|
the Perf workflow completes (`workflow_run`), with your repository's
|
|
229
231
|
own copy of the workflow and a token that can write, so a pull request
|
|
230
232
|
from a fork gets a comment too. It checks nothing out and runs none of
|
|
231
|
-
the pull request's code.
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
`github-actions[bot]`, it can't mention anyone, link anywhere or show an
|
|
240
|
-
image. A report too long for a comment is cut short. When the Duel job fails before it compares, there's
|
|
233
|
+
the pull request's code. It installs Duel from RubyGems (the newest
|
|
234
|
+
0.2 release, 0.2.0 or later) and runs `duel comment`, so a fix to how it comments comes
|
|
235
|
+
with a release of Duel, without a change to the workflow. The pull
|
|
236
|
+
request's code ran in the Duel job, and could have written anything
|
|
237
|
+
into the artifact, so `duel comment` trusts only what Duel could have
|
|
238
|
+
said there: a number that names the pull request whose head the run
|
|
239
|
+
compared, and a `comment` and a `status` that Duel uses. The report's
|
|
240
|
+
Markdown, a fork's too, is the comment's body. When the Duel job fails before it compares, there's
|
|
241
241
|
no report, and no comment. GitHub runs a `workflow_run` workflow only
|
|
242
242
|
from your default branch, so the pull request that adds the workflow
|
|
243
243
|
gets no comment.
|
|
@@ -249,14 +249,15 @@ name: Perf comment
|
|
|
249
249
|
# Duel's report, as a new comment on the pull request, when Duel says to:
|
|
250
250
|
# always (post), or only when its last comment's status differs from the
|
|
251
251
|
# run's (change), so the latest comment is never stale. It never edits or
|
|
252
|
-
# deletes a comment
|
|
253
|
-
#
|
|
254
|
-
#
|
|
255
|
-
#
|
|
256
|
-
#
|
|
257
|
-
#
|
|
258
|
-
#
|
|
259
|
-
#
|
|
252
|
+
# deletes a comment, but hides Duel's earlier ones as outdated, so the
|
|
253
|
+
# newest is the one in view. It runs once the Perf workflow has, with
|
|
254
|
+
# this repository's own workflow and a token that can write, so a pull
|
|
255
|
+
# request from a fork gets a comment too. It checks nothing out and runs
|
|
256
|
+
# none of the pull request's code, only `duel comment` (see Duel's
|
|
257
|
+
# docs/ci.md), which trusts only what the Duel job could have said in
|
|
258
|
+
# what it kept: the pull request's number, if it names the pull request
|
|
259
|
+
# the run compared, and the comment and the status, if they're ones Duel
|
|
260
|
+
# says. Its report, a fork's too, is the comment, as Markdown.
|
|
260
261
|
#
|
|
261
262
|
# And when the label that accepts a regression (c.accept_label in
|
|
262
263
|
# .duel.rb) is added to a pull request, or taken off, it runs the Duel
|
|
@@ -294,46 +295,20 @@ jobs:
|
|
|
294
295
|
path: ${{ runner.temp }}/report
|
|
295
296
|
run-id: ${{ github.event.workflow_run.id }}
|
|
296
297
|
github-token: ${{ github.token }}
|
|
298
|
+
- uses: ruby/setup-ruby@v1
|
|
299
|
+
if: steps.report.outcome == 'success'
|
|
300
|
+
with:
|
|
301
|
+
ruby-version: ruby
|
|
302
|
+
# Duel's own code, from RubyGems (none of the pull request's), so a
|
|
303
|
+
# fix to how it comments comes with a release of Duel.
|
|
297
304
|
- name: Comment on the pull request
|
|
298
305
|
if: steps.report.outcome == 'success'
|
|
299
306
|
env:
|
|
300
307
|
GH_TOKEN: ${{ github.token }}
|
|
301
308
|
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
302
309
|
run: |
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
COMMENT=$(said comment 'post|change')
|
|
306
|
-
STATUS=$(said status 'passed|failed|neutral')
|
|
307
|
-
[ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
|
|
308
|
-
# (The pull request's head commit, and the repository it's from.)
|
|
309
|
-
head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq '"\(.head.sha) \(.head.repo.full_name)"')
|
|
310
|
-
[ "${head%% *}" = "$HEAD_SHA" ] || exit 0
|
|
311
|
-
# The report, as Markdown for a branch of this repository (whose
|
|
312
|
-
# author can push here anyway), but for a fork's, as text: its run
|
|
313
|
-
# ran the fork's code, which could have written anything in it. In
|
|
314
|
-
# a code block, fenced with more backticks than any run of them in
|
|
315
|
-
# the report, it can't mention anyone, link anywhere or show an
|
|
316
|
-
# image. It's cut short where a comment couldn't hold it, and loses
|
|
317
|
-
# any NUL bytes, which GitHub wouldn't take, and grep reads it as
|
|
318
|
-
# text whatever's in it, so it finds every run of backticks.
|
|
319
|
-
report() {
|
|
320
|
-
if [ "${head#* }" = "$GITHUB_REPOSITORY" ]; then cat "$RUNNER_TEMP/report/duel.md" && return; fi
|
|
321
|
-
text="$RUNNER_TEMP/report.txt"
|
|
322
|
-
LC_ALL=C tr -d '\000' < "$RUNNER_TEMP/report/duel.md" | head -c 60000 > "$text"
|
|
323
|
-
fence=$(LC_ALL=C grep -ao '``*' "$text" | awk '{ if (length > n) n = length } END { n = (n < 3) ? 3 : n + 1; while (n-- > 0) printf "`" }')
|
|
324
|
-
printf 'From a fork, so shown as text:\n\n%s\n' "$fence"
|
|
325
|
-
cat "$text"
|
|
326
|
-
printf '\n%s\n' "$fence"
|
|
327
|
-
}
|
|
328
|
-
marker="<!-- duel"
|
|
329
|
-
# The first line of the last comment Duel posted here: its marker,
|
|
330
|
-
# with its status (none in Duel 0.1.0's, which counts as another).
|
|
331
|
-
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
|
|
332
|
-
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
|
|
333
|
-
if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
|
|
334
|
-
{ echo "$marker status=$STATUS -->"; report; } > "$RUNNER_TEMP/comment.md"
|
|
335
|
-
gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
|
|
336
|
-
fi
|
|
310
|
+
gem install duel --version '~> 0.2.0' --no-document
|
|
311
|
+
duel comment --report "$RUNNER_TEMP/report" --head "$HEAD_SHA" --marker 'duel'
|
|
337
312
|
|
|
338
313
|
again:
|
|
339
314
|
name: Run Duel again
|
|
@@ -442,7 +417,10 @@ label is added to a pull request or taken off. For the label `.duel.rb`
|
|
|
442
417
|
names, it waits for the Perf run of the pull request's head commit to
|
|
443
418
|
finish. Then it runs the failed jobs again where the label was added
|
|
444
419
|
to a run that failed, or the whole run again where the label was
|
|
445
|
-
taken off one that passed.
|
|
420
|
+
taken off one that passed. The run the label accepts is commented on
|
|
421
|
+
as one that passed would be: with the default `c.comment = :failures`,
|
|
422
|
+
only where the last comment's status isn't its own, so the report that
|
|
423
|
+
failed isn't posted again. It checks nothing out and runs none of the
|
|
446
424
|
pull request's code, and its only permission is `actions: write`. The
|
|
447
425
|
workflows name the label as `.duel.rb` did when `duel init` wrote them,
|
|
448
426
|
so if you rename it, rename it in both workflows too.
|
|
@@ -740,6 +718,120 @@ that branch, which a branch of its own marks: `duel/passed/2-x-stable`.
|
|
|
740
718
|
A pull request into the branch is compared with the branch, whatever
|
|
741
719
|
the workflow guards.
|
|
742
720
|
|
|
721
|
+
## Keep the workflows up to date
|
|
722
|
+
|
|
723
|
+
A release of Duel can change the workflows that `duel init` writes. To
|
|
724
|
+
see how yours differ from what the installed Duel writes, run it with
|
|
725
|
+
`--check`, and the options you ran it with (such as `--badge`):
|
|
726
|
+
|
|
727
|
+
```sh
|
|
728
|
+
bundle exec duel init --check
|
|
729
|
+
```
|
|
730
|
+
|
|
731
|
+
It says whether each workflow is the same, differs (and shows how, as
|
|
732
|
+
`diff -u` does), or is missing, and writes nothing. It exits with 1
|
|
733
|
+
where one isn't the same. Copy in the changes you want, so that steps
|
|
734
|
+
you added to the Duel job (to install a database server, say) stay.
|
|
735
|
+
With `--gitlab`, it checks the GitLab CI jobs. In a Rails app, run
|
|
736
|
+
duel-rails's generator with `--check` instead:
|
|
737
|
+
|
|
738
|
+
```sh
|
|
739
|
+
bin/rails generate duel:install --check
|
|
740
|
+
```
|
|
741
|
+
|
|
742
|
+
## Call Duel's reusable workflows
|
|
743
|
+
|
|
744
|
+
Rather than copy Duel's whole workflows into your repository, you can
|
|
745
|
+
call Duel's reusable ones, which are the same workflows with each
|
|
746
|
+
setting as an input. `duel init --reusable` writes a Perf workflow and a
|
|
747
|
+
Perf comment workflow that call them, at the tag of the Duel release you
|
|
748
|
+
run it with, and pass on what your `.duel.rb` sets:
|
|
749
|
+
|
|
750
|
+
```yaml
|
|
751
|
+
# .github/workflows/perf.yml
|
|
752
|
+
jobs:
|
|
753
|
+
duel:
|
|
754
|
+
uses: sferik/duel-ruby/.github/workflows/duel.yml@v0.2.0
|
|
755
|
+
permissions:
|
|
756
|
+
contents: read
|
|
757
|
+
pull-requests: read # for the pull request's labels (see c.accept_label)
|
|
758
|
+
with:
|
|
759
|
+
ruby-version: "3.4"
|
|
760
|
+
accept-label: "duel: accept"
|
|
761
|
+
```
|
|
762
|
+
|
|
763
|
+
Your workflows keep their triggers, their concurrency and their
|
|
764
|
+
permissions, which Duel's jobs can only narrow. The job that runs the
|
|
765
|
+
pull request's code only reads, as it does in the whole workflow.
|
|
766
|
+
|
|
767
|
+
Duel's Perf workflow (`.github/workflows/duel.yml`) takes these inputs:
|
|
768
|
+
|
|
769
|
+
| Input | Default | What it is |
|
|
770
|
+
| --- | --- | --- |
|
|
771
|
+
| `ruby-version` | `default` | The Ruby the Duel job runs (`default`: the one `.ruby-version` names). |
|
|
772
|
+
| `version-file` | `false` | Whether each version runs on the Ruby its own `.ruby-version` names ([`c.ruby = :version_file`](configuration.md)). |
|
|
773
|
+
| `yjit` | `false` | Whether the benchmarks run with YJIT on (`RUBY_YJIT_ENABLE=1`), as a Rails app runs, say. |
|
|
774
|
+
| `runs-on` | `ubuntu-latest` | The runner the Duel job runs on, Linux or macOS: a larger one, say, for steadier timings. |
|
|
775
|
+
| `timeout-minutes` | `60` | How long the Duel job may take before it's cancelled. |
|
|
776
|
+
| `working-directory` | `.` | The project's directory, from the repository's root. |
|
|
777
|
+
| `gemfile` | none | A Gemfile of Duel's own, from the project's directory (`gemfiles/duel.gemfile`, say), which keeps Duel out of the project's other bundles. Duel and both versions' workers run in its bundle, so it needs what the benchmarks load (`eval_gemfile "../Gemfile"`, say). |
|
|
778
|
+
| `submodules` | `false` | Whether to check out the repository's submodules too (`true`, or `recursive`), as [actions/checkout](https://github.com/actions/checkout) takes it. |
|
|
779
|
+
| `setup` | none | Shell commands that set up whatever else your tests need (packages, services), before Ruby is set up. |
|
|
780
|
+
| `report` | none | A name for the call's report and check (`yjit`, say), where your workflow calls Duel's more than once: see below. |
|
|
781
|
+
| `accept-label` | none | The label that [accepts a regression](#accept-a-regression-with-a-label) (`c.accept_label`). |
|
|
782
|
+
| `release-tags` | `v*` | The glob your release tags match (`c.release_tags`). |
|
|
783
|
+
| `comment-on-pushes` | `false` | Whether to [comment on a pushed commit](#comment-on-a-pushed-commit) (`c.fail_on = :pull_requests`). |
|
|
784
|
+
| `mark-passed` | `false` | Whether to [mark a push passed](#compare-a-push-with-the-last-commit-that-passed) (`c.push_base = :last_passed`). |
|
|
785
|
+
| `badge` | `false` | Whether to [keep a badge](#show-whether-the-last-push-passed) for the README. |
|
|
786
|
+
| `passed-branch`, `badge-branch` | `duel/passed`, `duel/badge` | The branches those jobs move. |
|
|
787
|
+
| `marker` | `duel` | What marks Duel's comments. |
|
|
788
|
+
|
|
789
|
+
It says what Duel said, as its outputs, for jobs of your own (a notice
|
|
790
|
+
in a channel, say): `comment` (`post` or `change`, or nothing),
|
|
791
|
+
`status` (`passed`, `failed` or `neutral`) and `passed` (`true`, or
|
|
792
|
+
nothing).
|
|
793
|
+
|
|
794
|
+
To measure each version more than one way, call Duel's Perf workflow
|
|
795
|
+
more than once, each call with a `report` of its own:
|
|
796
|
+
|
|
797
|
+
```yaml
|
|
798
|
+
jobs:
|
|
799
|
+
yjit:
|
|
800
|
+
uses: sferik/duel-ruby/.github/workflows/duel.yml@v0.2.0
|
|
801
|
+
permissions:
|
|
802
|
+
contents: read
|
|
803
|
+
pull-requests: read
|
|
804
|
+
with:
|
|
805
|
+
yjit: true
|
|
806
|
+
report: yjit
|
|
807
|
+
interpreter:
|
|
808
|
+
uses: sferik/duel-ruby/.github/workflows/duel.yml@v0.2.0
|
|
809
|
+
permissions:
|
|
810
|
+
contents: read
|
|
811
|
+
pull-requests: read
|
|
812
|
+
with:
|
|
813
|
+
report: interpreter
|
|
814
|
+
```
|
|
815
|
+
|
|
816
|
+
Each call is a check of its own (`yjit / Duel (yjit)`), and the Perf
|
|
817
|
+
comment workflow comments with every call's report, as one comment,
|
|
818
|
+
each under its name. The comment's status is the most any call's
|
|
819
|
+
says: failed, if any failed, and passed only if every one passed. Turn
|
|
820
|
+
on `mark-passed`, `comment-on-pushes` or `badge` in one call at most,
|
|
821
|
+
as each call's jobs run on their own.
|
|
822
|
+
|
|
823
|
+
Duel's Perf comment workflow (`.github/workflows/duel-comment.yml`)
|
|
824
|
+
takes `accept-label`, `marker`, and `perf-workflow` (the name of your
|
|
825
|
+
Perf workflow, `Perf` by default).
|
|
826
|
+
|
|
827
|
+
A job that `mark-passed`, `comment-on-pushes` or `badge` turns on writes
|
|
828
|
+
to the repository, so your Perf workflow grants `contents: write` for
|
|
829
|
+
it. With an accept label, your Perf comment workflow grants
|
|
830
|
+
`actions: write`, to run the Duel job again. The checks are named for
|
|
831
|
+
your job and Duel's, as in `duel / Duel`. To update Duel's workflows,
|
|
832
|
+
change the tag. `duel init --check --reusable` compares the workflows
|
|
833
|
+
that call them.
|
|
834
|
+
|
|
743
835
|
## Run Duel on GitLab CI
|
|
744
836
|
|
|
745
837
|
`duel init --gitlab` writes GitLab CI jobs, in
|
|
@@ -784,9 +876,22 @@ log and its `duel.md` artifact have the report.
|
|
|
784
876
|
protected, so only your protected branches' pipelines have it, to
|
|
785
877
|
comment on a pushed commit and to mark a push passed.
|
|
786
878
|
|
|
787
|
-
The comment on a
|
|
788
|
-
|
|
789
|
-
|
|
879
|
+
The comment on a merge request comes from `duel comment --gitlab`,
|
|
880
|
+
installed from RubyGems in a `ruby:slim` image, as on GitHub, so a fix
|
|
881
|
+
to it comes with a release of Duel. It trusts only a `DUEL_COMMENT` and
|
|
882
|
+
a `DUEL_STATUS` that Duel says, and comments only if the merge
|
|
883
|
+
request's head is still the one the pipeline compared. In a merged
|
|
884
|
+
results pipeline, the report names the merge request's own commit,
|
|
885
|
+
"merged with `main`", as it does on GitHub.
|
|
886
|
+
|
|
887
|
+
Three things GitLab does differently. Duel can't collapse its earlier
|
|
888
|
+
comments there, because only the Developer role can resolve a thread,
|
|
889
|
+
and the merge request's token has the Reporter role. Adding the label
|
|
890
|
+
that accepts a regression doesn't run the pipeline again, as GitLab has
|
|
891
|
+
no pipeline for a label, so run it again yourself (or push). And a
|
|
892
|
+
merge request from a fork runs its pipeline in the fork, without your
|
|
893
|
+
variables, so it gets no comment. The comment on a pushed commit looks
|
|
894
|
+
only at the comments on the commit before it for its last status.
|
|
790
895
|
|
|
791
896
|
```yaml
|
|
792
897
|
# .gitlab/duel.gitlab-ci.yml
|
|
@@ -853,9 +958,11 @@ duel:
|
|
|
853
958
|
# The report, as a new comment on the merge request, when Duel says to:
|
|
854
959
|
# always (post), or only when its last comment's status differs from this
|
|
855
960
|
# run's (change), so the latest comment is never stale. It never edits or
|
|
856
|
-
# deletes a comment.
|
|
961
|
+
# deletes a comment. It runs `duel comment`, from RubyGems (none of the
|
|
962
|
+
# merge request's code), so a fix to how it comments comes with a release
|
|
963
|
+
# of Duel.
|
|
857
964
|
duel:comment:
|
|
858
|
-
image:
|
|
965
|
+
image: ruby:slim
|
|
859
966
|
stage: .post
|
|
860
967
|
needs: [duel]
|
|
861
968
|
rules:
|
|
@@ -864,23 +971,10 @@ duel:comment:
|
|
|
864
971
|
variables:
|
|
865
972
|
GIT_STRATEGY: none # it runs none of the code
|
|
866
973
|
script:
|
|
867
|
-
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
[ -f "$report" ] || exit 0
|
|
872
|
-
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
873
|
-
token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
|
|
874
|
-
marker="<!-- duel"
|
|
875
|
-
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
876
|
-
# The first line of the last comment Duel posted here: its marker,
|
|
877
|
-
# with its status.
|
|
878
|
-
last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
|
|
879
|
-
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
|
|
880
|
-
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
881
|
-
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
|
|
882
|
-
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
|
|
883
|
-
fi
|
|
974
|
+
- gem install duel --version '~> 0.2.0' --no-document
|
|
975
|
+
# (The merge request's head: in a merged results pipeline, not the
|
|
976
|
+
# commit that merges it.)
|
|
977
|
+
- duel comment --gitlab --report . --head "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --marker 'duel'
|
|
884
978
|
|
|
885
979
|
# The report, as a new comment on a pushed commit, when Duel says to, as
|
|
886
980
|
# for a merge request: only where a regression in a push passes (see
|