duel 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +146 -1
- data/README.md +35 -0
- data/docs/ci.md +96 -3
- data/docs/configuration.md +35 -1
- data/docs/how-it-works.md +14 -8
- data/docs/plugins.md +16 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +45 -19
- data/lib/duel/comment.rb +71 -10
- data/lib/duel/config.rb +113 -2
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +142 -18
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +13 -1039
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +10 -10
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +45 -3
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +27 -3
- data/sig/duel/cli.rbs +8 -0
- data/sig/duel/comment.rbs +16 -1
- data/sig/duel/config.rbs +52 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +56 -0
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +3 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +3 -2
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -107
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3914db6930c721c09f8b53b0b0e8b9f15e04fa2106d7a385a2f773d36545e2f1
|
|
4
|
+
data.tar.gz: ab23ea38c226c83ba1da5145caee9548f48a22ffd76fb225e3ad05f5b0da5f27
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e2e23a16ee7d7d893ce503667b17f1fc677cbd91cda676401cb514aec820cf6bcea97294af7c3f9aceadc2b34e249b402ec751641bcc3dad4496aed58f1f764d
|
|
7
|
+
data.tar.gz: ae3d97e628cce46263ef4a255c937ff7f53e26c2582ec999bed4ed030881e3c6b3dc69fad0dfe5bb59616821ed774ec9444a182712f03fecd4e7e8a9544f868e
|
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,150 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.2.0] - 2026-11-16
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- Duel's workflows are reusable: `.github/workflows/duel.yml` and
|
|
15
|
+
`duel-comment.yml`, in Duel's repository, are the Perf and Perf
|
|
16
|
+
comment workflows that `duel init` writes, with each setting of
|
|
17
|
+
`.duel.rb` they need as an input, plus a `setup` input, for what else a
|
|
18
|
+
project's tests need, a `gemfile` input, for a bundle of Duel's own
|
|
19
|
+
that keeps it out of the project's other bundles, a `yjit` input, to
|
|
20
|
+
measure with YJIT on, and `runs-on`, `timeout-minutes` and
|
|
21
|
+
`submodules` inputs for the Duel job. A workflow can call Duel's more
|
|
22
|
+
than once (with YJIT and without, say), each call with a `report`
|
|
23
|
+
input of its own, and the Perf comment workflow comments with every
|
|
24
|
+
call's report, as one comment (`duel comment` merges the reports it's
|
|
25
|
+
given as `duel-report*` directories). Duel's Perf workflow says what
|
|
26
|
+
Duel said as its outputs. `duel init --reusable` writes short workflows
|
|
27
|
+
that call them, at the release's tag, passing on what `.duel.rb` sets,
|
|
28
|
+
and `--check --reusable` compares those. `rake workflows` renders the
|
|
29
|
+
reusable workflows from the templates, and a test keeps them the
|
|
30
|
+
same. See docs/ci.md.
|
|
31
|
+
- The start of Windows support (CRuby). CPU time there is the cycles
|
|
32
|
+
the thread (or the process) was charged, in nanoseconds at a rate
|
|
33
|
+
the run measures once for every worker, as Windows's own CPU times
|
|
34
|
+
move in ticks of 15.6 ms, more than a sample takes. A process
|
|
35
|
+
benchmark's CPU time and peak memory are its process's own (not what
|
|
36
|
+
it starts, as on Unix), from a handle on it, as Windows keeps no
|
|
37
|
+
account of a child's for its parent. Resident memory comes from the
|
|
38
|
+
process's working set, the memory available from
|
|
39
|
+
`GlobalMemoryStatusEx`, a process benchmark's process that hangs is
|
|
40
|
+
killed with what it started by `taskkill /T`, and a resource's
|
|
41
|
+
processes are killed as it stops, with what they started, as Windows
|
|
42
|
+
can send them no other signal (an `at_stop` block can ask one to stop
|
|
43
|
+
first). A run's lock
|
|
44
|
+
works there too, though Windows can't read a held lock, so another
|
|
45
|
+
run's isn't named. There are no instructions there. The tests run on
|
|
46
|
+
macOS and Windows in CI too.
|
|
47
|
+
- `.duel.rb` sets a run's settings too, as their options do, for a run
|
|
48
|
+
in CI, where the workflow runs `duel` with no options: `c.limit`,
|
|
49
|
+
`c.confidence`, `c.processes`, `c.timeout`, `c.time_limit` and
|
|
50
|
+
`c.warm`, each nil for Duel's own default, and an option given over
|
|
51
|
+
it. The confidence of every interval (0.99) has an option of its own,
|
|
52
|
+
`--confidence`. An option given a value out of its range (a
|
|
53
|
+
confidence of 1.5, or a limit below 1) fails as an invalid argument.
|
|
54
|
+
- JRuby 10 on Windows. Its `jruby.exe` starts the JVM in a process of
|
|
55
|
+
its own, so Duel starts each worker's JVM, and a process benchmark's
|
|
56
|
+
of `ruby:` code, itself, with the command `jruby -Xcommand` prints:
|
|
57
|
+
what's measured, waited for and killed is the JVM, not the launcher.
|
|
58
|
+
A worker talks to the runner over a port of 127.0.0.1's where there
|
|
59
|
+
are no Unix sockets (JRuby has none there). Where Process.spawn
|
|
60
|
+
redirects nothing and doesn't wait (JRuby's there), workers, a
|
|
61
|
+
resource's processes, and a process benchmark's Ruby process and its
|
|
62
|
+
runs start through Open3, their output copied to their logs, and
|
|
63
|
+
every command Duel runs through Open3 gets ENV (JRuby's gives the
|
|
64
|
+
JVM's own environment, and ignores `unsetenv_others`), each variable by
|
|
65
|
+
ENV's name for it (JRuby's PATH given doesn't replace ENV's Path), its
|
|
66
|
+
program found by the PATH and PATHEXT (JRuby's runs only an .exe by
|
|
67
|
+
name, so `gem` and `bundle`, which are .bat files, didn't run), and
|
|
68
|
+
its arguments quoted as Windows takes them (Java took out the quotes
|
|
69
|
+
in one, so `ruby: %(require "x")` ran `require x`). CPU time there is the cycles Windows charged,
|
|
70
|
+
as on CRuby, not the JVM's CPU time, in Windows's ticks of 15.6 ms
|
|
71
|
+
(and allocated bytes are read inside it, as reading the cycles, through
|
|
72
|
+
Fiddle, allocates). A
|
|
73
|
+
run's lock held by another run is told from the Java exception JRuby
|
|
74
|
+
raises for one, and a process that's ended from its EPERM. The tests
|
|
75
|
+
run on JRuby 10.0 and 10.1 on Windows in CI.
|
|
76
|
+
- `w.port`, in a prepare block: a port of 127.0.0.1's that's free, for
|
|
77
|
+
a server to listen on where it can't on a Unix socket (the plugins'
|
|
78
|
+
servers, on Windows). The Redis, memcached, MySQL and PostgreSQL
|
|
79
|
+
plugins use it, in place of a method each of their own. And
|
|
80
|
+
`Duel.unix_sockets?`, for a plugin's default: whether its server can
|
|
81
|
+
listen on a Unix socket that a worker can reach (not on Windows, nor
|
|
82
|
+
where Ruby has none), or should listen on that port.
|
|
83
|
+
- `Duel.windows?`, for a plugin that does something differently on
|
|
84
|
+
Windows. The PostgreSQL plugin uses it for its `pg_ctl:` default.
|
|
85
|
+
- `Duel.load_config` is public: it loads a `.duel.rb` and returns its
|
|
86
|
+
configuration, for a tool that reads a project's settings, as the
|
|
87
|
+
Rails plugin's `duel:install` generator does.
|
|
88
|
+
|
|
89
|
+
### Changed
|
|
90
|
+
|
|
91
|
+
- On macOS, instructions are the benchmark thread's
|
|
92
|
+
(`thread_selfcounts`), as they are on Linux, so they're no longer
|
|
93
|
+
counted for benchmarks with threads of their own. Where the thread's
|
|
94
|
+
can't be counted (in a virtual machine, say), they're the whole
|
|
95
|
+
process's, as before.
|
|
96
|
+
- The Perf comment workflow and GitLab CI's comment job that `duel
|
|
97
|
+
init` writes install Duel 0.2 (`~> 0.2.0`), whose `duel comment`
|
|
98
|
+
merges the reports of a workflow that called Duel's reusable one more
|
|
99
|
+
than once. `duel init --check` says to update an older one.
|
|
100
|
+
|
|
101
|
+
### Fixed
|
|
102
|
+
|
|
103
|
+
- `c.measure`'s `calibrate: false`, or `kind: :ratio`, for a measure of
|
|
104
|
+
running totals (`config.metric`'s, as a plugin's Redis commands or
|
|
105
|
+
queries are) is what decides whether reading it is calibrated. It was
|
|
106
|
+
what the metric declared, so what reading it cost was taken away
|
|
107
|
+
still.
|
|
108
|
+
- A command a prepare block runs (`w.run`) that a signal Ruby has no
|
|
109
|
+
name for killed fails saying the signal's number, as a process
|
|
110
|
+
benchmark's does. It said "SIG" with nothing after it, and on JRuby,
|
|
111
|
+
raised an ArgumentError instead.
|
|
112
|
+
- A resource's program given its arguments, in a path with a space in
|
|
113
|
+
it (`C:/Program Files/PostgreSQL/17/bin/initdb`, say), has its log
|
|
114
|
+
named for the program (`initdb.log`), not the path's first word
|
|
115
|
+
(`Program.log`).
|
|
116
|
+
- A process benchmark's Ruby process takes what it runs with (the
|
|
117
|
+
environment, the directory, the log) as its input's first line, not
|
|
118
|
+
as an argument: on Windows, an environment could be longer than a
|
|
119
|
+
command may be, and JRuby's spawn there split the argument at its
|
|
120
|
+
spaces.
|
|
121
|
+
- On a Mac that counts no instructions (a virtual machine, say),
|
|
122
|
+
resident memory, and so peak memory, is measured all the same: it
|
|
123
|
+
was left out with the instructions.
|
|
124
|
+
- `duel init` writes its files with LF line endings on Windows too,
|
|
125
|
+
and `duel init --check` takes a workflow whose lines end with CRLF
|
|
126
|
+
(as git can check it out on Windows) for the same.
|
|
127
|
+
- A process benchmark's process that doesn't end once its input is
|
|
128
|
+
closed is killed ten seconds later, with what it started, rather
|
|
129
|
+
than waited for without end.
|
|
130
|
+
- A process benchmark's run that doesn't end within the timeout is
|
|
131
|
+
killed, where its reply could be waited for without end once the pipe
|
|
132
|
+
said it was ready (as JRuby's on Windows says before it is).
|
|
133
|
+
- A benchmark's counts are taken once it's timed, not right after its
|
|
134
|
+
warm-up: on JRuby, the bytes it allocates fall as the JIT compiles it,
|
|
135
|
+
at different times in different workers, so a short warm-up (`--warm`
|
|
136
|
+
or `c.warm`) could find one version's workers allocating a small
|
|
137
|
+
object or two more than the other's, and fail it
|
|
138
|
+
(https://github.com/sferik/duel-ruby/issues/6).
|
|
139
|
+
- A memory sample's peak can't count memory outside an iteration: a
|
|
140
|
+
reading could land between a pause's changes, on JRuby, whose threads
|
|
141
|
+
run at once.
|
|
142
|
+
- `duel init` writes LF line endings from templates checked out with
|
|
143
|
+
CRLF, and a failure shows a log's lines without a Windows program's CR.
|
|
144
|
+
- On Windows, a resource's log is added to, as it is elsewhere: a
|
|
145
|
+
command's output overwrote the start of it, as a child given a file
|
|
146
|
+
opened to append writes from where the file's at.
|
|
147
|
+
- A process benchmark's command that can't start (there's no such
|
|
148
|
+
program, say) fails saying why ("couldn't start: No such file or
|
|
149
|
+
directory - ..."), not "the process running ... exited", and the next
|
|
150
|
+
run goes on in the same Ruby process. It was that process that
|
|
151
|
+
ended. (JRuby runs a program it can't find through the shell, which
|
|
152
|
+
says why, and exits with 127.)
|
|
153
|
+
|
|
10
154
|
## [0.1.8] - 2026-11-12
|
|
11
155
|
|
|
12
156
|
### Added
|
|
@@ -762,7 +906,8 @@ needs Ruby 3.4 or later (or JRuby 10, or TruffleRuby 40).
|
|
|
762
906
|
Anything marked `@api private`, in lib/ and sig/, may change in any
|
|
763
907
|
release.
|
|
764
908
|
|
|
765
|
-
[Unreleased]: https://github.com/sferik/duel-ruby/compare/v0.
|
|
909
|
+
[Unreleased]: https://github.com/sferik/duel-ruby/compare/v0.2.0...HEAD
|
|
910
|
+
[0.2.0]: https://github.com/sferik/duel-ruby/compare/v0.1.8...v0.2.0
|
|
766
911
|
[0.1.8]: https://github.com/sferik/duel-ruby/compare/v0.1.7...v0.1.8
|
|
767
912
|
[0.1.7]: https://github.com/sferik/duel-ruby/compare/v0.1.6...v0.1.7
|
|
768
913
|
[0.1.6]: https://github.com/sferik/duel-ruby/compare/v0.1.5...v0.1.6
|
data/README.md
CHANGED
|
@@ -69,6 +69,38 @@ out are Duel's.
|
|
|
69
69
|
It adds StackProf to your `Gemfile`, for profiles of regressions, so
|
|
70
70
|
run `bundle install` again.
|
|
71
71
|
|
|
72
|
+
Duel needs Ruby 3.4 or later. It runs on Linux, macOS and Windows, on
|
|
73
|
+
CRuby and JRuby 10 (and on Linux, TruffleRuby).
|
|
74
|
+
|
|
75
|
+
### Windows
|
|
76
|
+
|
|
77
|
+
Duel runs on Windows with CRuby and JRuby 10. Its plugins run there on
|
|
78
|
+
CRuby, and start their servers (MySQL, PostgreSQL, Redis, memcached) on
|
|
79
|
+
a port of 127.0.0.1's rather than on a Unix socket. A few things work
|
|
80
|
+
differently:
|
|
81
|
+
|
|
82
|
+
- **CPU time** is the cycles Windows charged the thread or the process,
|
|
83
|
+
as its own CPU times move in ticks of 15.6 ms, longer than a sample
|
|
84
|
+
takes. There are no instruction counts, so CPU time judges alone.
|
|
85
|
+
- **A process benchmark** measures its own process's CPU time and peak
|
|
86
|
+
memory, but not that of the processes it starts, as Windows keeps no
|
|
87
|
+
account of a child's for its parent.
|
|
88
|
+
- **Stopping:** Windows can send a process no signal but to kill it,
|
|
89
|
+
so a resource's processes are killed as it stops. An `at_stop` block
|
|
90
|
+
can ask one to stop first.
|
|
91
|
+
- **JRuby** starts its JVM in a process of its own there (`jruby.exe`
|
|
92
|
+
starts `javaw.exe`), so Duel starts each worker's JVM itself, and a
|
|
93
|
+
process benchmark's of `ruby:` code, with the command `jruby -Xcommand`
|
|
94
|
+
prints: what it measures and stops is the JVM. (A process benchmark of
|
|
95
|
+
a command of its own that starts JRuby is measured as that command's
|
|
96
|
+
process, not the JVM it starts.) JRuby has no Unix sockets there, so a
|
|
97
|
+
worker talks to the runner over a port of 127.0.0.1's, and its spawn
|
|
98
|
+
redirects no output, so Duel starts processes through Open3, copying
|
|
99
|
+
their output to their logs.
|
|
100
|
+
|
|
101
|
+
The workflow that `duel init` writes runs on Linux, and Duel's reusable
|
|
102
|
+
one on Linux or macOS.
|
|
103
|
+
|
|
72
104
|
### Set up a project
|
|
73
105
|
|
|
74
106
|
`duel init` sets Duel up in a gem or an app, and writes the following:
|
|
@@ -86,6 +118,9 @@ out are Duel's.
|
|
|
86
118
|
use `--dry-run`. For a badge in your README that turns red when Duel
|
|
87
119
|
fails a push, use `--badge` (see
|
|
88
120
|
[Show that a project is guarded](#show-that-a-project-is-guarded)).
|
|
121
|
+
To write short workflows that call Duel's reusable ones instead of
|
|
122
|
+
copies of them, use `--reusable` (see
|
|
123
|
+
[Call Duel's reusable workflows](docs/ci.md#call-duels-reusable-workflows)).
|
|
89
124
|
When it's done, `duel init` lists the next steps.
|
|
90
125
|
|
|
91
126
|
For a Rails app, use
|
data/docs/ci.md
CHANGED
|
@@ -231,7 +231,7 @@ the Perf workflow completes (`workflow_run`), with your repository's
|
|
|
231
231
|
own copy of the workflow and a token that can write, so a pull request
|
|
232
232
|
from a fork gets a comment too. It checks nothing out and runs none of
|
|
233
233
|
the pull request's code. It installs Duel from RubyGems (the newest
|
|
234
|
-
0.
|
|
234
|
+
0.2 release, 0.2.0 or later) and runs `duel comment`, so a fix to how it comments comes
|
|
235
235
|
with a release of Duel, without a change to the workflow. The pull
|
|
236
236
|
request's code ran in the Duel job, and could have written anything
|
|
237
237
|
into the artifact, so `duel comment` trusts only what Duel could have
|
|
@@ -307,7 +307,7 @@ jobs:
|
|
|
307
307
|
GH_TOKEN: ${{ github.token }}
|
|
308
308
|
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
309
309
|
run: |
|
|
310
|
-
gem install duel --version '~> 0.
|
|
310
|
+
gem install duel --version '~> 0.2.0' --no-document
|
|
311
311
|
duel comment --report "$RUNNER_TEMP/report" --head "$HEAD_SHA" --marker 'duel'
|
|
312
312
|
|
|
313
313
|
again:
|
|
@@ -739,6 +739,99 @@ duel-rails's generator with `--check` instead:
|
|
|
739
739
|
bin/rails generate duel:install --check
|
|
740
740
|
```
|
|
741
741
|
|
|
742
|
+
## Call Duel's reusable workflows
|
|
743
|
+
|
|
744
|
+
Rather than copy Duel's whole workflows into your repository, you can
|
|
745
|
+
call Duel's reusable ones, which are the same workflows with each
|
|
746
|
+
setting as an input. `duel init --reusable` writes a Perf workflow and a
|
|
747
|
+
Perf comment workflow that call them, at the tag of the Duel release you
|
|
748
|
+
run it with, and pass on what your `.duel.rb` sets:
|
|
749
|
+
|
|
750
|
+
```yaml
|
|
751
|
+
# .github/workflows/perf.yml
|
|
752
|
+
jobs:
|
|
753
|
+
duel:
|
|
754
|
+
uses: sferik/duel-ruby/.github/workflows/duel.yml@v0.2.0
|
|
755
|
+
permissions:
|
|
756
|
+
contents: read
|
|
757
|
+
pull-requests: read # for the pull request's labels (see c.accept_label)
|
|
758
|
+
with:
|
|
759
|
+
ruby-version: "3.4"
|
|
760
|
+
accept-label: "duel: accept"
|
|
761
|
+
```
|
|
762
|
+
|
|
763
|
+
Your workflows keep their triggers, their concurrency and their
|
|
764
|
+
permissions, which Duel's jobs can only narrow. The job that runs the
|
|
765
|
+
pull request's code only reads, as it does in the whole workflow.
|
|
766
|
+
|
|
767
|
+
Duel's Perf workflow (`.github/workflows/duel.yml`) takes these inputs:
|
|
768
|
+
|
|
769
|
+
| Input | Default | What it is |
|
|
770
|
+
| --- | --- | --- |
|
|
771
|
+
| `ruby-version` | `default` | The Ruby the Duel job runs (`default`: the one `.ruby-version` names). |
|
|
772
|
+
| `version-file` | `false` | Whether each version runs on the Ruby its own `.ruby-version` names ([`c.ruby = :version_file`](configuration.md)). |
|
|
773
|
+
| `yjit` | `false` | Whether the benchmarks run with YJIT on (`RUBY_YJIT_ENABLE=1`), as a Rails app runs, say. |
|
|
774
|
+
| `runs-on` | `ubuntu-latest` | The runner the Duel job runs on, Linux or macOS: a larger one, say, for steadier timings. |
|
|
775
|
+
| `timeout-minutes` | `60` | How long the Duel job may take before it's cancelled. |
|
|
776
|
+
| `working-directory` | `.` | The project's directory, from the repository's root. |
|
|
777
|
+
| `gemfile` | none | A Gemfile of Duel's own, from the project's directory (`gemfiles/duel.gemfile`, say), which keeps Duel out of the project's other bundles. Duel and both versions' workers run in its bundle, so it needs what the benchmarks load (`eval_gemfile "../Gemfile"`, say). |
|
|
778
|
+
| `submodules` | `false` | Whether to check out the repository's submodules too (`true`, or `recursive`), as [actions/checkout](https://github.com/actions/checkout) takes it. |
|
|
779
|
+
| `setup` | none | Shell commands that set up whatever else your tests need (packages, services), before Ruby is set up. |
|
|
780
|
+
| `report` | none | A name for the call's report and check (`yjit`, say), where your workflow calls Duel's more than once: see below. |
|
|
781
|
+
| `accept-label` | none | The label that [accepts a regression](#accept-a-regression-with-a-label) (`c.accept_label`). |
|
|
782
|
+
| `release-tags` | `v*` | The glob your release tags match (`c.release_tags`). |
|
|
783
|
+
| `comment-on-pushes` | `false` | Whether to [comment on a pushed commit](#comment-on-a-pushed-commit) (`c.fail_on = :pull_requests`). |
|
|
784
|
+
| `mark-passed` | `false` | Whether to [mark a push passed](#compare-a-push-with-the-last-commit-that-passed) (`c.push_base = :last_passed`). |
|
|
785
|
+
| `badge` | `false` | Whether to [keep a badge](#show-whether-the-last-push-passed) for the README. |
|
|
786
|
+
| `passed-branch`, `badge-branch` | `duel/passed`, `duel/badge` | The branches those jobs move. |
|
|
787
|
+
| `marker` | `duel` | What marks Duel's comments. |
|
|
788
|
+
|
|
789
|
+
It says what Duel said, as its outputs, for jobs of your own (a notice
|
|
790
|
+
in a channel, say): `comment` (`post` or `change`, or nothing),
|
|
791
|
+
`status` (`passed`, `failed` or `neutral`) and `passed` (`true`, or
|
|
792
|
+
nothing).
|
|
793
|
+
|
|
794
|
+
To measure each version more than one way, call Duel's Perf workflow
|
|
795
|
+
more than once, each call with a `report` of its own:
|
|
796
|
+
|
|
797
|
+
```yaml
|
|
798
|
+
jobs:
|
|
799
|
+
yjit:
|
|
800
|
+
uses: sferik/duel-ruby/.github/workflows/duel.yml@v0.2.0
|
|
801
|
+
permissions:
|
|
802
|
+
contents: read
|
|
803
|
+
pull-requests: read
|
|
804
|
+
with:
|
|
805
|
+
yjit: true
|
|
806
|
+
report: yjit
|
|
807
|
+
interpreter:
|
|
808
|
+
uses: sferik/duel-ruby/.github/workflows/duel.yml@v0.2.0
|
|
809
|
+
permissions:
|
|
810
|
+
contents: read
|
|
811
|
+
pull-requests: read
|
|
812
|
+
with:
|
|
813
|
+
report: interpreter
|
|
814
|
+
```
|
|
815
|
+
|
|
816
|
+
Each call is a check of its own (`yjit / Duel (yjit)`), and the Perf
|
|
817
|
+
comment workflow comments with every call's report, as one comment,
|
|
818
|
+
each under its name. The comment's status is the most any call's
|
|
819
|
+
says: failed, if any failed, and passed only if every one passed. Turn
|
|
820
|
+
on `mark-passed`, `comment-on-pushes` or `badge` in one call at most,
|
|
821
|
+
as each call's jobs run on their own.
|
|
822
|
+
|
|
823
|
+
Duel's Perf comment workflow (`.github/workflows/duel-comment.yml`)
|
|
824
|
+
takes `accept-label`, `marker`, and `perf-workflow` (the name of your
|
|
825
|
+
Perf workflow, `Perf` by default).
|
|
826
|
+
|
|
827
|
+
A job that `mark-passed`, `comment-on-pushes` or `badge` turns on writes
|
|
828
|
+
to the repository, so your Perf workflow grants `contents: write` for
|
|
829
|
+
it. With an accept label, your Perf comment workflow grants
|
|
830
|
+
`actions: write`, to run the Duel job again. The checks are named for
|
|
831
|
+
your job and Duel's, as in `duel / Duel`. To update Duel's workflows,
|
|
832
|
+
change the tag. `duel init --check --reusable` compares the workflows
|
|
833
|
+
that call them.
|
|
834
|
+
|
|
742
835
|
## Run Duel on GitLab CI
|
|
743
836
|
|
|
744
837
|
`duel init --gitlab` writes GitLab CI jobs, in
|
|
@@ -878,7 +971,7 @@ duel:comment:
|
|
|
878
971
|
variables:
|
|
879
972
|
GIT_STRATEGY: none # it runs none of the code
|
|
880
973
|
script:
|
|
881
|
-
- gem install duel --version '~> 0.
|
|
974
|
+
- gem install duel --version '~> 0.2.0' --no-document
|
|
882
975
|
# (The merge request's head: in a merged results pipeline, not the
|
|
883
976
|
# commit that merges it.)
|
|
884
977
|
- duel comment --gitlab --report . --head "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --marker 'duel'
|
data/docs/configuration.md
CHANGED
|
@@ -20,6 +20,7 @@ Duel.configure do |c|
|
|
|
20
20
|
c.fail_on = :pull_requests # which runs a regression fails (default: :every_run)
|
|
21
21
|
c.push_base = :last_passed # what CI compares a push with (default: :before)
|
|
22
22
|
c.accept_label = "duel: accept" # a pull request label that accepts a regression
|
|
23
|
+
c.limit = 1.05 # a run's settings, as their options (see Set a run's settings)
|
|
23
24
|
c.use :redis # a plugin (see Use plugins)
|
|
24
25
|
|
|
25
26
|
# How a worker loads one version of your code, instead of the library.
|
|
@@ -179,6 +180,37 @@ a regression, every push fails until one is no slower, or until you
|
|
|
179
180
|
move the branch to accept the regression:
|
|
180
181
|
`git push origin main:duel/passed/main`.
|
|
181
182
|
|
|
183
|
+
## Set a run's settings
|
|
184
|
+
|
|
185
|
+
The settings that `duel`'s options set can be set in `.duel.rb` too, so
|
|
186
|
+
that a run in CI, where the workflow runs `duel` with no options, uses
|
|
187
|
+
them. An option given still wins: `duel --limit 1.1` runs with 1.1,
|
|
188
|
+
whatever `c.limit` says.
|
|
189
|
+
|
|
190
|
+
```ruby
|
|
191
|
+
Duel.configure do |c|
|
|
192
|
+
c.limit = 1.05 # --limit: the ratio above which a timing is slower (default: 1.02)
|
|
193
|
+
c.confidence = 0.995 # --confidence: of every interval (default: 0.99)
|
|
194
|
+
c.processes = 6 # --processes: workers of each version (default: 12, 8 on JRuby, as many as fit)
|
|
195
|
+
c.timeout = 120 # --timeout: seconds a worker may take to load or answer (default: 60, 300 on JRuby)
|
|
196
|
+
c.time_limit = 3600 # --time-limit: seconds a run measures for (default: 7200)
|
|
197
|
+
c.warm = 10 # --warm: seconds of warm-up for each benchmark (default: until steady)
|
|
198
|
+
end
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Each is `nil` by default, for Duel's own default, and one that isn't a
|
|
202
|
+
number in its range fails the run, as its option does. A measure's own
|
|
203
|
+
limit (`c.measure("cpu", limit: 1.1)`: see [Metric settings](#metric-settings))
|
|
204
|
+
is that measure's, whatever `c.limit` says.
|
|
205
|
+
|
|
206
|
+
A higher `c.confidence` fails less code that isn't slower, but takes
|
|
207
|
+
more samples to fail code that is, and more workers to judge each
|
|
208
|
+
benchmark over the workers: 5 of each version at 0.99, 6 at 0.995, and 7
|
|
209
|
+
at 0.999. With fewer, Duel judges by pairs of samples instead. A number
|
|
210
|
+
of processes set (`c.processes`, or `--processes`) is that many,
|
|
211
|
+
however much memory the machine has, where Duel's own default is as
|
|
212
|
+
many as fit.
|
|
213
|
+
|
|
182
214
|
## Prepare each worker
|
|
183
215
|
|
|
184
216
|
Each worker can get things of its own, such as a server, so that the
|
|
@@ -212,6 +244,7 @@ A prepare block can use the following methods:
|
|
|
212
244
|
| `w.cached(key) { \|dir\| ... }` | Makes something once, for every worker. See the following example. |
|
|
213
245
|
| `w.name` | The worker's name, for example `"base-1"`. |
|
|
214
246
|
| `w.dir` | The worker's directory. `w.path("name")` returns the path of a file in it. |
|
|
247
|
+
| `w.port` | A free port of 127.0.0.1's, for a server to listen on, rather than on a Unix socket in `w.dir`. |
|
|
215
248
|
| `w.root` | The tree of the version that the worker runs. |
|
|
216
249
|
| `w.ruby` | The path of the Ruby that the version runs on. See [Run each version on its own Ruby](#run-each-version-on-its-own-ruby). |
|
|
217
250
|
| `w.env` | The version's environment variables. See the following paragraph. |
|
|
@@ -226,7 +259,8 @@ command that sets its whole environment should include `w.env`.
|
|
|
226
259
|
When a worker is done, Duel runs the blocks that its prepare blocks
|
|
227
260
|
gave `w.at_stop`, last first. Then Duel stops what they spawned, first
|
|
228
261
|
with `TERM` (or the `stop_signal:` that the process was spawned with),
|
|
229
|
-
and then with `KILL`.
|
|
262
|
+
and then with `KILL`. On Windows, which can send a process no other
|
|
263
|
+
signal, Duel kills each one, with what it started.
|
|
230
264
|
|
|
231
265
|
Some things take long to make, and are the same for every worker, such
|
|
232
266
|
as a database's initial files. `w.cached` makes them once. The first
|
data/docs/how-it-works.md
CHANGED
|
@@ -129,13 +129,17 @@ Instruction counts come from the CPU's counters, where the operating
|
|
|
129
129
|
system lets Duel read them:
|
|
130
130
|
|
|
131
131
|
- On Linux, they're the benchmark thread's, through `perf_event_open`,
|
|
132
|
-
which many virtual machines, including CI runners, don't offer.
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
132
|
+
which many virtual machines, including CI runners, don't offer.
|
|
133
|
+
- On macOS, they're the benchmark thread's too, through
|
|
134
|
+
`thread_selfcounts`. Where a thread's can't be counted (in a virtual
|
|
135
|
+
machine, say), they're the whole process's.
|
|
136
136
|
- Elsewhere, and on JRuby and TruffleRuby, there's no `instructions`
|
|
137
137
|
metric, and CPU time judges alone.
|
|
138
138
|
|
|
139
|
+
Where they're the thread's, Duel doesn't count instructions for a
|
|
140
|
+
benchmark with threads of its own, as the count would leave out their
|
|
141
|
+
work.
|
|
142
|
+
|
|
139
143
|
## Where the extra time went
|
|
140
144
|
|
|
141
145
|
Duel shows what regressed per iteration, on each version, with what a
|
|
@@ -318,10 +322,12 @@ a few more. And an allocation the JIT does away with in every worker
|
|
|
318
322
|
isn't counted at all, as it isn't made. Turning escape analysis off
|
|
319
323
|
would count every allocation, but makes CPU time some three times
|
|
320
324
|
noisier, so Duel leaves it on. The count falls as the JIT compiles the
|
|
321
|
-
code, so it's taken after the
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
+
code, at different times in different workers, so it's taken after the
|
|
326
|
+
warm-up and the timing both, when the JIT has had longest. With fewer
|
|
327
|
+
than five workers of each version, it's each version's fewest in 10
|
|
328
|
+
samples, as a sample now and then is some 31 bytes an iteration more
|
|
329
|
+
than the rest. TruffleRuby has no allocation count. Neither has an
|
|
330
|
+
instruction count or profiles.
|
|
325
331
|
TruffleRuby has peak memory only on Linux. JRuby measures peak memory in
|
|
326
332
|
the JVM's heap in use, on any operating system, because the JVM takes
|
|
327
333
|
memory for its heap ahead of time, so resident memory barely moves as
|
data/docs/plugins.md
CHANGED
|
@@ -84,6 +84,22 @@ When you write a plugin, follow these guidelines:
|
|
|
84
84
|
memory available by what one of each version takes as it starts, and
|
|
85
85
|
a server takes more as it's used, so Duel counts at least this much
|
|
86
86
|
for each worker. A version of Duel before 0.1.8 ignores it.
|
|
87
|
+
- If a worker's server listens on a Unix socket in `w.dir`, give the
|
|
88
|
+
plugin an option to listen on `w.port` instead, and default it to
|
|
89
|
+
where `Duel.unix_sockets?` is false, for example
|
|
90
|
+
`def self.configure(config, tcp: !Duel.unix_sockets?)`. It's false on
|
|
91
|
+
Windows, where a server's own build often has Unix sockets that Ruby
|
|
92
|
+
can't reach, and where Ruby has no Unix sockets at all (JRuby on
|
|
93
|
+
Windows).
|
|
94
|
+
- If the plugin does something differently on Windows, ask
|
|
95
|
+
`Duel.windows?`, for example
|
|
96
|
+
`def self.configure(config, pg_ctl: Duel.windows?)`, not
|
|
97
|
+
`Duel::Platform` or `RbConfig` yourself: Duel's internals can change
|
|
98
|
+
in any release, and `Duel.windows?` won't. A version of Duel before
|
|
99
|
+
0.2.0 doesn't have it.
|
|
100
|
+
- To read a project's settings outside a run, as a generator that writes
|
|
101
|
+
its files might, load its `.duel.rb` with `Duel.load_config(path)`,
|
|
102
|
+
which returns its configuration. Before 0.2.0, it was private.
|
|
87
103
|
|
|
88
104
|
> [!IMPORTANT]
|
|
89
105
|
> A helper that's meant for setup, such as one that runs a query, must
|
data/lib/duel/across.rb
CHANGED
|
@@ -58,12 +58,7 @@ module Duel
|
|
|
58
58
|
end
|
|
59
59
|
return by_process_or_benchmark(medians, limits) if @by_process
|
|
60
60
|
|
|
61
|
-
medians.to_h
|
|
62
|
-
limit = limits.fetch(metric)
|
|
63
|
-
interval = Stats.median_interval(list, @confidence)
|
|
64
|
-
[metric, {"median" => Stats.median(list), "interval" => interval, "limit" => limit, "benchmarks" => list.size,
|
|
65
|
-
"verdict" => Stats.decide(interval, limit).to_s}]
|
|
66
|
-
end
|
|
61
|
+
medians.to_h { |metric, list| [metric, by_benchmark(list, limits.fetch(metric))] }
|
|
67
62
|
end
|
|
68
63
|
|
|
69
64
|
private
|
data/lib/duel/baseline.rb
CHANGED
|
@@ -5,6 +5,7 @@ require "fileutils"
|
|
|
5
5
|
require "open3"
|
|
6
6
|
require_relative "../duel"
|
|
7
7
|
require_relative "rubies"
|
|
8
|
+
require_relative "spawn"
|
|
8
9
|
|
|
9
10
|
module Duel
|
|
10
11
|
# The version to compare against: a directory (Settings#base_dir), or a
|
|
@@ -72,7 +73,8 @@ module Duel
|
|
|
72
73
|
# branch pushed (Settings#pushed), for Config#push_base:
|
|
73
74
|
# duel/passed/main, or for a project in a subdirectory of its
|
|
74
75
|
# repository, with its path (duel/passed-gems-parser/main), so each
|
|
75
|
-
# branch, and each of a monorepo's projects, has its own.
|
|
76
|
+
# branch, and each of a monorepo's projects, has its own. (As
|
|
77
|
+
# Init#passed_branch names them, which the workflow is given.)
|
|
76
78
|
def passed_branch = "#{prefix.empty? ? "duel/passed" : "duel/passed-#{prefix.tr("/", "-")}"}/#{@settings.pushed}"
|
|
77
79
|
|
|
78
80
|
# The last commit Duel passed in a push, as checked out from GitHub
|
|
@@ -86,8 +88,7 @@ module Duel
|
|
|
86
88
|
|
|
87
89
|
# (There's a ref: CLI#run checks first.)
|
|
88
90
|
commit = ref #: String
|
|
89
|
-
|
|
90
|
-
status.success?
|
|
91
|
+
!git("diff", "--quiet", commit, "--", *watched, allow_failure: true).nil?
|
|
91
92
|
end
|
|
92
93
|
|
|
93
94
|
# What a change must touch to be compared, from the root: the code
|
|
@@ -176,7 +177,7 @@ module Duel
|
|
|
176
177
|
command = @config.build
|
|
177
178
|
return unless command
|
|
178
179
|
|
|
179
|
-
output, status =
|
|
180
|
+
output, status = Spawn.open3(:capture2e, command, chdir: dir)
|
|
180
181
|
raise Error, "building #{what} (#{command}) failed:\n#{output}" unless status.success?
|
|
181
182
|
end
|
|
182
183
|
|
|
@@ -205,7 +206,7 @@ module Duel
|
|
|
205
206
|
"or compare with a directory (--base DIR)"
|
|
206
207
|
end
|
|
207
208
|
branch = main_branch
|
|
208
|
-
out, err, status =
|
|
209
|
+
out, err, status = Spawn.open3(:capture3, "git", "-C", @config.root, "merge-base", "HEAD", branch)
|
|
209
210
|
return out.chomp if status.success?
|
|
210
211
|
raise Error, "git merge-base HEAD #{branch} failed: #{err.chomp}" unless err.empty?
|
|
211
212
|
|
|
@@ -222,7 +223,7 @@ module Duel
|
|
|
222
223
|
# push, or nil where that isn't here (a branch's first push, or one
|
|
223
224
|
# that rewrote history).
|
|
224
225
|
def pushed
|
|
225
|
-
return passed if @config.push_base == :last_passed &&
|
|
226
|
+
return passed if @config.push_base == :last_passed && ancestor?(passed, "HEAD")
|
|
226
227
|
|
|
227
228
|
before = @settings.base #: String
|
|
228
229
|
before if git("cat-file", "-e", "#{before}^{commit}", allow_failure: true)
|
|
@@ -236,7 +237,7 @@ module Duel
|
|
|
236
237
|
def common_dir(dir) = git("-C", dir, "rev-parse", "--path-format=absolute", "--git-common-dir", allow_failure: true)
|
|
237
238
|
|
|
238
239
|
def git(*args, allow_failure: false)
|
|
239
|
-
out, err, status =
|
|
240
|
+
out, err, status = Spawn.open3(:capture3, "git", "-C", @config.root, *args)
|
|
240
241
|
return out.chomp if status.success?
|
|
241
242
|
raise Error, "git #{args.join(" ")} failed: #{err.chomp}" unless allow_failure
|
|
242
243
|
end
|