duel 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +135 -1
- data/README.md +5 -5
- data/docs/ci.md +145 -42
- data/docs/configuration.md +8 -4
- data/docs/how-it-works.md +123 -35
- data/docs/json.md +18 -12
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +25 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +19 -3
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/memory.rb +54 -0
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +383 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/perf-comment.yml.erb +53 -5
- data/lib/duel/templates/perf.yml.erb +85 -42
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +6 -0
- data/sig/duel/memory.rbs +30 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +129 -6
- data/sig/duel/stats.rbs +25 -0
- metadata +3 -1
data/docs/how-it-works.md
CHANGED
|
@@ -17,25 +17,54 @@ never on noise, so that it can gate every pull request:
|
|
|
17
17
|
only ever adds time. Garbage collection is kept out of samples, and
|
|
18
18
|
its cost shows in the allocation count instead, which is exact.
|
|
19
19
|
- **Side by side, many times over.** The two versions run in worker
|
|
20
|
-
processes of their own,
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
- **A confidence interval, not an average.**
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
20
|
+
processes of their own, at the same time on the same machine: twelve
|
|
21
|
+
of each by default, or eight on JRuby and TruffleRuby, whose workers
|
|
22
|
+
are JVMs. They take turns, one sample from each in alternating order,
|
|
23
|
+
in rounds. Each round takes a sample from every worker, and pairs
|
|
24
|
+
each baseline worker with a different candidate worker than the
|
|
25
|
+
last round did. That way, a busy moment weighs on both versions
|
|
26
|
+
alike. There are no stored numbers and no thresholds to maintain,
|
|
27
|
+
and any machine will do.
|
|
28
|
+
- **A confidence interval over processes, not an average.** Where
|
|
29
|
+
things land in a process's memory can make the very same code a few
|
|
30
|
+
percent faster or slower for the process's whole life, and on JRuby,
|
|
31
|
+
so can what its JIT compiled. So a sample from one process isn't
|
|
32
|
+
independent of the next sample from it, and the unit Duel judges is
|
|
33
|
+
the process: each worker's median. A timing is slower only when a
|
|
34
|
+
99% distribution-free confidence interval for the ratio of the
|
|
35
|
+
candidate's workers to the baseline's (the Mann-Whitney interval,
|
|
36
|
+
over every candidate worker's median against every baseline
|
|
37
|
+
worker's) lies entirely above the limit, which is 1.02 by default.
|
|
38
|
+
No one outlier sample can decide it, and no one lucky or unlucky
|
|
39
|
+
worker can either: the interval is as wide as the workers make it.
|
|
40
|
+
An interval over the pairs would be as narrow as if every pair had
|
|
41
|
+
processes of its own, and with identical code on both sides, it
|
|
42
|
+
would often leave out 1. Duel takes rounds until the interval
|
|
43
|
+
settles the question, from the fourth round on, and up to some 120
|
|
44
|
+
pairs. Duel reports a timing that's still too close to the limit to
|
|
45
|
+
call, and doesn't fail on it.
|
|
46
|
+
|
|
47
|
+
The interval over workers takes five of each at least. With fewer
|
|
48
|
+
(`--processes`), Duel judges over the pairs, as it does a process
|
|
49
|
+
benchmark, and its first line says so. A process benchmark's
|
|
50
|
+
sample is the fastest of a few fresh processes, so its pairs are
|
|
51
|
+
independent already. It can also stop after any pair once the
|
|
35
52
|
interval shows that it isn't slower, which only ever ends a
|
|
36
53
|
measurement before it could find a slowdown.
|
|
37
|
-
|
|
38
|
-
|
|
54
|
+
|
|
55
|
+
Duel measures one benchmark at a time for every two CPUs
|
|
56
|
+
(`--jobs`), each with workers of its own, so a run can start many
|
|
57
|
+
workers, and each one's resources (a database server, say) with it.
|
|
58
|
+
Where they wouldn't fit in the memory available (Linux's
|
|
59
|
+
MemAvailable, or a container's cgroup limit, or macOS's free and
|
|
60
|
+
inactive pages), Duel measures fewer benchmarks at once, and where
|
|
61
|
+
even one benchmark's workers wouldn't fit, it starts fewer workers.
|
|
62
|
+
Running out would make the machine swap, or stop a CI runner from
|
|
63
|
+
answering, so its job is lost. To find what they take, Duel starts
|
|
64
|
+
one worker of each version first, and allows each pair a quarter
|
|
65
|
+
more than that, and 1 GB for everything else. Its first line says
|
|
66
|
+
when it lowered either. A `--jobs` or `--processes` you give is
|
|
67
|
+
kept as it is.
|
|
39
68
|
- **Measured again before failing.** Duel measures whatever looks
|
|
40
69
|
slower again from scratch, up to three times, in fresh processes. It
|
|
41
70
|
fails only if every measurement finds the same metric worse. A
|
|
@@ -54,15 +83,27 @@ never on noise, so that it can gate every pull request:
|
|
|
54
83
|
than it otherwise would.
|
|
55
84
|
- **Exact counts.** Allocations (on CRuby), and any counted metric of
|
|
56
85
|
your own, fail on any increase: one more allocation per two
|
|
57
|
-
iterations.
|
|
86
|
+
iterations. Each worker's count is its fewest in three rounds of a
|
|
87
|
+
sample from every worker, and a count fails when an interval over the
|
|
88
|
+
workers, for how many more the candidate's are, lies above the
|
|
89
|
+
tolerance. Where every worker counts the same, as CRuby's workers
|
|
90
|
+
do, that's exactly how many more. Where workers differ, it's as few
|
|
91
|
+
as they can tell apart, so one worker's luck can't fail a benchmark.
|
|
92
|
+
With fewer than five workers of each version, a count is each
|
|
93
|
+
version's fewest in three samples instead.
|
|
58
94
|
- **Across the board, too.** A slowdown that's too small for any one
|
|
59
95
|
benchmark to settle, such as 1% everywhere, still shows in all of the
|
|
60
|
-
benchmarks together.
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
96
|
+
benchmarks together. The same workers measure every benchmark, so a
|
|
97
|
+
worker whose luck makes it slower is slower at all of them, and the
|
|
98
|
+
unit is the worker again. Each worker has a score: the median, over
|
|
99
|
+
the benchmarks it measured, of how much faster or slower it was than
|
|
100
|
+
the other workers of its lane. The candidate fails when a confidence
|
|
101
|
+
interval for the ratio of its workers' scores to the baseline's lies
|
|
102
|
+
above a quarter of the metric's margin: 1.005, for CPU time's 1.02.
|
|
103
|
+
Where the benchmarks were judged over pairs, every benchmark's median
|
|
104
|
+
ratio is a sample instead, and the interval is for their median.
|
|
105
|
+
Either takes 8 benchmarks, so with fewer, Duel doesn't report it.
|
|
106
|
+
Duel measures again before it fails, as it does for the rest.
|
|
66
107
|
|
|
67
108
|
## What Duel measures
|
|
68
109
|
|
|
@@ -73,6 +114,7 @@ Duel judges every benchmark on the following metrics:
|
|
|
73
114
|
| `cpu` | The benchmark thread's CPU time (or the process's) | More than 2% slower, with 99% confidence |
|
|
74
115
|
| `instructions` | Instructions retired, from the CPU's own counters | More than 1% more, with 99% confidence |
|
|
75
116
|
| `allocations` | Ruby objects allocated (on CRuby) | One more per two iterations |
|
|
117
|
+
| `allocated_bytes` | Bytes allocated (on JRuby), by the JVM's count | More than 32 bytes more per iteration |
|
|
76
118
|
| `peak_memory` | The most resident memory (on JRuby, the JVM's heap in use), above the start, that measured work reached (not `once` or `untimed`), with garbage collection running as usual (when asked for) | More than 10% and 1 MB more |
|
|
77
119
|
| Yours | See [Add your own metrics](configuration.md#add-your-own-metrics) | |
|
|
78
120
|
|
|
@@ -216,21 +258,67 @@ when the gems are installed in the project's directory, as
|
|
|
216
258
|
|
|
217
259
|
## JIT compilers
|
|
218
260
|
|
|
219
|
-
If YJIT is on in a worker, Duel runs each benchmark
|
|
220
|
-
before it measures it. A small library's methods
|
|
221
|
-
Duel's first samples, but in a big app, YJIT
|
|
222
|
-
120 calls. YJIT can be on because you
|
|
223
|
-
for example), or because your code
|
|
224
|
-
|
|
261
|
+
If YJIT is on in a worker, Duel runs each benchmark until it's steady
|
|
262
|
+
before it measures it, for up to ten seconds. A small library's methods
|
|
263
|
+
are compiled within Duel's first samples, but in a big app, YJIT
|
|
264
|
+
compiles a method only after 120 calls. YJIT can be on because you
|
|
265
|
+
turned it on (`RUBY_YJIT_ENABLE=1`, for example), or because your code
|
|
266
|
+
did as it loaded, as Rails 7.2 and later do.
|
|
267
|
+
|
|
268
|
+
A worker is steady once the median of its samples over each of three
|
|
269
|
+
half seconds in a row is no more than 1% faster than over the last half
|
|
270
|
+
second that was, so a pause (a collection, say) doesn't end its warm-up,
|
|
271
|
+
and code that creeps faster by less a half second isn't steady. Duel
|
|
272
|
+
measures once every worker is steady, or the time is up. To warm up for
|
|
273
|
+
a set time instead, use `--warm`.
|
|
225
274
|
|
|
226
275
|
### JRuby and TruffleRuby
|
|
227
276
|
|
|
228
277
|
JRuby's and TruffleRuby's JIT compilers keep compiling for seconds, so
|
|
229
|
-
Duel runs each benchmark for
|
|
230
|
-
(`--warm`).
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
278
|
+
Duel runs each benchmark until it's steady, for up to a minute, before
|
|
279
|
+
it measures it (`--warm`). Code that a JIT compiles quickly is steady
|
|
280
|
+
in a few seconds, but some code is still several times faster after
|
|
281
|
+
five seconds than it was, and steady only after 10 or 20.
|
|
282
|
+
|
|
283
|
+
Each JVM's JIT also compiles the same code a little differently, so one
|
|
284
|
+
worker can run a benchmark faster or slower than another for its whole
|
|
285
|
+
life, however long it warms up. On JRuby, Duel starts each worker's JVM
|
|
286
|
+
with `-Xbatch`, which compiles in the foreground, so what it compiles is
|
|
287
|
+
less a matter of timing, and with the serial garbage collector, which
|
|
288
|
+
runs no threads of its own beside the code. That makes the workers about
|
|
289
|
+
twice as alike, but a CPU-time interval on JRuby is still some 5% to 8%
|
|
290
|
+
wide, as wide as the workers make it, so Duel leaves a timing regression
|
|
291
|
+
of a few percent unsettled there, where it would find it on CRuby. It
|
|
292
|
+
doesn't fail identical code on that noise: an interval over the pairs
|
|
293
|
+
would be narrower, but it would often leave out 1 for identical code.
|
|
294
|
+
The options come after any in `JRUBY_OPTS`, and Duel doesn't add the
|
|
295
|
+
serial collector where `JRUBY_OPTS` chooses another. The processes a
|
|
296
|
+
process benchmark starts don't get them, as their boot is what's
|
|
297
|
+
measured.
|
|
298
|
+
|
|
299
|
+
Their workers are big JVMs, and their compilers keep other CPUs busy,
|
|
300
|
+
so Duel starts eight workers of each version rather than twelve, and
|
|
301
|
+
measures one benchmark at a time for every four CPUs (`--jobs`), not
|
|
302
|
+
every two. More at once is no faster there, and takes more memory: some
|
|
303
|
+
500 MB a worker.
|
|
304
|
+
|
|
305
|
+
JRuby counts no objects allocated, but the JVM counts the bytes each
|
|
306
|
+
thread allocates, and Duel judges those instead (`allocated_bytes`):
|
|
307
|
+
more than 32 bytes more an iteration fails, about a small object's, and
|
|
308
|
+
a change that makes objects bigger counts too. Where every worker counts
|
|
309
|
+
the same, Duel finds one more small object an iteration. But the JIT can
|
|
310
|
+
do away with an allocation that never leaves a method (escape analysis),
|
|
311
|
+
and each JVM's JIT does away with different ones, so on some code,
|
|
312
|
+
workers' counts differ by thousands of bytes, and an interval over the
|
|
313
|
+
workers (see How Duel tells a regression from noise) is too wide to find
|
|
314
|
+
a few more. And an allocation the JIT does away with in every worker
|
|
315
|
+
isn't counted at all, as it isn't made. Turning escape analysis off
|
|
316
|
+
would count every allocation, but makes CPU time some three times
|
|
317
|
+
noisier, so Duel leaves it on. The count falls as the JIT compiles the
|
|
318
|
+
code, so it's taken after the warm-up. With fewer than five workers of
|
|
319
|
+
each version, it's each version's fewest in 10 samples, as a sample now
|
|
320
|
+
and then is some 31 bytes an iteration more than the rest. TruffleRuby
|
|
321
|
+
has no allocation count. Neither has an instruction count or profiles.
|
|
234
322
|
TruffleRuby has peak memory only on Linux. JRuby measures peak memory in
|
|
235
323
|
the JVM's heap in use, on any operating system, because the JVM takes
|
|
236
324
|
memory for its heap ahead of time, so resident memory barely moves as
|
data/docs/json.md
CHANGED
|
@@ -53,7 +53,7 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
53
53
|
"limit": 1.02,
|
|
54
54
|
"filter": "",
|
|
55
55
|
"jobs": 4,
|
|
56
|
-
"processes":
|
|
56
|
+
"processes": 8,
|
|
57
57
|
"timeout": 60.0,
|
|
58
58
|
"warm": null,
|
|
59
59
|
"force": false
|
|
@@ -87,7 +87,8 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
87
87
|
"ratio": 1.071,
|
|
88
88
|
"interval": [1.069, 1.073],
|
|
89
89
|
"limit": 1.01,
|
|
90
|
-
"pairs":
|
|
90
|
+
"pairs": 16,
|
|
91
|
+
"processes": 8,
|
|
91
92
|
"verdict": "slower",
|
|
92
93
|
"baseline": 1521000.0,
|
|
93
94
|
"candidate": 1629000.0,
|
|
@@ -99,7 +100,8 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
99
100
|
"ratio": 1.081,
|
|
100
101
|
"interval": [1.062, 1.097],
|
|
101
102
|
"limit": 1.02,
|
|
102
|
-
"pairs":
|
|
103
|
+
"pairs": 16,
|
|
104
|
+
"processes": 8,
|
|
103
105
|
"verdict": "slower",
|
|
104
106
|
"baseline": 112400.0,
|
|
105
107
|
"candidate": 121500.0,
|
|
@@ -143,7 +145,8 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
143
145
|
"ratio": 1.002,
|
|
144
146
|
"interval": [0.994, 1.009],
|
|
145
147
|
"limit": 1.02,
|
|
146
|
-
"pairs":
|
|
148
|
+
"pairs": 16,
|
|
149
|
+
"processes": 8,
|
|
147
150
|
"verdict": "same",
|
|
148
151
|
"baseline": 410.5,
|
|
149
152
|
"candidate": 411.2,
|
|
@@ -174,6 +177,7 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
174
177
|
"interval": [0.998, 1.012],
|
|
175
178
|
"limit": 1.005,
|
|
176
179
|
"benchmarks": 12,
|
|
180
|
+
"processes": 8,
|
|
177
181
|
"verdict": "undecided",
|
|
178
182
|
"confirmed": false
|
|
179
183
|
}
|
|
@@ -192,7 +196,7 @@ A run with nothing to compare has the same keys, with nothing compared:
|
|
|
192
196
|
"exit": 0,
|
|
193
197
|
"message": "duel: lib, ext, duel, .duel.rb unchanged since the baseline, so there's nothing to compare (--force to anyway)",
|
|
194
198
|
"comment": "change",
|
|
195
|
-
"settings": {"confidence": 0.99, "limit": 1.02, "filter": "", "jobs": 4, "processes":
|
|
199
|
+
"settings": {"confidence": 0.99, "limit": 1.02, "filter": "", "jobs": 4, "processes": 8, "timeout": 60.0, "warm": null, "force": false},
|
|
196
200
|
"seconds": null,
|
|
197
201
|
"baseline": null,
|
|
198
202
|
"candidate": null,
|
|
@@ -239,11 +243,11 @@ was slower across the board and confirmed (`"confirmed": true`).
|
|
|
239
243
|
| `confidence` | The confidence of each interval, such as `0.99`. |
|
|
240
244
|
| `limit` | The ratio above which a timing is slower (`--limit`). A metric can have its own limit, which its timings show. |
|
|
241
245
|
| `filter` | The regular expression that picked the benchmarks (`--filter`), as source. `""` matches every one. |
|
|
242
|
-
| `jobs` | Benchmarks measured at once (`--jobs`). |
|
|
243
|
-
| `processes` | Worker processes for each version (`--processes`). |
|
|
246
|
+
| `jobs` | Benchmarks measured at once (`--jobs`), or fewer, where that many wouldn't fit in the memory available. |
|
|
247
|
+
| `processes` | Worker processes for each version (`--processes`), or fewer, where even one benchmark's wouldn't fit in the memory available. |
|
|
244
248
|
| `timeout` | The seconds that a worker may take to load or to answer (`--timeout`). |
|
|
245
249
|
| `time_limit` | The seconds that a run measures for, at most (`--time-limit`). |
|
|
246
|
-
| `warm` | The seconds of warm-up for each benchmark (`--warm`), or `null` for the default,
|
|
250
|
+
| `warm` | The seconds of warm-up for each benchmark (`--warm`), or `null` for the default: until steady, for up to as long as the Ruby takes. |
|
|
247
251
|
| `force` | Whether `--force` was given. |
|
|
248
252
|
|
|
249
253
|
### The versions
|
|
@@ -280,10 +284,11 @@ Each timing in `timed`, such as `cpu`:
|
|
|
280
284
|
|
|
281
285
|
| Key | What it is |
|
|
282
286
|
|---|---|
|
|
283
|
-
| `ratio` | The
|
|
284
|
-
| `interval` | The confidence interval for that
|
|
287
|
+
| `ratio` | The candidate's ratio to the baseline: the median of every candidate worker's median over every baseline worker's, or with `processes` `null`, the median of the paired ratios. Above 1 is the candidate slower. |
|
|
288
|
+
| `interval` | The confidence interval for that ratio, as `[low, high]`. |
|
|
285
289
|
| `limit` | The ratio above which it's slower. |
|
|
286
290
|
| `pairs` | The pairs of samples measured. |
|
|
291
|
+
| `processes` | The worker processes of each version that the interval is over, or `null` where it's over the pairs: for a process benchmark, or with fewer workers than the interval takes (see [How it works](how-it-works.md)). |
|
|
287
292
|
| `verdict` | `"slower"`, `"faster"`, `"same"`, or `"undecided"` (too close to the limit to settle). |
|
|
288
293
|
| `baseline`, `candidate` | Each version's median for one iteration, in `unit`. |
|
|
289
294
|
| `unit`, `plural` | The metric's unit, singular and plural, such as `"ns"`, or `null` for a plain number. |
|
|
@@ -303,9 +308,10 @@ Each metric in `across`, such as `cpu`:
|
|
|
303
308
|
|
|
304
309
|
| Key | What it is |
|
|
305
310
|
|---|---|
|
|
306
|
-
| `ratio` | The median of every benchmark's ratio. |
|
|
307
|
-
| `interval` | The confidence interval for that
|
|
311
|
+
| `ratio` | The median of every benchmark's ratio, or where they were judged by process, the ratio of the candidate's workers to the baseline's over every benchmark. |
|
|
312
|
+
| `interval` | The confidence interval for that ratio, as `[low, high]`, or `null` with too few benchmarks to tell. |
|
|
308
313
|
| `limit` | The ratio above which it's slower across the board. |
|
|
309
314
|
| `benchmarks` | The benchmarks that it's from. |
|
|
315
|
+
| `processes` | The worker processes of each version that the interval is over, or `null` where it's over the benchmarks. |
|
|
310
316
|
| `verdict` | `"slower"`, `"faster"`, `"same"`, or `"undecided"`. |
|
|
311
317
|
| `confirmed` | Whether it was slower again when every benchmark was measured a second time, which fails the run. |
|
data/docs/plugins.md
CHANGED
|
@@ -385,7 +385,9 @@ with every setting filled in. For example,
|
|
|
385
385
|
`{kind: :ratio, unit: "µs", floor: 300, ...}`.
|
|
386
386
|
|
|
387
387
|
`compare` takes `duel`'s options as keywords: `processes:` (two unless
|
|
388
|
-
given
|
|
388
|
+
given, too few for an interval over workers, so each timing is judged
|
|
389
|
+
over its pairs: enough to test a plugin, but not to judge how calibrated
|
|
390
|
+
its intervals are), `timeout:`, `limit:`, `warm:`, and `filter:` (a Regexp). It
|
|
389
391
|
returns the comparison: `passed?`, its `output` (what `duel` would have
|
|
390
392
|
printed), and its `results`, one for each benchmark.
|
|
391
393
|
`comparison["name"]` finds one. A result has the following:
|
data/docs/troubleshooting.md
CHANGED
|
@@ -42,15 +42,15 @@ that crashes leaves nothing behind that blocks the next run.
|
|
|
42
42
|
|
|
43
43
|
## A benchmark is reported as an error after a minute
|
|
44
44
|
|
|
45
|
-
If a worker takes more than a minute
|
|
46
|
-
benchmark that hangs or that waits on a
|
|
47
|
-
worker and reports the benchmark as an
|
|
48
|
-
`--timeout`. If a benchmark's iterations
|
|
49
|
-
smaller benchmarks.
|
|
45
|
+
If a worker takes more than a minute (five on JRuby and TruffleRuby) to
|
|
46
|
+
load or to answer, such as for a benchmark that hangs or that waits on a
|
|
47
|
+
server that died, Duel stops the worker and reports the benchmark as an
|
|
48
|
+
error. To change the limit, use `--timeout`. If a benchmark's iterations
|
|
49
|
+
are slow, split it into smaller benchmarks.
|
|
50
50
|
|
|
51
51
|
## The run reached its time limit
|
|
52
52
|
|
|
53
|
-
A run measures for
|
|
53
|
+
A run measures for two hours at most (`--time-limit`). Past that, Duel
|
|
54
54
|
starts no more measurements: it fails the benchmarks it hasn't measured,
|
|
55
55
|
and judges each benchmark that looked worse on the measurements it has.
|
|
56
56
|
The report starts the list of what it left undone with `TIME LIMIT`.
|
data/lib/duel/across.rb
CHANGED
|
@@ -5,24 +5,43 @@ require_relative "stats"
|
|
|
5
5
|
module Duel
|
|
6
6
|
# Whether the candidate is slower across the board: a slowdown smaller
|
|
7
7
|
# than any one benchmark can settle (1% everywhere, say) still shows in
|
|
8
|
-
# all of them together. For each ratio metric,
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
8
|
+
# all of them together. For each ratio metric, the candidate is slower
|
|
9
|
+
# when a distribution-free confidence interval lies entirely above a
|
|
10
|
+
# quarter of the metric's own margin (1.005, for a limit of 1.02). It
|
|
11
|
+
# takes 8 benchmarks at least, at 99%. One that says across: false is
|
|
12
|
+
# left out.
|
|
13
|
+
#
|
|
14
|
+
# Where the benchmarks were judged by process (Lane#judge_processes),
|
|
15
|
+
# the interval is over the workers: each has a score, how much faster
|
|
16
|
+
# or slower it was than its lane's other workers on the same
|
|
17
|
+
# benchmarks (see #scores), and the interval is for the ratio of the
|
|
18
|
+
# candidate's workers' scores to the baseline's (Stats.shift_interval).
|
|
19
|
+
# The same workers measure every benchmark in their lane, so a worker
|
|
20
|
+
# whose luck makes it slower is slower at all of them: an interval
|
|
21
|
+
# over the benchmarks would take that for a slowdown across the board.
|
|
22
|
+
# Otherwise, every benchmark judged on the metric is a sample (its
|
|
23
|
+
# median ratio), and the interval is for their median.
|
|
14
24
|
# @api private
|
|
15
25
|
class Across
|
|
26
|
+
# The fewest benchmarks across the board is judged on by process.
|
|
27
|
+
MIN_BENCHMARKS = 8
|
|
28
|
+
|
|
16
29
|
# The across-the-board limit for a metric whose own limit is +limit+.
|
|
17
30
|
def self.limit(limit) = 1 + ((limit - 1) / 4.0)
|
|
18
31
|
|
|
19
|
-
|
|
32
|
+
# +by_process+: whether the run judged benchmarks over workers (see
|
|
33
|
+
# Lane.by_process?), which they're judged across the board by too.
|
|
34
|
+
# Without, it looks for none, so that a run over pairs costs nothing
|
|
35
|
+
# more for them.
|
|
36
|
+
def initialize(results, confidence:, by_process: false)
|
|
20
37
|
@results = results
|
|
21
38
|
@confidence = confidence
|
|
39
|
+
@by_process = by_process
|
|
22
40
|
end
|
|
23
41
|
|
|
24
|
-
# {metric => {"median", "interval", "limit", "benchmarks", "verdict"}}
|
|
25
|
-
#
|
|
42
|
+
# {metric => {"median", "interval", "limit", "benchmarks", "verdict"}},
|
|
43
|
+
# and where it's judged by process, "processes" (the workers of each
|
|
44
|
+
# version), for every ratio metric any benchmark was judged on.
|
|
26
45
|
def verdicts
|
|
27
46
|
medians = Hash.new { |h, k| h[k] = [] }
|
|
28
47
|
# @type var limits: Hash[String, Float]
|
|
@@ -37,6 +56,8 @@ module Duel
|
|
|
37
56
|
limits[metric] ||= Across.limit(timed.fetch("limit"))
|
|
38
57
|
end
|
|
39
58
|
end
|
|
59
|
+
return by_process_or_benchmark(medians, limits) if @by_process
|
|
60
|
+
|
|
40
61
|
medians.to_h do |metric, list|
|
|
41
62
|
limit = limits.fetch(metric)
|
|
42
63
|
interval = Stats.median_interval(list, @confidence)
|
|
@@ -44,5 +65,75 @@ module Duel
|
|
|
44
65
|
"verdict" => Stats.decide(interval, limit).to_s}]
|
|
45
66
|
end
|
|
46
67
|
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
# The verdicts of a run that judged benchmarks over workers: over the
|
|
72
|
+
# workers' scores, for a metric with MIN_BENCHMARKS judged by process
|
|
73
|
+
# at least, and else over the benchmarks (+medians+, with +limits+).
|
|
74
|
+
def by_process_or_benchmark(medians, limits)
|
|
75
|
+
found = workers
|
|
76
|
+
medians.to_h do |metric, list|
|
|
77
|
+
limit = limits.fetch(metric)
|
|
78
|
+
processes = found.fetch(metric, [])
|
|
79
|
+
[metric, (processes.size >= MIN_BENCHMARKS) ? by_process(processes, limit) : by_benchmark(list, limit)]
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Each benchmark's workers' medians (Result#processes), of those
|
|
84
|
+
# judged by process, by metric.
|
|
85
|
+
def workers
|
|
86
|
+
# @type var found: Hash[String, Array[Result::processes]]
|
|
87
|
+
found = Hash.new { |h, k| h[k] = [] }
|
|
88
|
+
@results.each do |result|
|
|
89
|
+
next if result.error || result.skipped
|
|
90
|
+
|
|
91
|
+
result.timed.each do |metric, timed|
|
|
92
|
+
each = result.processes[metric]
|
|
93
|
+
found[metric] << each if each && timed.fetch("across")
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
found
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# The verdict over every benchmark's median ratio (+list+).
|
|
100
|
+
def by_benchmark(list, limit)
|
|
101
|
+
interval = Stats.median_interval(list, @confidence)
|
|
102
|
+
{"median" => Stats.median(list), "interval" => interval, "limit" => limit, "benchmarks" => list.size,
|
|
103
|
+
"verdict" => Stats.decide(interval, limit).to_s}
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# The verdict over the workers' scores (see #scores), from each
|
|
107
|
+
# benchmark's workers' medians (+processes+: Result#processes).
|
|
108
|
+
def by_process(processes, limit)
|
|
109
|
+
scored = scores(processes)
|
|
110
|
+
bases = scored.fetch("baseline")
|
|
111
|
+
candidates = scored.fetch("candidate")
|
|
112
|
+
interval = Stats.shift_interval(bases, candidates, @confidence)
|
|
113
|
+
{"median" => Stats.shift(bases, candidates), "interval" => interval, "limit" => limit, "benchmarks" => processes.size,
|
|
114
|
+
"processes" => bases.size, "verdict" => Stats.decide(interval, limit).to_s}
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Each worker's score, by version: the median, over the benchmarks it
|
|
118
|
+
# measured, of how much its median was above or below the median of
|
|
119
|
+
# every worker's in its lane (each version's), as a ratio. A worker's
|
|
120
|
+
# luck (where things landed in its memory) is the same for every
|
|
121
|
+
# benchmark, so it shows in the score, and the scores of two versions
|
|
122
|
+
# whose code is the same are alike.
|
|
123
|
+
def scores(processes)
|
|
124
|
+
deviations = Hash.new { |h, k| h[k] = [] }
|
|
125
|
+
processes.each do |each|
|
|
126
|
+
logs = {"baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}.transform_values do |values|
|
|
127
|
+
values.map { |value| Math.log([value, 1].max) }
|
|
128
|
+
end
|
|
129
|
+
center = Stats.median(logs.values.flatten)
|
|
130
|
+
logs.each do |side, list|
|
|
131
|
+
list.each_with_index { |log, i| deviations[[side, each.fetch("lane"), i]] << (log - center) }
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
%w[baseline candidate].to_h do |side|
|
|
135
|
+
[side, deviations.select { |(of, _, _), _| of == side }.values.map { |list| Math.exp(Stats.median(list)) }]
|
|
136
|
+
end
|
|
137
|
+
end
|
|
47
138
|
end
|
|
48
139
|
end
|
data/lib/duel/baseline.rb
CHANGED
|
@@ -17,7 +17,8 @@ module Duel
|
|
|
17
17
|
# origin/master, or (a repository without a remote) main, or master
|
|
18
18
|
# - "last-release": the newest release tag (Config#release_tags) before
|
|
19
19
|
# HEAD, to catch slowdowns too small for any one change to fail but
|
|
20
|
-
# that add up between releases
|
|
20
|
+
# that add up between releases, or a commit since then that merged a
|
|
21
|
+
# pull request with Config#accept_label (see #last_release)
|
|
21
22
|
# - a push's (Settings#pushed, the branch pushed): the commit before it, or where
|
|
22
23
|
# Config#push_base says, the last commit Duel passed (#passed)
|
|
23
24
|
# - anything else: a git ref
|
|
@@ -39,11 +40,34 @@ module Duel
|
|
|
39
40
|
|
|
40
41
|
@ref = case @settings.base
|
|
41
42
|
when nil then merge_base
|
|
42
|
-
when LAST_RELEASE then
|
|
43
|
+
when LAST_RELEASE then last_release
|
|
43
44
|
else @settings.pushed ? pushed : @settings.base
|
|
44
45
|
end
|
|
45
46
|
end
|
|
46
47
|
|
|
48
|
+
# The newest release tag (Config#release_tags) before HEAD, or nil.
|
|
49
|
+
def release
|
|
50
|
+
return @release if instance_variable_defined?(:@release)
|
|
51
|
+
|
|
52
|
+
@release = git("describe", "--tags", "--abbrev=0", "--match", @config.release_tags, "HEAD^", allow_failure: true)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The last release, or where a pull request merged since then was
|
|
56
|
+
# accepted with Config#accept_label, the commit that merged the last
|
|
57
|
+
# one (Settings#accepted, which the workflow finds): what it slowed
|
|
58
|
+
# down was accepted, so it's compared with instead. Nil with no
|
|
59
|
+
# release.
|
|
60
|
+
def last_release
|
|
61
|
+
found = release or return
|
|
62
|
+
accepted = @settings.accepted
|
|
63
|
+
return found unless accepted && !@config.accept_label.nil?
|
|
64
|
+
return found unless ancestor?(found, accepted) && !ancestor?(accepted, found) && ancestor?(accepted, "HEAD")
|
|
65
|
+
|
|
66
|
+
accepted
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def ancestor?(commit, of) = !git("merge-base", "--is-ancestor", commit, of, allow_failure: true).nil?
|
|
70
|
+
|
|
47
71
|
# The branch that marks the last commit Duel passed in a push to the
|
|
48
72
|
# branch pushed (Settings#pushed), for Config#push_base:
|
|
49
73
|
# duel/passed/main, or for a project in a subdirectory of its
|
data/lib/duel/cli.rb
CHANGED
|
@@ -123,13 +123,15 @@ module Duel
|
|
|
123
123
|
# The settings, from the environment and then the options.
|
|
124
124
|
def parse
|
|
125
125
|
settings = Settings.defaults
|
|
126
|
-
settings.jobs = [Etc.nprocessors /
|
|
126
|
+
settings.jobs = [Etc.nprocessors / CPUS_PER_LANE, 1].max
|
|
127
|
+
settings.fit = %i[jobs processes]
|
|
127
128
|
parser(settings).parse!(env_options + @argv)
|
|
128
129
|
branch = variable("GITHUB_REF_NAME", "CI_COMMIT_BRANCH")
|
|
129
130
|
base = variable("DUEL_BASE")
|
|
130
131
|
settings.pushed = branch if pushed? && !base.nil? && settings.base == base
|
|
131
132
|
target = variable("GITHUB_BASE_REF", "CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
|
|
132
133
|
settings.target = "origin/#{target}" if target && pull_request?
|
|
134
|
+
settings.accepted = variable("DUEL_ACCEPTED")
|
|
133
135
|
settings.config_path ||= (".duel.rb" if File.exist?(".duel.rb"))
|
|
134
136
|
settings.config_path &&= File.expand_path(settings.config_path)
|
|
135
137
|
settings
|
|
@@ -360,7 +362,7 @@ module Duel
|
|
|
360
362
|
commit = git(config.root, "rev-parse", "--short", "#{ref}^{commit}").first.chomp
|
|
361
363
|
chosen = case settings.base
|
|
362
364
|
when nil then "where HEAD left #{Summary.code(baseline.main_branch)}"
|
|
363
|
-
when Baseline::LAST_RELEASE then
|
|
365
|
+
when Baseline::LAST_RELEASE then last_release(config, baseline)
|
|
364
366
|
else
|
|
365
367
|
if settings.pushed then (ref == baseline.passed) ? "the last commit Duel passed" : "the commit before the push"
|
|
366
368
|
elsif queued?(settings) then "the merge queue's base"
|
|
@@ -371,6 +373,16 @@ module Duel
|
|
|
371
373
|
"#{commit} (#{chosen})"
|
|
372
374
|
end
|
|
373
375
|
|
|
376
|
+
# What chose a baseline of the last release: the release, or a change
|
|
377
|
+
# since it that was accepted with the label (Settings#accepted).
|
|
378
|
+
def last_release(config, baseline)
|
|
379
|
+
release = baseline.release #: String
|
|
380
|
+
return "the last release, #{Summary.code(release)}" if baseline.ref == release
|
|
381
|
+
|
|
382
|
+
label = config.accept_label #: String
|
|
383
|
+
"the last change accepted with the #{Summary.code(label)} label, since the last release, #{Summary.code(release)}"
|
|
384
|
+
end
|
|
385
|
+
|
|
374
386
|
# The candidate, in Markdown: its directory, or its commit, bare (see
|
|
375
387
|
# #described_baseline), and whether what's watched (Baseline#watched)
|
|
376
388
|
# has changes not committed.
|
|
@@ -448,14 +460,20 @@ module Duel
|
|
|
448
460
|
end
|
|
449
461
|
o.on("--candidate-dir DIR", "Candidate tree (default: this one)") { |v| settings.candidate_dir = v }
|
|
450
462
|
o.on("--filter REGEXP", "Only benchmarks whose names match") { |v| settings.filter = CLI.regexp(v) }
|
|
451
|
-
o.on("--jobs N", Integer, "Benchmarks measured at once (default:
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
463
|
+
o.on("--jobs N", Integer, "Benchmarks measured at once (default: one for every #{CPUS_PER_LANE} CPUs, as many as fit in memory)") do |v|
|
|
464
|
+
settings.jobs = v
|
|
465
|
+
settings.fit -= [:jobs]
|
|
466
|
+
end
|
|
467
|
+
o.on("--processes N", Integer, "Worker processes per version (default: #{PROCESSES}, as many as fit in memory)") do |v|
|
|
468
|
+
settings.processes = v
|
|
469
|
+
settings.fit -= [:processes]
|
|
470
|
+
end
|
|
471
|
+
o.on("--timeout SECONDS", Float, "Most a worker may take to load or answer (default: #{TIMEOUT.round})") { |v| settings.timeout = v }
|
|
472
|
+
o.on("--time-limit SECONDS", Float, "Most a run measures for, then fails what's unmeasured or looks worse (default: 7200)") do |v|
|
|
455
473
|
settings.time_limit = v
|
|
456
474
|
end
|
|
457
475
|
o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: 1.02)") { |v| settings.limit = v }
|
|
458
|
-
o.on("--warm SECONDS", Float, "Warm-up per benchmark (default:
|
|
476
|
+
o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT on, else 0)") { |v| settings.warm = v }
|
|
459
477
|
o.on("--ruby version-file|PATH", "Each version's Ruby: the one its version file names (.ruby-version, say), or a",
|
|
460
478
|
"path (default: c.ruby, or this one)") { |v| settings.ruby = (v == "version-file") ? :version_file : v }
|
|
461
479
|
o.on("--force", "Compare even if nothing watched changed") { settings.force = true }
|
data/lib/duel/config.rb
CHANGED
|
@@ -134,7 +134,10 @@ module Duel
|
|
|
134
134
|
# warning, and the report says it was accepted. For a push, it's the
|
|
135
135
|
# labels of the pull request that merged it, so where #push_base is
|
|
136
136
|
# :last_passed, the push is marked passed too. The labels are read
|
|
137
|
-
# when the workflow runs, so
|
|
137
|
+
# when the workflow runs, so the comment workflow runs the Duel job
|
|
138
|
+
# again when the label is added or taken off. On a schedule, the
|
|
139
|
+
# workflow compares with the last commit since the last release that
|
|
140
|
+
# merged a pull request with the label (see Baseline#last_release).
|
|
138
141
|
#
|
|
139
142
|
# c.accept_label = "duel: accept"
|
|
140
143
|
attr_reader :accept_label
|