duel 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -1
- data/README.md +5 -5
- data/docs/ci.md +120 -25
- data/docs/configuration.md +8 -4
- data/docs/how-it-works.md +109 -35
- data/docs/json.md +16 -10
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +18 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +19 -3
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +337 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/perf-comment.yml.erb +53 -5
- data/lib/duel/templates/perf.yml.erb +71 -33
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +6 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +112 -6
- data/sig/duel/stats.rbs +25 -0
- metadata +1 -1
data/docs/json.md
CHANGED
|
@@ -53,7 +53,7 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
53
53
|
"limit": 1.02,
|
|
54
54
|
"filter": "",
|
|
55
55
|
"jobs": 4,
|
|
56
|
-
"processes":
|
|
56
|
+
"processes": 8,
|
|
57
57
|
"timeout": 60.0,
|
|
58
58
|
"warm": null,
|
|
59
59
|
"force": false
|
|
@@ -87,7 +87,8 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
87
87
|
"ratio": 1.071,
|
|
88
88
|
"interval": [1.069, 1.073],
|
|
89
89
|
"limit": 1.01,
|
|
90
|
-
"pairs":
|
|
90
|
+
"pairs": 16,
|
|
91
|
+
"processes": 8,
|
|
91
92
|
"verdict": "slower",
|
|
92
93
|
"baseline": 1521000.0,
|
|
93
94
|
"candidate": 1629000.0,
|
|
@@ -99,7 +100,8 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
99
100
|
"ratio": 1.081,
|
|
100
101
|
"interval": [1.062, 1.097],
|
|
101
102
|
"limit": 1.02,
|
|
102
|
-
"pairs":
|
|
103
|
+
"pairs": 16,
|
|
104
|
+
"processes": 8,
|
|
103
105
|
"verdict": "slower",
|
|
104
106
|
"baseline": 112400.0,
|
|
105
107
|
"candidate": 121500.0,
|
|
@@ -143,7 +145,8 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
143
145
|
"ratio": 1.002,
|
|
144
146
|
"interval": [0.994, 1.009],
|
|
145
147
|
"limit": 1.02,
|
|
146
|
-
"pairs":
|
|
148
|
+
"pairs": 16,
|
|
149
|
+
"processes": 8,
|
|
147
150
|
"verdict": "same",
|
|
148
151
|
"baseline": 410.5,
|
|
149
152
|
"candidate": 411.2,
|
|
@@ -174,6 +177,7 @@ A run that found one regression, abridged to two benchmarks:
|
|
|
174
177
|
"interval": [0.998, 1.012],
|
|
175
178
|
"limit": 1.005,
|
|
176
179
|
"benchmarks": 12,
|
|
180
|
+
"processes": 8,
|
|
177
181
|
"verdict": "undecided",
|
|
178
182
|
"confirmed": false
|
|
179
183
|
}
|
|
@@ -192,7 +196,7 @@ A run with nothing to compare has the same keys, with nothing compared:
|
|
|
192
196
|
"exit": 0,
|
|
193
197
|
"message": "duel: lib, ext, duel, .duel.rb unchanged since the baseline, so there's nothing to compare (--force to anyway)",
|
|
194
198
|
"comment": "change",
|
|
195
|
-
"settings": {"confidence": 0.99, "limit": 1.02, "filter": "", "jobs": 4, "processes":
|
|
199
|
+
"settings": {"confidence": 0.99, "limit": 1.02, "filter": "", "jobs": 4, "processes": 8, "timeout": 60.0, "warm": null, "force": false},
|
|
196
200
|
"seconds": null,
|
|
197
201
|
"baseline": null,
|
|
198
202
|
"candidate": null,
|
|
@@ -243,7 +247,7 @@ was slower across the board and confirmed (`"confirmed": true`).
|
|
|
243
247
|
| `processes` | Worker processes for each version (`--processes`). |
|
|
244
248
|
| `timeout` | The seconds that a worker may take to load or to answer (`--timeout`). |
|
|
245
249
|
| `time_limit` | The seconds that a run measures for, at most (`--time-limit`). |
|
|
246
|
-
| `warm` | The seconds of warm-up for each benchmark (`--warm`), or `null` for the default,
|
|
250
|
+
| `warm` | The seconds of warm-up for each benchmark (`--warm`), or `null` for the default: until steady, for up to as long as the Ruby takes. |
|
|
247
251
|
| `force` | Whether `--force` was given. |
|
|
248
252
|
|
|
249
253
|
### The versions
|
|
@@ -280,10 +284,11 @@ Each timing in `timed`, such as `cpu`:
|
|
|
280
284
|
|
|
281
285
|
| Key | What it is |
|
|
282
286
|
|---|---|
|
|
283
|
-
| `ratio` | The
|
|
284
|
-
| `interval` | The confidence interval for that
|
|
287
|
+
| `ratio` | The candidate's ratio to the baseline: the median of every candidate worker's median over every baseline worker's, or with `processes` `null`, the median of the paired ratios. Above 1 is the candidate slower. |
|
|
288
|
+
| `interval` | The confidence interval for that ratio, as `[low, high]`. |
|
|
285
289
|
| `limit` | The ratio above which it's slower. |
|
|
286
290
|
| `pairs` | The pairs of samples measured. |
|
|
291
|
+
| `processes` | The worker processes of each version that the interval is over, or `null` where it's over the pairs: for a process benchmark, or with fewer workers than the interval takes (see [How it works](how-it-works.md)). |
|
|
287
292
|
| `verdict` | `"slower"`, `"faster"`, `"same"`, or `"undecided"` (too close to the limit to settle). |
|
|
288
293
|
| `baseline`, `candidate` | Each version's median for one iteration, in `unit`. |
|
|
289
294
|
| `unit`, `plural` | The metric's unit, singular and plural, such as `"ns"`, or `null` for a plain number. |
|
|
@@ -303,9 +308,10 @@ Each metric in `across`, such as `cpu`:
|
|
|
303
308
|
|
|
304
309
|
| Key | What it is |
|
|
305
310
|
|---|---|
|
|
306
|
-
| `ratio` | The median of every benchmark's ratio. |
|
|
307
|
-
| `interval` | The confidence interval for that
|
|
311
|
+
| `ratio` | The median of every benchmark's ratio, or where they were judged by process, the ratio of the candidate's workers to the baseline's over every benchmark. |
|
|
312
|
+
| `interval` | The confidence interval for that ratio, as `[low, high]`, or `null` with too few benchmarks to tell. |
|
|
308
313
|
| `limit` | The ratio above which it's slower across the board. |
|
|
309
314
|
| `benchmarks` | The benchmarks that it's from. |
|
|
315
|
+
| `processes` | The worker processes of each version that the interval is over, or `null` where it's over the benchmarks. |
|
|
310
316
|
| `verdict` | `"slower"`, `"faster"`, `"same"`, or `"undecided"`. |
|
|
311
317
|
| `confirmed` | Whether it was slower again when every benchmark was measured a second time, which fails the run. |
|
data/docs/plugins.md
CHANGED
|
@@ -385,7 +385,9 @@ with every setting filled in. For example,
|
|
|
385
385
|
`{kind: :ratio, unit: "µs", floor: 300, ...}`.
|
|
386
386
|
|
|
387
387
|
`compare` takes `duel`'s options as keywords: `processes:` (two unless
|
|
388
|
-
given
|
|
388
|
+
given, too few for an interval over workers, so each timing is judged
|
|
389
|
+
over its pairs: enough to test a plugin, but not to judge how calibrated
|
|
390
|
+
its intervals are), `timeout:`, `limit:`, `warm:`, and `filter:` (a Regexp). It
|
|
389
391
|
returns the comparison: `passed?`, its `output` (what `duel` would have
|
|
390
392
|
printed), and its `results`, one for each benchmark.
|
|
391
393
|
`comparison["name"]` finds one. A result has the following:
|
data/docs/troubleshooting.md
CHANGED
|
@@ -42,15 +42,15 @@ that crashes leaves nothing behind that blocks the next run.
|
|
|
42
42
|
|
|
43
43
|
## A benchmark is reported as an error after a minute
|
|
44
44
|
|
|
45
|
-
If a worker takes more than a minute
|
|
46
|
-
benchmark that hangs or that waits on a
|
|
47
|
-
worker and reports the benchmark as an
|
|
48
|
-
`--timeout`. If a benchmark's iterations
|
|
49
|
-
smaller benchmarks.
|
|
45
|
+
If a worker takes more than a minute (five on JRuby and TruffleRuby) to
|
|
46
|
+
load or to answer, such as for a benchmark that hangs or that waits on a
|
|
47
|
+
server that died, Duel stops the worker and reports the benchmark as an
|
|
48
|
+
error. To change the limit, use `--timeout`. If a benchmark's iterations
|
|
49
|
+
are slow, split it into smaller benchmarks.
|
|
50
50
|
|
|
51
51
|
## The run reached its time limit
|
|
52
52
|
|
|
53
|
-
A run measures for
|
|
53
|
+
A run measures for two hours at most (`--time-limit`). Past that, Duel
|
|
54
54
|
starts no more measurements: it fails the benchmarks it hasn't measured,
|
|
55
55
|
and judges each benchmark that looked worse on the measurements it has.
|
|
56
56
|
The report starts the list of what it left undone with `TIME LIMIT`.
|
data/lib/duel/across.rb
CHANGED
|
@@ -5,24 +5,43 @@ require_relative "stats"
|
|
|
5
5
|
module Duel
|
|
6
6
|
# Whether the candidate is slower across the board: a slowdown smaller
|
|
7
7
|
# than any one benchmark can settle (1% everywhere, say) still shows in
|
|
8
|
-
# all of them together. For each ratio metric,
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
8
|
+
# all of them together. For each ratio metric, the candidate is slower
|
|
9
|
+
# when a distribution-free confidence interval lies entirely above a
|
|
10
|
+
# quarter of the metric's own margin (1.005, for a limit of 1.02). It
|
|
11
|
+
# takes 8 benchmarks at least, at 99%. One that says across: false is
|
|
12
|
+
# left out.
|
|
13
|
+
#
|
|
14
|
+
# Where the benchmarks were judged by process (Lane#judge_processes),
|
|
15
|
+
# the interval is over the workers: each has a score, how much faster
|
|
16
|
+
# or slower it was than its lane's other workers on the same
|
|
17
|
+
# benchmarks (see #scores), and the interval is for the ratio of the
|
|
18
|
+
# candidate's workers' scores to the baseline's (Stats.shift_interval).
|
|
19
|
+
# The same workers measure every benchmark in their lane, so a worker
|
|
20
|
+
# whose luck makes it slower is slower at all of them: an interval
|
|
21
|
+
# over the benchmarks would take that for a slowdown across the board.
|
|
22
|
+
# Otherwise, every benchmark judged on the metric is a sample (its
|
|
23
|
+
# median ratio), and the interval is for their median.
|
|
14
24
|
# @api private
|
|
15
25
|
class Across
|
|
26
|
+
# The fewest benchmarks across the board is judged on by process.
|
|
27
|
+
MIN_BENCHMARKS = 8
|
|
28
|
+
|
|
16
29
|
# The across-the-board limit for a metric whose own limit is +limit+.
|
|
17
30
|
def self.limit(limit) = 1 + ((limit - 1) / 4.0)
|
|
18
31
|
|
|
19
|
-
|
|
32
|
+
# +by_process+: whether the run judged benchmarks over workers (see
|
|
33
|
+
# Lane.by_process?), which they're judged across the board by too.
|
|
34
|
+
# Without, it looks for none, so that a run over pairs costs nothing
|
|
35
|
+
# more for them.
|
|
36
|
+
def initialize(results, confidence:, by_process: false)
|
|
20
37
|
@results = results
|
|
21
38
|
@confidence = confidence
|
|
39
|
+
@by_process = by_process
|
|
22
40
|
end
|
|
23
41
|
|
|
24
|
-
# {metric => {"median", "interval", "limit", "benchmarks", "verdict"}}
|
|
25
|
-
#
|
|
42
|
+
# {metric => {"median", "interval", "limit", "benchmarks", "verdict"}},
|
|
43
|
+
# and where it's judged by process, "processes" (the workers of each
|
|
44
|
+
# version), for every ratio metric any benchmark was judged on.
|
|
26
45
|
def verdicts
|
|
27
46
|
medians = Hash.new { |h, k| h[k] = [] }
|
|
28
47
|
# @type var limits: Hash[String, Float]
|
|
@@ -37,6 +56,8 @@ module Duel
|
|
|
37
56
|
limits[metric] ||= Across.limit(timed.fetch("limit"))
|
|
38
57
|
end
|
|
39
58
|
end
|
|
59
|
+
return by_process_or_benchmark(medians, limits) if @by_process
|
|
60
|
+
|
|
40
61
|
medians.to_h do |metric, list|
|
|
41
62
|
limit = limits.fetch(metric)
|
|
42
63
|
interval = Stats.median_interval(list, @confidence)
|
|
@@ -44,5 +65,75 @@ module Duel
|
|
|
44
65
|
"verdict" => Stats.decide(interval, limit).to_s}]
|
|
45
66
|
end
|
|
46
67
|
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
# The verdicts of a run that judged benchmarks over workers: over the
|
|
72
|
+
# workers' scores, for a metric with MIN_BENCHMARKS judged by process
|
|
73
|
+
# at least, and else over the benchmarks (+medians+, with +limits+).
|
|
74
|
+
def by_process_or_benchmark(medians, limits)
|
|
75
|
+
found = workers
|
|
76
|
+
medians.to_h do |metric, list|
|
|
77
|
+
limit = limits.fetch(metric)
|
|
78
|
+
processes = found.fetch(metric, [])
|
|
79
|
+
[metric, (processes.size >= MIN_BENCHMARKS) ? by_process(processes, limit) : by_benchmark(list, limit)]
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Each benchmark's workers' medians (Result#processes), of those
|
|
84
|
+
# judged by process, by metric.
|
|
85
|
+
def workers
|
|
86
|
+
# @type var found: Hash[String, Array[Result::processes]]
|
|
87
|
+
found = Hash.new { |h, k| h[k] = [] }
|
|
88
|
+
@results.each do |result|
|
|
89
|
+
next if result.error || result.skipped
|
|
90
|
+
|
|
91
|
+
result.timed.each do |metric, timed|
|
|
92
|
+
each = result.processes[metric]
|
|
93
|
+
found[metric] << each if each && timed.fetch("across")
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
found
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# The verdict over every benchmark's median ratio (+list+).
|
|
100
|
+
def by_benchmark(list, limit)
|
|
101
|
+
interval = Stats.median_interval(list, @confidence)
|
|
102
|
+
{"median" => Stats.median(list), "interval" => interval, "limit" => limit, "benchmarks" => list.size,
|
|
103
|
+
"verdict" => Stats.decide(interval, limit).to_s}
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# The verdict over the workers' scores (see #scores), from each
|
|
107
|
+
# benchmark's workers' medians (+processes+: Result#processes).
|
|
108
|
+
def by_process(processes, limit)
|
|
109
|
+
scored = scores(processes)
|
|
110
|
+
bases = scored.fetch("baseline")
|
|
111
|
+
candidates = scored.fetch("candidate")
|
|
112
|
+
interval = Stats.shift_interval(bases, candidates, @confidence)
|
|
113
|
+
{"median" => Stats.shift(bases, candidates), "interval" => interval, "limit" => limit, "benchmarks" => processes.size,
|
|
114
|
+
"processes" => bases.size, "verdict" => Stats.decide(interval, limit).to_s}
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Each worker's score, by version: the median, over the benchmarks it
|
|
118
|
+
# measured, of how much its median was above or below the median of
|
|
119
|
+
# every worker's in its lane (each version's), as a ratio. A worker's
|
|
120
|
+
# luck (where things landed in its memory) is the same for every
|
|
121
|
+
# benchmark, so it shows in the score, and the scores of two versions
|
|
122
|
+
# whose code is the same are alike.
|
|
123
|
+
def scores(processes)
|
|
124
|
+
deviations = Hash.new { |h, k| h[k] = [] }
|
|
125
|
+
processes.each do |each|
|
|
126
|
+
logs = {"baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}.transform_values do |values|
|
|
127
|
+
values.map { |value| Math.log([value, 1].max) }
|
|
128
|
+
end
|
|
129
|
+
center = Stats.median(logs.values.flatten)
|
|
130
|
+
logs.each do |side, list|
|
|
131
|
+
list.each_with_index { |log, i| deviations[[side, each.fetch("lane"), i]] << (log - center) }
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
%w[baseline candidate].to_h do |side|
|
|
135
|
+
[side, deviations.select { |(of, _, _), _| of == side }.values.map { |list| Math.exp(Stats.median(list)) }]
|
|
136
|
+
end
|
|
137
|
+
end
|
|
47
138
|
end
|
|
48
139
|
end
|
data/lib/duel/baseline.rb
CHANGED
|
@@ -17,7 +17,8 @@ module Duel
|
|
|
17
17
|
# origin/master, or (a repository without a remote) main, or master
|
|
18
18
|
# - "last-release": the newest release tag (Config#release_tags) before
|
|
19
19
|
# HEAD, to catch slowdowns too small for any one change to fail but
|
|
20
|
-
# that add up between releases
|
|
20
|
+
# that add up between releases, or a commit since then that merged a
|
|
21
|
+
# pull request with Config#accept_label (see #last_release)
|
|
21
22
|
# - a push's (Settings#pushed, the branch pushed): the commit before it, or where
|
|
22
23
|
# Config#push_base says, the last commit Duel passed (#passed)
|
|
23
24
|
# - anything else: a git ref
|
|
@@ -39,11 +40,34 @@ module Duel
|
|
|
39
40
|
|
|
40
41
|
@ref = case @settings.base
|
|
41
42
|
when nil then merge_base
|
|
42
|
-
when LAST_RELEASE then
|
|
43
|
+
when LAST_RELEASE then last_release
|
|
43
44
|
else @settings.pushed ? pushed : @settings.base
|
|
44
45
|
end
|
|
45
46
|
end
|
|
46
47
|
|
|
48
|
+
# The newest release tag (Config#release_tags) before HEAD, or nil.
|
|
49
|
+
def release
|
|
50
|
+
return @release if instance_variable_defined?(:@release)
|
|
51
|
+
|
|
52
|
+
@release = git("describe", "--tags", "--abbrev=0", "--match", @config.release_tags, "HEAD^", allow_failure: true)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The last release, or where a pull request merged since then was
|
|
56
|
+
# accepted with Config#accept_label, the commit that merged the last
|
|
57
|
+
# one (Settings#accepted, which the workflow finds): what it slowed
|
|
58
|
+
# down was accepted, so it's compared with instead. Nil with no
|
|
59
|
+
# release.
|
|
60
|
+
def last_release
|
|
61
|
+
found = release or return
|
|
62
|
+
accepted = @settings.accepted
|
|
63
|
+
return found unless accepted && !@config.accept_label.nil?
|
|
64
|
+
return found unless ancestor?(found, accepted) && !ancestor?(accepted, found) && ancestor?(accepted, "HEAD")
|
|
65
|
+
|
|
66
|
+
accepted
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def ancestor?(commit, of) = !git("merge-base", "--is-ancestor", commit, of, allow_failure: true).nil?
|
|
70
|
+
|
|
47
71
|
# The branch that marks the last commit Duel passed in a push to the
|
|
48
72
|
# branch pushed (Settings#pushed), for Config#push_base:
|
|
49
73
|
# duel/passed/main, or for a project in a subdirectory of its
|
data/lib/duel/cli.rb
CHANGED
|
@@ -123,13 +123,14 @@ module Duel
|
|
|
123
123
|
# The settings, from the environment and then the options.
|
|
124
124
|
def parse
|
|
125
125
|
settings = Settings.defaults
|
|
126
|
-
settings.jobs = [Etc.nprocessors /
|
|
126
|
+
settings.jobs = [Etc.nprocessors / CPUS_PER_LANE, 1].max
|
|
127
127
|
parser(settings).parse!(env_options + @argv)
|
|
128
128
|
branch = variable("GITHUB_REF_NAME", "CI_COMMIT_BRANCH")
|
|
129
129
|
base = variable("DUEL_BASE")
|
|
130
130
|
settings.pushed = branch if pushed? && !base.nil? && settings.base == base
|
|
131
131
|
target = variable("GITHUB_BASE_REF", "CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
|
|
132
132
|
settings.target = "origin/#{target}" if target && pull_request?
|
|
133
|
+
settings.accepted = variable("DUEL_ACCEPTED")
|
|
133
134
|
settings.config_path ||= (".duel.rb" if File.exist?(".duel.rb"))
|
|
134
135
|
settings.config_path &&= File.expand_path(settings.config_path)
|
|
135
136
|
settings
|
|
@@ -360,7 +361,7 @@ module Duel
|
|
|
360
361
|
commit = git(config.root, "rev-parse", "--short", "#{ref}^{commit}").first.chomp
|
|
361
362
|
chosen = case settings.base
|
|
362
363
|
when nil then "where HEAD left #{Summary.code(baseline.main_branch)}"
|
|
363
|
-
when Baseline::LAST_RELEASE then
|
|
364
|
+
when Baseline::LAST_RELEASE then last_release(config, baseline)
|
|
364
365
|
else
|
|
365
366
|
if settings.pushed then (ref == baseline.passed) ? "the last commit Duel passed" : "the commit before the push"
|
|
366
367
|
elsif queued?(settings) then "the merge queue's base"
|
|
@@ -371,6 +372,16 @@ module Duel
|
|
|
371
372
|
"#{commit} (#{chosen})"
|
|
372
373
|
end
|
|
373
374
|
|
|
375
|
+
# What chose a baseline of the last release: the release, or a change
|
|
376
|
+
# since it that was accepted with the label (Settings#accepted).
|
|
377
|
+
def last_release(config, baseline)
|
|
378
|
+
release = baseline.release #: String
|
|
379
|
+
return "the last release, #{Summary.code(release)}" if baseline.ref == release
|
|
380
|
+
|
|
381
|
+
label = config.accept_label #: String
|
|
382
|
+
"the last change accepted with the #{Summary.code(label)} label, since the last release, #{Summary.code(release)}"
|
|
383
|
+
end
|
|
384
|
+
|
|
374
385
|
# The candidate, in Markdown: its directory, or its commit, bare (see
|
|
375
386
|
# #described_baseline), and whether what's watched (Baseline#watched)
|
|
376
387
|
# has changes not committed.
|
|
@@ -448,14 +459,14 @@ module Duel
|
|
|
448
459
|
end
|
|
449
460
|
o.on("--candidate-dir DIR", "Candidate tree (default: this one)") { |v| settings.candidate_dir = v }
|
|
450
461
|
o.on("--filter REGEXP", "Only benchmarks whose names match") { |v| settings.filter = CLI.regexp(v) }
|
|
451
|
-
o.on("--jobs N", Integer, "Benchmarks measured at once (default:
|
|
452
|
-
o.on("--processes N", Integer, "Worker processes per version (default:
|
|
453
|
-
o.on("--timeout SECONDS", Float, "Most a worker may take to load or answer (default:
|
|
454
|
-
o.on("--time-limit SECONDS", Float, "Most a run measures for, then fails what's unmeasured or looks worse (default:
|
|
462
|
+
o.on("--jobs N", Integer, "Benchmarks measured at once (default: one for every #{CPUS_PER_LANE} CPUs)") { |v| settings.jobs = v }
|
|
463
|
+
o.on("--processes N", Integer, "Worker processes per version (default: #{PROCESSES})") { |v| settings.processes = v }
|
|
464
|
+
o.on("--timeout SECONDS", Float, "Most a worker may take to load or answer (default: #{TIMEOUT.round})") { |v| settings.timeout = v }
|
|
465
|
+
o.on("--time-limit SECONDS", Float, "Most a run measures for, then fails what's unmeasured or looks worse (default: 7200)") do |v|
|
|
455
466
|
settings.time_limit = v
|
|
456
467
|
end
|
|
457
468
|
o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: 1.02)") { |v| settings.limit = v }
|
|
458
|
-
o.on("--warm SECONDS", Float, "Warm-up per benchmark (default:
|
|
469
|
+
o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT on, else 0)") { |v| settings.warm = v }
|
|
459
470
|
o.on("--ruby version-file|PATH", "Each version's Ruby: the one its version file names (.ruby-version, say), or a",
|
|
460
471
|
"path (default: c.ruby, or this one)") { |v| settings.ruby = (v == "version-file") ? :version_file : v }
|
|
461
472
|
o.on("--force", "Compare even if nothing watched changed") { settings.force = true }
|
data/lib/duel/config.rb
CHANGED
|
@@ -134,7 +134,10 @@ module Duel
|
|
|
134
134
|
# warning, and the report says it was accepted. For a push, it's the
|
|
135
135
|
# labels of the pull request that merged it, so where #push_base is
|
|
136
136
|
# :last_passed, the push is marked passed too. The labels are read
|
|
137
|
-
# when the workflow runs, so
|
|
137
|
+
# when the workflow runs, so the comment workflow runs the Duel job
|
|
138
|
+
# again when the label is added or taken off. On a schedule, the
|
|
139
|
+
# workflow compares with the last commit since the last release that
|
|
140
|
+
# merged a pull request with the label (see Baseline#last_release).
|
|
138
141
|
#
|
|
139
142
|
# c.accept_label = "duel: accept"
|
|
140
143
|
attr_reader :accept_label
|
data/lib/duel/init.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "erb"
|
|
4
4
|
require "fileutils"
|
|
5
|
+
require "tmpdir"
|
|
5
6
|
require "open3"
|
|
6
7
|
require "optparse"
|
|
7
8
|
require "pathname"
|
|
@@ -182,21 +183,36 @@ module Duel
|
|
|
182
183
|
def config = Config.new(@root)
|
|
183
184
|
|
|
184
185
|
# The project's settings, which decide the workflow's jobs: its
|
|
185
|
-
# .duel.rb's, or
|
|
186
|
-
#
|
|
186
|
+
# .duel.rb's, or where there's none yet or --force replaces it, those
|
|
187
|
+
# of the one it writes. Nil where it doesn't load (one
|
|
187
188
|
# of its plugins isn't installed, say), and then the workflow has
|
|
188
189
|
# both jobs for pushes, as if .duel.rb asked for them, and says so.
|
|
189
190
|
def settings = instance_variable_defined?(:@settings) ? @settings : (@settings = load_settings)
|
|
190
191
|
|
|
191
192
|
def load_settings
|
|
192
193
|
path = File.join(@root, ".duel.rb")
|
|
193
|
-
return
|
|
194
|
+
return written_settings if @force || !File.exist?(path)
|
|
194
195
|
|
|
195
196
|
Duel.load_config(path)
|
|
196
197
|
rescue ScriptError, StandardError => e
|
|
197
198
|
@err.puts "duel init: .duel.rb didn't load (#{e.class}: #{e}), so the workflow has both jobs for pushes, as if it asked for them"
|
|
198
199
|
end
|
|
199
200
|
|
|
201
|
+
# The settings of the .duel.rb it writes.
|
|
202
|
+
def written_settings
|
|
203
|
+
Dir.mktmpdir do |dir|
|
|
204
|
+
path = File.join(dir, ".duel.rb")
|
|
205
|
+
File.write(path, render("duel.rb"))
|
|
206
|
+
Duel.load_config(path)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# The label that accepts a regression (Config#accept_label), for the
|
|
211
|
+
# workflows: the Duel job's, to find what was accepted since the last
|
|
212
|
+
# release, and the comment's, to run the Duel job again when it's
|
|
213
|
+
# added or taken off. Nil without one, or where .duel.rb doesn't load.
|
|
214
|
+
def accept_label = settings&.accept_label
|
|
215
|
+
|
|
200
216
|
# Whether the workflow comments on a pushed commit: where a regression
|
|
201
217
|
# in a push passes (c.fail_on = :pull_requests), the only time Duel
|
|
202
218
|
# says to.
|
data/lib/duel/json_report.rb
CHANGED
|
@@ -77,7 +77,7 @@ module Duel
|
|
|
77
77
|
result.timed.to_h do |metric, t|
|
|
78
78
|
n = result.n #: Integer
|
|
79
79
|
[metric, {"ratio" => t.fetch("median"), "interval" => t.fetch("interval"), "limit" => t.fetch("limit"),
|
|
80
|
-
"pairs" => t.fetch("pairs"), "verdict" => t.fetch("verdict"), "baseline" => t.fetch("baseline").fdiv(n),
|
|
80
|
+
"pairs" => t.fetch("pairs"), "processes" => t["processes"], "verdict" => t.fetch("verdict"), "baseline" => t.fetch("baseline").fdiv(n),
|
|
81
81
|
"candidate" => t.fetch("candidate").fdiv(n), "unit" => t.fetch("unit"), "plural" => t.fetch("plural"),
|
|
82
82
|
"across" => t.fetch("across")}]
|
|
83
83
|
end
|
|
@@ -103,7 +103,7 @@ module Duel
|
|
|
103
103
|
def across
|
|
104
104
|
@runner.across.transform_values do |v|
|
|
105
105
|
{"ratio" => v.fetch("median"), "interval" => v.fetch("interval"), "limit" => v.fetch("limit"), "benchmarks" => v.fetch("benchmarks"),
|
|
106
|
-
"verdict" => v.fetch("verdict"), "confirmed" => v.fetch("confirmed", false)}
|
|
106
|
+
"processes" => v["processes"], "verdict" => v.fetch("verdict"), "confirmed" => v.fetch("confirmed", false)}
|
|
107
107
|
end
|
|
108
108
|
end
|
|
109
109
|
end
|
data/lib/duel/metric.rb
CHANGED
|
@@ -424,11 +424,21 @@ module Duel
|
|
|
424
424
|
|
|
425
425
|
def self.process = java.lang.management.ManagementFactory.getOperatingSystemMXBean.getProcessCpuTime
|
|
426
426
|
end
|
|
427
|
+
|
|
428
|
+
# Bytes allocated (JRuby: the JVM's count, as it has none of objects):
|
|
429
|
+
# by the benchmark thread, or by every thread.
|
|
430
|
+
# @api private
|
|
431
|
+
module Allocated
|
|
432
|
+
def self.thread = java.lang.management.ManagementFactory.getThreadMXBean.getCurrentThreadAllocatedBytes
|
|
433
|
+
|
|
434
|
+
def self.process = java.lang.management.ManagementFactory.getThreadMXBean.getTotalThreadAllocatedBytes
|
|
435
|
+
end
|
|
427
436
|
end
|
|
428
437
|
# simplecov:enable
|
|
429
438
|
|
|
430
439
|
# The built-in metrics: their measures are named ("cpu",
|
|
431
|
-
# "instructions", "allocations"), as Config#measure
|
|
440
|
+
# "instructions", "allocations", "allocated_bytes"), as Config#measure
|
|
441
|
+
# takes them.
|
|
432
442
|
# @api private
|
|
433
443
|
module Metrics
|
|
434
444
|
# CPU time (see Definition#cpu), in nanoseconds: what other
|
|
@@ -477,6 +487,27 @@ module Duel
|
|
|
477
487
|
def stop_sample = (@counter.read - @started if @counting)
|
|
478
488
|
end
|
|
479
489
|
|
|
490
|
+
# Bytes allocated (JRuby, which counts no objects): the benchmark
|
|
491
|
+
# thread's, or for a benchmark with threads of its own, every
|
|
492
|
+
# thread's (see Definition#cpu), as the JVM counts them. It's the same
|
|
493
|
+
# in every worker, once the JIT has compiled the code (it can do away
|
|
494
|
+
# with an allocation, so the count falls as it warms up), but for a
|
|
495
|
+
# sample here and there some 31 bytes an iteration apart: so more by
|
|
496
|
+
# 32 bytes an iteration, about a small object's, isn't more yet.
|
|
497
|
+
class AllocatedBytes < Metric
|
|
498
|
+
def self.measures = {"allocated_bytes" => {tolerance: 32, unit: %w[byte bytes]}}
|
|
499
|
+
|
|
500
|
+
# mutant:disable -- mutant runs on CRuby, where this is always false
|
|
501
|
+
def self.available? = RUBY_ENGINE == "jruby"
|
|
502
|
+
|
|
503
|
+
def start_sample(sample)
|
|
504
|
+
@scope = sample.benchmark.cpu
|
|
505
|
+
@started = Allocated.public_send(@scope)
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
def stop_sample = Allocated.public_send(@scope) - @started
|
|
509
|
+
end
|
|
510
|
+
|
|
480
511
|
# Ruby objects allocated (CRuby). The same code's count can differ by
|
|
481
512
|
# a few from run to run (ids of random lengths, say), so it's one
|
|
482
513
|
# more per two iterations, not one more, that's a regression.
|
data/lib/duel/rubies.rb
CHANGED
|
@@ -35,6 +35,21 @@ module Duel
|
|
|
35
35
|
VARIABLES = %w[GEM_HOME GEM_PATH GEM_ROOT RUBY_ROOT RUBY_ENGINE RUBY_VERSION RUBY_PATCHLEVEL RBENV_VERSION
|
|
36
36
|
MISE_RUBY_VERSION ASDF_RUBY_VERSION].freeze
|
|
37
37
|
|
|
38
|
+
# The JVM's options for a JRuby worker. Each JVM's JIT compiles the
|
|
39
|
+
# same code a little differently, as its compiling in the background
|
|
40
|
+
# races its running, so one worker can be some 10% faster or slower
|
|
41
|
+
# than another for its whole life. -Xbatch compiles in the
|
|
42
|
+
# foreground, so what's compiled is less a matter of timing, and the
|
|
43
|
+
# serial collector runs no threads of its own beside the code:
|
|
44
|
+
# together they make JRuby's intervals over workers some half as
|
|
45
|
+
# wide.
|
|
46
|
+
JRUBY_WORKER = %w[-J-Xbatch -J-XX:+UseSerialGC].freeze
|
|
47
|
+
|
|
48
|
+
# A JVM option that chooses its garbage collector: the JVM won't start
|
|
49
|
+
# with two, so where JRUBY_OPTS chooses one, the serial one isn't
|
|
50
|
+
# added.
|
|
51
|
+
GARBAGE_COLLECTOR = /\A-J-XX:\+Use\w+GC\z/
|
|
52
|
+
|
|
38
53
|
class << self
|
|
39
54
|
# The Ruby (its executable) that +choice+ gives the version +name+d
|
|
40
55
|
# ("baseline") at +root+: one that isn't there fails.
|
|
@@ -50,6 +65,30 @@ module Duel
|
|
|
50
65
|
ruby
|
|
51
66
|
end
|
|
52
67
|
|
|
68
|
+
# The engine of +ruby+ (RUBY_ENGINE): the runner's, or what +ruby+
|
|
69
|
+
# says, without the runner's gems (or "ruby", where it can't say).
|
|
70
|
+
def engine(ruby)
|
|
71
|
+
return RUBY_ENGINE if runner?(ruby)
|
|
72
|
+
|
|
73
|
+
out = IO.popen(environment(ruby).merge("RUBYOPT" => nil), [ruby, "--disable-gems", "-e", "print RUBY_ENGINE"], err: File::NULL, &:read)
|
|
74
|
+
status = $? #: Process::Status
|
|
75
|
+
status.success? ? out : "ruby"
|
|
76
|
+
rescue SystemCallError
|
|
77
|
+
"ruby"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# What a worker on +engine+ starts with: +env+, and on JRuby, the
|
|
81
|
+
# JVM's options in JRUBY_WORKER (+env+'s JRUBY_OPTS, or else
|
|
82
|
+
# +inherited+, the runner's, before them).
|
|
83
|
+
def worker_environment(env, engine, inherited = ENV["JRUBY_OPTS"])
|
|
84
|
+
return env unless engine == "jruby"
|
|
85
|
+
|
|
86
|
+
options = env.fetch("JRUBY_OPTS") { inherited }.to_s.split
|
|
87
|
+
collector = options.any? { |option| option.match?(GARBAGE_COLLECTOR) }
|
|
88
|
+
added = JRUBY_WORKER.reject { |option| options.include?(option) || (collector && option.match?(GARBAGE_COLLECTOR)) }
|
|
89
|
+
env.merge("JRUBY_OPTS" => (options + added).join(" "))
|
|
90
|
+
end
|
|
91
|
+
|
|
53
92
|
# Whether +ruby+ is the runner's own.
|
|
54
93
|
def runner?(ruby) = File.realpath(ruby) == File.realpath(RbConfig.ruby)
|
|
55
94
|
|