duel 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +203 -1
- data/README.md +41 -1
- data/docs/ci.md +170 -76
- data/docs/configuration.md +38 -2
- data/docs/how-it-works.md +18 -9
- data/docs/plugins.md +23 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +96 -28
- data/lib/duel/comment.rb +296 -0
- data/lib/duel/config.rb +124 -4
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +255 -20
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +33 -1044
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +21 -15
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +61 -44
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +35 -3
- data/sig/duel/cli.rbs +19 -0
- data/sig/duel/comment.rbs +97 -0
- data/sig/duel/config.rbs +57 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +95 -1
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +6 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +6 -3
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -105
data/docs/configuration.md
CHANGED
|
@@ -20,6 +20,7 @@ Duel.configure do |c|
|
|
|
20
20
|
c.fail_on = :pull_requests # which runs a regression fails (default: :every_run)
|
|
21
21
|
c.push_base = :last_passed # what CI compares a push with (default: :before)
|
|
22
22
|
c.accept_label = "duel: accept" # a pull request label that accepts a regression
|
|
23
|
+
c.limit = 1.05 # a run's settings, as their options (see Set a run's settings)
|
|
23
24
|
c.use :redis # a plugin (see Use plugins)
|
|
24
25
|
|
|
25
26
|
# How a worker loads one version of your code, instead of the library.
|
|
@@ -139,7 +140,9 @@ end
|
|
|
139
140
|
On GitHub Actions, the [workflow](ci.md#accept-a-regression-with-a-label)
|
|
140
141
|
that `duel init` writes passes Duel the pull request's labels. With
|
|
141
142
|
that label, a regression passes, with a warning, and the report says
|
|
142
|
-
that the label accepted it.
|
|
143
|
+
that the label accepted it. It comments on that run as it would on one
|
|
144
|
+
that passed, so by default, after a comment on the run that failed, it
|
|
145
|
+
doesn't comment again. The labels are read when the workflow runs,
|
|
143
146
|
so the comment workflow runs the Duel job again when you add the label
|
|
144
147
|
to a pull request whose run failed, or take it off one whose run
|
|
145
148
|
passed. For a push, Duel reads the labels of the pull request that
|
|
@@ -177,6 +180,37 @@ a regression, every push fails until one is no slower, or until you
|
|
|
177
180
|
move the branch to accept the regression:
|
|
178
181
|
`git push origin main:duel/passed/main`.
|
|
179
182
|
|
|
183
|
+
## Set a run's settings
|
|
184
|
+
|
|
185
|
+
The settings that `duel`'s options set can be set in `.duel.rb` too, so
|
|
186
|
+
that a run in CI, where the workflow runs `duel` with no options, uses
|
|
187
|
+
them. An option given still wins: `duel --limit 1.1` runs with 1.1,
|
|
188
|
+
whatever `c.limit` says.
|
|
189
|
+
|
|
190
|
+
```ruby
|
|
191
|
+
Duel.configure do |c|
|
|
192
|
+
c.limit = 1.05 # --limit: the ratio above which a timing is slower (default: 1.02)
|
|
193
|
+
c.confidence = 0.995 # --confidence: of every interval (default: 0.99)
|
|
194
|
+
c.processes = 6 # --processes: workers of each version (default: 12, 8 on JRuby, as many as fit)
|
|
195
|
+
c.timeout = 120 # --timeout: seconds a worker may take to load or answer (default: 60, 300 on JRuby)
|
|
196
|
+
c.time_limit = 3600 # --time-limit: seconds a run measures for (default: 7200)
|
|
197
|
+
c.warm = 10 # --warm: seconds of warm-up for each benchmark (default: until steady)
|
|
198
|
+
end
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Each is `nil` by default, for Duel's own default, and one that isn't a
|
|
202
|
+
number in its range fails the run, as its option does. A measure's own
|
|
203
|
+
limit (`c.measure("cpu", limit: 1.1)`: see [Metric settings](#metric-settings))
|
|
204
|
+
is that measure's, whatever `c.limit` says.
|
|
205
|
+
|
|
206
|
+
A higher `c.confidence` fails less code that isn't slower, but takes
|
|
207
|
+
more samples to fail code that is, and more workers to judge each
|
|
208
|
+
benchmark over the workers: 5 of each version at 0.99, 6 at 0.995, and 7
|
|
209
|
+
at 0.999. With fewer, Duel judges by pairs of samples instead. A number
|
|
210
|
+
of processes set (`c.processes`, or `--processes`) is that many,
|
|
211
|
+
however much memory the machine has, where Duel's own default is as
|
|
212
|
+
many as fit.
|
|
213
|
+
|
|
180
214
|
## Prepare each worker
|
|
181
215
|
|
|
182
216
|
Each worker can get things of its own, such as a server, so that the
|
|
@@ -210,6 +244,7 @@ A prepare block can use the following methods:
|
|
|
210
244
|
| `w.cached(key) { \|dir\| ... }` | Makes something once, for every worker. See the following example. |
|
|
211
245
|
| `w.name` | The worker's name, for example `"base-1"`. |
|
|
212
246
|
| `w.dir` | The worker's directory. `w.path("name")` returns the path of a file in it. |
|
|
247
|
+
| `w.port` | A free port of 127.0.0.1's, for a server to listen on, rather than on a Unix socket in `w.dir`. |
|
|
213
248
|
| `w.root` | The tree of the version that the worker runs. |
|
|
214
249
|
| `w.ruby` | The path of the Ruby that the version runs on. See [Run each version on its own Ruby](#run-each-version-on-its-own-ruby). |
|
|
215
250
|
| `w.env` | The version's environment variables. See the following paragraph. |
|
|
@@ -224,7 +259,8 @@ command that sets its whole environment should include `w.env`.
|
|
|
224
259
|
When a worker is done, Duel runs the blocks that its prepare blocks
|
|
225
260
|
gave `w.at_stop`, last first. Then Duel stops what they spawned, first
|
|
226
261
|
with `TERM` (or the `stop_signal:` that the process was spawned with),
|
|
227
|
-
and then with `KILL`.
|
|
262
|
+
and then with `KILL`. On Windows, which can send a process no other
|
|
263
|
+
signal, Duel kills each one, with what it started.
|
|
228
264
|
|
|
229
265
|
Some things take long to make, and are the same for every worker, such
|
|
230
266
|
as a database's initial files. `w.cached` makes them once. The first
|
data/docs/how-it-works.md
CHANGED
|
@@ -62,7 +62,10 @@ never on noise, so that it can gate every pull request:
|
|
|
62
62
|
Running out would make the machine swap, or stop a CI runner from
|
|
63
63
|
answering, so its job is lost. To find what they take, Duel starts
|
|
64
64
|
one worker of each version first, and allows each pair a quarter
|
|
65
|
-
more than that
|
|
65
|
+
more than that (on TruffleRuby, whose workers grow as its JIT warms
|
|
66
|
+
up, 2.5 times that), or what its plugins say each worker's resources
|
|
67
|
+
take at most, if that's more (a server takes more as it's used), and
|
|
68
|
+
1 GB for everything else. Its first line says
|
|
66
69
|
when it lowered either. A `--jobs` or `--processes` you give is
|
|
67
70
|
kept as it is.
|
|
68
71
|
- **Measured again before failing.** Duel measures whatever looks
|
|
@@ -126,13 +129,17 @@ Instruction counts come from the CPU's counters, where the operating
|
|
|
126
129
|
system lets Duel read them:
|
|
127
130
|
|
|
128
131
|
- On Linux, they're the benchmark thread's, through `perf_event_open`,
|
|
129
|
-
which many virtual machines, including CI runners, don't offer.
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
132
|
+
which many virtual machines, including CI runners, don't offer.
|
|
133
|
+
- On macOS, they're the benchmark thread's too, through
|
|
134
|
+
`thread_selfcounts`. Where a thread's can't be counted (in a virtual
|
|
135
|
+
machine, say), they're the whole process's.
|
|
133
136
|
- Elsewhere, and on JRuby and TruffleRuby, there's no `instructions`
|
|
134
137
|
metric, and CPU time judges alone.
|
|
135
138
|
|
|
139
|
+
Where they're the thread's, Duel doesn't count instructions for a
|
|
140
|
+
benchmark with threads of its own, as the count would leave out their
|
|
141
|
+
work.
|
|
142
|
+
|
|
136
143
|
## Where the extra time went
|
|
137
144
|
|
|
138
145
|
Duel shows what regressed per iteration, on each version, with what a
|
|
@@ -315,10 +322,12 @@ a few more. And an allocation the JIT does away with in every worker
|
|
|
315
322
|
isn't counted at all, as it isn't made. Turning escape analysis off
|
|
316
323
|
would count every allocation, but makes CPU time some three times
|
|
317
324
|
noisier, so Duel leaves it on. The count falls as the JIT compiles the
|
|
318
|
-
code, so it's taken after the
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
325
|
+
code, at different times in different workers, so it's taken after the
|
|
326
|
+
warm-up and the timing both, when the JIT has had longest. With fewer
|
|
327
|
+
than five workers of each version, it's each version's fewest in 10
|
|
328
|
+
samples, as a sample now and then is some 31 bytes an iteration more
|
|
329
|
+
than the rest. TruffleRuby has no allocation count. Neither has an
|
|
330
|
+
instruction count or profiles.
|
|
322
331
|
TruffleRuby has peak memory only on Linux. JRuby measures peak memory in
|
|
323
332
|
the JVM's heap in use, on any operating system, because the JVM takes
|
|
324
333
|
memory for its heap ahead of time, so resident memory barely moves as
|
data/docs/plugins.md
CHANGED
|
@@ -77,6 +77,29 @@ When you write a plugin, follow these guidelines:
|
|
|
77
77
|
- Set up what a whole version needs once, such as its bundle, with
|
|
78
78
|
`c.prepare(:version)`, not in each worker's `c.prepare(:worker)`. Run
|
|
79
79
|
its commands on the version's Ruby: `v.env` puts it first on `PATH`.
|
|
80
|
+
- If each worker's `c.prepare(:worker)` starts something that takes a
|
|
81
|
+
lot of memory once it's used, such as a database server, say how much
|
|
82
|
+
at most, in bytes, as the module's `worker_memory`, for example
|
|
83
|
+
`def self.worker_memory = 450 << 20`. Duel fits its workers in the
|
|
84
|
+
memory available by what one of each version takes as it starts, and
|
|
85
|
+
a server takes more as it's used, so Duel counts at least this much
|
|
86
|
+
for each worker. A version of Duel before 0.1.8 ignores it.
|
|
87
|
+
- If a worker's server listens on a Unix socket in `w.dir`, give the
|
|
88
|
+
plugin an option to listen on `w.port` instead, and default it to
|
|
89
|
+
where `Duel.unix_sockets?` is false, for example
|
|
90
|
+
`def self.configure(config, tcp: !Duel.unix_sockets?)`. It's false on
|
|
91
|
+
Windows, where a server's own build often has Unix sockets that Ruby
|
|
92
|
+
can't reach, and where Ruby has no Unix sockets at all (JRuby on
|
|
93
|
+
Windows).
|
|
94
|
+
- If the plugin does something differently on Windows, ask
|
|
95
|
+
`Duel.windows?`, for example
|
|
96
|
+
`def self.configure(config, pg_ctl: Duel.windows?)`, not
|
|
97
|
+
`Duel::Platform` or `RbConfig` yourself: Duel's internals can change
|
|
98
|
+
in any release, and `Duel.windows?` won't. A version of Duel before
|
|
99
|
+
0.2.0 doesn't have it.
|
|
100
|
+
- To read a project's settings outside a run, as a generator that writes
|
|
101
|
+
its files might, load its `.duel.rb` with `Duel.load_config(path)`,
|
|
102
|
+
which returns its configuration. Before 0.2.0, it was private.
|
|
80
103
|
|
|
81
104
|
> [!IMPORTANT]
|
|
82
105
|
> A helper that's meant for setup, such as one that runs a query, must
|
data/lib/duel/across.rb
CHANGED
|
@@ -58,12 +58,7 @@ module Duel
|
|
|
58
58
|
end
|
|
59
59
|
return by_process_or_benchmark(medians, limits) if @by_process
|
|
60
60
|
|
|
61
|
-
medians.to_h
|
|
62
|
-
limit = limits.fetch(metric)
|
|
63
|
-
interval = Stats.median_interval(list, @confidence)
|
|
64
|
-
[metric, {"median" => Stats.median(list), "interval" => interval, "limit" => limit, "benchmarks" => list.size,
|
|
65
|
-
"verdict" => Stats.decide(interval, limit).to_s}]
|
|
66
|
-
end
|
|
61
|
+
medians.to_h { |metric, list| [metric, by_benchmark(list, limits.fetch(metric))] }
|
|
67
62
|
end
|
|
68
63
|
|
|
69
64
|
private
|
data/lib/duel/baseline.rb
CHANGED
|
@@ -5,6 +5,7 @@ require "fileutils"
|
|
|
5
5
|
require "open3"
|
|
6
6
|
require_relative "../duel"
|
|
7
7
|
require_relative "rubies"
|
|
8
|
+
require_relative "spawn"
|
|
8
9
|
|
|
9
10
|
module Duel
|
|
10
11
|
# The version to compare against: a directory (Settings#base_dir), or a
|
|
@@ -72,7 +73,8 @@ module Duel
|
|
|
72
73
|
# branch pushed (Settings#pushed), for Config#push_base:
|
|
73
74
|
# duel/passed/main, or for a project in a subdirectory of its
|
|
74
75
|
# repository, with its path (duel/passed-gems-parser/main), so each
|
|
75
|
-
# branch, and each of a monorepo's projects, has its own.
|
|
76
|
+
# branch, and each of a monorepo's projects, has its own. (As
|
|
77
|
+
# Init#passed_branch names them, which the workflow is given.)
|
|
76
78
|
def passed_branch = "#{prefix.empty? ? "duel/passed" : "duel/passed-#{prefix.tr("/", "-")}"}/#{@settings.pushed}"
|
|
77
79
|
|
|
78
80
|
# The last commit Duel passed in a push, as checked out from GitHub
|
|
@@ -86,8 +88,7 @@ module Duel
|
|
|
86
88
|
|
|
87
89
|
# (There's a ref: CLI#run checks first.)
|
|
88
90
|
commit = ref #: String
|
|
89
|
-
|
|
90
|
-
status.success?
|
|
91
|
+
!git("diff", "--quiet", commit, "--", *watched, allow_failure: true).nil?
|
|
91
92
|
end
|
|
92
93
|
|
|
93
94
|
# What a change must touch to be compared, from the root: the code
|
|
@@ -176,7 +177,7 @@ module Duel
|
|
|
176
177
|
command = @config.build
|
|
177
178
|
return unless command
|
|
178
179
|
|
|
179
|
-
output, status =
|
|
180
|
+
output, status = Spawn.open3(:capture2e, command, chdir: dir)
|
|
180
181
|
raise Error, "building #{what} (#{command}) failed:\n#{output}" unless status.success?
|
|
181
182
|
end
|
|
182
183
|
|
|
@@ -205,7 +206,7 @@ module Duel
|
|
|
205
206
|
"or compare with a directory (--base DIR)"
|
|
206
207
|
end
|
|
207
208
|
branch = main_branch
|
|
208
|
-
out, err, status =
|
|
209
|
+
out, err, status = Spawn.open3(:capture3, "git", "-C", @config.root, "merge-base", "HEAD", branch)
|
|
209
210
|
return out.chomp if status.success?
|
|
210
211
|
raise Error, "git merge-base HEAD #{branch} failed: #{err.chomp}" unless err.empty?
|
|
211
212
|
|
|
@@ -222,7 +223,7 @@ module Duel
|
|
|
222
223
|
# push, or nil where that isn't here (a branch's first push, or one
|
|
223
224
|
# that rewrote history).
|
|
224
225
|
def pushed
|
|
225
|
-
return passed if @config.push_base == :last_passed &&
|
|
226
|
+
return passed if @config.push_base == :last_passed && ancestor?(passed, "HEAD")
|
|
226
227
|
|
|
227
228
|
before = @settings.base #: String
|
|
228
229
|
before if git("cat-file", "-e", "#{before}^{commit}", allow_failure: true)
|
|
@@ -236,7 +237,7 @@ module Duel
|
|
|
236
237
|
def common_dir(dir) = git("-C", dir, "rev-parse", "--path-format=absolute", "--git-common-dir", allow_failure: true)
|
|
237
238
|
|
|
238
239
|
def git(*args, allow_failure: false)
|
|
239
|
-
out, err, status =
|
|
240
|
+
out, err, status = Spawn.open3(:capture3, "git", "-C", @config.root, *args)
|
|
240
241
|
return out.chomp if status.success?
|
|
241
242
|
raise Error, "git #{args.join(" ")} failed: #{err.chomp}" unless allow_failure
|
|
242
243
|
end
|
data/lib/duel/cli.rb
CHANGED
|
@@ -10,11 +10,14 @@ require_relative "lock"
|
|
|
10
10
|
require_relative "summary"
|
|
11
11
|
require_relative "json_report"
|
|
12
12
|
require_relative "init"
|
|
13
|
+
require_relative "comment"
|
|
14
|
+
require_relative "spawn"
|
|
13
15
|
|
|
14
16
|
module Duel
|
|
15
|
-
# The `duel` command (and `duel init`: see Init
|
|
16
|
-
# rake task or a CI expression can't pass
|
|
17
|
-
# the environment (DUEL_BASE, DUEL_FILTER,
|
|
17
|
+
# The `duel` command (and `duel init` and `duel comment`: see Init and
|
|
18
|
+
# Comment). What a run behind a rake task or a CI expression can't pass
|
|
19
|
+
# as an option can be set in the environment (DUEL_BASE, DUEL_FILTER,
|
|
20
|
+
# DUEL_FORCE), and options win.
|
|
18
21
|
# On GitHub Actions (GITHUB_STEP_SUMMARY), it adds its report to the
|
|
19
22
|
# job's summary, in Markdown (see Summary), and tells the step's
|
|
20
23
|
# outputs (GITHUB_OUTPUT) what to do about the comment on a pull
|
|
@@ -24,8 +27,9 @@ module Duel
|
|
|
24
27
|
# outcome, but not for a command line that isn't valid.
|
|
25
28
|
# @api private
|
|
26
29
|
class CLI
|
|
27
|
-
USAGE = "Usage: duel [options]\n duel init [options]\n\n" \
|
|
28
|
-
"Fails if this tree is slower than a baseline at any benchmark. `duel init` sets Duel up in a project
|
|
30
|
+
USAGE = "Usage: duel [options]\n duel init [options]\n duel comment [options]\n\n" \
|
|
31
|
+
"Fails if this tree is slower than a baseline at any benchmark. `duel init` sets Duel up in a project, and\n" \
|
|
32
|
+
"`duel comment` is what its Perf comment workflow runs.\n\n"
|
|
29
33
|
|
|
30
34
|
# Where GitHub Actions has a step add to its job's summary.
|
|
31
35
|
STEP_SUMMARY = "GITHUB_STEP_SUMMARY"
|
|
@@ -73,9 +77,12 @@ module Duel
|
|
|
73
77
|
# warned (see WARNED).
|
|
74
78
|
def run
|
|
75
79
|
return Init.new(@argv.drop(1), out: @out, err: @err).run if @argv.first == "init"
|
|
80
|
+
return Comment.new(@argv.drop(1), env: @env, out: @out, err: @err).run if @argv.first == "comment"
|
|
76
81
|
|
|
77
82
|
settings = @settings = catch(:version) { parse } or return 0
|
|
78
83
|
config = @config = settings.config_path ? Duel.load_config(settings.config_path) : Config.new
|
|
84
|
+
# (.duel.rb's, but where an option says: c.limit, unless --limit.)
|
|
85
|
+
config.run_settings.each { |name, value| set(settings, name, value) unless @given.include?(name) }
|
|
79
86
|
Lock.hold(config.state) { compare(config, settings) }
|
|
80
87
|
rescue Error => e
|
|
81
88
|
@err.puts "duel: #{e}"
|
|
@@ -122,6 +129,7 @@ module Duel
|
|
|
122
129
|
|
|
123
130
|
# The settings, from the environment and then the options.
|
|
124
131
|
def parse
|
|
132
|
+
@given = [] # (the RUN_SETTINGS options given: see Config)
|
|
125
133
|
settings = Settings.defaults
|
|
126
134
|
settings.jobs = [Etc.nprocessors / CPUS_PER_LANE, 1].max
|
|
127
135
|
settings.fit = %i[jobs processes]
|
|
@@ -171,7 +179,7 @@ module Duel
|
|
|
171
179
|
passed ? "passed" : "failed",
|
|
172
180
|
-> {
|
|
173
181
|
Summary.new(runner, results, settings: settings, baseline: described_baseline(config, settings, baseline),
|
|
174
|
-
candidate: described_candidate(config, settings), note: @note).markdown
|
|
182
|
+
candidate: described_candidate(config, settings), note: @note, accepted: @accepted).markdown
|
|
175
183
|
},
|
|
176
184
|
->(comment) {
|
|
177
185
|
JSONReport.new(runner, results, settings: settings, exit: status, comment: comment,
|
|
@@ -190,6 +198,7 @@ module Duel
|
|
|
190
198
|
if accepted?(config)
|
|
191
199
|
# (Set: see #accepted?.)
|
|
192
200
|
label = config.accept_label #: String
|
|
201
|
+
@accepted = true
|
|
193
202
|
return passing("accepted with the #{label} label", "Accepted with the #{Summary.code(label)} label.")
|
|
194
203
|
end
|
|
195
204
|
return 1 unless pull_requests_only?(config, settings)
|
|
@@ -222,11 +231,10 @@ module Duel
|
|
|
222
231
|
# (CI_MERGE_REQUEST_LABELS, by commas), or none, where neither is
|
|
223
232
|
# set, or DUEL_LABELS isn't a JSON array.
|
|
224
233
|
def labels
|
|
225
|
-
|
|
226
|
-
return
|
|
227
|
-
return [] unless set.key?("DUEL_LABELS")
|
|
234
|
+
json = variable("DUEL_LABELS")
|
|
235
|
+
return variable("CI_MERGE_REQUEST_LABELS").to_s.split(",") unless json
|
|
228
236
|
|
|
229
|
-
names = JSON.parse(
|
|
237
|
+
names = JSON.parse(json)
|
|
230
238
|
names.instance_of?(Array) ? names : []
|
|
231
239
|
rescue JSON::ParserError
|
|
232
240
|
[]
|
|
@@ -303,14 +311,17 @@ module Duel
|
|
|
303
311
|
# JSON report's: as --comment says, or else the configuration, or the
|
|
304
312
|
# default where the configuration didn't load. It comments on a push
|
|
305
313
|
# only where a regression in one passes (Config#fail_on), so the
|
|
306
|
-
# comment says what the check doesn't.
|
|
314
|
+
# comment says what the check doesn't. A regression the label
|
|
315
|
+
# accepts (Config#accept_label) is commented on as a run that passed
|
|
316
|
+
# would be: whoever added the label knows, so by default it posts
|
|
317
|
+
# only where the last comment's status isn't its own.
|
|
307
318
|
def commenting(status)
|
|
308
319
|
# (Parsed, so set: see #run.)
|
|
309
320
|
settings = @settings #: Settings
|
|
310
321
|
config = @config || Config.new
|
|
311
322
|
return "none" if pushed? && !pull_requests_only?(config, settings)
|
|
312
323
|
|
|
313
|
-
CLI.comment(settings.comment || config.comment, status)
|
|
324
|
+
CLI.comment(settings.comment || config.comment, @accepted ? "passed" : status)
|
|
314
325
|
end
|
|
315
326
|
|
|
316
327
|
# Writes the reports of a run whose +status+ is the JSON report's
|
|
@@ -326,14 +337,15 @@ module Duel
|
|
|
326
337
|
# writes them there too, as variables.
|
|
327
338
|
def publish(status, markdown, json)
|
|
328
339
|
comment = commenting(status)
|
|
329
|
-
set = variables
|
|
330
340
|
outputs = "comment=#{comment}\nstatus=#{MARKED.fetch(status)}\n#{"passed=true\n" if @passed}"
|
|
331
|
-
|
|
332
|
-
File.write(
|
|
341
|
+
step_output = variable(STEP_OUTPUT)
|
|
342
|
+
File.write(step_output, outputs, mode: "a") if step_output
|
|
343
|
+
output = variable(OUTPUT)
|
|
344
|
+
File.write(output, outputs.gsub(/^\w+=/) { |name| "DUEL_#{name.upcase}" }, mode: "a") if output
|
|
333
345
|
path = @json
|
|
334
346
|
write(path, "#{JSON.pretty_generate(json.call(comment))}\n") if path
|
|
335
347
|
path = @markdown
|
|
336
|
-
summary = (
|
|
348
|
+
summary = variable(STEP_SUMMARY)
|
|
337
349
|
return unless path || summary
|
|
338
350
|
|
|
339
351
|
text = markdown.call
|
|
@@ -392,7 +404,41 @@ module Duel
|
|
|
392
404
|
return Summary.code(dir) unless found.success?
|
|
393
405
|
|
|
394
406
|
_, clean = git(dir, "diff", "--quiet", "HEAD", "--", *Baseline.new(config, settings).watched)
|
|
395
|
-
"#{head.chomp}
|
|
407
|
+
return "#{head.chomp} (with changes not committed)" unless clean.success?
|
|
408
|
+
|
|
409
|
+
merged(dir) || head.chomp
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
# On GitHub Actions, a pull request's run checks out the commit that
|
|
413
|
+
# merges it with the branch it targets, which isn't in the pull
|
|
414
|
+
# request, so its head is named instead, as merged (or nil, where HEAD
|
|
415
|
+
# is no such commit). So does a merge request's merged results
|
|
416
|
+
# pipeline, on GitLab CI.
|
|
417
|
+
def merged(dir)
|
|
418
|
+
pull = (gitlab? ? merge_request : pull_request) or return
|
|
419
|
+
sha, branch = pull.dig("head", "sha"), pull.dig("base", "ref")
|
|
420
|
+
return unless branch.instance_of?(String) && git(dir, "rev-parse", "HEAD^2").first.chomp == sha
|
|
421
|
+
|
|
422
|
+
"#{git(dir, "rev-parse", "--short", sha).first.chomp} (merged with #{Summary.code(branch)})"
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
# A GitLab merge request's merged results pipeline's head and the
|
|
426
|
+
# branch it targets, as #pull_request has a pull request's (each nil
|
|
427
|
+
# in a merge request's other pipelines, whose HEAD is its head).
|
|
428
|
+
def merge_request = {"head" => {"sha" => variable("CI_MERGE_REQUEST_SOURCE_BRANCH_SHA")},
|
|
429
|
+
"base" => {"ref" => variable("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")}}
|
|
430
|
+
|
|
431
|
+
# The pull request a pull_request event's run is for, as GitHub
|
|
432
|
+
# Actions describes it (GITHUB_EVENT_PATH), or nil.
|
|
433
|
+
def pull_request
|
|
434
|
+
path = variable("GITHUB_EVENT_PATH") if event == "pull_request"
|
|
435
|
+
return unless path && File.file?(path)
|
|
436
|
+
|
|
437
|
+
payload = JSON.parse(File.read(path))
|
|
438
|
+
pull = payload["pull_request"] if payload.instance_of?(Hash)
|
|
439
|
+
pull if pull.instance_of?(Hash)
|
|
440
|
+
rescue JSON::ParserError
|
|
441
|
+
nil
|
|
396
442
|
end
|
|
397
443
|
|
|
398
444
|
# The baseline, for JSON: the ref that named it (the main branch,
|
|
@@ -422,7 +468,7 @@ module Duel
|
|
|
422
468
|
|
|
423
469
|
# Git's output, and its status, in +dir+.
|
|
424
470
|
def git(dir, *args)
|
|
425
|
-
out, _, status =
|
|
471
|
+
out, _, status = Spawn.open3(:capture3, "git", "-C", dir, *args)
|
|
426
472
|
[out, status]
|
|
427
473
|
end
|
|
428
474
|
|
|
@@ -448,6 +494,25 @@ module Duel
|
|
|
448
494
|
# The environment's variables that are set (not empty).
|
|
449
495
|
def variables = @env.reject { |_, value| value.empty? }
|
|
450
496
|
|
|
497
|
+
# Sets +settings+' +name+d setting (one of Config::RUN_SETTINGS) to
|
|
498
|
+
# +value+, an option's, over .duel.rb's: one it doesn't take is an
|
|
499
|
+
# invalid option.
|
|
500
|
+
def given(settings, name, value)
|
|
501
|
+
problem = Config.problem(name, value)
|
|
502
|
+
# (OptionParser says which option.)
|
|
503
|
+
raise OptionParser::InvalidArgument, "#{value}: give #{problem}" if problem
|
|
504
|
+
|
|
505
|
+
set(settings, name, value)
|
|
506
|
+
@given << name
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
# Sets +settings+' +name+d setting to +value+: a number of processes,
|
|
510
|
+
# that many, not as many as fit in memory.
|
|
511
|
+
def set(settings, name, value)
|
|
512
|
+
settings[name] = value
|
|
513
|
+
settings.fit -= [name]
|
|
514
|
+
end
|
|
515
|
+
|
|
451
516
|
def parser(settings)
|
|
452
517
|
OptionParser.new do |o|
|
|
453
518
|
o.banner = USAGE
|
|
@@ -461,19 +526,22 @@ module Duel
|
|
|
461
526
|
o.on("--candidate-dir DIR", "Candidate tree (default: this one)") { |v| settings.candidate_dir = v }
|
|
462
527
|
o.on("--filter REGEXP", "Only benchmarks whose names match") { |v| settings.filter = CLI.regexp(v) }
|
|
463
528
|
o.on("--jobs N", Integer, "Benchmarks measured at once (default: one for every #{CPUS_PER_LANE} CPUs, as many as fit in memory)") do |v|
|
|
464
|
-
settings
|
|
465
|
-
|
|
529
|
+
set(settings, :jobs, v)
|
|
530
|
+
end
|
|
531
|
+
o.on("--processes N", Integer, "Worker processes per version (default: c.processes, or #{PROCESSES}, as many as fit in memory)") do |v|
|
|
532
|
+
given(settings, :processes, v)
|
|
533
|
+
end
|
|
534
|
+
o.on("--timeout SECONDS", Float, "Most a worker may take to load or answer (default: c.timeout, or #{TIMEOUT.round})") do |v|
|
|
535
|
+
given(settings, :timeout, v)
|
|
466
536
|
end
|
|
467
|
-
o.on("--
|
|
468
|
-
settings
|
|
469
|
-
settings.fit -= [:processes]
|
|
537
|
+
o.on("--time-limit SECONDS", Float, "Most a run measures for, then fails what's unmeasured or looks worse (default: c.time_limit, or 7200)") do |v|
|
|
538
|
+
given(settings, :time_limit, v)
|
|
470
539
|
end
|
|
471
|
-
o.on("--
|
|
472
|
-
o.on("--
|
|
473
|
-
|
|
540
|
+
o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: c.limit, or 1.02)") { |v| given(settings, :limit, v) }
|
|
541
|
+
o.on("--confidence LEVEL", Float, "Confidence of every interval (default: c.confidence, or 0.99)") { |v| given(settings, :confidence, v) }
|
|
542
|
+
o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: c.warm, or until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT on, else 0)") do |v|
|
|
543
|
+
given(settings, :warm, v)
|
|
474
544
|
end
|
|
475
|
-
o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: 1.02)") { |v| settings.limit = v }
|
|
476
|
-
o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT on, else 0)") { |v| settings.warm = v }
|
|
477
545
|
o.on("--ruby version-file|PATH", "Each version's Ruby: the one its version file names (.ruby-version, say), or a",
|
|
478
546
|
"path (default: c.ruby, or this one)") { |v| settings.ruby = (v == "version-file") ? :version_file : v }
|
|
479
547
|
o.on("--force", "Compare even if nothing watched changed") { settings.force = true }
|