duel 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +2 -2
  3. data/CHANGELOG.md +203 -1
  4. data/README.md +41 -1
  5. data/docs/ci.md +170 -76
  6. data/docs/configuration.md +38 -2
  7. data/docs/how-it-works.md +18 -9
  8. data/docs/plugins.md +23 -0
  9. data/lib/duel/across.rb +1 -6
  10. data/lib/duel/baseline.rb +8 -7
  11. data/lib/duel/cli.rb +96 -28
  12. data/lib/duel/comment.rb +296 -0
  13. data/lib/duel/config.rb +124 -4
  14. data/lib/duel/context.rb +33 -34
  15. data/lib/duel/control.rb +52 -0
  16. data/lib/duel/init.rb +255 -20
  17. data/lib/duel/lane.rb +734 -0
  18. data/lib/duel/lock.rb +34 -4
  19. data/lib/duel/memory.rb +14 -5
  20. data/lib/duel/metric.rb +97 -26
  21. data/lib/duel/native.rb +199 -32
  22. data/lib/duel/platform.rb +31 -0
  23. data/lib/duel/process_sample.rb +74 -49
  24. data/lib/duel/resource.rb +71 -42
  25. data/lib/duel/result.rb +165 -0
  26. data/lib/duel/rubies.rb +21 -3
  27. data/lib/duel/runner.rb +33 -1044
  28. data/lib/duel/spawn.rb +327 -0
  29. data/lib/duel/summary.rb +21 -15
  30. data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
  31. data/lib/duel/templates/perf-caller.yml.erb +43 -0
  32. data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
  33. data/lib/duel/templates/perf-comment.yml.erb +61 -44
  34. data/lib/duel/templates/perf.yml.erb +143 -7
  35. data/lib/duel/testing.rb +3 -0
  36. data/lib/duel/version.rb +1 -1
  37. data/lib/duel/worker.rb +13 -4
  38. data/lib/duel/worker_process.rb +142 -0
  39. data/lib/duel.rb +35 -3
  40. data/sig/duel/cli.rbs +19 -0
  41. data/sig/duel/comment.rbs +97 -0
  42. data/sig/duel/config.rbs +57 -2
  43. data/sig/duel/context.rbs +4 -31
  44. data/sig/duel/control.rbs +27 -0
  45. data/sig/duel/init.rbs +95 -1
  46. data/sig/duel/lane.rbs +202 -0
  47. data/sig/duel/lock.rbs +8 -1
  48. data/sig/duel/memory.rbs +1 -1
  49. data/sig/duel/metric.rbs +53 -10
  50. data/sig/duel/native.rbs +109 -9
  51. data/sig/duel/platform.rbs +13 -0
  52. data/sig/duel/process_sample.rbs +7 -9
  53. data/sig/duel/resource.rbs +28 -9
  54. data/sig/duel/result.rbs +164 -0
  55. data/sig/duel/rubies.rbs +4 -0
  56. data/sig/duel/runner.rbs +6 -460
  57. data/sig/duel/spawn.rbs +111 -0
  58. data/sig/duel/summary.rbs +6 -3
  59. data/sig/duel/worker_process.rbs +90 -0
  60. data/sig/duel.rbs +17 -3
  61. metadata +121 -105
@@ -20,6 +20,7 @@ Duel.configure do |c|
20
20
  c.fail_on = :pull_requests # which runs a regression fails (default: :every_run)
21
21
  c.push_base = :last_passed # what CI compares a push with (default: :before)
22
22
  c.accept_label = "duel: accept" # a pull request label that accepts a regression
23
+ c.limit = 1.05 # a run's settings, as their options (see Set a run's settings)
23
24
  c.use :redis # a plugin (see Use plugins)
24
25
 
25
26
  # How a worker loads one version of your code, instead of the library.
@@ -139,7 +140,9 @@ end
139
140
  On GitHub Actions, the [workflow](ci.md#accept-a-regression-with-a-label)
140
141
  that `duel init` writes passes Duel the pull request's labels. With
141
142
  that label, a regression passes, with a warning, and the report says
142
- that the label accepted it. The labels are read when the workflow runs,
143
+ that the label accepted it. It comments on that run as it would on one
144
+ that passed, so by default, after a comment on the run that failed, it
145
+ doesn't comment again. The labels are read when the workflow runs,
143
146
  so the comment workflow runs the Duel job again when you add the label
144
147
  to a pull request whose run failed, or take it off one whose run
145
148
  passed. For a push, Duel reads the labels of the pull request that
@@ -177,6 +180,37 @@ a regression, every push fails until one is no slower, or until you
177
180
  move the branch to accept the regression:
178
181
  `git push origin main:duel/passed/main`.
179
182
 
183
+ ## Set a run's settings
184
+
185
+ The settings that `duel`'s options set can be set in `.duel.rb` too, so
186
+ that a run in CI, where the workflow runs `duel` with no options, uses
187
+ them. An option given still wins: `duel --limit 1.1` runs with 1.1,
188
+ whatever `c.limit` says.
189
+
190
+ ```ruby
191
+ Duel.configure do |c|
192
+ c.limit = 1.05 # --limit: the ratio above which a timing is slower (default: 1.02)
193
+ c.confidence = 0.995 # --confidence: of every interval (default: 0.99)
194
+ c.processes = 6 # --processes: workers of each version (default: 12, 8 on JRuby, as many as fit)
195
+ c.timeout = 120 # --timeout: seconds a worker may take to load or answer (default: 60, 300 on JRuby)
196
+ c.time_limit = 3600 # --time-limit: seconds a run measures for (default: 7200)
197
+ c.warm = 10 # --warm: seconds of warm-up for each benchmark (default: until steady)
198
+ end
199
+ ```
200
+
201
+ Each is `nil` by default, for Duel's own default, and one that isn't a
202
+ number in its range fails the run, as its option does. A measure's own
203
+ limit (`c.measure("cpu", limit: 1.1)`: see [Metric settings](#metric-settings))
204
+ is that measure's, whatever `c.limit` says.
205
+
206
+ A higher `c.confidence` fails less code that isn't slower, but takes
207
+ more samples to fail code that is, and more workers to judge each
208
+ benchmark over the workers: 5 of each version at 0.99, 6 at 0.995, and 7
209
+ at 0.999. With fewer, Duel judges by pairs of samples instead. A number
210
+ of processes set (`c.processes`, or `--processes`) is that many,
211
+ however much memory the machine has, where Duel's own default is as
212
+ many as fit.
213
+
180
214
  ## Prepare each worker
181
215
 
182
216
  Each worker can get things of its own, such as a server, so that the
@@ -210,6 +244,7 @@ A prepare block can use the following methods:
210
244
  | `w.cached(key) { \|dir\| ... }` | Makes something once, for every worker. See the following example. |
211
245
  | `w.name` | The worker's name, for example `"base-1"`. |
212
246
  | `w.dir` | The worker's directory. `w.path("name")` returns the path of a file in it. |
247
+ | `w.port` | A free port of 127.0.0.1's, for a server to listen on, rather than on a Unix socket in `w.dir`. |
213
248
  | `w.root` | The tree of the version that the worker runs. |
214
249
  | `w.ruby` | The path of the Ruby that the version runs on. See [Run each version on its own Ruby](#run-each-version-on-its-own-ruby). |
215
250
  | `w.env` | The version's environment variables. See the following paragraph. |
@@ -224,7 +259,8 @@ command that sets its whole environment should include `w.env`.
224
259
  When a worker is done, Duel runs the blocks that its prepare blocks
225
260
  gave `w.at_stop`, last first. Then Duel stops what they spawned, first
226
261
  with `TERM` (or the `stop_signal:` that the process was spawned with),
227
- and then with `KILL`.
262
+ and then with `KILL`. On Windows, which can send a process no other
263
+ signal, Duel kills each one, with what it started.
228
264
 
229
265
  Some things take long to make, and are the same for every worker, such
230
266
  as a database's initial files. `w.cached` makes them once. The first
data/docs/how-it-works.md CHANGED
@@ -62,7 +62,10 @@ never on noise, so that it can gate every pull request:
62
62
  Running out would make the machine swap, or stop a CI runner from
63
63
  answering, so its job is lost. To find what they take, Duel starts
64
64
  one worker of each version first, and allows each pair a quarter
65
- more than that, and 1 GB for everything else. Its first line says
65
+ more than that (on TruffleRuby, whose workers grow as its JIT warms
66
+ up, 2.5 times that), or what its plugins say each worker's resources
67
+ take at most, if that's more (a server takes more as it's used), and
68
+ 1 GB for everything else. Its first line says
66
69
  when it lowered either. A `--jobs` or `--processes` you give is
67
70
  kept as it is.
68
71
  - **Measured again before failing.** Duel measures whatever looks
@@ -126,13 +129,17 @@ Instruction counts come from the CPU's counters, where the operating
126
129
  system lets Duel read them:
127
130
 
128
131
  - On Linux, they're the benchmark thread's, through `perf_event_open`,
129
- which many virtual machines, including CI runners, don't offer. So
130
- Linux doesn't count instructions for benchmarks with threads of their
131
- own.
132
- - On macOS, they're the whole process's.
132
+ which many virtual machines, including CI runners, don't offer.
133
+ - On macOS, they're the benchmark thread's too, through
134
+ `thread_selfcounts`. Where a thread's can't be counted (in a virtual
135
+ machine, say), they're the whole process's.
133
136
  - Elsewhere, and on JRuby and TruffleRuby, there's no `instructions`
134
137
  metric, and CPU time judges alone.
135
138
 
139
+ Where they're the thread's, Duel doesn't count instructions for a
140
+ benchmark with threads of its own, as the count would leave out their
141
+ work.
142
+
136
143
  ## Where the extra time went
137
144
 
138
145
  Duel shows what regressed per iteration, on each version, with what a
@@ -315,10 +322,12 @@ a few more. And an allocation the JIT does away with in every worker
315
322
  isn't counted at all, as it isn't made. Turning escape analysis off
316
323
  would count every allocation, but makes CPU time some three times
317
324
  noisier, so Duel leaves it on. The count falls as the JIT compiles the
318
- code, so it's taken after the warm-up. With fewer than five workers of
319
- each version, it's each version's fewest in 10 samples, as a sample now
320
- and then is some 31 bytes an iteration more than the rest. TruffleRuby
321
- has no allocation count. Neither has an instruction count or profiles.
325
+ code, at different times in different workers, so it's taken after the
326
+ warm-up and the timing both, when the JIT has had longest. With fewer
327
+ than five workers of each version, it's each version's fewest in 10
328
+ samples, as a sample now and then is some 31 bytes an iteration more
329
+ than the rest. TruffleRuby has no allocation count. Neither has an
330
+ instruction count or profiles.
322
331
  TruffleRuby has peak memory only on Linux. JRuby measures peak memory in
323
332
  the JVM's heap in use, on any operating system, because the JVM takes
324
333
  memory for its heap ahead of time, so resident memory barely moves as
data/docs/plugins.md CHANGED
@@ -77,6 +77,29 @@ When you write a plugin, follow these guidelines:
77
77
  - Set up what a whole version needs once, such as its bundle, with
78
78
  `c.prepare(:version)`, not in each worker's `c.prepare(:worker)`. Run
79
79
  its commands on the version's Ruby: `v.env` puts it first on `PATH`.
80
+ - If each worker's `c.prepare(:worker)` starts something that takes a
81
+ lot of memory once it's used, such as a database server, say how much
82
+ at most, in bytes, as the module's `worker_memory`, for example
83
+ `def self.worker_memory = 450 << 20`. Duel fits its workers in the
84
+ memory available by what one of each version takes as it starts, and
85
+ a server takes more as it's used, so Duel counts at least this much
86
+ for each worker. A version of Duel before 0.1.8 ignores it.
87
+ - If a worker's server listens on a Unix socket in `w.dir`, give the
88
+ plugin an option to listen on `w.port` instead, and default it to
89
+ where `Duel.unix_sockets?` is false, for example
90
+ `def self.configure(config, tcp: !Duel.unix_sockets?)`. It's false on
91
+ Windows, where a server's own build often has Unix sockets that Ruby
92
+ can't reach, and where Ruby has no Unix sockets at all (JRuby on
93
+ Windows).
94
+ - If the plugin does something differently on Windows, ask
95
+ `Duel.windows?`, for example
96
+ `def self.configure(config, pg_ctl: Duel.windows?)`, not
97
+ `Duel::Platform` or `RbConfig` yourself: Duel's internals can change
98
+ in any release, and `Duel.windows?` won't. A version of Duel before
99
+ 0.2.0 doesn't have it.
100
+ - To read a project's settings outside a run, as a generator that writes
101
+ its files might, load its `.duel.rb` with `Duel.load_config(path)`,
102
+ which returns its configuration. Before 0.2.0, it was private.
80
103
 
81
104
  > [!IMPORTANT]
82
105
  > A helper that's meant for setup, such as one that runs a query, must
data/lib/duel/across.rb CHANGED
@@ -58,12 +58,7 @@ module Duel
58
58
  end
59
59
  return by_process_or_benchmark(medians, limits) if @by_process
60
60
 
61
- medians.to_h do |metric, list|
62
- limit = limits.fetch(metric)
63
- interval = Stats.median_interval(list, @confidence)
64
- [metric, {"median" => Stats.median(list), "interval" => interval, "limit" => limit, "benchmarks" => list.size,
65
- "verdict" => Stats.decide(interval, limit).to_s}]
66
- end
61
+ medians.to_h { |metric, list| [metric, by_benchmark(list, limits.fetch(metric))] }
67
62
  end
68
63
 
69
64
  private
data/lib/duel/baseline.rb CHANGED
@@ -5,6 +5,7 @@ require "fileutils"
5
5
  require "open3"
6
6
  require_relative "../duel"
7
7
  require_relative "rubies"
8
+ require_relative "spawn"
8
9
 
9
10
  module Duel
10
11
  # The version to compare against: a directory (Settings#base_dir), or a
@@ -72,7 +73,8 @@ module Duel
72
73
  # branch pushed (Settings#pushed), for Config#push_base:
73
74
  # duel/passed/main, or for a project in a subdirectory of its
74
75
  # repository, with its path (duel/passed-gems-parser/main), so each
75
- # branch, and each of a monorepo's projects, has its own.
76
+ # branch, and each of a monorepo's projects, has its own. (As
77
+ # Init#passed_branch names them, which the workflow is given.)
76
78
  def passed_branch = "#{prefix.empty? ? "duel/passed" : "duel/passed-#{prefix.tr("/", "-")}"}/#{@settings.pushed}"
77
79
 
78
80
  # The last commit Duel passed in a push, as checked out from GitHub
@@ -86,8 +88,7 @@ module Duel
86
88
 
87
89
  # (There's a ref: CLI#run checks first.)
88
90
  commit = ref #: String
89
- _, status = Open3.capture2e("git", "-C", @config.root, "diff", "--quiet", commit, "--", *watched)
90
- status.success?
91
+ !git("diff", "--quiet", commit, "--", *watched, allow_failure: true).nil?
91
92
  end
92
93
 
93
94
  # What a change must touch to be compared, from the root: the code
@@ -176,7 +177,7 @@ module Duel
176
177
  command = @config.build
177
178
  return unless command
178
179
 
179
- output, status = Open3.capture2e(command, chdir: dir)
180
+ output, status = Spawn.open3(:capture2e, command, chdir: dir)
180
181
  raise Error, "building #{what} (#{command}) failed:\n#{output}" unless status.success?
181
182
  end
182
183
 
@@ -205,7 +206,7 @@ module Duel
205
206
  "or compare with a directory (--base DIR)"
206
207
  end
207
208
  branch = main_branch
208
- out, err, status = Open3.capture3("git", "-C", @config.root, "merge-base", "HEAD", branch)
209
+ out, err, status = Spawn.open3(:capture3, "git", "-C", @config.root, "merge-base", "HEAD", branch)
209
210
  return out.chomp if status.success?
210
211
  raise Error, "git merge-base HEAD #{branch} failed: #{err.chomp}" unless err.empty?
211
212
 
@@ -222,7 +223,7 @@ module Duel
222
223
  # push, or nil where that isn't here (a branch's first push, or one
223
224
  # that rewrote history).
224
225
  def pushed
225
- return passed if @config.push_base == :last_passed && git("merge-base", "--is-ancestor", passed, "HEAD", allow_failure: true)
226
+ return passed if @config.push_base == :last_passed && ancestor?(passed, "HEAD")
226
227
 
227
228
  before = @settings.base #: String
228
229
  before if git("cat-file", "-e", "#{before}^{commit}", allow_failure: true)
@@ -236,7 +237,7 @@ module Duel
236
237
  def common_dir(dir) = git("-C", dir, "rev-parse", "--path-format=absolute", "--git-common-dir", allow_failure: true)
237
238
 
238
239
  def git(*args, allow_failure: false)
239
- out, err, status = Open3.capture3("git", "-C", @config.root, *args)
240
+ out, err, status = Spawn.open3(:capture3, "git", "-C", @config.root, *args)
240
241
  return out.chomp if status.success?
241
242
  raise Error, "git #{args.join(" ")} failed: #{err.chomp}" unless allow_failure
242
243
  end
data/lib/duel/cli.rb CHANGED
@@ -10,11 +10,14 @@ require_relative "lock"
10
10
  require_relative "summary"
11
11
  require_relative "json_report"
12
12
  require_relative "init"
13
+ require_relative "comment"
14
+ require_relative "spawn"
13
15
 
14
16
  module Duel
15
- # The `duel` command (and `duel init`: see Init). What a run behind a
16
- # rake task or a CI expression can't pass as an option can be set in
17
- # the environment (DUEL_BASE, DUEL_FILTER, DUEL_FORCE), and options win.
17
+ # The `duel` command (and `duel init` and `duel comment`: see Init and
18
+ # Comment). What a run behind a rake task or a CI expression can't pass
19
+ # as an option can be set in the environment (DUEL_BASE, DUEL_FILTER,
20
+ # DUEL_FORCE), and options win.
18
21
  # On GitHub Actions (GITHUB_STEP_SUMMARY), it adds its report to the
19
22
  # job's summary, in Markdown (see Summary), and tells the step's
20
23
  # outputs (GITHUB_OUTPUT) what to do about the comment on a pull
@@ -24,8 +27,9 @@ module Duel
24
27
  # outcome, but not for a command line that isn't valid.
25
28
  # @api private
26
29
  class CLI
27
- USAGE = "Usage: duel [options]\n duel init [options]\n\n" \
28
- "Fails if this tree is slower than a baseline at any benchmark. `duel init` sets Duel up in a project.\n\n"
30
+ USAGE = "Usage: duel [options]\n duel init [options]\n duel comment [options]\n\n" \
31
+ "Fails if this tree is slower than a baseline at any benchmark. `duel init` sets Duel up in a project, and\n" \
32
+ "`duel comment` is what its Perf comment workflow runs.\n\n"
29
33
 
30
34
  # Where GitHub Actions has a step add to its job's summary.
31
35
  STEP_SUMMARY = "GITHUB_STEP_SUMMARY"
@@ -73,9 +77,12 @@ module Duel
73
77
  # warned (see WARNED).
74
78
  def run
75
79
  return Init.new(@argv.drop(1), out: @out, err: @err).run if @argv.first == "init"
80
+ return Comment.new(@argv.drop(1), env: @env, out: @out, err: @err).run if @argv.first == "comment"
76
81
 
77
82
  settings = @settings = catch(:version) { parse } or return 0
78
83
  config = @config = settings.config_path ? Duel.load_config(settings.config_path) : Config.new
84
+ # (.duel.rb's, but where an option says: c.limit, unless --limit.)
85
+ config.run_settings.each { |name, value| set(settings, name, value) unless @given.include?(name) }
79
86
  Lock.hold(config.state) { compare(config, settings) }
80
87
  rescue Error => e
81
88
  @err.puts "duel: #{e}"
@@ -122,6 +129,7 @@ module Duel
122
129
 
123
130
  # The settings, from the environment and then the options.
124
131
  def parse
132
+ @given = [] # (the RUN_SETTINGS options given: see Config)
125
133
  settings = Settings.defaults
126
134
  settings.jobs = [Etc.nprocessors / CPUS_PER_LANE, 1].max
127
135
  settings.fit = %i[jobs processes]
@@ -171,7 +179,7 @@ module Duel
171
179
  passed ? "passed" : "failed",
172
180
  -> {
173
181
  Summary.new(runner, results, settings: settings, baseline: described_baseline(config, settings, baseline),
174
- candidate: described_candidate(config, settings), note: @note).markdown
182
+ candidate: described_candidate(config, settings), note: @note, accepted: @accepted).markdown
175
183
  },
176
184
  ->(comment) {
177
185
  JSONReport.new(runner, results, settings: settings, exit: status, comment: comment,
@@ -190,6 +198,7 @@ module Duel
190
198
  if accepted?(config)
191
199
  # (Set: see #accepted?.)
192
200
  label = config.accept_label #: String
201
+ @accepted = true
193
202
  return passing("accepted with the #{label} label", "Accepted with the #{Summary.code(label)} label.")
194
203
  end
195
204
  return 1 unless pull_requests_only?(config, settings)
@@ -222,11 +231,10 @@ module Duel
222
231
  # (CI_MERGE_REQUEST_LABELS, by commas), or none, where neither is
223
232
  # set, or DUEL_LABELS isn't a JSON array.
224
233
  def labels
225
- set = variables
226
- return set.fetch("CI_MERGE_REQUEST_LABELS").split(",") if !set.key?("DUEL_LABELS") && set.key?("CI_MERGE_REQUEST_LABELS")
227
- return [] unless set.key?("DUEL_LABELS")
234
+ json = variable("DUEL_LABELS")
235
+ return variable("CI_MERGE_REQUEST_LABELS").to_s.split(",") unless json
228
236
 
229
- names = JSON.parse(set.fetch("DUEL_LABELS"))
237
+ names = JSON.parse(json)
230
238
  names.instance_of?(Array) ? names : []
231
239
  rescue JSON::ParserError
232
240
  []
@@ -303,14 +311,17 @@ module Duel
303
311
  # JSON report's: as --comment says, or else the configuration, or the
304
312
  # default where the configuration didn't load. It comments on a push
305
313
  # only where a regression in one passes (Config#fail_on), so the
306
- # comment says what the check doesn't.
314
+ # comment says what the check doesn't. A regression the label
315
+ # accepts (Config#accept_label) is commented on as a run that passed
316
+ # would be: whoever added the label knows, so by default it posts
317
+ # only where the last comment's status isn't its own.
307
318
  def commenting(status)
308
319
  # (Parsed, so set: see #run.)
309
320
  settings = @settings #: Settings
310
321
  config = @config || Config.new
311
322
  return "none" if pushed? && !pull_requests_only?(config, settings)
312
323
 
313
- CLI.comment(settings.comment || config.comment, status)
324
+ CLI.comment(settings.comment || config.comment, @accepted ? "passed" : status)
314
325
  end
315
326
 
316
327
  # Writes the reports of a run whose +status+ is the JSON report's
@@ -326,14 +337,15 @@ module Duel
326
337
  # writes them there too, as variables.
327
338
  def publish(status, markdown, json)
328
339
  comment = commenting(status)
329
- set = variables
330
340
  outputs = "comment=#{comment}\nstatus=#{MARKED.fetch(status)}\n#{"passed=true\n" if @passed}"
331
- File.write(set.fetch(STEP_OUTPUT), outputs, mode: "a") if set.key?(STEP_OUTPUT)
332
- File.write(set.fetch(OUTPUT), outputs.gsub(/^\w+=/) { |name| "DUEL_#{name.upcase}" }, mode: "a") if set.key?(OUTPUT)
341
+ step_output = variable(STEP_OUTPUT)
342
+ File.write(step_output, outputs, mode: "a") if step_output
343
+ output = variable(OUTPUT)
344
+ File.write(output, outputs.gsub(/^\w+=/) { |name| "DUEL_#{name.upcase}" }, mode: "a") if output
333
345
  path = @json
334
346
  write(path, "#{JSON.pretty_generate(json.call(comment))}\n") if path
335
347
  path = @markdown
336
- summary = (set.fetch(STEP_SUMMARY) if set.key?(STEP_SUMMARY))
348
+ summary = variable(STEP_SUMMARY)
337
349
  return unless path || summary
338
350
 
339
351
  text = markdown.call
@@ -392,7 +404,41 @@ module Duel
392
404
  return Summary.code(dir) unless found.success?
393
405
 
394
406
  _, clean = git(dir, "diff", "--quiet", "HEAD", "--", *Baseline.new(config, settings).watched)
395
- "#{head.chomp}#{" (with changes not committed)" unless clean.success?}"
407
+ return "#{head.chomp} (with changes not committed)" unless clean.success?
408
+
409
+ merged(dir) || head.chomp
410
+ end
411
+
412
+ # On GitHub Actions, a pull request's run checks out the commit that
413
+ # merges it with the branch it targets, which isn't in the pull
414
+ # request, so its head is named instead, as merged (or nil, where HEAD
415
+ # is no such commit). So does a merge request's merged results
416
+ # pipeline, on GitLab CI.
417
+ def merged(dir)
418
+ pull = (gitlab? ? merge_request : pull_request) or return
419
+ sha, branch = pull.dig("head", "sha"), pull.dig("base", "ref")
420
+ return unless branch.instance_of?(String) && git(dir, "rev-parse", "HEAD^2").first.chomp == sha
421
+
422
+ "#{git(dir, "rev-parse", "--short", sha).first.chomp} (merged with #{Summary.code(branch)})"
423
+ end
424
+
425
+ # A GitLab merge request's merged results pipeline's head and the
426
+ # branch it targets, as #pull_request has a pull request's (each nil
427
+ # in a merge request's other pipelines, whose HEAD is its head).
428
+ def merge_request = {"head" => {"sha" => variable("CI_MERGE_REQUEST_SOURCE_BRANCH_SHA")},
429
+ "base" => {"ref" => variable("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")}}
430
+
431
+ # The pull request a pull_request event's run is for, as GitHub
432
+ # Actions describes it (GITHUB_EVENT_PATH), or nil.
433
+ def pull_request
434
+ path = variable("GITHUB_EVENT_PATH") if event == "pull_request"
435
+ return unless path && File.file?(path)
436
+
437
+ payload = JSON.parse(File.read(path))
438
+ pull = payload["pull_request"] if payload.instance_of?(Hash)
439
+ pull if pull.instance_of?(Hash)
440
+ rescue JSON::ParserError
441
+ nil
396
442
  end
397
443
 
398
444
  # The baseline, for JSON: the ref that named it (the main branch,
@@ -422,7 +468,7 @@ module Duel
422
468
 
423
469
  # Git's output, and its status, in +dir+.
424
470
  def git(dir, *args)
425
- out, _, status = Open3.capture3("git", "-C", dir, *args)
471
+ out, _, status = Spawn.open3(:capture3, "git", "-C", dir, *args)
426
472
  [out, status]
427
473
  end
428
474
 
@@ -448,6 +494,25 @@ module Duel
448
494
  # The environment's variables that are set (not empty).
449
495
  def variables = @env.reject { |_, value| value.empty? }
450
496
 
497
+ # Sets +settings+' +name+d setting (one of Config::RUN_SETTINGS) to
498
+ # +value+, an option's, over .duel.rb's: one it doesn't take is an
499
+ # invalid option.
500
+ def given(settings, name, value)
501
+ problem = Config.problem(name, value)
502
+ # (OptionParser says which option.)
503
+ raise OptionParser::InvalidArgument, "#{value}: give #{problem}" if problem
504
+
505
+ set(settings, name, value)
506
+ @given << name
507
+ end
508
+
509
+ # Sets +settings+' +name+d setting to +value+: a number of processes,
510
+ # that many, not as many as fit in memory.
511
+ def set(settings, name, value)
512
+ settings[name] = value
513
+ settings.fit -= [name]
514
+ end
515
+
451
516
  def parser(settings)
452
517
  OptionParser.new do |o|
453
518
  o.banner = USAGE
@@ -461,19 +526,22 @@ module Duel
461
526
  o.on("--candidate-dir DIR", "Candidate tree (default: this one)") { |v| settings.candidate_dir = v }
462
527
  o.on("--filter REGEXP", "Only benchmarks whose names match") { |v| settings.filter = CLI.regexp(v) }
463
528
  o.on("--jobs N", Integer, "Benchmarks measured at once (default: one for every #{CPUS_PER_LANE} CPUs, as many as fit in memory)") do |v|
464
- settings.jobs = v
465
- settings.fit -= [:jobs]
529
+ set(settings, :jobs, v)
530
+ end
531
+ o.on("--processes N", Integer, "Worker processes per version (default: c.processes, or #{PROCESSES}, as many as fit in memory)") do |v|
532
+ given(settings, :processes, v)
533
+ end
534
+ o.on("--timeout SECONDS", Float, "Most a worker may take to load or answer (default: c.timeout, or #{TIMEOUT.round})") do |v|
535
+ given(settings, :timeout, v)
466
536
  end
467
- o.on("--processes N", Integer, "Worker processes per version (default: #{PROCESSES}, as many as fit in memory)") do |v|
468
- settings.processes = v
469
- settings.fit -= [:processes]
537
+ o.on("--time-limit SECONDS", Float, "Most a run measures for, then fails what's unmeasured or looks worse (default: c.time_limit, or 7200)") do |v|
538
+ given(settings, :time_limit, v)
470
539
  end
471
- o.on("--timeout SECONDS", Float, "Most a worker may take to load or answer (default: #{TIMEOUT.round})") { |v| settings.timeout = v }
472
- o.on("--time-limit SECONDS", Float, "Most a run measures for, then fails what's unmeasured or looks worse (default: 7200)") do |v|
473
- settings.time_limit = v
540
+ o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: c.limit, or 1.02)") { |v| given(settings, :limit, v) }
541
+ o.on("--confidence LEVEL", Float, "Confidence of every interval (default: c.confidence, or 0.99)") { |v| given(settings, :confidence, v) }
542
+ o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: c.warm, or until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT on, else 0)") do |v|
543
+ given(settings, :warm, v)
474
544
  end
475
- o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: 1.02)") { |v| settings.limit = v }
476
- o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT on, else 0)") { |v| settings.warm = v }
477
545
  o.on("--ruby version-file|PATH", "Each version's Ruby: the one its version file names (.ruby-version, say), or a",
478
546
  "path (default: c.ruby, or this one)") { |v| settings.ruby = (v == "version-file") ? :version_file : v }
479
547
  o.on("--force", "Compare even if nothing watched changed") { settings.force = true }