duel 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +51 -27
  3. data/CHANGELOG.md +247 -1
  4. data/README.md +30 -14
  5. data/docs/benchmarks.md +28 -24
  6. data/docs/ci.md +328 -362
  7. data/docs/configuration.md +40 -37
  8. data/docs/how-it-works.md +23 -10
  9. data/docs/json.md +21 -25
  10. data/docs/plugins.md +40 -22
  11. data/docs/troubleshooting.md +3 -2
  12. data/lib/duel/across.rb +18 -18
  13. data/lib/duel/badge.rb +174 -0
  14. data/lib/duel/baseline.rb +38 -15
  15. data/lib/duel/cli.rb +175 -107
  16. data/lib/duel/comment.rb +80 -71
  17. data/lib/duel/config.rb +145 -63
  18. data/lib/duel/context.rb +29 -24
  19. data/lib/duel/forge.rb +87 -0
  20. data/lib/duel/init.rb +44 -87
  21. data/lib/duel/json_report.rb +34 -25
  22. data/lib/duel/lane.rb +107 -61
  23. data/lib/duel/lock.rb +1 -1
  24. data/lib/duel/mark_passed.rb +132 -0
  25. data/lib/duel/metric.rb +28 -27
  26. data/lib/duel/native.rb +7 -5
  27. data/lib/duel/platform.rb +4 -0
  28. data/lib/duel/process_sample.rb +9 -6
  29. data/lib/duel/rake_task.rb +4 -5
  30. data/lib/duel/resource.rb +10 -11
  31. data/lib/duel/result.rb +22 -14
  32. data/lib/duel/rubies.rb +6 -2
  33. data/lib/duel/runner.rb +27 -15
  34. data/lib/duel/spawn.rb +29 -4
  35. data/lib/duel/stats.rb +14 -12
  36. data/lib/duel/summary.rb +31 -29
  37. data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
  38. data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
  39. data/lib/duel/templates/duel.rb.erb +7 -7
  40. data/lib/duel/templates/perf-caller.yml.erb +22 -5
  41. data/lib/duel/templates/perf-comment.yml.erb +11 -11
  42. data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
  43. data/lib/duel/templates/perf-push.yml.erb +50 -0
  44. data/lib/duel/templates/perf.yml.erb +113 -335
  45. data/lib/duel/testing.rb +20 -25
  46. data/lib/duel/version.rb +1 -1
  47. data/lib/duel/worker.rb +16 -9
  48. data/lib/duel/worker_process.rb +6 -6
  49. data/lib/duel.rb +88 -33
  50. data/sig/duel/across.rbs +4 -0
  51. data/sig/duel/badge.rbs +51 -0
  52. data/sig/duel/baseline.rbs +7 -2
  53. data/sig/duel/cli.rbs +31 -20
  54. data/sig/duel/comment.rbs +14 -15
  55. data/sig/duel/config.rbs +71 -29
  56. data/sig/duel/context.rbs +4 -4
  57. data/sig/duel/forge.rbs +42 -0
  58. data/sig/duel/init.rbs +8 -26
  59. data/sig/duel/json_report.rbs +7 -6
  60. data/sig/duel/lane.rbs +29 -6
  61. data/sig/duel/mark_passed.rbs +39 -0
  62. data/sig/duel/metric.rbs +10 -10
  63. data/sig/duel/native.rbs +3 -0
  64. data/sig/duel/platform.rbs +1 -0
  65. data/sig/duel/process_sample.rbs +3 -0
  66. data/sig/duel/resource.rbs +3 -3
  67. data/sig/duel/result.rbs +12 -5
  68. data/sig/duel/rubies.rbs +3 -0
  69. data/sig/duel/runner.rbs +23 -16
  70. data/sig/duel/spawn.rbs +7 -0
  71. data/sig/duel/stats.rbs +4 -1
  72. data/sig/duel/summary.rbs +8 -0
  73. data/sig/duel/testing.rbs +2 -2
  74. data/sig/duel/worker.rbs +6 -5
  75. data/sig/duel/worker_process.rbs +3 -3
  76. data/sig/duel.rbs +50 -31
  77. metadata +9 -1
data/lib/duel/testing.rb CHANGED
@@ -96,7 +96,7 @@ module Duel
96
96
 
97
97
  # What +config+'s metrics (a plugin's: not Duel's own) measure of the
98
98
  # block, as one counted sample of +n+ iterations of a benchmark would,
99
- # with +only:+ (:timing, say) and its own +hooks+ (around_iteration:,
99
+ # with +judge:+ (:timed, say) and its own +hooks+ (around_iteration:,
100
100
  # say), as DSL#duel takes them: {name => measured}, with what they
101
101
  # said of it (Measurement#details). It runs with the configuration's
102
102
  # hooks too, as a worker runs a benchmark: its setup hooks, then the
@@ -105,25 +105,21 @@ module Duel
105
105
  # it), then its teardown hooks. The block gets the benchmark's
106
106
  # Context (with its #untimed), and keeps its own self.
107
107
  #
108
- # The block runs once, but with around_iteration hooks (the
109
- # configuration's, or its own), it's one iteration, run +n+ times,
110
- # each inside the hooks, as a worker runs a benchmark with them: what
111
- # they do isn't measured, and what they add to metrics costly to read
112
- # (Metric.costly?) is learned first, and taken away.
108
+ # The block is one iteration, run +n+ times, as a benchmark's block
109
+ # that takes no +n+ is: with around_iteration hooks (the
110
+ # configuration's, or its own), each inside them, as a worker runs a
111
+ # benchmark with them: what they do isn't measured, and what they add
112
+ # to metrics costly to read (Metric.costly?) is learned first, and
113
+ # taken away.
113
114
  #
114
115
  # Inside #in_worker, it's that worker's. Outside, it's a worker of its
115
116
  # own, ended after (its metrics closed, and what its blocks gave
116
117
  # at_stop run).
117
- def measure(config, n: 1, only: nil, **hooks, &block)
118
- return in_worker(config) { measure(config, n: n, only: only, **hooks, &block) } unless STOPS.key?(config)
119
-
120
- # @type var body: Duel::body
121
- body = if hooks.key?(:around_iteration) || config.hooks.fetch(:around_iteration).any?
122
- -> { block.call(self) } #: ^() [self: Context] -> void
123
- else
124
- ->(_n) { block.call(self) } #: ^(Integer) [self: Context] -> void
125
- end
126
- benchmark = Definition.new("test", only, false, __FILE__, body, DSL.hooks("test", hooks, body))
118
+ def measure(config, n: 1, judge: JUDGE, **hooks, &block)
119
+ return in_worker(config) { measure(config, n: n, judge: judge, **hooks, &block) } unless STOPS.key?(config)
120
+
121
+ body = -> { block.call(self) } #: ^() [self: Context] -> void
122
+ benchmark = Definition.new("test", DSL.judged("test", judge), __FILE__, body, DSL.hooks("test", hooks, body))
127
123
  harness = Harness.new(benchmark, config.hooks)
128
124
  metrics = metrics(config)
129
125
  sample = harness.sample(metrics, n: n)
@@ -151,8 +147,7 @@ module Duel
151
147
  stops = Stops.new
152
148
  STOPS[config] = stops
153
149
  config.in_workers.each { |block| block.call(stops) }
154
- MADE[config] = [config.measures.transform_values(&:dup), Worker.made(config.metrics, config.measures).first]
155
- MADE.fetch(config).last
150
+ (MADE[config] = [config.measures.transform_values(&:dup), Worker.made(config.metrics, config.measures).first]).last
156
151
  end
157
152
 
158
153
  # Runs the block as a worker of +config+ would run (its in_worker
@@ -180,14 +175,14 @@ module Duel
180
175
  MADE.compare_by_identity
181
176
 
182
177
  # A stand-in Sample of +n+ iterations of a benchmark named +name+
183
- # (with +only:+ and +peak_memory:+, as DSL#duel takes them), measuring
178
+ # (judged on what +judge:+ says, as DSL#duel takes it), measuring
184
179
  # nothing: what a metric's #start_sample, and its details block
185
180
  # (Snapshot.details), are given.
186
181
  #
187
182
  # metric.start_sample(Duel::Testing.sample(n: 10))
188
- def sample(n: 1, name: "test", only: nil, peak_memory: false)
183
+ def sample(n: 1, name: "test", judge: JUDGE)
189
184
  nothing = -> {} #: ^() [self: Context] -> void
190
- Sample.new(Definition.new(name, only, peak_memory, __FILE__, nothing), [], n: n)
185
+ Sample.new(Definition.new(name, DSL.judged(name, judge), __FILE__, nothing), [], n: n)
191
186
  end
192
187
 
193
188
  # A stand-in reading of running totals (Snapshot::Reading): +totals+,
@@ -211,16 +206,16 @@ module Duel
211
206
  # version unless they say.
212
207
  def compare(base, candidate, **settings)
213
208
  config_path = File.join(candidate, ".duel.rb")
209
+ configured = File.exist?(config_path)
214
210
  saved = Duel.config
215
211
  begin
216
- config = File.exist?(config_path) ? Duel.load_config(config_path) : Config.new(candidate)
212
+ config = configured ? Duel.load_config(config_path) : Config.new(candidate)
217
213
  run = Settings.defaults
218
- run.config_path = config_path if File.exist?(config_path)
214
+ run.config_path = config_path if configured
219
215
  run.processes = 2
220
216
  # (.duel.rb's run settings, c.limit say, but where one's given, and
221
217
  # its processes: a test's are few.)
222
- config.run_settings.except(:processes).each { |name, value| run[name] = value }
223
- settings.each { |setting, value| run[setting] = value }
218
+ config.run_settings.except(:processes).merge(settings).each { |name, value| run[name] = value }
224
219
  out = StringIO.new
225
220
  runner = Runner.new(config, run, base, out: out)
226
221
  results = runner.run
data/lib/duel/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Duel
4
- VERSION = "0.2.0"
4
+ VERSION = "0.3.0"
5
5
  end
data/lib/duel/worker.rb CHANGED
@@ -86,7 +86,7 @@ module Duel
86
86
  def self.made(metrics, measures)
87
87
  metrics.each { |metric| metric.measures.each { |name, settings| Metric.check(name, settings) } }
88
88
  names = metrics.flat_map { |metric| metric.measures.keys }
89
- twice = names.select { |name| names.count(name) > 1 }.uniq
89
+ twice = names.tally.select { |_, count| count > 1 }.keys
90
90
  raise Error, "more than one metric measures #{twice.join(", ")}" unless twice.empty?
91
91
  unknown = measures.keys - names
92
92
  raise Error, "no metric measures #{unknown.join(", ")} (c.measure)" unless unknown.empty?
@@ -101,10 +101,11 @@ module Duel
101
101
  end
102
102
 
103
103
  # What the runner learns first: the benchmarks, the metrics, whether
104
- # peak memory and profiles can be measured here, and whether YJIT is on.
104
+ # peak memory and profiles can be measured here, and which of CRuby's
105
+ # JITs is on.
105
106
  def hello
106
107
  {"benchmarks" => @benchmarks.values.map(&:to_h), "metrics" => @specs, "ruby" => RUBY_DESCRIPTION,
107
- "memory" => !Native::Memory.used.nil?, "profiler" => Worker.profiler?, "yjit" => Worker.yjit?,
108
+ "memory" => !Native::Memory.used.nil?, "profiler" => Worker.profiler?, "jit" => Worker.jit,
108
109
  "extensions" => Worker.extensions, "twice" => Worker.twice(@before + $LOADED_FEATURES), "gem_dirs" => Worker.gem_dirs}
109
110
  end
110
111
 
@@ -144,13 +145,19 @@ module Duel
144
145
  dirs.select { |dir| File.directory?(dir) }.map { |dir| File.join(File.realpath(dir), "") }.uniq
145
146
  end
146
147
 
147
- # Whether YJIT is on, by Ruby: JRuby and TruffleRuby have none, and a
148
- # CRuby built without it has no RubyVM::YJIT.
149
- YJIT = {"ruby" => -> { RubyVM.const_defined?(:YJIT) && RubyVM::YJIT.enabled? }}.fetch(RUBY_ENGINE, -> { false })
148
+ # CRuby's JITs, as RubyVM names them: YJIT, and since Ruby 4.0, ZJIT.
149
+ # Ruby runs one at most.
150
+ JITS = %w[YJIT ZJIT].freeze
150
151
 
151
- # Whether YJIT is on (which the version may have turned on as it
152
- # loaded), for a longer warm-up (YJIT_WARM).
153
- def self.yjit? = YJIT.call
152
+ # Which of CRuby's JITs is on ("yjit" or "zjit"), which the version may
153
+ # have turned on as it loaded, for a longer warm-up (CRUBY_JIT_WARM),
154
+ # or nil: none, or a Ruby (+engine+) other than CRuby, which has
155
+ # neither. (A CRuby built without one has no RubyVM constant for it.)
156
+ def self.jit(engine: RUBY_ENGINE)
157
+ return unless engine == "ruby"
158
+
159
+ JITS.find { |name| RubyVM.const_defined?(name) && RubyVM.const_get(name).enabled? }&.downcase
160
+ end
154
161
 
155
162
  # Whether StackProf is installed, for profiles: on CRuby (+engine+)
156
163
  # only. TruffleRuby installs it, but it fails to start there, and
@@ -20,7 +20,7 @@ module Duel
20
20
  # The gems that are Duel's: Duel, and its plugins (duel-redis, say).
21
21
  DUEL_GEM = /\Aduel(-|\z)/
22
22
 
23
- attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :yjit
23
+ attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :jit
24
24
 
25
25
  # The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
26
26
  # loaded (see Worker.extensions), those it loaded twice (see
@@ -70,7 +70,7 @@ module Duel
70
70
  @metrics = hello.fetch("metrics")
71
71
  @memory = hello.fetch("memory")
72
72
  @profiler = hello.fetch("profiler")
73
- @yjit = hello.fetch("yjit")
73
+ @jit = hello.fetch("jit")
74
74
  @description = hello.fetch("ruby")
75
75
  @extensions = hello.fetch("extensions")
76
76
  @twice = hello.fetch("twice")
@@ -96,7 +96,7 @@ module Duel
96
96
  # ProcessSample#run).
97
97
  def run(process)
98
98
  # (Code, without a command.)
99
- code = process.fetch("ruby") #: String
99
+ code = process.fetch("code") #: String
100
100
  command = process.fetch("command") || [*(@launch ||= Rubies.launch(@ruby, @engine, @env)), "-I", File.join(@root, "lib"), "-e", code]
101
101
  runs = @runs ||= ProcessSample.new(@label, env: @env, chdir: @root, log: File.join(@dir, "#{@label}.log"), timeout: @timeout)
102
102
  runs.run(command)
@@ -113,8 +113,8 @@ module Duel
113
113
  private
114
114
 
115
115
  # The worker's connection, once it has loaded (or Failed if it died,
116
- # or took too long, first).
117
- def accept(server, path)
116
+ # or took too long, first), to +server+, at +address+ (see Control).
117
+ def accept(server, address)
118
118
  deadline = monotonic + @timeout
119
119
  until server.wait_readable(POLL)
120
120
  raise Failed, "#{@label} worker exited while loading" unless @child.alive?
@@ -123,7 +123,7 @@ module Duel
123
123
  server.accept
124
124
  ensure
125
125
  server.close
126
- Control.remove(path)
126
+ Control.remove(address)
127
127
  end
128
128
 
129
129
  def read
data/lib/duel.rb CHANGED
@@ -16,7 +16,7 @@ module Duel
16
16
  class Error < StandardError; end
17
17
 
18
18
  # What Duel raises when something a run needs fails: a resource that
19
- # didn't start (Resource#failed), two that set a variable differently,
19
+ # didn't start (Resource#fail!), two that set a variable differently,
20
20
  # a worker that died or hung.
21
21
  class Failed < Error; end
22
22
 
@@ -41,23 +41,33 @@ module Duel
41
41
  # @api private
42
42
  NO_HOOKS = HOOKS.to_h { |hook| [hook, Array.new(0).freeze] }.freeze #: Hash[Symbol, Array[hook]]
43
43
 
44
- # A benchmark (see DSL#duel), defined in +file+, whose hooks (those of
45
- # its file, and its own, +hooks+, by name) it gets. Or a benchmark of a
46
- # whole +process+ (see DSL#duel_process), with no +body+. What a
47
- # metric sees of it (Sample#benchmark), and its hooks are given, is
48
- # its +name+, +only+ and +peak_memory+. Its +file+, +body+, +hooks+
49
- # and +process+, and its writers, are @api private.
50
- Definition = Struct.new(:name, :only, :peak_memory, :file, :body, :hooks, :process)
44
+ # A benchmark (see DSL#duel), defined in +file+, judged on what
45
+ # +judge+ says (see JUDGED), whose hooks (those of its file, and its
46
+ # own, +hooks+, by name) it gets. Or a benchmark of a whole +process+
47
+ # (see DSL#duel_process), with no +body+. What a metric sees of it
48
+ # (Sample#benchmark) is its +name+ and +judge+, and #timed?, #counted?
49
+ # and #peak_memory?. Its +file+, +body+, +hooks+ and +process+, and its
50
+ # writers, are @api private.
51
+ Definition = Struct.new(:name, :judge, :file, :body, :hooks, :process)
51
52
 
52
53
  class Definition
54
+ # Whether it's judged on its timings.
55
+ def timed? = judge.include?(:timed)
56
+
57
+ # Whether it's judged on its counts.
58
+ def counted? = judge.include?(:counted)
59
+
60
+ # Whether it's judged on its peak memory too.
61
+ def peak_memory? = judge.include?(:peak_memory)
62
+
53
63
  # Whose CPU time it takes: the benchmark thread's, or (judged on its
54
64
  # timings alone, as work on threads of its own is) the process's.
55
65
  # @api private
56
- def cpu = (only == :timing) ? :process : :thread
66
+ def cpu = counted? ? :thread : :process
57
67
 
58
68
  # What the runner learns of it.
59
69
  # @api private
60
- def to_h = {"name" => name, "only" => only&.to_s, "peak_memory" => peak_memory, "process" => process}
70
+ def to_h = {"name" => name, "judge" => judge.map(&:to_s), "process" => process}
61
71
  end
62
72
 
63
73
  # The iterations of a benchmark judged on its counts alone (see
@@ -65,9 +75,18 @@ module Duel
65
75
  # @api private
66
76
  COUNTS_ONLY = 4
67
77
 
68
- # What a benchmark may be judged on (see DSL#duel): nil for both.
78
+ # What a benchmark can be judged on (see DSL#duel), in this order: its
79
+ # timings, its counts, and its peak memory.
80
+ JUDGED = %i[timed counted peak_memory].freeze #: Array[judged]
81
+
82
+ # What a benchmark is judged on, unless it says (see DSL#duel): its
83
+ # timings and its counts.
84
+ JUDGE = %i[timed counted].freeze #: Array[judged]
85
+
86
+ # What a process benchmark is judged on (see DSL#duel_process): its CPU
87
+ # time, and its peak memory.
69
88
  # @api private
70
- ONLY = [nil, :count, :timing] #: Array[only?]
89
+ PROCESS_JUDGE = %i[timed peak_memory].freeze #: Array[judged]
71
90
 
72
91
  # What a worker's blocks (Config#in_worker) are given: blocks to run
73
92
  # when the worker ends, last first.
@@ -124,8 +143,8 @@ module Duel
124
143
  # plugin's helper meant for setting up, +name+d, calls it first, since
125
144
  # what it does then would be counted.
126
145
  #
127
- # def self.flush
128
- # Duel.unmeasured!("Duel::Redis.flush")
146
+ # def self.reset
147
+ # Duel.unmeasured!("Duel::Redis.reset")
129
148
  # ...
130
149
  def unmeasured!(name)
131
150
  raise Error, "#{name}: in setup or untimed only (it would be counted)" if measuring?
@@ -244,12 +263,30 @@ module Duel
244
263
  # Whether a server that a plugin starts for a worker can listen on a
245
264
  # Unix socket (in the worker's directory) that a worker can reach: not
246
265
  # on Windows, nor where Ruby has none (see Control.unix_sockets?).
247
- # Where it can't, the server listens on a port (see Resource#port).
266
+ # Where it can't, the server listens on a port (see Resource#free_port).
248
267
  # For a plugin's default:
249
268
  #
250
269
  # def self.configure(config, tcp: !Duel.unix_sockets?)
251
270
  def unix_sockets? = Control.unix_sockets?(windows: windows?)
252
271
 
272
+ # Whether +program+ (a plugin's server: postgres, mysqld) is a path,
273
+ # with a / or a \ in it (as Windows's have), not a name to look for on
274
+ # the PATH (see .find_program): for a plugin's error that it isn't
275
+ # there.
276
+ def path?(program) = program.match?(%r{[/\\]})
277
+
278
+ # A plugin's server program: +program+ if it's a path (see .path?),
279
+ # expanded (~/mysql/bin/mysqld, say), or the one by that name in one
280
+ # of +path+'s directories, the first one's, as it is or ending with
281
+ # +exe+ (on Windows, .exe: mysqld for mysqld.exe). Nil if there's
282
+ # none.
283
+ #
284
+ # program = Duel.find_program(server) || w.fail!("no #{server}")
285
+ def find_program(program, path = ENV.fetch("PATH", ""), exe: Platform::EXE)
286
+ files = path?(program) ? [File.expand_path(program)] : path.split(File::PATH_SEPARATOR).map { |dir| File.join(dir, program) }
287
+ files.flat_map { |file| [file, "#{file}#{exe}"] }.find { |file| File.file?(file) && File.executable?(file) }
288
+ end
289
+
253
290
  # The file of benchmarks that called the DSL method running (see
254
291
  # DSL.add), if one is: the file that what it defines is filed
255
292
  # under, so it gets that file's hooks, not those of the file the
@@ -330,12 +367,13 @@ module Duel
330
367
  # queue.drain
331
368
  # end
332
369
  #
333
- # Options: only: what it's judged on, if not both its timings and its
334
- # counts: :count for work too slow and light to time (waiting, say),
335
- # run a few iterations at a time, or :timing for work done by
336
- # threads of its own, which make its counts vary (and whose CPU time
337
- # is the process's, not the benchmark thread's). peak_memory: true to
338
- # judge its peak memory too. And its own hooks (see HOOKS), lambdas
370
+ # Options: judge: what it's judged on (see JUDGED), one or a list, if
371
+ # not its timings and its counts: :counted alone for work too slow and
372
+ # light to time (waiting, say), run a few iterations at a time,
373
+ # :timed alone for work done by threads of its own, which make its
374
+ # counts vary (and whose CPU time is the process's, not the benchmark
375
+ # thread's), and :peak_memory too to judge its peak memory. And its
376
+ # own hooks (see HOOKS), lambdas
339
377
  # run in its context, after its configuration's and its file's: an
340
378
  # around_iteration one takes the iteration (an Iteration) and runs
341
379
  # it, and only the iteration is measured, so what it does before and
@@ -344,21 +382,36 @@ module Duel
344
382
  # duel "posts: create", around_iteration: ->(iteration) { rolled_back { iteration.call } } do
345
383
  # Post.create!(title: "Hi")
346
384
  # end
347
- def duel(name, only: nil, peak_memory: false, **hooks, &body)
385
+ #
386
+ # duel "redis: PING", judge: :counted do ... end
387
+ # duel "server: 10 workers", judge: %i[timed peak_memory] do ... end
388
+ def duel(name, judge: JUDGE, **hooks, &body)
348
389
  raise ArgumentError, "duplicate benchmark #{name.inspect}" if Duel.benchmarks.key?(name)
349
- raise ArgumentError, "only: must be :count or :timing, not #{only.inspect}" unless ONLY.include?(only)
350
390
 
351
- Duel.benchmarks[name] = Definition.new(name, only, peak_memory, Duel.defining_file || body.source_location.first, body,
391
+ Duel.benchmarks[name] = Definition.new(name, DSL.judged(name, judge), Duel.defining_file || body.source_location.first, body,
352
392
  DSL.hooks(name, hooks, body))
353
393
  end
354
394
 
395
+ # What a benchmark +name+d is judged on, given +judge+ (one of JUDGED,
396
+ # or a list of them: :timed, :counted, or both, and :peak_memory, with
397
+ # :timed, as it's measured at the timings' number of iterations): a
398
+ # list of them, in JUDGED's order.
399
+ # @api private
400
+ def self.judged(name, judge)
401
+ given = Array(judge)
402
+ valid = (given - JUDGED).empty? && given.intersect?(%i[timed counted]) && (given.include?(:timed) || !given.include?(:peak_memory))
403
+ raise ArgumentError, "duel #{name.inspect}: judge: #{judge.inspect}: give :timed, :counted, or both, and :peak_memory with :timed" unless valid
404
+
405
+ JUDGED & given
406
+ end
407
+
355
408
  # A benchmark's own hooks (+hooks+, by name), each one a hook there
356
409
  # is (see Duel.hook): an around_iteration one runs around each
357
410
  # iteration, so +body+ (the benchmark's block) can't take them.
358
411
  # @api private
359
412
  def self.hooks(name, hooks, body)
360
413
  unknown = hooks.keys - HOOKS
361
- raise ArgumentError, "duel #{name.inspect}: no option #{unknown.join(", ")} (only, peak_memory, #{HOOKS.join(", ")})" unless unknown.empty?
414
+ raise ArgumentError, "duel #{name.inspect}: no option #{unknown.join(", ")} (judge, #{HOOKS.join(", ")})" unless unknown.empty?
362
415
 
363
416
  hooks.each { |hook, block| Duel.hook(hook, block) }
364
417
  if hooks.key?(:around_iteration) && body.arity == 1
@@ -376,10 +429,10 @@ module Duel
376
429
  # it, its peak memory.
377
430
  #
378
431
  # duel_process "boot: the app", "bin/rails", "runner", "1"
379
- # duel_process "boot: require the library", ruby: %(require "my_gem")
432
+ # duel_process "boot: require the library", code: %(require "my_gem")
380
433
  #
381
434
  # A +command+ is run as Process.spawn takes it (one String with
382
- # spaces is a shell command). Code given as +ruby+ is run by the
435
+ # spaces is a shell command). Code given as +code+ is run by the
383
436
  # version's Ruby, with the version's lib/ first on the load path. A
384
437
  # process that fails (a status other than 0) is an error. Each sample
385
438
  # is the fastest of +runs+ processes: by default, as many as take about
@@ -389,9 +442,12 @@ module Duel
389
442
  # hooks (the configuration's, and its file's), and plugins' metrics.
390
443
  # Nor do #duel's options: given one (around_iteration:, say), it
391
444
  # fails.
392
- def duel_process(name, *command, ruby: nil, runs: nil, **options)
445
+ def duel_process(name, *command, code: nil, runs: nil, **options)
393
446
  raise ArgumentError, "duplicate benchmark #{name.inspect}" if Duel.benchmarks.key?(name)
394
- raise ArgumentError, "duel_process #{name.inspect}: give a command, or ruby: code, not both" if command.empty? == ruby.nil?
447
+ if options.key?(:ruby)
448
+ raise ArgumentError, "duel_process #{name.inspect}: give the code to run as code:, not ruby: (the Ruby it runs on is c.ruby's)"
449
+ end
450
+ raise ArgumentError, "duel_process #{name.inspect}: give a command, or code:, not both" if command.empty? == code.nil?
395
451
  counted = runs.nil? || (runs.instance_of?(Integer) && runs.positive?)
396
452
  raise ArgumentError, "duel_process #{name.inspect}: runs: is a number of processes a sample, 1 or more, not #{runs.inspect}" unless counted
397
453
  unless options.empty?
@@ -399,8 +455,8 @@ module Duel
399
455
  "for a benchmark run in a worker: a process benchmark runs no hooks, and is judged on its CPU time and peak memory alone"
400
456
  end
401
457
 
402
- process = {"command" => (command unless command.empty?), "ruby" => ruby, "runs" => runs} #: Definition::process
403
- Duel.benchmarks[name] = Definition.new(name, nil, false, Duel.defining_file || caller_locations.fetch(0).path, nil, {}, process)
458
+ process = {"command" => (command unless command.empty?), "code" => code, "runs" => runs} #: Definition::process
459
+ Duel.benchmarks[name] = Definition.new(name, PROCESS_JUDGE, Duel.defining_file || caller_locations.fetch(0).path, nil, {}, process)
404
460
  end
405
461
 
406
462
  # En garde.
@@ -415,8 +471,7 @@ module Duel
415
471
  # benchmark's samples, and teardown after them (last first).
416
472
  # before_sample runs before every sample (the warm-up's too), not
417
473
  # measured: to put back what earlier samples changed (data they
418
- # wrote, say), so that every sample starts alike. The first three
419
- # are given the benchmark, if they take it.
474
+ # wrote, say), so that every sample starts alike.
420
475
  def setup(&block) = Duel.file_hook(:setup, block, DSL.caller_file)
421
476
 
422
477
  def teardown(&block) = Duel.file_hook(:teardown, block, DSL.caller_file)
data/sig/duel/across.rbs CHANGED
@@ -24,6 +24,10 @@ module Duel
24
24
 
25
25
  private
26
26
 
27
+ # Each timed measure that counts across the board, of each benchmark
28
+ # judged: its result, its metric and the measure.
29
+ def each_across: () { (Result, String, Result::timed) -> void } -> void
30
+
27
31
  # The verdicts of a run that judged benchmarks over workers.
28
32
  def by_process_or_benchmark: (Hash[String, Array[Float]] medians, Hash[String, Float] limits) -> Hash[String, verdict]
29
33
 
@@ -0,0 +1,51 @@
1
+ module Duel
2
+ # duel badge BRANCH --status STATUS --result RESULT: writes Duel's
3
+ # badge, as the push left it, to its branch, on GitHub.
4
+ # @api private
5
+ class Badge
6
+ include Forge
7
+
8
+ USAGE: String
9
+
10
+ # Each state's words, width, colors and right edge, by name.
11
+ STATES: Hash[String, [String, Integer, String, String, String]]
12
+
13
+ # The badge's image, to format.
14
+ SVG: String
15
+
16
+ # What its commit says, before the push it names.
17
+ MESSAGE: String
18
+
19
+ @argv: Array[String]
20
+ @status: String?
21
+ @result: String?
22
+
23
+ def initialize: (Array[String] argv, env: CLI::_Env, out: _Output, err: _Output) -> void
24
+
25
+ # Returns the exit status: 0 written (or left), 1 GitHub wouldn't
26
+ # take it, 2 a bad option.
27
+ def run: () -> Integer
28
+
29
+ # What the badge says, given the Duel job's status and result.
30
+ def self.state: (String? status, String? result) -> String
31
+
32
+ # The badge's image, saying a state.
33
+ def self.svg: (String state) -> String
34
+
35
+ private
36
+
37
+ def parser: () -> OptionParser
38
+
39
+ # Whether the badge on the branch is a newer push's.
40
+ def newer?: (String branch, String sha) -> bool
41
+
42
+ # Writes the image, alone, to the branch, in a commit naming the push.
43
+ def write: (String branch, String svg, String sha) -> Integer
44
+
45
+ def left: (String branch) -> Integer
46
+
47
+ def command: () -> String
48
+
49
+ def gitlab_token: () -> String
50
+ end
51
+ end
@@ -29,6 +29,10 @@ module Duel
29
29
  # Whether +commit+ is +of+ or an ancestor of it.
30
30
  def ancestor?: (String commit, String of) -> bool
31
31
 
32
+ # The last commit since +release+ that merged a pull request with the
33
+ # label, on GitHub, as gh finds them, or nil.
34
+ def merged_accepted: (String release) -> String?
35
+
32
36
  # The branch that marks the last commit Duel passed in a push to the
33
37
  # branch pushed.
34
38
  def passed_branch: () -> String
@@ -69,8 +73,9 @@ module Duel
69
73
 
70
74
  private
71
75
 
72
- # The benchmarks' directory, or for benchmarks at the root, their glob.
73
- def benchmarks: () -> String
76
+ # Each of the benchmarks' globs' directory, or for benchmarks at the
77
+ # root, its glob.
78
+ def benchmarks: () -> Array[String]
74
79
 
75
80
  # The configuration's file, from the root.
76
81
  def config_file: () -> String
data/sig/duel/cli.rbs CHANGED
@@ -41,26 +41,30 @@ module Duel
41
41
  # Where else to write the step's outputs, as variables.
42
42
  OUTPUT: String
43
43
 
44
- # The variables that set options, to their options.
44
+ # The variables that set options, to their options: DUEL_ and the
45
+ # option's name.
45
46
  ENV_OPTIONS: Hash[String, String]
46
47
 
47
- # The variables that set a flag by being set at all.
48
- ENV_FLAGS: Hash[String, String]
48
+ # What DUEL_FORCE's values say: to compare whatever changed, or not.
49
+ FORCE: Hash[String, bool]
49
50
 
50
51
  @argv: Array[String]
51
52
  @env: _Env
52
53
  @out: _Output
53
54
  @err: _Output
54
- @clean: bool?
55
55
  @markdown: String?
56
56
  @json: String?
57
57
  @settings: Settings?
58
58
  @config: Config?
59
59
  # The Config::RUN_SETTINGS options given.
60
60
  @given: Array[Symbol]
61
- @passed: bool?
61
+ # The branch that marks the push passed (see Baseline#passed_branch),
62
+ # where the run marks it.
63
+ @passed: String?
62
64
  @note: String?
63
65
  @accepted: bool?
66
+ # Why a run that failed passed anyway, for the JSON report.
67
+ @passed_anyway: String?
64
68
 
65
69
  def initialize: (Array[String] argv, ?env: _Env, ?out: _Output, ?err: _Output) -> void
66
70
 
@@ -76,26 +80,23 @@ module Duel
76
80
  # or "none".
77
81
  def self.comment: (Config::comment mode, String status) -> String
78
82
 
79
- # The mode --comment names, or an invalid argument.
80
- def self.comment_mode: (String name) -> Config::comment
81
-
82
- # The runs --fail-on names, or an invalid argument.
83
- def self.fail_on: (String name) -> Config::fail_on
84
-
85
- # Which of +values+ an option's argument names, or an invalid
86
- # argument.
87
- def self.named: [T < Symbol] (String name, Array[T] values) -> T
88
83
 
89
84
  def parse: () -> Settings
90
85
 
91
86
  private
92
87
 
88
+ # duel base: prints the commit a run compares with, or nothing.
89
+ def show_base: (Array[String] argv) -> Integer
90
+
91
+ # duel clean: removes what Duel keeps, and the baseline's worktree.
92
+ def clean: (Array[String] argv) -> Integer
93
+
93
94
  # What #run does, holding the project's lock.
94
95
  def compare: (Config config, Settings settings) -> Integer
95
96
 
96
97
  # The exit status of a run that was slower: 1, or 0 where only a pull
97
98
  # request fails, outside one.
98
- def slower: (Config config, Settings settings) -> Integer
99
+ def slower: (Config config) -> Integer
99
100
 
100
101
  # Passes a run that was slower, with a warning and a note.
101
102
  def passing: (String why, String note, ?Integer exit) -> Integer
@@ -122,7 +123,7 @@ module Duel
122
123
  def variable: (*String names) -> String?
123
124
 
124
125
  # Whether a regression fails only a pull request.
125
- def pull_requests_only?: (Config config, Settings settings) -> bool
126
+ def pull_requests_only?: (Config config) -> bool
126
127
 
127
128
  # Whether this is a run on GitHub Actions for something other than a
128
129
  # pull request.
@@ -132,7 +133,14 @@ module Duel
132
133
  def pull_request?: () -> bool
133
134
 
134
135
  # Whether the baseline is a merge queue's base.
135
- def queued?: (Settings settings) -> bool
136
+ # Whether the baseline is the one the run's event says.
137
+ def from_event?: (Settings settings) -> bool
138
+
139
+ # The baseline a run on CI is for, as its event says, or nil.
140
+ def event_base: () -> String?
141
+
142
+ # The event a run on GitHub Actions is for, as its file describes it.
143
+ def event_payload: () -> Hash[String, untyped]
136
144
 
137
145
  # Whether this is a run on GitHub Actions for a push.
138
146
  def pushed?: () -> bool
@@ -149,7 +157,7 @@ module Duel
149
157
 
150
158
  # Writes the reports where they're asked for, each made only if it's
151
159
  # written, and on GitHub Actions, the step's outputs.
152
- def publish: (String status, ^() -> String markdown, ^(String comment) -> JSONReport::report json) -> void
160
+ def publish: (String status, ^() -> String markdown, ^() -> JSONReport::report json) -> void
153
161
 
154
162
  # Writes +text+ to +path+, in a directory made for it.
155
163
  def write: (String path, String text) -> void
@@ -158,7 +166,7 @@ module Duel
158
166
  def described_baseline: (Config config, Settings settings, Baseline baseline) -> String
159
167
 
160
168
  # The candidate, in Markdown.
161
- def described_candidate: (Config config, Settings settings) -> String
169
+ def described_candidate: (Config config, Settings settings, Baseline baseline) -> String
162
170
 
163
171
  def merged: (String dir) -> String?
164
172
 
@@ -176,7 +184,10 @@ module Duel
176
184
 
177
185
  # The candidate, for JSON: its commit, whether it has changes not
178
186
  # committed, and its tree.
179
- def candidate_json: (Config config, Settings settings) -> Hash[String, String | bool | nil]
187
+ def candidate_json: (Config config, Settings settings, Baseline baseline) -> Hash[String, String | bool | nil]
188
+
189
+ # Whether what's watched has changes not committed in the directory.
190
+ def uncommitted?: (String dir, Baseline baseline) -> bool
180
191
 
181
192
  # Git's output, and its status, in +dir+.
182
193
  def git: (String dir, *String args) -> [String, Process::Status]