duel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +51 -27
- data/CHANGELOG.md +247 -1
- data/README.md +30 -14
- data/docs/benchmarks.md +28 -24
- data/docs/ci.md +328 -362
- data/docs/configuration.md +40 -37
- data/docs/how-it-works.md +23 -10
- data/docs/json.md +21 -25
- data/docs/plugins.md +40 -22
- data/docs/troubleshooting.md +3 -2
- data/lib/duel/across.rb +18 -18
- data/lib/duel/badge.rb +174 -0
- data/lib/duel/baseline.rb +38 -15
- data/lib/duel/cli.rb +175 -107
- data/lib/duel/comment.rb +80 -71
- data/lib/duel/config.rb +145 -63
- data/lib/duel/context.rb +29 -24
- data/lib/duel/forge.rb +87 -0
- data/lib/duel/init.rb +44 -87
- data/lib/duel/json_report.rb +34 -25
- data/lib/duel/lane.rb +107 -61
- data/lib/duel/lock.rb +1 -1
- data/lib/duel/mark_passed.rb +132 -0
- data/lib/duel/metric.rb +28 -27
- data/lib/duel/native.rb +7 -5
- data/lib/duel/platform.rb +4 -0
- data/lib/duel/process_sample.rb +9 -6
- data/lib/duel/rake_task.rb +4 -5
- data/lib/duel/resource.rb +10 -11
- data/lib/duel/result.rb +22 -14
- data/lib/duel/rubies.rb +6 -2
- data/lib/duel/runner.rb +27 -15
- data/lib/duel/spawn.rb +29 -4
- data/lib/duel/stats.rb +14 -12
- data/lib/duel/summary.rb +31 -29
- data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
- data/lib/duel/templates/duel.rb.erb +7 -7
- data/lib/duel/templates/perf-caller.yml.erb +22 -5
- data/lib/duel/templates/perf-comment.yml.erb +11 -11
- data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
- data/lib/duel/templates/perf-push.yml.erb +50 -0
- data/lib/duel/templates/perf.yml.erb +113 -335
- data/lib/duel/testing.rb +20 -25
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +16 -9
- data/lib/duel/worker_process.rb +6 -6
- data/lib/duel.rb +88 -33
- data/sig/duel/across.rbs +4 -0
- data/sig/duel/badge.rbs +51 -0
- data/sig/duel/baseline.rbs +7 -2
- data/sig/duel/cli.rbs +31 -20
- data/sig/duel/comment.rbs +14 -15
- data/sig/duel/config.rbs +71 -29
- data/sig/duel/context.rbs +4 -4
- data/sig/duel/forge.rbs +42 -0
- data/sig/duel/init.rbs +8 -26
- data/sig/duel/json_report.rbs +7 -6
- data/sig/duel/lane.rbs +29 -6
- data/sig/duel/mark_passed.rbs +39 -0
- data/sig/duel/metric.rbs +10 -10
- data/sig/duel/native.rbs +3 -0
- data/sig/duel/platform.rbs +1 -0
- data/sig/duel/process_sample.rbs +3 -0
- data/sig/duel/resource.rbs +3 -3
- data/sig/duel/result.rbs +12 -5
- data/sig/duel/rubies.rbs +3 -0
- data/sig/duel/runner.rbs +23 -16
- data/sig/duel/spawn.rbs +7 -0
- data/sig/duel/stats.rbs +4 -1
- data/sig/duel/summary.rbs +8 -0
- data/sig/duel/testing.rbs +2 -2
- data/sig/duel/worker.rbs +6 -5
- data/sig/duel/worker_process.rbs +3 -3
- data/sig/duel.rbs +50 -31
- metadata +9 -1
data/lib/duel/testing.rb
CHANGED
|
@@ -96,7 +96,7 @@ module Duel
|
|
|
96
96
|
|
|
97
97
|
# What +config+'s metrics (a plugin's: not Duel's own) measure of the
|
|
98
98
|
# block, as one counted sample of +n+ iterations of a benchmark would,
|
|
99
|
-
# with +
|
|
99
|
+
# with +judge:+ (:timed, say) and its own +hooks+ (around_iteration:,
|
|
100
100
|
# say), as DSL#duel takes them: {name => measured}, with what they
|
|
101
101
|
# said of it (Measurement#details). It runs with the configuration's
|
|
102
102
|
# hooks too, as a worker runs a benchmark: its setup hooks, then the
|
|
@@ -105,25 +105,21 @@ module Duel
|
|
|
105
105
|
# it), then its teardown hooks. The block gets the benchmark's
|
|
106
106
|
# Context (with its #untimed), and keeps its own self.
|
|
107
107
|
#
|
|
108
|
-
# The block
|
|
109
|
-
#
|
|
110
|
-
# each inside
|
|
111
|
-
# they do isn't measured, and what they add
|
|
112
|
-
# (Metric.costly?) is learned first, and
|
|
108
|
+
# The block is one iteration, run +n+ times, as a benchmark's block
|
|
109
|
+
# that takes no +n+ is: with around_iteration hooks (the
|
|
110
|
+
# configuration's, or its own), each inside them, as a worker runs a
|
|
111
|
+
# benchmark with them: what they do isn't measured, and what they add
|
|
112
|
+
# to metrics costly to read (Metric.costly?) is learned first, and
|
|
113
|
+
# taken away.
|
|
113
114
|
#
|
|
114
115
|
# Inside #in_worker, it's that worker's. Outside, it's a worker of its
|
|
115
116
|
# own, ended after (its metrics closed, and what its blocks gave
|
|
116
117
|
# at_stop run).
|
|
117
|
-
def measure(config, n: 1,
|
|
118
|
-
return in_worker(config) { measure(config, n: n,
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
-> { block.call(self) } #: ^() [self: Context] -> void
|
|
123
|
-
else
|
|
124
|
-
->(_n) { block.call(self) } #: ^(Integer) [self: Context] -> void
|
|
125
|
-
end
|
|
126
|
-
benchmark = Definition.new("test", only, false, __FILE__, body, DSL.hooks("test", hooks, body))
|
|
118
|
+
def measure(config, n: 1, judge: JUDGE, **hooks, &block)
|
|
119
|
+
return in_worker(config) { measure(config, n: n, judge: judge, **hooks, &block) } unless STOPS.key?(config)
|
|
120
|
+
|
|
121
|
+
body = -> { block.call(self) } #: ^() [self: Context] -> void
|
|
122
|
+
benchmark = Definition.new("test", DSL.judged("test", judge), __FILE__, body, DSL.hooks("test", hooks, body))
|
|
127
123
|
harness = Harness.new(benchmark, config.hooks)
|
|
128
124
|
metrics = metrics(config)
|
|
129
125
|
sample = harness.sample(metrics, n: n)
|
|
@@ -151,8 +147,7 @@ module Duel
|
|
|
151
147
|
stops = Stops.new
|
|
152
148
|
STOPS[config] = stops
|
|
153
149
|
config.in_workers.each { |block| block.call(stops) }
|
|
154
|
-
MADE[config] = [config.measures.transform_values(&:dup), Worker.made(config.metrics, config.measures).first]
|
|
155
|
-
MADE.fetch(config).last
|
|
150
|
+
(MADE[config] = [config.measures.transform_values(&:dup), Worker.made(config.metrics, config.measures).first]).last
|
|
156
151
|
end
|
|
157
152
|
|
|
158
153
|
# Runs the block as a worker of +config+ would run (its in_worker
|
|
@@ -180,14 +175,14 @@ module Duel
|
|
|
180
175
|
MADE.compare_by_identity
|
|
181
176
|
|
|
182
177
|
# A stand-in Sample of +n+ iterations of a benchmark named +name+
|
|
183
|
-
# (
|
|
178
|
+
# (judged on what +judge:+ says, as DSL#duel takes it), measuring
|
|
184
179
|
# nothing: what a metric's #start_sample, and its details block
|
|
185
180
|
# (Snapshot.details), are given.
|
|
186
181
|
#
|
|
187
182
|
# metric.start_sample(Duel::Testing.sample(n: 10))
|
|
188
|
-
def sample(n: 1, name: "test",
|
|
183
|
+
def sample(n: 1, name: "test", judge: JUDGE)
|
|
189
184
|
nothing = -> {} #: ^() [self: Context] -> void
|
|
190
|
-
Sample.new(Definition.new(name,
|
|
185
|
+
Sample.new(Definition.new(name, DSL.judged(name, judge), __FILE__, nothing), [], n: n)
|
|
191
186
|
end
|
|
192
187
|
|
|
193
188
|
# A stand-in reading of running totals (Snapshot::Reading): +totals+,
|
|
@@ -211,16 +206,16 @@ module Duel
|
|
|
211
206
|
# version unless they say.
|
|
212
207
|
def compare(base, candidate, **settings)
|
|
213
208
|
config_path = File.join(candidate, ".duel.rb")
|
|
209
|
+
configured = File.exist?(config_path)
|
|
214
210
|
saved = Duel.config
|
|
215
211
|
begin
|
|
216
|
-
config =
|
|
212
|
+
config = configured ? Duel.load_config(config_path) : Config.new(candidate)
|
|
217
213
|
run = Settings.defaults
|
|
218
|
-
run.config_path = config_path if
|
|
214
|
+
run.config_path = config_path if configured
|
|
219
215
|
run.processes = 2
|
|
220
216
|
# (.duel.rb's run settings, c.limit say, but where one's given, and
|
|
221
217
|
# its processes: a test's are few.)
|
|
222
|
-
config.run_settings.except(:processes).each { |name, value| run[name] = value }
|
|
223
|
-
settings.each { |setting, value| run[setting] = value }
|
|
218
|
+
config.run_settings.except(:processes).merge(settings).each { |name, value| run[name] = value }
|
|
224
219
|
out = StringIO.new
|
|
225
220
|
runner = Runner.new(config, run, base, out: out)
|
|
226
221
|
results = runner.run
|
data/lib/duel/version.rb
CHANGED
data/lib/duel/worker.rb
CHANGED
|
@@ -86,7 +86,7 @@ module Duel
|
|
|
86
86
|
def self.made(metrics, measures)
|
|
87
87
|
metrics.each { |metric| metric.measures.each { |name, settings| Metric.check(name, settings) } }
|
|
88
88
|
names = metrics.flat_map { |metric| metric.measures.keys }
|
|
89
|
-
twice = names.select { |
|
|
89
|
+
twice = names.tally.select { |_, count| count > 1 }.keys
|
|
90
90
|
raise Error, "more than one metric measures #{twice.join(", ")}" unless twice.empty?
|
|
91
91
|
unknown = measures.keys - names
|
|
92
92
|
raise Error, "no metric measures #{unknown.join(", ")} (c.measure)" unless unknown.empty?
|
|
@@ -101,10 +101,11 @@ module Duel
|
|
|
101
101
|
end
|
|
102
102
|
|
|
103
103
|
# What the runner learns first: the benchmarks, the metrics, whether
|
|
104
|
-
# peak memory and profiles can be measured here, and
|
|
104
|
+
# peak memory and profiles can be measured here, and which of CRuby's
|
|
105
|
+
# JITs is on.
|
|
105
106
|
def hello
|
|
106
107
|
{"benchmarks" => @benchmarks.values.map(&:to_h), "metrics" => @specs, "ruby" => RUBY_DESCRIPTION,
|
|
107
|
-
"memory" => !Native::Memory.used.nil?, "profiler" => Worker.profiler?, "
|
|
108
|
+
"memory" => !Native::Memory.used.nil?, "profiler" => Worker.profiler?, "jit" => Worker.jit,
|
|
108
109
|
"extensions" => Worker.extensions, "twice" => Worker.twice(@before + $LOADED_FEATURES), "gem_dirs" => Worker.gem_dirs}
|
|
109
110
|
end
|
|
110
111
|
|
|
@@ -144,13 +145,19 @@ module Duel
|
|
|
144
145
|
dirs.select { |dir| File.directory?(dir) }.map { |dir| File.join(File.realpath(dir), "") }.uniq
|
|
145
146
|
end
|
|
146
147
|
|
|
147
|
-
#
|
|
148
|
-
#
|
|
149
|
-
|
|
148
|
+
# CRuby's JITs, as RubyVM names them: YJIT, and since Ruby 4.0, ZJIT.
|
|
149
|
+
# Ruby runs one at most.
|
|
150
|
+
JITS = %w[YJIT ZJIT].freeze
|
|
150
151
|
|
|
151
|
-
#
|
|
152
|
-
# loaded
|
|
153
|
-
|
|
152
|
+
# Which of CRuby's JITs is on ("yjit" or "zjit"), which the version may
|
|
153
|
+
# have turned on as it loaded, for a longer warm-up (CRUBY_JIT_WARM),
|
|
154
|
+
# or nil: none, or a Ruby (+engine+) other than CRuby, which has
|
|
155
|
+
# neither. (A CRuby built without one has no RubyVM constant for it.)
|
|
156
|
+
def self.jit(engine: RUBY_ENGINE)
|
|
157
|
+
return unless engine == "ruby"
|
|
158
|
+
|
|
159
|
+
JITS.find { |name| RubyVM.const_defined?(name) && RubyVM.const_get(name).enabled? }&.downcase
|
|
160
|
+
end
|
|
154
161
|
|
|
155
162
|
# Whether StackProf is installed, for profiles: on CRuby (+engine+)
|
|
156
163
|
# only. TruffleRuby installs it, but it fails to start there, and
|
data/lib/duel/worker_process.rb
CHANGED
|
@@ -20,7 +20,7 @@ module Duel
|
|
|
20
20
|
# The gems that are Duel's: Duel, and its plugins (duel-redis, say).
|
|
21
21
|
DUEL_GEM = /\Aduel(-|\z)/
|
|
22
22
|
|
|
23
|
-
attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :
|
|
23
|
+
attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :jit
|
|
24
24
|
|
|
25
25
|
# The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
|
|
26
26
|
# loaded (see Worker.extensions), those it loaded twice (see
|
|
@@ -70,7 +70,7 @@ module Duel
|
|
|
70
70
|
@metrics = hello.fetch("metrics")
|
|
71
71
|
@memory = hello.fetch("memory")
|
|
72
72
|
@profiler = hello.fetch("profiler")
|
|
73
|
-
@
|
|
73
|
+
@jit = hello.fetch("jit")
|
|
74
74
|
@description = hello.fetch("ruby")
|
|
75
75
|
@extensions = hello.fetch("extensions")
|
|
76
76
|
@twice = hello.fetch("twice")
|
|
@@ -96,7 +96,7 @@ module Duel
|
|
|
96
96
|
# ProcessSample#run).
|
|
97
97
|
def run(process)
|
|
98
98
|
# (Code, without a command.)
|
|
99
|
-
code = process.fetch("
|
|
99
|
+
code = process.fetch("code") #: String
|
|
100
100
|
command = process.fetch("command") || [*(@launch ||= Rubies.launch(@ruby, @engine, @env)), "-I", File.join(@root, "lib"), "-e", code]
|
|
101
101
|
runs = @runs ||= ProcessSample.new(@label, env: @env, chdir: @root, log: File.join(@dir, "#{@label}.log"), timeout: @timeout)
|
|
102
102
|
runs.run(command)
|
|
@@ -113,8 +113,8 @@ module Duel
|
|
|
113
113
|
private
|
|
114
114
|
|
|
115
115
|
# The worker's connection, once it has loaded (or Failed if it died,
|
|
116
|
-
# or took too long, first).
|
|
117
|
-
def accept(server,
|
|
116
|
+
# or took too long, first), to +server+, at +address+ (see Control).
|
|
117
|
+
def accept(server, address)
|
|
118
118
|
deadline = monotonic + @timeout
|
|
119
119
|
until server.wait_readable(POLL)
|
|
120
120
|
raise Failed, "#{@label} worker exited while loading" unless @child.alive?
|
|
@@ -123,7 +123,7 @@ module Duel
|
|
|
123
123
|
server.accept
|
|
124
124
|
ensure
|
|
125
125
|
server.close
|
|
126
|
-
Control.remove(
|
|
126
|
+
Control.remove(address)
|
|
127
127
|
end
|
|
128
128
|
|
|
129
129
|
def read
|
data/lib/duel.rb
CHANGED
|
@@ -16,7 +16,7 @@ module Duel
|
|
|
16
16
|
class Error < StandardError; end
|
|
17
17
|
|
|
18
18
|
# What Duel raises when something a run needs fails: a resource that
|
|
19
|
-
# didn't start (Resource#
|
|
19
|
+
# didn't start (Resource#fail!), two that set a variable differently,
|
|
20
20
|
# a worker that died or hung.
|
|
21
21
|
class Failed < Error; end
|
|
22
22
|
|
|
@@ -41,23 +41,33 @@ module Duel
|
|
|
41
41
|
# @api private
|
|
42
42
|
NO_HOOKS = HOOKS.to_h { |hook| [hook, Array.new(0).freeze] }.freeze #: Hash[Symbol, Array[hook]]
|
|
43
43
|
|
|
44
|
-
# A benchmark (see DSL#duel), defined in +file+,
|
|
45
|
-
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
# its +name
|
|
49
|
-
# and +
|
|
50
|
-
|
|
44
|
+
# A benchmark (see DSL#duel), defined in +file+, judged on what
|
|
45
|
+
# +judge+ says (see JUDGED), whose hooks (those of its file, and its
|
|
46
|
+
# own, +hooks+, by name) it gets. Or a benchmark of a whole +process+
|
|
47
|
+
# (see DSL#duel_process), with no +body+. What a metric sees of it
|
|
48
|
+
# (Sample#benchmark) is its +name+ and +judge+, and #timed?, #counted?
|
|
49
|
+
# and #peak_memory?. Its +file+, +body+, +hooks+ and +process+, and its
|
|
50
|
+
# writers, are @api private.
|
|
51
|
+
Definition = Struct.new(:name, :judge, :file, :body, :hooks, :process)
|
|
51
52
|
|
|
52
53
|
class Definition
|
|
54
|
+
# Whether it's judged on its timings.
|
|
55
|
+
def timed? = judge.include?(:timed)
|
|
56
|
+
|
|
57
|
+
# Whether it's judged on its counts.
|
|
58
|
+
def counted? = judge.include?(:counted)
|
|
59
|
+
|
|
60
|
+
# Whether it's judged on its peak memory too.
|
|
61
|
+
def peak_memory? = judge.include?(:peak_memory)
|
|
62
|
+
|
|
53
63
|
# Whose CPU time it takes: the benchmark thread's, or (judged on its
|
|
54
64
|
# timings alone, as work on threads of its own is) the process's.
|
|
55
65
|
# @api private
|
|
56
|
-
def cpu =
|
|
66
|
+
def cpu = counted? ? :thread : :process
|
|
57
67
|
|
|
58
68
|
# What the runner learns of it.
|
|
59
69
|
# @api private
|
|
60
|
-
def to_h = {"name" => name, "
|
|
70
|
+
def to_h = {"name" => name, "judge" => judge.map(&:to_s), "process" => process}
|
|
61
71
|
end
|
|
62
72
|
|
|
63
73
|
# The iterations of a benchmark judged on its counts alone (see
|
|
@@ -65,9 +75,18 @@ module Duel
|
|
|
65
75
|
# @api private
|
|
66
76
|
COUNTS_ONLY = 4
|
|
67
77
|
|
|
68
|
-
# What a benchmark
|
|
78
|
+
# What a benchmark can be judged on (see DSL#duel), in this order: its
|
|
79
|
+
# timings, its counts, and its peak memory.
|
|
80
|
+
JUDGED = %i[timed counted peak_memory].freeze #: Array[judged]
|
|
81
|
+
|
|
82
|
+
# What a benchmark is judged on, unless it says (see DSL#duel): its
|
|
83
|
+
# timings and its counts.
|
|
84
|
+
JUDGE = %i[timed counted].freeze #: Array[judged]
|
|
85
|
+
|
|
86
|
+
# What a process benchmark is judged on (see DSL#duel_process): its CPU
|
|
87
|
+
# time, and its peak memory.
|
|
69
88
|
# @api private
|
|
70
|
-
|
|
89
|
+
PROCESS_JUDGE = %i[timed peak_memory].freeze #: Array[judged]
|
|
71
90
|
|
|
72
91
|
# What a worker's blocks (Config#in_worker) are given: blocks to run
|
|
73
92
|
# when the worker ends, last first.
|
|
@@ -124,8 +143,8 @@ module Duel
|
|
|
124
143
|
# plugin's helper meant for setting up, +name+d, calls it first, since
|
|
125
144
|
# what it does then would be counted.
|
|
126
145
|
#
|
|
127
|
-
# def self.
|
|
128
|
-
# Duel.unmeasured!("Duel::Redis.
|
|
146
|
+
# def self.reset
|
|
147
|
+
# Duel.unmeasured!("Duel::Redis.reset")
|
|
129
148
|
# ...
|
|
130
149
|
def unmeasured!(name)
|
|
131
150
|
raise Error, "#{name}: in setup or untimed only (it would be counted)" if measuring?
|
|
@@ -244,12 +263,30 @@ module Duel
|
|
|
244
263
|
# Whether a server that a plugin starts for a worker can listen on a
|
|
245
264
|
# Unix socket (in the worker's directory) that a worker can reach: not
|
|
246
265
|
# on Windows, nor where Ruby has none (see Control.unix_sockets?).
|
|
247
|
-
# Where it can't, the server listens on a port (see Resource#
|
|
266
|
+
# Where it can't, the server listens on a port (see Resource#free_port).
|
|
248
267
|
# For a plugin's default:
|
|
249
268
|
#
|
|
250
269
|
# def self.configure(config, tcp: !Duel.unix_sockets?)
|
|
251
270
|
def unix_sockets? = Control.unix_sockets?(windows: windows?)
|
|
252
271
|
|
|
272
|
+
# Whether +program+ (a plugin's server: postgres, mysqld) is a path,
|
|
273
|
+
# with a / or a \ in it (as Windows's have), not a name to look for on
|
|
274
|
+
# the PATH (see .find_program): for a plugin's error that it isn't
|
|
275
|
+
# there.
|
|
276
|
+
def path?(program) = program.match?(%r{[/\\]})
|
|
277
|
+
|
|
278
|
+
# A plugin's server program: +program+ if it's a path (see .path?),
|
|
279
|
+
# expanded (~/mysql/bin/mysqld, say), or the one by that name in one
|
|
280
|
+
# of +path+'s directories, the first one's, as it is or ending with
|
|
281
|
+
# +exe+ (on Windows, .exe: mysqld for mysqld.exe). Nil if there's
|
|
282
|
+
# none.
|
|
283
|
+
#
|
|
284
|
+
# program = Duel.find_program(server) || w.fail!("no #{server}")
|
|
285
|
+
def find_program(program, path = ENV.fetch("PATH", ""), exe: Platform::EXE)
|
|
286
|
+
files = path?(program) ? [File.expand_path(program)] : path.split(File::PATH_SEPARATOR).map { |dir| File.join(dir, program) }
|
|
287
|
+
files.flat_map { |file| [file, "#{file}#{exe}"] }.find { |file| File.file?(file) && File.executable?(file) }
|
|
288
|
+
end
|
|
289
|
+
|
|
253
290
|
# The file of benchmarks that called the DSL method running (see
|
|
254
291
|
# DSL.add), if one is: the file that what it defines is filed
|
|
255
292
|
# under, so it gets that file's hooks, not those of the file the
|
|
@@ -330,12 +367,13 @@ module Duel
|
|
|
330
367
|
# queue.drain
|
|
331
368
|
# end
|
|
332
369
|
#
|
|
333
|
-
# Options:
|
|
334
|
-
# counts: :
|
|
335
|
-
# run a few iterations at a time,
|
|
336
|
-
# threads of its own, which make its
|
|
337
|
-
# is the process's, not the benchmark
|
|
338
|
-
# judge its peak memory
|
|
370
|
+
# Options: judge: what it's judged on (see JUDGED), one or a list, if
|
|
371
|
+
# not its timings and its counts: :counted alone for work too slow and
|
|
372
|
+
# light to time (waiting, say), run a few iterations at a time,
|
|
373
|
+
# :timed alone for work done by threads of its own, which make its
|
|
374
|
+
# counts vary (and whose CPU time is the process's, not the benchmark
|
|
375
|
+
# thread's), and :peak_memory too to judge its peak memory. And its
|
|
376
|
+
# own hooks (see HOOKS), lambdas
|
|
339
377
|
# run in its context, after its configuration's and its file's: an
|
|
340
378
|
# around_iteration one takes the iteration (an Iteration) and runs
|
|
341
379
|
# it, and only the iteration is measured, so what it does before and
|
|
@@ -344,21 +382,36 @@ module Duel
|
|
|
344
382
|
# duel "posts: create", around_iteration: ->(iteration) { rolled_back { iteration.call } } do
|
|
345
383
|
# Post.create!(title: "Hi")
|
|
346
384
|
# end
|
|
347
|
-
|
|
385
|
+
#
|
|
386
|
+
# duel "redis: PING", judge: :counted do ... end
|
|
387
|
+
# duel "server: 10 workers", judge: %i[timed peak_memory] do ... end
|
|
388
|
+
def duel(name, judge: JUDGE, **hooks, &body)
|
|
348
389
|
raise ArgumentError, "duplicate benchmark #{name.inspect}" if Duel.benchmarks.key?(name)
|
|
349
|
-
raise ArgumentError, "only: must be :count or :timing, not #{only.inspect}" unless ONLY.include?(only)
|
|
350
390
|
|
|
351
|
-
Duel.benchmarks[name] = Definition.new(name,
|
|
391
|
+
Duel.benchmarks[name] = Definition.new(name, DSL.judged(name, judge), Duel.defining_file || body.source_location.first, body,
|
|
352
392
|
DSL.hooks(name, hooks, body))
|
|
353
393
|
end
|
|
354
394
|
|
|
395
|
+
# What a benchmark +name+d is judged on, given +judge+ (one of JUDGED,
|
|
396
|
+
# or a list of them: :timed, :counted, or both, and :peak_memory, with
|
|
397
|
+
# :timed, as it's measured at the timings' number of iterations): a
|
|
398
|
+
# list of them, in JUDGED's order.
|
|
399
|
+
# @api private
|
|
400
|
+
def self.judged(name, judge)
|
|
401
|
+
given = Array(judge)
|
|
402
|
+
valid = (given - JUDGED).empty? && given.intersect?(%i[timed counted]) && (given.include?(:timed) || !given.include?(:peak_memory))
|
|
403
|
+
raise ArgumentError, "duel #{name.inspect}: judge: #{judge.inspect}: give :timed, :counted, or both, and :peak_memory with :timed" unless valid
|
|
404
|
+
|
|
405
|
+
JUDGED & given
|
|
406
|
+
end
|
|
407
|
+
|
|
355
408
|
# A benchmark's own hooks (+hooks+, by name), each one a hook there
|
|
356
409
|
# is (see Duel.hook): an around_iteration one runs around each
|
|
357
410
|
# iteration, so +body+ (the benchmark's block) can't take them.
|
|
358
411
|
# @api private
|
|
359
412
|
def self.hooks(name, hooks, body)
|
|
360
413
|
unknown = hooks.keys - HOOKS
|
|
361
|
-
raise ArgumentError, "duel #{name.inspect}: no option #{unknown.join(", ")} (
|
|
414
|
+
raise ArgumentError, "duel #{name.inspect}: no option #{unknown.join(", ")} (judge, #{HOOKS.join(", ")})" unless unknown.empty?
|
|
362
415
|
|
|
363
416
|
hooks.each { |hook, block| Duel.hook(hook, block) }
|
|
364
417
|
if hooks.key?(:around_iteration) && body.arity == 1
|
|
@@ -376,10 +429,10 @@ module Duel
|
|
|
376
429
|
# it, its peak memory.
|
|
377
430
|
#
|
|
378
431
|
# duel_process "boot: the app", "bin/rails", "runner", "1"
|
|
379
|
-
# duel_process "boot: require the library",
|
|
432
|
+
# duel_process "boot: require the library", code: %(require "my_gem")
|
|
380
433
|
#
|
|
381
434
|
# A +command+ is run as Process.spawn takes it (one String with
|
|
382
|
-
# spaces is a shell command). Code given as +
|
|
435
|
+
# spaces is a shell command). Code given as +code+ is run by the
|
|
383
436
|
# version's Ruby, with the version's lib/ first on the load path. A
|
|
384
437
|
# process that fails (a status other than 0) is an error. Each sample
|
|
385
438
|
# is the fastest of +runs+ processes: by default, as many as take about
|
|
@@ -389,9 +442,12 @@ module Duel
|
|
|
389
442
|
# hooks (the configuration's, and its file's), and plugins' metrics.
|
|
390
443
|
# Nor do #duel's options: given one (around_iteration:, say), it
|
|
391
444
|
# fails.
|
|
392
|
-
def duel_process(name, *command,
|
|
445
|
+
def duel_process(name, *command, code: nil, runs: nil, **options)
|
|
393
446
|
raise ArgumentError, "duplicate benchmark #{name.inspect}" if Duel.benchmarks.key?(name)
|
|
394
|
-
|
|
447
|
+
if options.key?(:ruby)
|
|
448
|
+
raise ArgumentError, "duel_process #{name.inspect}: give the code to run as code:, not ruby: (the Ruby it runs on is c.ruby's)"
|
|
449
|
+
end
|
|
450
|
+
raise ArgumentError, "duel_process #{name.inspect}: give a command, or code:, not both" if command.empty? == code.nil?
|
|
395
451
|
counted = runs.nil? || (runs.instance_of?(Integer) && runs.positive?)
|
|
396
452
|
raise ArgumentError, "duel_process #{name.inspect}: runs: is a number of processes a sample, 1 or more, not #{runs.inspect}" unless counted
|
|
397
453
|
unless options.empty?
|
|
@@ -399,8 +455,8 @@ module Duel
|
|
|
399
455
|
"for a benchmark run in a worker: a process benchmark runs no hooks, and is judged on its CPU time and peak memory alone"
|
|
400
456
|
end
|
|
401
457
|
|
|
402
|
-
process = {"command" => (command unless command.empty?), "
|
|
403
|
-
Duel.benchmarks[name] = Definition.new(name,
|
|
458
|
+
process = {"command" => (command unless command.empty?), "code" => code, "runs" => runs} #: Definition::process
|
|
459
|
+
Duel.benchmarks[name] = Definition.new(name, PROCESS_JUDGE, Duel.defining_file || caller_locations.fetch(0).path, nil, {}, process)
|
|
404
460
|
end
|
|
405
461
|
|
|
406
462
|
# En garde.
|
|
@@ -415,8 +471,7 @@ module Duel
|
|
|
415
471
|
# benchmark's samples, and teardown after them (last first).
|
|
416
472
|
# before_sample runs before every sample (the warm-up's too), not
|
|
417
473
|
# measured: to put back what earlier samples changed (data they
|
|
418
|
-
# wrote, say), so that every sample starts alike.
|
|
419
|
-
# are given the benchmark, if they take it.
|
|
474
|
+
# wrote, say), so that every sample starts alike.
|
|
420
475
|
def setup(&block) = Duel.file_hook(:setup, block, DSL.caller_file)
|
|
421
476
|
|
|
422
477
|
def teardown(&block) = Duel.file_hook(:teardown, block, DSL.caller_file)
|
data/sig/duel/across.rbs
CHANGED
|
@@ -24,6 +24,10 @@ module Duel
|
|
|
24
24
|
|
|
25
25
|
private
|
|
26
26
|
|
|
27
|
+
# Each timed measure that counts across the board, of each benchmark
|
|
28
|
+
# judged: its result, its metric and the measure.
|
|
29
|
+
def each_across: () { (Result, String, Result::timed) -> void } -> void
|
|
30
|
+
|
|
27
31
|
# The verdicts of a run that judged benchmarks over workers.
|
|
28
32
|
def by_process_or_benchmark: (Hash[String, Array[Float]] medians, Hash[String, Float] limits) -> Hash[String, verdict]
|
|
29
33
|
|
data/sig/duel/badge.rbs
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
module Duel
|
|
2
|
+
# duel badge BRANCH --status STATUS --result RESULT: writes Duel's
|
|
3
|
+
# badge, as the push left it, to its branch, on GitHub.
|
|
4
|
+
# @api private
|
|
5
|
+
class Badge
|
|
6
|
+
include Forge
|
|
7
|
+
|
|
8
|
+
USAGE: String
|
|
9
|
+
|
|
10
|
+
# Each state's words, width, colors and right edge, by name.
|
|
11
|
+
STATES: Hash[String, [String, Integer, String, String, String]]
|
|
12
|
+
|
|
13
|
+
# The badge's image, to format.
|
|
14
|
+
SVG: String
|
|
15
|
+
|
|
16
|
+
# What its commit says, before the push it names.
|
|
17
|
+
MESSAGE: String
|
|
18
|
+
|
|
19
|
+
@argv: Array[String]
|
|
20
|
+
@status: String?
|
|
21
|
+
@result: String?
|
|
22
|
+
|
|
23
|
+
def initialize: (Array[String] argv, env: CLI::_Env, out: _Output, err: _Output) -> void
|
|
24
|
+
|
|
25
|
+
# Returns the exit status: 0 written (or left), 1 GitHub wouldn't
|
|
26
|
+
# take it, 2 a bad option.
|
|
27
|
+
def run: () -> Integer
|
|
28
|
+
|
|
29
|
+
# What the badge says, given the Duel job's status and result.
|
|
30
|
+
def self.state: (String? status, String? result) -> String
|
|
31
|
+
|
|
32
|
+
# The badge's image, saying a state.
|
|
33
|
+
def self.svg: (String state) -> String
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def parser: () -> OptionParser
|
|
38
|
+
|
|
39
|
+
# Whether the badge on the branch is a newer push's.
|
|
40
|
+
def newer?: (String branch, String sha) -> bool
|
|
41
|
+
|
|
42
|
+
# Writes the image, alone, to the branch, in a commit naming the push.
|
|
43
|
+
def write: (String branch, String svg, String sha) -> Integer
|
|
44
|
+
|
|
45
|
+
def left: (String branch) -> Integer
|
|
46
|
+
|
|
47
|
+
def command: () -> String
|
|
48
|
+
|
|
49
|
+
def gitlab_token: () -> String
|
|
50
|
+
end
|
|
51
|
+
end
|
data/sig/duel/baseline.rbs
CHANGED
|
@@ -29,6 +29,10 @@ module Duel
|
|
|
29
29
|
# Whether +commit+ is +of+ or an ancestor of it.
|
|
30
30
|
def ancestor?: (String commit, String of) -> bool
|
|
31
31
|
|
|
32
|
+
# The last commit since +release+ that merged a pull request with the
|
|
33
|
+
# label, on GitHub, as gh finds them, or nil.
|
|
34
|
+
def merged_accepted: (String release) -> String?
|
|
35
|
+
|
|
32
36
|
# The branch that marks the last commit Duel passed in a push to the
|
|
33
37
|
# branch pushed.
|
|
34
38
|
def passed_branch: () -> String
|
|
@@ -69,8 +73,9 @@ module Duel
|
|
|
69
73
|
|
|
70
74
|
private
|
|
71
75
|
|
|
72
|
-
#
|
|
73
|
-
|
|
76
|
+
# Each of the benchmarks' globs' directory, or for benchmarks at the
|
|
77
|
+
# root, its glob.
|
|
78
|
+
def benchmarks: () -> Array[String]
|
|
74
79
|
|
|
75
80
|
# The configuration's file, from the root.
|
|
76
81
|
def config_file: () -> String
|
data/sig/duel/cli.rbs
CHANGED
|
@@ -41,26 +41,30 @@ module Duel
|
|
|
41
41
|
# Where else to write the step's outputs, as variables.
|
|
42
42
|
OUTPUT: String
|
|
43
43
|
|
|
44
|
-
# The variables that set options, to their options
|
|
44
|
+
# The variables that set options, to their options: DUEL_ and the
|
|
45
|
+
# option's name.
|
|
45
46
|
ENV_OPTIONS: Hash[String, String]
|
|
46
47
|
|
|
47
|
-
#
|
|
48
|
-
|
|
48
|
+
# What DUEL_FORCE's values say: to compare whatever changed, or not.
|
|
49
|
+
FORCE: Hash[String, bool]
|
|
49
50
|
|
|
50
51
|
@argv: Array[String]
|
|
51
52
|
@env: _Env
|
|
52
53
|
@out: _Output
|
|
53
54
|
@err: _Output
|
|
54
|
-
@clean: bool?
|
|
55
55
|
@markdown: String?
|
|
56
56
|
@json: String?
|
|
57
57
|
@settings: Settings?
|
|
58
58
|
@config: Config?
|
|
59
59
|
# The Config::RUN_SETTINGS options given.
|
|
60
60
|
@given: Array[Symbol]
|
|
61
|
-
|
|
61
|
+
# The branch that marks the push passed (see Baseline#passed_branch),
|
|
62
|
+
# where the run marks it.
|
|
63
|
+
@passed: String?
|
|
62
64
|
@note: String?
|
|
63
65
|
@accepted: bool?
|
|
66
|
+
# Why a run that failed passed anyway, for the JSON report.
|
|
67
|
+
@passed_anyway: String?
|
|
64
68
|
|
|
65
69
|
def initialize: (Array[String] argv, ?env: _Env, ?out: _Output, ?err: _Output) -> void
|
|
66
70
|
|
|
@@ -76,26 +80,23 @@ module Duel
|
|
|
76
80
|
# or "none".
|
|
77
81
|
def self.comment: (Config::comment mode, String status) -> String
|
|
78
82
|
|
|
79
|
-
# The mode --comment names, or an invalid argument.
|
|
80
|
-
def self.comment_mode: (String name) -> Config::comment
|
|
81
|
-
|
|
82
|
-
# The runs --fail-on names, or an invalid argument.
|
|
83
|
-
def self.fail_on: (String name) -> Config::fail_on
|
|
84
|
-
|
|
85
|
-
# Which of +values+ an option's argument names, or an invalid
|
|
86
|
-
# argument.
|
|
87
|
-
def self.named: [T < Symbol] (String name, Array[T] values) -> T
|
|
88
83
|
|
|
89
84
|
def parse: () -> Settings
|
|
90
85
|
|
|
91
86
|
private
|
|
92
87
|
|
|
88
|
+
# duel base: prints the commit a run compares with, or nothing.
|
|
89
|
+
def show_base: (Array[String] argv) -> Integer
|
|
90
|
+
|
|
91
|
+
# duel clean: removes what Duel keeps, and the baseline's worktree.
|
|
92
|
+
def clean: (Array[String] argv) -> Integer
|
|
93
|
+
|
|
93
94
|
# What #run does, holding the project's lock.
|
|
94
95
|
def compare: (Config config, Settings settings) -> Integer
|
|
95
96
|
|
|
96
97
|
# The exit status of a run that was slower: 1, or 0 where only a pull
|
|
97
98
|
# request fails, outside one.
|
|
98
|
-
def slower: (Config config
|
|
99
|
+
def slower: (Config config) -> Integer
|
|
99
100
|
|
|
100
101
|
# Passes a run that was slower, with a warning and a note.
|
|
101
102
|
def passing: (String why, String note, ?Integer exit) -> Integer
|
|
@@ -122,7 +123,7 @@ module Duel
|
|
|
122
123
|
def variable: (*String names) -> String?
|
|
123
124
|
|
|
124
125
|
# Whether a regression fails only a pull request.
|
|
125
|
-
def pull_requests_only?: (Config config
|
|
126
|
+
def pull_requests_only?: (Config config) -> bool
|
|
126
127
|
|
|
127
128
|
# Whether this is a run on GitHub Actions for something other than a
|
|
128
129
|
# pull request.
|
|
@@ -132,7 +133,14 @@ module Duel
|
|
|
132
133
|
def pull_request?: () -> bool
|
|
133
134
|
|
|
134
135
|
# Whether the baseline is a merge queue's base.
|
|
135
|
-
|
|
136
|
+
# Whether the baseline is the one the run's event says.
|
|
137
|
+
def from_event?: (Settings settings) -> bool
|
|
138
|
+
|
|
139
|
+
# The baseline a run on CI is for, as its event says, or nil.
|
|
140
|
+
def event_base: () -> String?
|
|
141
|
+
|
|
142
|
+
# The event a run on GitHub Actions is for, as its file describes it.
|
|
143
|
+
def event_payload: () -> Hash[String, untyped]
|
|
136
144
|
|
|
137
145
|
# Whether this is a run on GitHub Actions for a push.
|
|
138
146
|
def pushed?: () -> bool
|
|
@@ -149,7 +157,7 @@ module Duel
|
|
|
149
157
|
|
|
150
158
|
# Writes the reports where they're asked for, each made only if it's
|
|
151
159
|
# written, and on GitHub Actions, the step's outputs.
|
|
152
|
-
def publish: (String status, ^() -> String markdown, ^(
|
|
160
|
+
def publish: (String status, ^() -> String markdown, ^() -> JSONReport::report json) -> void
|
|
153
161
|
|
|
154
162
|
# Writes +text+ to +path+, in a directory made for it.
|
|
155
163
|
def write: (String path, String text) -> void
|
|
@@ -158,7 +166,7 @@ module Duel
|
|
|
158
166
|
def described_baseline: (Config config, Settings settings, Baseline baseline) -> String
|
|
159
167
|
|
|
160
168
|
# The candidate, in Markdown.
|
|
161
|
-
def described_candidate: (Config config, Settings settings) -> String
|
|
169
|
+
def described_candidate: (Config config, Settings settings, Baseline baseline) -> String
|
|
162
170
|
|
|
163
171
|
def merged: (String dir) -> String?
|
|
164
172
|
|
|
@@ -176,7 +184,10 @@ module Duel
|
|
|
176
184
|
|
|
177
185
|
# The candidate, for JSON: its commit, whether it has changes not
|
|
178
186
|
# committed, and its tree.
|
|
179
|
-
def candidate_json: (Config config, Settings settings) -> Hash[String, String | bool | nil]
|
|
187
|
+
def candidate_json: (Config config, Settings settings, Baseline baseline) -> Hash[String, String | bool | nil]
|
|
188
|
+
|
|
189
|
+
# Whether what's watched has changes not committed in the directory.
|
|
190
|
+
def uncommitted?: (String dir, Baseline baseline) -> bool
|
|
180
191
|
|
|
181
192
|
# Git's output, and its status, in +dir+.
|
|
182
193
|
def git: (String dir, *String args) -> [String, Process::Status]
|