duel 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +203 -1
- data/README.md +41 -1
- data/docs/ci.md +170 -76
- data/docs/configuration.md +38 -2
- data/docs/how-it-works.md +18 -9
- data/docs/plugins.md +23 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +96 -28
- data/lib/duel/comment.rb +296 -0
- data/lib/duel/config.rb +124 -4
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +255 -20
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +33 -1044
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +21 -15
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +61 -44
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +35 -3
- data/sig/duel/cli.rbs +19 -0
- data/sig/duel/comment.rbs +97 -0
- data/sig/duel/config.rbs +57 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +95 -1
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +6 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +6 -3
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -105
data/lib/duel/runner.rb
CHANGED
|
@@ -2,17 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
require "fileutils"
|
|
4
4
|
require "rbconfig"
|
|
5
|
-
require "socket"
|
|
6
|
-
require "tmpdir"
|
|
7
5
|
require_relative "../duel"
|
|
8
|
-
require_relative "stats"
|
|
9
|
-
require_relative "profile"
|
|
10
6
|
require_relative "across"
|
|
11
|
-
require_relative "
|
|
7
|
+
require_relative "lane"
|
|
8
|
+
require_relative "memory"
|
|
9
|
+
require_relative "profile"
|
|
12
10
|
require_relative "resource"
|
|
11
|
+
require_relative "result"
|
|
13
12
|
require_relative "rubies"
|
|
14
|
-
require_relative "process_sample"
|
|
15
|
-
require_relative "memory"
|
|
16
13
|
|
|
17
14
|
module Duel
|
|
18
15
|
# The most seconds each benchmark runs before it's measured, unless
|
|
@@ -56,6 +53,7 @@ module Duel
|
|
|
56
53
|
# The seconds a run measures for, at most, by default (see
|
|
57
54
|
# Settings#time_limit and Runner#out_of_time?): two hours, as on JRuby,
|
|
58
55
|
# a benchmark can take half a minute to warm up.
|
|
56
|
+
# @api private
|
|
59
57
|
TIME_LIMIT = 7200.0
|
|
60
58
|
|
|
61
59
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
@@ -100,1028 +98,6 @@ module Duel
|
|
|
100
98
|
def label = LABELS.fetch(name)
|
|
101
99
|
end
|
|
102
100
|
|
|
103
|
-
# A worker process (Duel::Worker) running one version of the code.
|
|
104
|
-
# @api private
|
|
105
|
-
class WorkerProcess
|
|
106
|
-
LIB = File.expand_path("..", __dir__)
|
|
107
|
-
# Seconds between looks at a worker that's loading, or ending.
|
|
108
|
-
POLL = 0.01
|
|
109
|
-
# Seconds a worker may take to end, once its connection is closed,
|
|
110
|
-
# before it's killed.
|
|
111
|
-
STOP_WAIT = 10
|
|
112
|
-
# The gems that are Duel's: Duel, and its plugins (duel-redis, say).
|
|
113
|
-
DUEL_GEM = /\Aduel(-|\z)/
|
|
114
|
-
|
|
115
|
-
attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :yjit
|
|
116
|
-
|
|
117
|
-
# The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
|
|
118
|
-
# loaded (see Worker.extensions), those it loaded twice (see
|
|
119
|
-
# Worker.twice), and the directories gems are installed in (see
|
|
120
|
-
# Worker.gem_dirs).
|
|
121
|
-
attr_reader :description, :extensions, :twice, :gem_dirs
|
|
122
|
-
|
|
123
|
-
# Where a worker on +ruby+ loads Duel from, ahead of anything else
|
|
124
|
-
# (a version's own bundle, which may not have Duel, or an older one):
|
|
125
|
-
# the runner's Duel, and every Duel gem (+specs+) the runner loaded
|
|
126
|
-
# (its plugins), so both versions are measured with the same. On the
|
|
127
|
-
# runner's own Ruby, Duel's dependencies (Fiddle) too: on another,
|
|
128
|
-
# their builds wouldn't load.
|
|
129
|
-
def self.load_path(ruby, specs: Gem.loaded_specs)
|
|
130
|
-
duel = specs.select { |name, _| name.match?(DUEL_GEM) }.values
|
|
131
|
-
none = [] #: Array[Gem::BasicSpecification]
|
|
132
|
-
dependencies = Rubies.runner?(ruby) ? specs["duel"]&.runtime_dependencies.to_a.filter_map { |d| specs[d.name] } : none
|
|
133
|
-
[LIB, *(duel + dependencies).flat_map(&:full_require_paths)].uniq
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
# +timeout+: the seconds it may take to load, or to answer a request,
|
|
137
|
-
# before it's taken for hung (and killed). It runs on +ruby+ (whose
|
|
138
|
-
# engine is +engine+), with +env+, and on JRuby, the JVM's options for
|
|
139
|
-
# a worker (see Rubies.worker_environment), not given to the processes
|
|
140
|
-
# it starts for a process benchmark (see #run): their boot is measured
|
|
141
|
-
# as it is.
|
|
142
|
-
def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:)
|
|
143
|
-
@label = label
|
|
144
|
-
@timeout = timeout
|
|
145
|
-
@root = root
|
|
146
|
-
@dir = dir
|
|
147
|
-
@env = env
|
|
148
|
-
@ruby = ruby
|
|
149
|
-
control = File.join(dir, "#{label}.sock")
|
|
150
|
-
server = UNIXServer.new(control)
|
|
151
|
-
# (A nil config_path is left out. And the worker gets no other
|
|
152
|
-
# worker's connection: TruffleRuby's sockets aren't close-on-exec,
|
|
153
|
-
# and one left open in another worker would keep it from ending.)
|
|
154
|
-
@pid = Process.spawn(Rubies.worker_environment(env, engine), ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
|
|
155
|
-
"-e", "Duel::Worker.main(*ARGV)", root, candidate_root, control, *[config_path].compact, in: File::NULL, close_others: true)
|
|
156
|
-
@socket = accept(server, control)
|
|
157
|
-
hello = read
|
|
158
|
-
@benchmarks = hello.fetch("benchmarks").to_h { |b| [b.fetch("name"), b] }
|
|
159
|
-
@metrics = hello.fetch("metrics")
|
|
160
|
-
@memory = hello.fetch("memory")
|
|
161
|
-
@profiler = hello.fetch("profiler")
|
|
162
|
-
@yjit = hello.fetch("yjit")
|
|
163
|
-
@description = hello.fetch("ruby")
|
|
164
|
-
@extensions = hello.fetch("extensions")
|
|
165
|
-
@twice = hello.fetch("twice")
|
|
166
|
-
@gem_dirs = hello.fetch("gem_dirs")
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
# Sends a request, and returns the reply, or raises Failed with its
|
|
170
|
-
# error.
|
|
171
|
-
def request(op, name, **args)
|
|
172
|
-
# (By name: +args+ come by keyword, as Symbols.)
|
|
173
|
-
Message.write(@socket, {"op" => op, "name" => name, **args.transform_keys(&:to_s)})
|
|
174
|
-
reply = read
|
|
175
|
-
raise Failed, "#{@label}: #{reply.fetch("error")}\n #{reply.fetch("backtrace").join("\n ")}" if reply.key?("error")
|
|
176
|
-
|
|
177
|
-
reply
|
|
178
|
-
end
|
|
179
|
-
|
|
180
|
-
def sample(name, n, runs: 1, count: false) = request("sample", name, n: n, runs: runs, count: count)
|
|
181
|
-
|
|
182
|
-
# A run of a process benchmark (see DSL#duel_process): +process+, a
|
|
183
|
-
# fresh process, started as this worker was (on its Ruby, in its
|
|
184
|
-
# version's tree, with its environment), and measured (see
|
|
185
|
-
# ProcessSample#run).
|
|
186
|
-
def run(process)
|
|
187
|
-
# (Code, without a command.)
|
|
188
|
-
code = process.fetch("ruby") #: String
|
|
189
|
-
command = process.fetch("command") || [@ruby, "-I", File.join(@root, "lib"), "-e", code]
|
|
190
|
-
runs = @runs ||= ProcessSample.new(@label, env: @env, chdir: @root, log: File.join(@dir, "#{@label}.log"), timeout: @timeout)
|
|
191
|
-
runs.run(command)
|
|
192
|
-
end
|
|
193
|
-
|
|
194
|
-
# Closing the connection ends the worker. One still running +wait+
|
|
195
|
-
# seconds later is killed.
|
|
196
|
-
def stop(wait: STOP_WAIT)
|
|
197
|
-
@runs&.stop
|
|
198
|
-
@socket.close
|
|
199
|
-
deadline = monotonic + wait
|
|
200
|
-
sleep POLL until (gone = Process.wait(@pid, Process::WNOHANG)) || monotonic > deadline
|
|
201
|
-
kill unless gone
|
|
202
|
-
rescue Errno::ECHILD # gone already
|
|
203
|
-
nil
|
|
204
|
-
end
|
|
205
|
-
|
|
206
|
-
private
|
|
207
|
-
|
|
208
|
-
# The worker's connection, once it has loaded (or Failed if it died,
|
|
209
|
-
# or took too long, first).
|
|
210
|
-
def accept(server, path)
|
|
211
|
-
deadline = monotonic + @timeout
|
|
212
|
-
until server.wait_readable(POLL)
|
|
213
|
-
raise Failed, "#{@label} worker exited while loading" if Process.wait(@pid, Process::WNOHANG)
|
|
214
|
-
hung("load") if monotonic > deadline
|
|
215
|
-
end
|
|
216
|
-
server.accept
|
|
217
|
-
ensure
|
|
218
|
-
server.close
|
|
219
|
-
File.delete(path)
|
|
220
|
-
end
|
|
221
|
-
|
|
222
|
-
def read
|
|
223
|
-
hung("answer") unless @socket.wait_readable(@timeout)
|
|
224
|
-
Message.read(@socket) or raise Failed, "#{@label} worker exited"
|
|
225
|
-
end
|
|
226
|
-
|
|
227
|
-
# Kills a worker that didn't +what+ within the timeout, and fails.
|
|
228
|
-
def hung(what)
|
|
229
|
-
kill
|
|
230
|
-
raise Failed, "#{@label} worker didn't #{what} within #{@timeout}s (killed: a benchmark that hangs?)"
|
|
231
|
-
end
|
|
232
|
-
|
|
233
|
-
def kill
|
|
234
|
-
Process.kill("KILL", @pid)
|
|
235
|
-
Process.wait(@pid)
|
|
236
|
-
end
|
|
237
|
-
|
|
238
|
-
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
239
|
-
end
|
|
240
|
-
|
|
241
|
-
# What Duel raises when something a run needs fails: a resource that
|
|
242
|
-
# didn't start (Resource#failed), two that set a variable differently,
|
|
243
|
-
# a worker that died or hung.
|
|
244
|
-
class Failed < Error; end
|
|
245
|
-
|
|
246
|
-
# Workers of both versions, taking turns: Settings#processes of each
|
|
247
|
-
# (PROCESSES, by default). Where things landed in one process's memory
|
|
248
|
-
# can make the very same code a few percent faster or slower for that
|
|
249
|
-
# process's whole life, so the interval is over the workers, where
|
|
250
|
-
# they're enough for one (see #judge_processes), and not over the
|
|
251
|
-
# pairs, which share them. Each worker has its own resources
|
|
252
|
-
# (Config#prepare(:worker)).
|
|
253
|
-
# @api private
|
|
254
|
-
class Lane
|
|
255
|
-
# CPU time (or a plugin's ratio metric's equivalent) a run aims for:
|
|
256
|
-
# long enough that a clock's resolution is lost in it. A sample is the
|
|
257
|
-
# fastest of RUNS runs.
|
|
258
|
-
TARGET_NS = 1_000_000
|
|
259
|
-
RUNS = 5
|
|
260
|
-
WARMUP = 5
|
|
261
|
-
# A process benchmark's pairs before any are measured (of one run
|
|
262
|
-
# each), and the most runs a sample of it takes (unless it says: see
|
|
263
|
-
# DSL#duel_process): as many as take PROCESS_SAMPLE_NS of CPU time,
|
|
264
|
-
# each sample the fastest of them.
|
|
265
|
-
PROCESS_WARMUP = 1
|
|
266
|
-
PROCESS_RUNS = 3
|
|
267
|
-
PROCESS_SAMPLE_NS = 1_000_000_000
|
|
268
|
-
# The most a run may allocate, in bytes (see Sample#heap): it all stays
|
|
269
|
-
# until the run is over, as the garbage collector is held off.
|
|
270
|
-
HEAP_CAP = 64 << 20
|
|
271
|
-
# Pairs are taken in rounds of STEP until the verdict is clear (a
|
|
272
|
-
# process benchmark's: see #judge).
|
|
273
|
-
STEP = 10
|
|
274
|
-
MAX_PAIRS = 60
|
|
275
|
-
# A benchmark judged by process (see #judge_processes) is decided from
|
|
276
|
-
# its MIN_ROUNDS-th round of a sample from every worker, a sample
|
|
277
|
-
# each takes, and up to PROCESS_PAIRS pairs.
|
|
278
|
-
MIN_ROUNDS = 4
|
|
279
|
-
PROCESS_PAIRS = 120
|
|
280
|
-
# Counts are each version's fewest in COUNTS samples, or where a
|
|
281
|
-
# version is on JRuby, JRUBY_COUNTS: its count of bytes allocated
|
|
282
|
-
# takes one of two values, some 31 bytes an iteration apart, sample by
|
|
283
|
-
# sample, and the fewest of more are the lower one.
|
|
284
|
-
COUNTS = 3
|
|
285
|
-
JRUBY_COUNTS = 10
|
|
286
|
-
# A count by process (see #count_by_process) is each worker's fewest
|
|
287
|
-
# in COUNT_ROUNDS rounds of a sample from every worker.
|
|
288
|
-
COUNT_ROUNDS = 3
|
|
289
|
-
# Peak memory is each version's least in MEMORY_SAMPLES fresh workers.
|
|
290
|
-
MEMORY_SAMPLES = 3
|
|
291
|
-
# A profile: PROFILE_SAMPLES samples of the benchmark (some 5,000 of
|
|
292
|
-
# StackProf's a version: enough to tell a few percent apart).
|
|
293
|
-
PROFILE_SAMPLES = 500
|
|
294
|
-
|
|
295
|
-
attr_reader :profiler
|
|
296
|
-
|
|
297
|
-
# Its workers start at once, each with its resources (a database
|
|
298
|
-
# server takes seconds), and stop at once. +sides+ are the versions as
|
|
299
|
-
# the run prepared them (Runner#prepare), by name.
|
|
300
|
-
def initialize(config, settings, sides, processes: settings.processes)
|
|
301
|
-
@id = Lane.next_id
|
|
302
|
-
@dir = Lane.tmpdir
|
|
303
|
-
@resources = []
|
|
304
|
-
@workers = []
|
|
305
|
-
@lock = Mutex.new
|
|
306
|
-
@config = config
|
|
307
|
-
@settings = settings
|
|
308
|
-
@sides = sides
|
|
309
|
-
workers = %w[baseline candidate].product(Array.new(processes) { |i| i }) #: Array[[String, Integer]]
|
|
310
|
-
started = in_parallel(workers) { |side, index| spawn(side, index) }
|
|
311
|
-
@bases = started.first(processes)
|
|
312
|
-
@candidates = started.drop(processes)
|
|
313
|
-
@profiler = @candidates.first.profiler
|
|
314
|
-
@warm = settings.warm || (started.any?(&:yjit) ? YJIT_WARM : JIT_WARM)
|
|
315
|
-
check_builds
|
|
316
|
-
rescue Exception # rubocop:disable Lint/RescueException
|
|
317
|
-
stop
|
|
318
|
-
raise
|
|
319
|
-
end
|
|
320
|
-
|
|
321
|
-
@ids = 0
|
|
322
|
-
@ids_lock = Mutex.new
|
|
323
|
-
|
|
324
|
-
# A number of its own for each lane, from 1: what tells one lane's
|
|
325
|
-
# workers from another's (see Result#processes).
|
|
326
|
-
# mutant:disable -- CRuby doesn't switch threads in the middle of += on an Integer, and JRuby can
|
|
327
|
-
def self.next_id = @ids_lock.synchronize { @ids += 1 }
|
|
328
|
-
|
|
329
|
-
# Whether +processes+ workers of each version are enough for a
|
|
330
|
-
# +confidence+ interval over them (see Stats.shift_rank): 5 at 99%.
|
|
331
|
-
def self.by_process?(processes, confidence) = !Stats.shift_rank(processes, processes, confidence).nil?
|
|
332
|
-
|
|
333
|
-
# A directory for a lane's sockets, short as a Unix socket's path must
|
|
334
|
-
# be (see .short_tmp).
|
|
335
|
-
def self.tmpdir = Dir.mktmpdir("duel", short_tmp)
|
|
336
|
-
|
|
337
|
-
# The longest +tmpdir+ a lane's directory goes in: a resource's socket
|
|
338
|
-
# there (Resource#path) must stay within a Unix socket path's 104 bytes.
|
|
339
|
-
SHORT = 40
|
|
340
|
-
|
|
341
|
-
# Where lanes' directories go: +tmpdir+ (TMPDIR) if it's that short,
|
|
342
|
-
# else /tmp (macOS's TMPDIR is long).
|
|
343
|
-
def self.short_tmp(tmpdir = Dir.tmpdir) = (tmpdir.size <= SHORT) ? tmpdir : "/tmp"
|
|
344
|
-
|
|
345
|
-
# Opens a lane, yields it and always closes it.
|
|
346
|
-
def self.open(config, settings, sides, processes: settings.processes)
|
|
347
|
-
# @type var lane: Lane
|
|
348
|
-
lane = new(config, settings, sides, processes:)
|
|
349
|
-
yield lane
|
|
350
|
-
ensure
|
|
351
|
-
lane&.stop
|
|
352
|
-
end
|
|
353
|
-
|
|
354
|
-
def benchmarks = @candidates.first.benchmarks
|
|
355
|
-
|
|
356
|
-
# The Ruby each version runs on (RUBY_DESCRIPTION), by version.
|
|
357
|
-
def rubies = {"baseline" => @bases.first.description, "candidate" => @candidates.first.description}
|
|
358
|
-
|
|
359
|
-
# Stops its workers, then their resources: each worker's last first
|
|
360
|
-
# (one may need another), the workers' at once.
|
|
361
|
-
def stop
|
|
362
|
-
in_parallel(@workers) { |worker| worker.stop }
|
|
363
|
-
in_parallel(@resources.group_by(&:name).values) { |resources| resources.reverse_each(&:stop) }
|
|
364
|
-
FileUtils.rm_rf(@dir)
|
|
365
|
-
end
|
|
366
|
-
|
|
367
|
-
# Measures one benchmark on both versions, and returns its Result. (A
|
|
368
|
-
# process benchmark's workers have nothing to set up: its samples are
|
|
369
|
-
# processes of their own.)
|
|
370
|
-
def compare(name)
|
|
371
|
-
result = Result.new(name)
|
|
372
|
-
started = monotonic
|
|
373
|
-
if benchmarks.fetch(name).fetch("process")
|
|
374
|
-
measure_process(name, result)
|
|
375
|
-
else
|
|
376
|
-
set_up(name, result)
|
|
377
|
-
return result if result.skipped
|
|
378
|
-
|
|
379
|
-
begin
|
|
380
|
-
measure(name, result)
|
|
381
|
-
ensure
|
|
382
|
-
each_worker { |worker| worker.request("teardown", name) }
|
|
383
|
-
end
|
|
384
|
-
end
|
|
385
|
-
result.seconds = monotonic - started
|
|
386
|
-
result
|
|
387
|
-
end
|
|
388
|
-
|
|
389
|
-
# Profiles of the benchmark on each version (+n+ iterations a
|
|
390
|
-
# sample): {"baseline" =>, "candidate" =>}, or nil without StackProf.
|
|
391
|
-
def profile(name, n)
|
|
392
|
-
return unless @profiler
|
|
393
|
-
|
|
394
|
-
# @type var sides: Hash[String, WorkerProcess]?
|
|
395
|
-
sides = {"baseline" => @bases.first, "candidate" => @candidates.first}
|
|
396
|
-
sides.each_value { |worker| worker.request("setup", name) }
|
|
397
|
-
sides.transform_values { |worker| worker.request("profile", name, n: n, runs: PROFILE_SAMPLES) }
|
|
398
|
-
ensure
|
|
399
|
-
sides&.each_value { |worker| worker.request("teardown", name) }
|
|
400
|
-
end
|
|
401
|
-
|
|
402
|
-
private
|
|
403
|
-
|
|
404
|
-
# A worker of +side+, with its own resources, and its version's Ruby
|
|
405
|
-
# and environment.
|
|
406
|
-
def spawn(side, index)
|
|
407
|
-
version = @sides.fetch(side)
|
|
408
|
-
label = "#{version.label}-#{index + 1}"
|
|
409
|
-
env = version.env
|
|
410
|
-
@config.prepares.fetch(:worker).each_with_index do |block, i|
|
|
411
|
-
resource = Resource.new(File.join(@dir, "#{label}-#{i + 1}"), label, version.root, timeout: @settings.timeout,
|
|
412
|
-
cache: File.join(@config.state, "cache"), ruby: version.ruby, env: version.env)
|
|
413
|
-
keep(@resources, resource)
|
|
414
|
-
env = Resource.environment(label, env, block.call(resource))
|
|
415
|
-
end
|
|
416
|
-
worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
|
|
417
|
-
config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
|
|
418
|
-
keep(@workers, worker)
|
|
419
|
-
worker
|
|
420
|
-
end
|
|
421
|
-
|
|
422
|
-
# Fails if a version's workers loaded a native extension that isn't a
|
|
423
|
-
# build of its own: one in the other version's tree, or one where the
|
|
424
|
-
# other version loaded the same from its own tree (an installed gem's,
|
|
425
|
-
# say). A checkout without a build of its own (see Config#build)
|
|
426
|
-
# quietly loads whichever build it finds, and the comparison wouldn't
|
|
427
|
-
# be of its code. An installed gem's extension (in the directories
|
|
428
|
-
# gems are installed in, by the worker that loaded it) is in no
|
|
429
|
-
# version's tree, even in a bundle there (vendor/bundle, say, which
|
|
430
|
-
# both versions may load from). And fails if a version's workers
|
|
431
|
-
# loaded one twice (see Worker.twice): the candidate's, that something
|
|
432
|
-
# required before Duel loaded the version, and then its own over it.
|
|
433
|
-
def check_builds
|
|
434
|
-
workers = {"baseline" => @bases.first, "candidate" => @candidates.first}
|
|
435
|
-
workers.each do |side, worker|
|
|
436
|
-
worker.twice.each do |feature, paths|
|
|
437
|
-
raise Failed, "the #{side} loaded #{feature} twice, from #{paths.join(" and ")}: something required it before " \
|
|
438
|
-
"Duel loaded the #{side}, and both stay loaded. Require what loads it in a block that runs once the version " \
|
|
439
|
-
"has loaded (c.in_worker, in .duel.rb), not as .duel.rb or a plugin loads"
|
|
440
|
-
end
|
|
441
|
-
other = (side == "baseline") ? "candidate" : "baseline"
|
|
442
|
-
theirs = workers.fetch(other).then { |w| w.extensions.select { |_, path| inside?(path, other, w) } }
|
|
443
|
-
worker.extensions.each do |feature, path|
|
|
444
|
-
next if inside?(path, side, worker) || !(inside?(path, other, worker) || theirs.key?(feature))
|
|
445
|
-
|
|
446
|
-
raise Failed, "the #{side} loaded #{path}, not a build of its own: #{feature} isn't built in " \
|
|
447
|
-
"#{@sides.fetch(side).root}. Set c.build in .duel.rb to build each version (by default, it's " \
|
|
448
|
-
"`bundle exec rake compile` for a project with ext/ and a Rakefile)"
|
|
449
|
-
end
|
|
450
|
-
end
|
|
451
|
-
end
|
|
452
|
-
|
|
453
|
-
# Whether +path+ (a real path, that +worker+ loaded) is in +side+'s
|
|
454
|
-
# tree, and not in an installed gem (in +worker+'s gem directories).
|
|
455
|
-
def inside?(path, side, worker)
|
|
456
|
-
root = @sides.fetch(side).root
|
|
457
|
-
path.start_with?(File.join(File.exist?(root) ? File.realpath(root) : root, "")) &&
|
|
458
|
-
worker.gem_dirs.none? { |dir| path.start_with?(dir) }
|
|
459
|
-
end
|
|
460
|
-
|
|
461
|
-
# Adds +item+ to +list+, from a thread of its own (see #in_parallel).
|
|
462
|
-
# mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
|
|
463
|
-
def keep(list, item) = @lock.synchronize { list << item }
|
|
464
|
-
|
|
465
|
-
def each_worker(&) = (@bases + @candidates).each(&)
|
|
466
|
-
|
|
467
|
-
# Sets the benchmark up in every worker, the first candidate first (a
|
|
468
|
-
# benchmark that fails there is broken) and the first baseline next
|
|
469
|
-
# (one that fails there is new: skipped), then the rest at once.
|
|
470
|
-
def set_up(name, result)
|
|
471
|
-
@candidates.first.request("setup", name)
|
|
472
|
-
begin
|
|
473
|
-
@bases.first.request("setup", name)
|
|
474
|
-
rescue Failed => e
|
|
475
|
-
@candidates.first.request("teardown", name)
|
|
476
|
-
result.skipped = "not on the baseline (#{e.message.lines.first.chomp})"
|
|
477
|
-
return
|
|
478
|
-
end
|
|
479
|
-
in_parallel(@bases.drop(1) + @candidates.drop(1)) { |worker| worker.request("setup", name) }
|
|
480
|
-
end
|
|
481
|
-
|
|
482
|
-
# What's measured: ratio metrics' measures are in a sample's "timed",
|
|
483
|
-
# counted ones' in its "counted".
|
|
484
|
-
def metrics = @candidates.first.metrics
|
|
485
|
-
|
|
486
|
-
# Whether workers start (and stop) at once: not on TruffleRuby, whose
|
|
487
|
-
# Process.spawn, closing other descriptors (see WorkerProcess), fails
|
|
488
|
-
# when another thread closes one meanwhile.
|
|
489
|
-
PARALLEL = RUBY_ENGINE != "truffleruby"
|
|
490
|
-
|
|
491
|
-
# What the block returns for each item, each on a thread of its own
|
|
492
|
-
# (one after another, unless PARALLEL). Every one finishes before the
|
|
493
|
-
# first that failed, if any, raises.
|
|
494
|
-
def in_parallel(items, &)
|
|
495
|
-
return items.map(&) unless PARALLEL
|
|
496
|
-
|
|
497
|
-
threads = items.map do |item|
|
|
498
|
-
Thread.new do
|
|
499
|
-
Thread.current.report_on_exception = false
|
|
500
|
-
yield item
|
|
501
|
-
end
|
|
502
|
-
end
|
|
503
|
-
threads.each do |thread|
|
|
504
|
-
thread.join
|
|
505
|
-
rescue
|
|
506
|
-
nil # (raised by #value, once they're all done)
|
|
507
|
-
end
|
|
508
|
-
threads.map(&:value)
|
|
509
|
-
end
|
|
510
|
-
|
|
511
|
-
def measure(name, result)
|
|
512
|
-
benchmark = benchmarks.fetch(name)
|
|
513
|
-
only = benchmark.fetch("only")
|
|
514
|
-
if only == "count"
|
|
515
|
-
count(name, result.n = COUNTS_ONLY, result)
|
|
516
|
-
return
|
|
517
|
-
end
|
|
518
|
-
|
|
519
|
-
timings_only = !only.nil? # (the only other choice)
|
|
520
|
-
target = timings_only ? TARGET_NS * 10 : TARGET_NS
|
|
521
|
-
n = calibrate(name, target, 1)
|
|
522
|
-
warm(name, n)
|
|
523
|
-
n = calibrate(name, target, n) if @warm.positive? # it may have got faster
|
|
524
|
-
floors = Array.new(WARMUP) { |i| pair(name, n, i) }
|
|
525
|
-
# (It got faster still, a JIT got to it, say: sized again, unless
|
|
526
|
-
# what it allocates holds it there.)
|
|
527
|
-
if below_cpu_floor?(floors) && (sized = calibrate(name, target, n)) > n
|
|
528
|
-
n = sized
|
|
529
|
-
floors = Array.new(WARMUP) { |i| pair(name, n, i) }
|
|
530
|
-
end
|
|
531
|
-
result.n = n
|
|
532
|
-
count(name, n, result) unless timings_only
|
|
533
|
-
peak_memory(name, n, result) if benchmark.fetch("peak_memory")
|
|
534
|
-
judge(name, n, floors, result)
|
|
535
|
-
end
|
|
536
|
-
|
|
537
|
-
# A process benchmark's samples (see DSL#duel_process): each the
|
|
538
|
-
# fastest of a few fresh processes (see #pair and #process_runs), as
|
|
539
|
-
# its one iteration, paired as any other's, after PROCESS_WARMUP pairs
|
|
540
|
-
# (the first runs read files the disk's cache hasn't yet), and each
|
|
541
|
-
# version's least peak memory, where it can be told.
|
|
542
|
-
def measure_process(name, result)
|
|
543
|
-
@peaks = {"baseline" => [], "candidate" => []}
|
|
544
|
-
result.n = 1
|
|
545
|
-
floors = Array.new(PROCESS_WARMUP) { |i| pair(name, 1, i) }
|
|
546
|
-
# (A process benchmark's.)
|
|
547
|
-
process = benchmarks.fetch(name).fetch("process") #: Definition::process
|
|
548
|
-
judge(name, process.fetch("runs") || process_runs(floors), floors, result)
|
|
549
|
-
return if @peaks.values.flatten.include?(nil)
|
|
550
|
-
|
|
551
|
-
# (Each told, so each least is.)
|
|
552
|
-
least = @peaks.transform_values(&:min) #: Hash[String, Integer]
|
|
553
|
-
result.memory = least
|
|
554
|
-
end
|
|
555
|
-
|
|
556
|
-
# The runs a process benchmark's sample takes, by its warm-up's
|
|
557
|
-
# samples (+floors+, of a run each): as many as take
|
|
558
|
-
# PROCESS_SAMPLE_NS of CPU time on the slower version, up to
|
|
559
|
-
# PROCESS_RUNS, and one at least. A short process varies the most, and
|
|
560
|
-
# the fastest of a few is steadier, but a long one's pairs take long
|
|
561
|
-
# enough as they are.
|
|
562
|
-
def process_runs(floors)
|
|
563
|
-
longest = floors.flatten.map { |sample| sample.fetch("timed").fetch("cpu") }.max #: Integer
|
|
564
|
-
# (Any number, for one that took none at all.)
|
|
565
|
-
PROCESS_SAMPLE_NS.fdiv(longest).clamp(1, PROCESS_RUNS).floor
|
|
566
|
-
end
|
|
567
|
-
|
|
568
|
-
# Every worker runs the benchmark a few times, and then, all at once
|
|
569
|
-
# (none of it is measured, so they needn't take turns), for the
|
|
570
|
-
# seconds --warm says (Settings#warm), or else until it's steady (see
|
|
571
|
-
# #steady), for JIT_WARM's or YJIT_WARM's seconds at most.
|
|
572
|
-
def warm(name, n)
|
|
573
|
-
warm_until = monotonic + @warm
|
|
574
|
-
in_parallel(@bases + @candidates) do |worker|
|
|
575
|
-
2.times { worker.sample(name, n) }
|
|
576
|
-
if @settings.warm
|
|
577
|
-
worker.sample(name, n) until monotonic > warm_until
|
|
578
|
-
else
|
|
579
|
-
steady(worker, name, n, warm_until)
|
|
580
|
-
end
|
|
581
|
-
end
|
|
582
|
-
end
|
|
583
|
-
|
|
584
|
-
# A worker's warm-up ends once STEADY_WINDOWS windows of STEADY
|
|
585
|
-
# seconds in a row each have a median of its samples no faster, by
|
|
586
|
-
# STEADY_GAIN, than the last window that was (so code that creeps
|
|
587
|
-
# faster by less a window isn't steady): a JIT has compiled what it
|
|
588
|
-
# will. (Not once its fastest sample stops getting
|
|
589
|
-
# faster: the more samples, the faster the fastest of them, by
|
|
590
|
-
# chance. And not once one window is no faster: a pause, a collection
|
|
591
|
-
# say, can slow one.) A small library's code is steady in a few
|
|
592
|
-
# seconds, where a big app's JIT keeps compiling, for up to the
|
|
593
|
-
# warm-up's seconds.
|
|
594
|
-
STEADY = 0.5
|
|
595
|
-
STEADY_GAIN = 0.01
|
|
596
|
-
STEADY_WINDOWS = 3
|
|
597
|
-
|
|
598
|
-
# Samples +worker+ (+n+ iterations each), STEADY seconds at a time,
|
|
599
|
-
# until it's steady, or it's +warm_until+.
|
|
600
|
-
def steady(worker, name, n, warm_until)
|
|
601
|
-
window, now = window(worker, name, n, warm_until)
|
|
602
|
-
return if now > warm_until
|
|
603
|
-
|
|
604
|
-
fastest = Stats.median(window)
|
|
605
|
-
calm = 0
|
|
606
|
-
loop do
|
|
607
|
-
window, now = window(worker, name, n, warm_until)
|
|
608
|
-
return if now > warm_until
|
|
609
|
-
|
|
610
|
-
median = Stats.median(window)
|
|
611
|
-
if median > fastest * (1 - STEADY_GAIN)
|
|
612
|
-
calm += 1
|
|
613
|
-
return if calm == STEADY_WINDOWS
|
|
614
|
-
else
|
|
615
|
-
calm = 0
|
|
616
|
-
fastest = median
|
|
617
|
-
end
|
|
618
|
-
end
|
|
619
|
-
end
|
|
620
|
-
|
|
621
|
-
# STEADY seconds of +worker+'s samples' CPU time (a sample at least),
|
|
622
|
-
# or until +warm_until+, and when they ended.
|
|
623
|
-
def window(worker, name, n, warm_until)
|
|
624
|
-
window_until = monotonic + STEADY
|
|
625
|
-
samples = [] #: Array[Metric::value]
|
|
626
|
-
loop do
|
|
627
|
-
samples << worker.sample(name, n).fetch("timed").fetch("cpu")
|
|
628
|
-
now = monotonic
|
|
629
|
-
return [samples, now] if now > window_until || now > warm_until
|
|
630
|
-
end
|
|
631
|
-
end
|
|
632
|
-
|
|
633
|
-
# Takes pairs until every ratio metric worth judging (measured on
|
|
634
|
-
# both versions in the warm-up, and above its floors on either: one
|
|
635
|
-
# that rises from next to nothing is judged too) is decided, or
|
|
636
|
-
# MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
|
|
637
|
-
# and each interval over the pairs. That's for a process benchmark, or
|
|
638
|
-
# too few workers for an interval over them: otherwise, see
|
|
639
|
-
# #judge_processes. A process benchmark's pairs take a process each,
|
|
640
|
-
# so it's decided not slower (Result::CLEARED) after any pair too.
|
|
641
|
-
# That can only stop it before it would have been found slower, never
|
|
642
|
-
# make it slower: it's found slower only at the end of a round, as any
|
|
643
|
-
# benchmark is.
|
|
644
|
-
def judge(name, n, floors, result)
|
|
645
|
-
judged = metrics.select do |spec|
|
|
646
|
-
floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
|
|
647
|
-
floors.transpose.any? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, spec) }
|
|
648
|
-
end
|
|
649
|
-
return judge_processes(name, n, judged, result) if by_process?(name)
|
|
650
|
-
|
|
651
|
-
ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
652
|
-
# @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
|
|
653
|
-
raw = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
654
|
-
pending = judged.dup
|
|
655
|
-
every = benchmarks.fetch(name).fetch("process")
|
|
656
|
-
(0...MAX_PAIRS).each do |i|
|
|
657
|
-
break if pending.empty?
|
|
658
|
-
|
|
659
|
-
base, candidate = pair(name, n, i)
|
|
660
|
-
result.note(base, candidate)
|
|
661
|
-
ratios.each do |metric, list|
|
|
662
|
-
values = [base.fetch("timed").fetch(metric), candidate.fetch("timed").fetch(metric)] #: [Metric::value, Metric::value]
|
|
663
|
-
raw.fetch(metric) << values
|
|
664
|
-
list << values.last.fdiv([values.first, 1].max)
|
|
665
|
-
end
|
|
666
|
-
round = ((i + 1) % STEP).zero?
|
|
667
|
-
next unless round || every
|
|
668
|
-
|
|
669
|
-
pending.reject! { |spec| decided?(verdict(ratios.fetch(spec.fetch("name")), spec), round) }
|
|
670
|
-
end
|
|
671
|
-
judged.each do |spec|
|
|
672
|
-
list = ratios.fetch(spec.fetch("name"))
|
|
673
|
-
medians = raw.fetch(spec.fetch("name")).transpose.map { |values| Stats.median(values) }
|
|
674
|
-
result.timed[spec.fetch("name")] = {"median" => Stats.median(list), "limit" => limit(spec), "across" => spec.fetch("across"),
|
|
675
|
-
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
|
|
676
|
-
"interval" => Stats.median_interval(list, @settings.confidence),
|
|
677
|
-
"pairs" => list.size, "verdict" => verdict(list, spec).to_s,
|
|
678
|
-
"baseline" => medians.fetch(0), "candidate" => medians.fetch(1)}
|
|
679
|
-
end
|
|
680
|
-
end
|
|
681
|
-
|
|
682
|
-
# Whether a benchmark's timings are judged by process (see
|
|
683
|
-
# #judge_processes): unless it's a process benchmark (each of its
|
|
684
|
-
# samples is a fresh process already, so its pairs are independent),
|
|
685
|
-
# or there are too few workers of each version for an interval over
|
|
686
|
-
# them (Lane.by_process?).
|
|
687
|
-
def by_process?(name) = !benchmarks.fetch(name).fetch("process") && Lane.by_process?(@bases.size, @settings.confidence)
|
|
688
|
-
|
|
689
|
-
# Takes rounds of pairs, each a sample from every worker of each
|
|
690
|
-
# version (the baseline's in order, paired with the candidate's in
|
|
691
|
-
# turn: each round pairs each with another), until every ratio metric
|
|
692
|
-
# judged (see #judge) is decided, from the second round on, or
|
|
693
|
-
# #rounds run out. Each worker's median is a sample, and the interval
|
|
694
|
-
# is over workers (Stats.shift_interval): where a process's luck
|
|
695
|
-
# makes it faster or slower than another for its whole life (where
|
|
696
|
-
# things landed in its memory, or what a JIT compiled), the interval
|
|
697
|
-
# is as wide as that makes it. An interval over pairs would be as
|
|
698
|
-
# narrow as if every pair had processes of its own. Each worker's
|
|
699
|
-
# medians are kept, by metric, for across the board (Result#processes).
|
|
700
|
-
def judge_processes(name, n, judged, result)
|
|
701
|
-
k = @bases.size
|
|
702
|
-
# @type var values: Hash[String, Hash[String, Array[Array[Metric::value]]]]
|
|
703
|
-
values = judged.to_h { |spec| [spec.fetch("name"), {"baseline" => Array.new(k) { [] }, "candidate" => Array.new(k) { [] }}] }
|
|
704
|
-
pending = judged.dup
|
|
705
|
-
pairs = 0
|
|
706
|
-
rounds(k).times do |round|
|
|
707
|
-
break if pending.empty?
|
|
708
|
-
|
|
709
|
-
k.times do |i|
|
|
710
|
-
j = (i + round) % k
|
|
711
|
-
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
|
|
712
|
-
result.note(base, candidate)
|
|
713
|
-
values.each do |metric, sides|
|
|
714
|
-
sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
|
|
715
|
-
sides.fetch("candidate").fetch(j) << candidate.fetch("timed").fetch(metric)
|
|
716
|
-
end
|
|
717
|
-
pairs += 1
|
|
718
|
-
end
|
|
719
|
-
next if round + 1 < MIN_ROUNDS
|
|
720
|
-
|
|
721
|
-
pending.reject! { |spec| decided?(process_verdict(medians(values.fetch(spec.fetch("name"))), spec), true) }
|
|
722
|
-
end
|
|
723
|
-
judged.each do |spec|
|
|
724
|
-
metric = spec.fetch("name")
|
|
725
|
-
each = medians(values.fetch(metric))
|
|
726
|
-
bases, candidates = clamped(each)
|
|
727
|
-
result.timed[metric] = {"median" => Stats.shift(bases, candidates), "limit" => limit(spec), "across" => spec.fetch("across"),
|
|
728
|
-
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
|
|
729
|
-
"interval" => Stats.shift_interval(bases, candidates, @settings.confidence),
|
|
730
|
-
"pairs" => pairs, "processes" => k, "verdict" => process_verdict(each, spec).to_s,
|
|
731
|
-
"baseline" => Stats.median(each.fetch("baseline")), "candidate" => Stats.median(each.fetch("candidate"))}
|
|
732
|
-
result.processes[metric] = {"lane" => @id, "baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}
|
|
733
|
-
end
|
|
734
|
-
end
|
|
735
|
-
|
|
736
|
-
# The most rounds of +k+ workers' pairs: as many as make
|
|
737
|
-
# PROCESS_PAIRS pairs, and MIN_ROUNDS at least.
|
|
738
|
-
def rounds(k) = [PROCESS_PAIRS / k, MIN_ROUNDS].max
|
|
739
|
-
|
|
740
|
-
# Each worker's median, by version, of +values+ (each worker's
|
|
741
|
-
# measures, by version).
|
|
742
|
-
def medians(values) = values.transform_values { |lists| lists.map { |list| Stats.median(list) } }
|
|
743
|
-
|
|
744
|
-
# The baseline's medians (of +each+) at least 1, as the ratio's
|
|
745
|
-
# denominator (so a ratio to nothing is one to 1, as a pair's is),
|
|
746
|
-
# and the candidate's.
|
|
747
|
-
def clamped(each) = [each.fetch("baseline").map { |value| [value, 1].max }, each.fetch("candidate")]
|
|
748
|
-
|
|
749
|
-
def process_verdict(each, spec)
|
|
750
|
-
bases, candidates = clamped(each)
|
|
751
|
-
Stats.decide(Stats.shift_interval(bases, candidates, @settings.confidence), limit(spec))
|
|
752
|
-
end
|
|
753
|
-
|
|
754
|
-
# Whether both versions' CPU time in +floors+ is below its floor: then
|
|
755
|
-
# it wouldn't be judged at all.
|
|
756
|
-
def below_cpu_floor?(floors)
|
|
757
|
-
cpu = metrics.find { |spec| spec.fetch("name") == "cpu" } #: Metric::spec
|
|
758
|
-
floors.transpose.none? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, cpu) }
|
|
759
|
-
end
|
|
760
|
-
|
|
761
|
-
# Whether +samples+ (one version's timed measures) are above +spec+'s
|
|
762
|
-
# floors (the median): its own, and its share of CPU time.
|
|
763
|
-
def above_floors?(samples, spec)
|
|
764
|
-
values = samples.map { |sample| sample.fetch(spec.fetch("name")) }
|
|
765
|
-
cpu = samples.map { |sample| sample.fetch("cpu") }
|
|
766
|
-
median = Stats.median(values)
|
|
767
|
-
median >= spec.fetch("floor") && median * spec.fetch("unit_ns") >= spec.fetch("cpu_floor") * Stats.median(cpu)
|
|
768
|
-
end
|
|
769
|
-
|
|
770
|
-
def limit(spec) = spec.fetch("limit") || @settings.limit
|
|
771
|
-
|
|
772
|
-
# The verdicts that decide a measure at the end of a round (see
|
|
773
|
-
# #judge), and after any other pair: only those that clear it.
|
|
774
|
-
DECISIVE = {true => [:slower, :faster, :same], false => [:faster, :same]}.freeze #: Hash[bool, Array[Stats::verdict]]
|
|
775
|
-
|
|
776
|
-
# Whether +verdict+ decides a measure, at the end of a +round+ or not.
|
|
777
|
-
def decided?(verdict, round) = DECISIVE.fetch(round).include?(verdict)
|
|
778
|
-
|
|
779
|
-
def verdict(ratios, spec) = Stats.verdict(ratios, limit: limit(spec), confidence: @settings.confidence)
|
|
780
|
-
|
|
781
|
-
# The iterations per sample (from +n+ up) whose fastest run takes
|
|
782
|
-
# about +target+ on the first candidate, by its largest timed metric
|
|
783
|
-
# that sizes samples (CPU time does), found while warming up (twice in
|
|
784
|
-
# a row: a cold run may look like enough), and that allocate no more
|
|
785
|
-
# than HEAP_CAP a run.
|
|
786
|
-
def calibrate(name, target, n)
|
|
787
|
-
units = metrics.select { |spec| spec.fetch("sizes_samples") }.to_h { |spec| [spec.fetch("name"), spec.fetch("unit_ns")] }
|
|
788
|
-
reached = 0
|
|
789
|
-
until reached == 2
|
|
790
|
-
sample = pair(name, n, 0, runs: 3).last
|
|
791
|
-
# (CPU time sizes samples, and is always measured.)
|
|
792
|
-
timed = sample.fetch("timed")
|
|
793
|
-
work = units.filter_map { |metric, unit| timed[metric]&.*(unit) }.max #: Metric::value
|
|
794
|
-
most = most_iterations(n, sample.fetch("heap"))
|
|
795
|
-
if most <= n
|
|
796
|
-
# (Finite, when it's at most n.)
|
|
797
|
-
return most #: Integer
|
|
798
|
-
end
|
|
799
|
-
|
|
800
|
-
reached = (work >= target) ? reached + 1 : 0
|
|
801
|
-
next unless reached.zero?
|
|
802
|
-
|
|
803
|
-
# (An Integer: most is finite, or more.)
|
|
804
|
-
n = [(n * (target * 1.2 / work).clamp(2, 100)).ceil, most].min #: Integer
|
|
805
|
-
end
|
|
806
|
-
n
|
|
807
|
-
end
|
|
808
|
-
|
|
809
|
-
# The most iterations whose allocations fit HEAP_CAP, by +heap+ bytes
|
|
810
|
-
# over +n+ (at least one, and any number when it allocated nothing).
|
|
811
|
-
def most_iterations(n, heap)
|
|
812
|
-
return Float::INFINITY unless heap.positive?
|
|
813
|
-
|
|
814
|
-
[n * HEAP_CAP / heap, 1].max
|
|
815
|
-
end
|
|
816
|
-
|
|
817
|
-
# The +index+th pair: a sample (of +n+ iterations) from a baseline
|
|
818
|
-
# worker and from a candidate worker, each pair of workers in turn,
|
|
819
|
-
# baseline first on even pairs. A process benchmark's sample is the
|
|
820
|
-
# fastest of +n+ runs (see #measure_process), the two versions' taking
|
|
821
|
-
# turns: so interference, which only ever adds time, is mostly left
|
|
822
|
-
# out, and weighs on both alike.
|
|
823
|
-
def pair(name, n, index, runs: RUNS, count: false)
|
|
824
|
-
k = @bases.size
|
|
825
|
-
sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
|
|
826
|
-
end
|
|
827
|
-
|
|
828
|
-
# A sample (of +n+ iterations) from +base+, a baseline worker, and
|
|
829
|
-
# from +candidate+, a candidate worker, the baseline's first if
|
|
830
|
-
# +base_first+ (see #pair).
|
|
831
|
-
def sampled(name, n, base, candidate, base_first:, runs: RUNS, count: false)
|
|
832
|
-
order = base_first ? [base, candidate] : [candidate, base]
|
|
833
|
-
process = benchmarks.fetch(name).fetch("process")
|
|
834
|
-
replies = if process
|
|
835
|
-
# (Each run of one version's next to one of the other's.)
|
|
836
|
-
each_run = Array.new(n) { order.map { |worker| worker.run(process) } }
|
|
837
|
-
order.zip(each_run.transpose).to_h do |worker, samples|
|
|
838
|
-
# (Never nil: each worker in order has its runs.)
|
|
839
|
-
ran = samples #: Array[ProcessSample::sampled]
|
|
840
|
-
[worker, ProcessSample.fastest(ran)]
|
|
841
|
-
end
|
|
842
|
-
else
|
|
843
|
-
order.to_h { |worker| [worker, worker.sample(name, n, runs: runs, count: count)] }
|
|
844
|
-
end
|
|
845
|
-
if process
|
|
846
|
-
@peaks.fetch("baseline") << replies.fetch(base).fetch("peak")
|
|
847
|
-
@peaks.fetch("candidate") << replies.fetch(candidate).fetch("peak")
|
|
848
|
-
end
|
|
849
|
-
[replies.fetch(base), replies.fetch(candidate)]
|
|
850
|
-
end
|
|
851
|
-
|
|
852
|
-
# Each counted measure's fewest, on each version, over COUNTS pairs of
|
|
853
|
-
# samples, and what its metric said of those samples.
|
|
854
|
-
def count(name, n, result)
|
|
855
|
-
return count_by_process(name, n, result) if Lane.by_process?(@bases.size, @settings.confidence)
|
|
856
|
-
|
|
857
|
-
samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
|
|
858
|
-
metrics.each do |spec|
|
|
859
|
-
metric = spec.fetch("name")
|
|
860
|
-
base, candidate = samples.transpose.map do |side|
|
|
861
|
-
side.select { |sample| sample.fetch("counted").key?(metric) }.min_by { |sample| sample.fetch("counted").fetch(metric) }
|
|
862
|
-
end
|
|
863
|
-
next unless base && candidate
|
|
864
|
-
|
|
865
|
-
result.counted[metric] = {"baseline" => base.fetch("counted").fetch(metric), "candidate" => candidate.fetch("counted").fetch(metric),
|
|
866
|
-
"n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
|
|
867
|
-
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
|
|
868
|
-
result.note(base, candidate, metric)
|
|
869
|
-
end
|
|
870
|
-
end
|
|
871
|
-
|
|
872
|
-
# Each counted measure, with workers enough for an interval over them
|
|
873
|
-
# (see #judge_processes): each worker's fewest in COUNT_ROUNDS rounds
|
|
874
|
-
# of a sample from every worker, and how much more the candidate's
|
|
875
|
-
# are than the baseline's, at least (the low end of an interval for
|
|
876
|
-
# it, Stats.difference_interval), which is what's judged (see
|
|
877
|
-
# Result#worse). Where every worker's is the same, as CRuby's counts
|
|
878
|
-
# of objects are, that's exactly how many more. Where they differ, as
|
|
879
|
-
# JRuby's bytes do where one JVM's JIT does away with an allocation
|
|
880
|
-
# and another's doesn't, it's as little as the workers can tell.
|
|
881
|
-
# Each version's own is the median of its workers'.
|
|
882
|
-
def count_by_process(name, n, result)
|
|
883
|
-
k = @bases.size
|
|
884
|
-
# @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
|
|
885
|
-
fewest = Hash.new do |h, metric|
|
|
886
|
-
none = {} #: Hash[Integer, Metric::value]
|
|
887
|
-
h[metric] = {"baseline" => none, "candidate" => none.dup}
|
|
888
|
-
end
|
|
889
|
-
COUNT_ROUNDS.times do |round|
|
|
890
|
-
k.times do |i|
|
|
891
|
-
j = (i + round) % k
|
|
892
|
-
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
|
|
893
|
-
fewest!(fewest, "baseline", i, base)
|
|
894
|
-
fewest!(fewest, "candidate", j, candidate)
|
|
895
|
-
result.note(base, candidate)
|
|
896
|
-
end
|
|
897
|
-
end
|
|
898
|
-
metrics.each do |spec|
|
|
899
|
-
metric = spec.fetch("name")
|
|
900
|
-
next unless fewest.key?(metric)
|
|
901
|
-
|
|
902
|
-
found = fewest.fetch(metric)
|
|
903
|
-
bases = found.fetch("baseline").values
|
|
904
|
-
candidates = found.fetch("candidate").values
|
|
905
|
-
next if bases.empty? || candidates.empty?
|
|
906
|
-
|
|
907
|
-
result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
|
|
908
|
-
"more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
|
|
909
|
-
"n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
|
|
910
|
-
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
|
|
911
|
-
end
|
|
912
|
-
end
|
|
913
|
-
|
|
914
|
-
# Keeps each count of +sample+ (from +side+'s +worker+th worker) in
|
|
915
|
-
# +fewest+, where it's that worker's fewest so far.
|
|
916
|
-
def fewest!(fewest, side, worker, sample)
|
|
917
|
-
sample.fetch("counted").each do |metric, value|
|
|
918
|
-
least = fewest[metric].fetch(side)
|
|
919
|
-
least[worker] = [least[worker], value].compact.min
|
|
920
|
-
end
|
|
921
|
-
end
|
|
922
|
-
|
|
923
|
-
# The samples a count takes: JRUBY_COUNTS where a version is on JRuby,
|
|
924
|
-
# or else COUNTS.
|
|
925
|
-
def counts = (@sides.each_value.any? { |side| side.engine == "jruby" }) ? JRUBY_COUNTS : COUNTS
|
|
926
|
-
|
|
927
|
-
# Each version's least peak memory over MEMORY_SAMPLES fresh workers
|
|
928
|
-
# (a long-lived one has room left over from earlier work), turn about.
|
|
929
|
-
# Fresh means no trial run either: the allocator keeps what that freed,
|
|
930
|
-
# and the measured run would reuse it rather than grow.
|
|
931
|
-
def peak_memory(name, n, result)
|
|
932
|
-
return unless @candidates.first.memory
|
|
933
|
-
|
|
934
|
-
# @type var peaks: Hash[String, Array[Integer]]
|
|
935
|
-
peaks = {"baseline" => [], "candidate" => []}
|
|
936
|
-
MEMORY_SAMPLES.times do |i|
|
|
937
|
-
sides = i.even? ? %w[baseline candidate] : %w[candidate baseline]
|
|
938
|
-
sides.each do |side|
|
|
939
|
-
peaks.fetch(side) << fresh(side) do |worker|
|
|
940
|
-
worker.request("setup", name, trial: false)
|
|
941
|
-
worker.request("memory", name, n: n).fetch("peak")
|
|
942
|
-
end
|
|
943
|
-
end
|
|
944
|
-
end
|
|
945
|
-
result.memory = peaks.transform_values(&:min)
|
|
946
|
-
end
|
|
947
|
-
|
|
948
|
-
# Yields a worker of +side+ of its own, set up for nothing yet, then
|
|
949
|
-
# stops it and its resources.
|
|
950
|
-
def fresh(side)
|
|
951
|
-
# @type var workers: Integer
|
|
952
|
-
# @type var resources: Integer
|
|
953
|
-
workers = @workers.size
|
|
954
|
-
resources = @resources.size
|
|
955
|
-
worker = spawn(side, @workers.size)
|
|
956
|
-
yield worker
|
|
957
|
-
ensure
|
|
958
|
-
@workers.pop(@workers.size - workers).each(&:stop)
|
|
959
|
-
@resources.pop(@resources.size - resources).reverse_each(&:stop)
|
|
960
|
-
end
|
|
961
|
-
|
|
962
|
-
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
963
|
-
end
|
|
964
|
-
|
|
965
|
-
# What one comparison of a benchmark found (Testing::Comparison#results
|
|
966
|
-
# has them): its #name, whether it #passed?, the measures it #regressed
|
|
967
|
-
# on, each #timed measure's verdict and each #counted one's counts,
|
|
968
|
-
# what metrics said of it (#details), its peak #memory, its iterations
|
|
969
|
-
# a sample (#n), the #seconds it took, why it was #skipped or its
|
|
970
|
-
# #error, and its #confirmations (the measurements that confirmed it
|
|
971
|
-
# was worse, not those too close to call: see #confirm). Its writers,
|
|
972
|
-
# and the rest, are @api private.
|
|
973
|
-
class Result
|
|
974
|
-
# Peak memory grows by more than MEMORY_RATIO, and by MEMORY_FLOOR
|
|
975
|
-
# bytes at least, before it's worse (it's measured as a whole, with the
|
|
976
|
-
# garbage collector working, so it isn't exact).
|
|
977
|
-
# @api private
|
|
978
|
-
MEMORY_RATIO = 1.1
|
|
979
|
-
|
|
980
|
-
# @api private
|
|
981
|
-
MEMORY_FLOOR = 1 << 20
|
|
982
|
-
|
|
983
|
-
# What a sample says when no metric said anything (and what a measure
|
|
984
|
-
# it said nothing of did).
|
|
985
|
-
# @api private
|
|
986
|
-
NOTHING_SAID = {} #: Hash[String, String]
|
|
987
|
-
|
|
988
|
-
# The verdicts that say a timing isn't slower.
|
|
989
|
-
# @api private
|
|
990
|
-
CLEARED = %w[same faster].freeze
|
|
991
|
-
|
|
992
|
-
attr_reader :name, :timed, :counted, :details, :confirmations, :n, :seconds, :skipped, :error, :memory
|
|
993
|
-
|
|
994
|
-
# Where a regression's time went (see Profile).
|
|
995
|
-
# @api private
|
|
996
|
-
attr_reader :profile
|
|
997
|
-
|
|
998
|
-
# Each worker's median of each measure judged by process (see
|
|
999
|
-
# Lane#judge_processes): {metric => {"lane" => its lane's number,
|
|
1000
|
-
# "baseline" => [each baseline worker's], "candidate" => [...]}},
|
|
1001
|
-
# for across the board.
|
|
1002
|
-
# @api private
|
|
1003
|
-
attr_reader :processes
|
|
1004
|
-
|
|
1005
|
-
# The measurements again that were too close to call (see #confirm).
|
|
1006
|
-
# @api private
|
|
1007
|
-
attr_reader :undecided
|
|
1008
|
-
|
|
1009
|
-
# @api private
|
|
1010
|
-
attr_writer :n, :seconds, :skipped, :error, :memory, :profile
|
|
1011
|
-
|
|
1012
|
-
# @api private
|
|
1013
|
-
def initialize(name)
|
|
1014
|
-
@name = name
|
|
1015
|
-
@timed = {}
|
|
1016
|
-
@counted = {}
|
|
1017
|
-
@details = {}
|
|
1018
|
-
@confirmations = []
|
|
1019
|
-
@undecided = []
|
|
1020
|
-
@processes = {}
|
|
1021
|
-
end
|
|
1022
|
-
|
|
1023
|
-
# Keeps what metrics said of a pair of samples (Metric#details): each
|
|
1024
|
-
# measure's latest, by version (only +measure+'s, if given).
|
|
1025
|
-
# @api private
|
|
1026
|
-
def note(base, candidate, measure = nil)
|
|
1027
|
-
{"baseline" => base, "candidate" => candidate}.each do |side, sample|
|
|
1028
|
-
said = sample["details"] || NOTHING_SAID
|
|
1029
|
-
said = said.slice(measure) if measure
|
|
1030
|
-
said.each { |name, text| (@details[name] ||= {}).store(side, text) }
|
|
1031
|
-
end
|
|
1032
|
-
end
|
|
1033
|
-
|
|
1034
|
-
# The metrics on which the candidate did worse: a timed one slower, or
|
|
1035
|
-
# a counted one up by more than its tolerance per iteration and its
|
|
1036
|
-
# allowance per sample (counted by process, up by that much at least:
|
|
1037
|
-
# see Lane#count_by_process).
|
|
1038
|
-
# @api private
|
|
1039
|
-
def worse
|
|
1040
|
-
more = @counted.select do |_, c|
|
|
1041
|
-
!Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), tolerance: c.fetch("tolerance"),
|
|
1042
|
-
allowance: c.fetch("allowance"))
|
|
1043
|
-
end
|
|
1044
|
-
slower + more.keys + (more_memory? ? ["peak_memory"] : [])
|
|
1045
|
-
end
|
|
1046
|
-
|
|
1047
|
-
# The timed metrics on which the candidate was slower.
|
|
1048
|
-
# @api private
|
|
1049
|
-
def slower = @timed.filter_map { |metric, t| metric if t.fetch("verdict") == "slower" }
|
|
1050
|
-
|
|
1051
|
-
# Whether its timings say slower: one is, and none says it isn't
|
|
1052
|
-
# (same, or faster).
|
|
1053
|
-
# @api private
|
|
1054
|
-
def slower_in_time? = slower.any? && @timed.none? { |_, t| CLEARED.include?(t.fetch("verdict")) }
|
|
1055
|
-
|
|
1056
|
-
# Whether it's to be measured again (see Runner#confirm): it looks
|
|
1057
|
-
# worse in every measurement so far, and there are fewer confirmations
|
|
1058
|
-
# than it takes to fail (Runner::CONFIRM, or one more where it's
|
|
1059
|
-
# #mixed?).
|
|
1060
|
-
# @api private
|
|
1061
|
-
def confirming?
|
|
1062
|
-
return false if regressed.empty?
|
|
1063
|
-
|
|
1064
|
-
@confirmations.size < (mixed? ? Runner::CONFIRM + 1 : Runner::CONFIRM)
|
|
1065
|
-
end
|
|
1066
|
-
|
|
1067
|
-
# Keeps +again+, a measurement of it again from scratch (see
|
|
1068
|
-
# Runner#confirm), as a confirmation, or sets it aside where it's too
|
|
1069
|
-
# close to call, up to Runner::CONFIRM times: every timing is still
|
|
1070
|
-
# undecided there (none slower, none the same or faster), and each
|
|
1071
|
-
# measure it regressed on so far is one of them. One that can't tell
|
|
1072
|
-
# neither confirms it nor clears it, so it's measured again instead.
|
|
1073
|
-
# Returns whether +again+ was set aside.
|
|
1074
|
-
# @api private
|
|
1075
|
-
def confirm(again)
|
|
1076
|
-
timed = again.timed
|
|
1077
|
-
close = timed.all? { |_, t| t.fetch("verdict") == "undecided" } && regressed.all? { |metric| timed.key?(metric) } &&
|
|
1078
|
-
@undecided.size < Runner::CONFIRM
|
|
1079
|
-
(close ? @undecided : @confirmations) << again
|
|
1080
|
-
close
|
|
1081
|
-
end
|
|
1082
|
-
|
|
1083
|
-
# Whether its peak memory grew by too much (see MEMORY_RATIO).
|
|
1084
|
-
# @api private
|
|
1085
|
-
def more_memory?
|
|
1086
|
-
memory = @memory
|
|
1087
|
-
return false unless memory
|
|
1088
|
-
|
|
1089
|
-
base = memory.fetch("baseline")
|
|
1090
|
-
memory.fetch("candidate") - base > [MEMORY_FLOOR, base * (MEMORY_RATIO - 1)].max
|
|
1091
|
-
end
|
|
1092
|
-
|
|
1093
|
-
# Worse on a metric in the first measurement and in every confirmation.
|
|
1094
|
-
# Or, where no one metric is, slower in time in all of them (see
|
|
1095
|
-
# #mixed?): the timed metrics slower in any.
|
|
1096
|
-
def regressed
|
|
1097
|
-
return [] if @error || @skipped
|
|
1098
|
-
|
|
1099
|
-
agreed = measurements.map(&:worse).reduce(:&)
|
|
1100
|
-
return agreed unless agreed.empty? && measurements.all?(&:slower_in_time?)
|
|
1101
|
-
|
|
1102
|
-
measurements.flat_map(&:slower).uniq
|
|
1103
|
-
end
|
|
1104
|
-
|
|
1105
|
-
# Whether it regressed in time on one timed metric in some
|
|
1106
|
-
# measurements and on another in the rest (CPU time in one,
|
|
1107
|
-
# instructions in the next, say), slower in every one with none saying
|
|
1108
|
-
# it isn't: that takes one measurement more than one metric does.
|
|
1109
|
-
# @api private
|
|
1110
|
-
def mixed? = measurements.map(&:worse).reduce(:&).empty? && !regressed.empty?
|
|
1111
|
-
|
|
1112
|
-
# Whether it regressed on nothing, and didn't fail.
|
|
1113
|
-
def passed? = regressed.empty? && !@error
|
|
1114
|
-
|
|
1115
|
-
# Whether a timed measure's verdict was too close to the limit to settle.
|
|
1116
|
-
# @api private
|
|
1117
|
-
def unsettled? = @timed.any? { |_, t| t.fetch("verdict") == "undecided" }
|
|
1118
|
-
|
|
1119
|
-
private
|
|
1120
|
-
|
|
1121
|
-
# The first measurement and its confirmations.
|
|
1122
|
-
def measurements = [self, *@confirmations]
|
|
1123
|
-
end
|
|
1124
|
-
|
|
1125
101
|
# Measures every benchmark, confirms what looks worse, and reports.
|
|
1126
102
|
# @api private
|
|
1127
103
|
class Runner
|
|
@@ -1140,6 +116,9 @@ module Duel
|
|
|
1140
116
|
# The least count #number shows as a whole number.
|
|
1141
117
|
NUMBER = 9_999.5
|
|
1142
118
|
|
|
119
|
+
# Bytes of memory, in megabytes to a tenth ("41.5MB").
|
|
120
|
+
def self.mb(bytes) = format("%.1fMB", bytes / 1_048_576.0)
|
|
121
|
+
|
|
1143
122
|
# Across the board (see Across): {metric => verdict}, "confirmed" in
|
|
1144
123
|
# those slower in a second measurement too. Set by #run.
|
|
1145
124
|
attr_reader :across
|
|
@@ -1175,6 +154,7 @@ module Duel
|
|
|
1175
154
|
|
|
1176
155
|
# Returns the Results.
|
|
1177
156
|
def run
|
|
157
|
+
CycleClock.prepare
|
|
1178
158
|
started = @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
1179
159
|
@dir = Lane.tmpdir
|
|
1180
160
|
begin
|
|
@@ -1182,7 +162,7 @@ module Duel
|
|
|
1182
162
|
ensure
|
|
1183
163
|
@prepared.reverse_each(&:stop)
|
|
1184
164
|
FileUtils.rm_rf(@dir)
|
|
1185
|
-
Resource.evict(
|
|
165
|
+
Resource.evict(@config.cache)
|
|
1186
166
|
end
|
|
1187
167
|
end
|
|
1188
168
|
|
|
@@ -1272,12 +252,8 @@ module Duel
|
|
|
1272
252
|
side.ruby = Rubies.find(choice, side.root, side.name)
|
|
1273
253
|
side.env = Rubies.environment(side.ruby)
|
|
1274
254
|
side.engine = Rubies.engine(side.ruby)
|
|
1275
|
-
@config.prepares.fetch(:version).
|
|
1276
|
-
|
|
1277
|
-
cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
|
|
1278
|
-
@prepared << resource
|
|
1279
|
-
side.env = Resource.environment(side.label, side.env, block.call(resource))
|
|
1280
|
-
end
|
|
255
|
+
side.env = Resource.prepare(@config.prepares.fetch(:version), File.join(@dir, side.label), side.label, side.root,
|
|
256
|
+
env: side.env, ruby: side.ruby, timeout: @settings.timeout, cache: @config.cache, so_far: true) { |resource| @prepared << resource }
|
|
1281
257
|
side
|
|
1282
258
|
end
|
|
1283
259
|
end
|
|
@@ -1291,21 +267,33 @@ module Duel
|
|
|
1291
267
|
# Runner#fit's reckoning.
|
|
1292
268
|
HEADROOM = 1.25
|
|
1293
269
|
|
|
270
|
+
# How many times what a worker took as it started one on a JIT's
|
|
271
|
+
# engine takes once it's warm, by Runner#fit's reckoning, as the JIT
|
|
272
|
+
# compiles: a TruffleRuby worker that started at some 510 MB grew to
|
|
273
|
+
# some 1.5 GB (and HEADROOM covers the rest).
|
|
274
|
+
JIT_GROWTH = {"truffleruby" => 2.5}.freeze
|
|
275
|
+
|
|
1294
276
|
# Lowers Settings#jobs, and then Settings#processes, as far as
|
|
1295
277
|
# Settings#fit lets it, where their workers (and each one's
|
|
1296
278
|
# resources, a database server, say) wouldn't fit in the memory
|
|
1297
279
|
# available: more than that and the machine swaps, or a CI runner
|
|
1298
280
|
# stops answering and its job is lost. What one worker of each
|
|
1299
281
|
# version takes is found by starting them (a lane of them), and every
|
|
1300
|
-
# pair is taken to need that, HEADROOM over
|
|
282
|
+
# pair is taken to need that, HEADROOM over (and on a JIT's engine,
|
|
283
|
+
# JIT_GROWTH times that, as its workers grow as they warm up), or what
|
|
284
|
+
# the plugins say each worker's resources take at most
|
|
285
|
+
# (Config#worker_memory), if that's more: a database server takes
|
|
286
|
+
# more as it's used.
|
|
1301
287
|
def fit
|
|
1302
288
|
return if @settings.fit.empty?
|
|
1303
289
|
|
|
1304
290
|
available = Memory.available or return
|
|
1305
291
|
taken = Lane.open(@config, @settings, @sides, processes: 1) { available - (Memory.available || available) }
|
|
1306
|
-
|
|
292
|
+
said = 2 * @config.worker_memory
|
|
293
|
+
growth = @sides.each_value.map { |side| JIT_GROWTH.fetch(side.engine, 1.0) }.max
|
|
294
|
+
pair = [(taken * HEADROOM * growth).ceil, said].max
|
|
295
|
+
return unless pair.positive?
|
|
1307
296
|
|
|
1308
|
-
pair = (taken * HEADROOM).ceil
|
|
1309
297
|
pairs = [(available - RESERVE) / pair, 1].max
|
|
1310
298
|
processes = @settings.fit.include?(:processes) ? [@settings.processes, pairs].min : @settings.processes
|
|
1311
299
|
# (More jobs than the settings' are never taken: only fewer are.)
|
|
@@ -1314,8 +302,11 @@ module Duel
|
|
|
1314
302
|
("#{Runner.count(processes, "worker")} of each version (not #{@settings.processes})" if processes < @settings.processes)].compact
|
|
1315
303
|
return if fewer.empty?
|
|
1316
304
|
|
|
1317
|
-
|
|
1318
|
-
|
|
305
|
+
takes = if said == pair then "#{gigabytes(said)} GB, with its resources, as its plugins say"
|
|
306
|
+
elsif growth > 1 then "#{gigabytes(taken)} GB, with its resources, as it starts, and some #{growth}x that once warm"
|
|
307
|
+
else "#{gigabytes(taken)} GB, with its resources"
|
|
308
|
+
end
|
|
309
|
+
@out.puts "A worker of each version takes #{takes}, of #{gigabytes(available)} GB available, so to fit: #{fewer.join(", and ")}"
|
|
1319
310
|
@settings.jobs = jobs
|
|
1320
311
|
@settings.processes = processes
|
|
1321
312
|
end
|
|
@@ -1482,13 +473,11 @@ module Duel
|
|
|
1482
473
|
[format("%s %s%s", metric, Runner.number(base), ("→#{Runner.number(candidate)}" unless base == candidate)),
|
|
1483
474
|
(unit(candidate, c) unless named)].compact.join(" ")
|
|
1484
475
|
end
|
|
1485
|
-
memory = result.memory&.then { |m| ["peak_memory #{mb(m.fetch("baseline"))}→#{mb(m.fetch("candidate"))}"] }
|
|
476
|
+
memory = result.memory&.then { |m| ["peak_memory #{Runner.mb(m.fetch("baseline"))}→#{Runner.mb(m.fetch("candidate"))}"] }
|
|
1486
477
|
format("%s %-48s %5.1fs n=%-6d %s", result.worse.empty? ? " " : "!", result.name, result.seconds, result.n,
|
|
1487
478
|
(timed + counted + memory.to_a).join(" "))
|
|
1488
479
|
end
|
|
1489
480
|
|
|
1490
|
-
def mb(bytes) = format("%.1fMB", bytes / 1_048_576.0)
|
|
1491
|
-
|
|
1492
481
|
# The unit of +measured+ (a result's measure), for +number+ of it: the
|
|
1493
482
|
# singular for one, the plural for any other.
|
|
1494
483
|
def unit(number, measured) = (number == 1) ? measured.fetch("unit") : measured.fetch("plural")
|