duel 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +203 -1
- data/README.md +41 -1
- data/docs/ci.md +170 -76
- data/docs/configuration.md +38 -2
- data/docs/how-it-works.md +18 -9
- data/docs/plugins.md +23 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +96 -28
- data/lib/duel/comment.rb +296 -0
- data/lib/duel/config.rb +124 -4
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +255 -20
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +33 -1044
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +21 -15
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +61 -44
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +35 -3
- data/sig/duel/cli.rbs +19 -0
- data/sig/duel/comment.rbs +97 -0
- data/sig/duel/config.rbs +57 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +95 -1
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +6 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +6 -3
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -105
data/lib/duel/lane.rb
ADDED
|
@@ -0,0 +1,734 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "tmpdir"
|
|
5
|
+
require_relative "../duel"
|
|
6
|
+
require_relative "platform"
|
|
7
|
+
require_relative "process_sample"
|
|
8
|
+
require_relative "resource"
|
|
9
|
+
require_relative "result"
|
|
10
|
+
require_relative "stats"
|
|
11
|
+
require_relative "worker_process"
|
|
12
|
+
|
|
13
|
+
module Duel
|
|
14
|
+
# Workers of both versions, taking turns: Settings#processes of each
|
|
15
|
+
# (PROCESSES, by default). Where things landed in one process's memory
|
|
16
|
+
# can make the very same code a few percent faster or slower for that
|
|
17
|
+
# process's whole life, so the interval is over the workers, where
|
|
18
|
+
# they're enough for one (see #judge_processes), and not over the
|
|
19
|
+
# pairs, which share them. Each worker has its own resources
|
|
20
|
+
# (Config#prepare(:worker)).
|
|
21
|
+
# @api private
|
|
22
|
+
class Lane
|
|
23
|
+
# CPU time (or a plugin's ratio metric's equivalent) a run aims for:
|
|
24
|
+
# long enough that a clock's resolution is lost in it. A sample is the
|
|
25
|
+
# fastest of RUNS runs.
|
|
26
|
+
TARGET_NS = 1_000_000
|
|
27
|
+
RUNS = 5
|
|
28
|
+
WARMUP = 5
|
|
29
|
+
# A process benchmark's pairs before any are measured (of one run
|
|
30
|
+
# each), and the most runs a sample of it takes (unless it says: see
|
|
31
|
+
# DSL#duel_process): as many as take PROCESS_SAMPLE_NS of CPU time,
|
|
32
|
+
# each sample the fastest of them.
|
|
33
|
+
PROCESS_WARMUP = 1
|
|
34
|
+
PROCESS_RUNS = 3
|
|
35
|
+
PROCESS_SAMPLE_NS = 1_000_000_000
|
|
36
|
+
# The most a run may allocate, in bytes (see Sample#heap): it all stays
|
|
37
|
+
# until the run is over, as the garbage collector is held off.
|
|
38
|
+
HEAP_CAP = 64 << 20
|
|
39
|
+
# Pairs are taken in rounds of STEP until the verdict is clear (a
|
|
40
|
+
# process benchmark's: see #judge).
|
|
41
|
+
STEP = 10
|
|
42
|
+
MAX_PAIRS = 60
|
|
43
|
+
# A benchmark judged by process (see #judge_processes) is decided from
|
|
44
|
+
# its MIN_ROUNDS-th round of a sample from every worker, a sample
|
|
45
|
+
# each takes, and up to PROCESS_PAIRS pairs.
|
|
46
|
+
MIN_ROUNDS = 4
|
|
47
|
+
PROCESS_PAIRS = 120
|
|
48
|
+
# Counts are each version's fewest in COUNTS samples, or where a
|
|
49
|
+
# version is on JRuby, JRUBY_COUNTS: its count of bytes allocated
|
|
50
|
+
# takes one of two values, some 31 bytes an iteration apart, sample by
|
|
51
|
+
# sample, and the fewest of more are the lower one.
|
|
52
|
+
COUNTS = 3
|
|
53
|
+
JRUBY_COUNTS = 10
|
|
54
|
+
# A count by process (see #count_by_process) is each worker's fewest
|
|
55
|
+
# in COUNT_ROUNDS rounds of a sample from every worker.
|
|
56
|
+
COUNT_ROUNDS = 3
|
|
57
|
+
# Peak memory is each version's least in MEMORY_SAMPLES fresh workers.
|
|
58
|
+
MEMORY_SAMPLES = 3
|
|
59
|
+
# A profile: PROFILE_SAMPLES samples of the benchmark (some 5,000 of
|
|
60
|
+
# StackProf's a version: enough to tell a few percent apart).
|
|
61
|
+
PROFILE_SAMPLES = 500
|
|
62
|
+
|
|
63
|
+
attr_reader :profiler
|
|
64
|
+
|
|
65
|
+
# Its workers start at once, each with its resources (a database
|
|
66
|
+
# server takes seconds), and stop at once. +sides+ are the versions as
|
|
67
|
+
# the run prepared them (Runner#prepare), by name.
|
|
68
|
+
def initialize(config, settings, sides, processes: settings.processes)
|
|
69
|
+
@id = Lane.next_id
|
|
70
|
+
@dir = Lane.tmpdir
|
|
71
|
+
@resources = []
|
|
72
|
+
@workers = []
|
|
73
|
+
@lock = Mutex.new
|
|
74
|
+
@config = config
|
|
75
|
+
@settings = settings
|
|
76
|
+
@sides = sides
|
|
77
|
+
workers = %w[baseline candidate].product(Array.new(processes) { |i| i }) #: Array[[String, Integer]]
|
|
78
|
+
started = in_parallel(workers) { |side, index| spawn(side, index) }
|
|
79
|
+
@bases = started.first(processes)
|
|
80
|
+
@candidates = started.drop(processes)
|
|
81
|
+
@profiler = @candidates.first.profiler
|
|
82
|
+
@warm = settings.warm || (started.any?(&:yjit) ? YJIT_WARM : JIT_WARM)
|
|
83
|
+
check_builds
|
|
84
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
85
|
+
stop
|
|
86
|
+
raise
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
@ids = 0
|
|
90
|
+
@ids_lock = Mutex.new
|
|
91
|
+
|
|
92
|
+
# A number of its own for each lane, from 1: what tells one lane's
|
|
93
|
+
# workers from another's (see Result#processes).
|
|
94
|
+
# mutant:disable -- CRuby doesn't switch threads in the middle of += on an Integer, and JRuby can
|
|
95
|
+
def self.next_id = @ids_lock.synchronize { @ids += 1 }
|
|
96
|
+
|
|
97
|
+
# Whether +processes+ workers of each version are enough for a
|
|
98
|
+
# +confidence+ interval over them (see Stats.shift_rank): 5 at 99%.
|
|
99
|
+
def self.by_process?(processes, confidence) = !Stats.shift_rank(processes, processes, confidence).nil?
|
|
100
|
+
|
|
101
|
+
# A directory for a lane's sockets, short as a Unix socket's path must
|
|
102
|
+
# be (see .short_tmp).
|
|
103
|
+
def self.tmpdir = Dir.mktmpdir("duel", short_tmp)
|
|
104
|
+
|
|
105
|
+
# The longest +tmpdir+ a lane's directory goes in: a resource's socket
|
|
106
|
+
# there (Resource#path) must stay within a Unix socket path's 104 bytes.
|
|
107
|
+
SHORT = 40
|
|
108
|
+
|
|
109
|
+
# Where lanes' directories go: +tmpdir+ (TMPDIR) if it's that short,
|
|
110
|
+
# else /tmp (macOS's TMPDIR is long). Windows has no /tmp, so its own
|
|
111
|
+
# TMP, however long.
|
|
112
|
+
def self.short_tmp(tmpdir = Dir.tmpdir, windows: Platform::WINDOWS) = (windows || tmpdir.size <= SHORT) ? tmpdir : "/tmp"
|
|
113
|
+
|
|
114
|
+
# Opens a lane, yields it and always closes it.
|
|
115
|
+
def self.open(config, settings, sides, processes: settings.processes)
|
|
116
|
+
# @type var lane: Lane
|
|
117
|
+
lane = new(config, settings, sides, processes:)
|
|
118
|
+
yield lane
|
|
119
|
+
ensure
|
|
120
|
+
lane&.stop
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def benchmarks = @candidates.first.benchmarks
|
|
124
|
+
|
|
125
|
+
# The Ruby each version runs on (RUBY_DESCRIPTION), by version.
|
|
126
|
+
def rubies = {"baseline" => @bases.first.description, "candidate" => @candidates.first.description}
|
|
127
|
+
|
|
128
|
+
# Stops its workers, then their resources: each worker's last first
|
|
129
|
+
# (one may need another), the workers' at once.
|
|
130
|
+
def stop
|
|
131
|
+
in_parallel(@workers) { |worker| worker.stop }
|
|
132
|
+
in_parallel(@resources.group_by(&:name).values) { |resources| resources.reverse_each(&:stop) }
|
|
133
|
+
FileUtils.rm_rf(@dir)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Measures one benchmark on both versions, and returns its Result. (A
|
|
137
|
+
# process benchmark's workers have nothing to set up: its samples are
|
|
138
|
+
# processes of their own.)
|
|
139
|
+
def compare(name)
|
|
140
|
+
result = Result.new(name)
|
|
141
|
+
started = monotonic
|
|
142
|
+
if benchmarks.fetch(name).fetch("process")
|
|
143
|
+
measure_process(name, result)
|
|
144
|
+
else
|
|
145
|
+
set_up(name, result)
|
|
146
|
+
return result if result.skipped
|
|
147
|
+
|
|
148
|
+
begin
|
|
149
|
+
measure(name, result)
|
|
150
|
+
ensure
|
|
151
|
+
each_worker { |worker| worker.request("teardown", name) }
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
result.seconds = monotonic - started
|
|
155
|
+
result
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Profiles of the benchmark on each version (+n+ iterations a
|
|
159
|
+
# sample): {"baseline" =>, "candidate" =>}, or nil without StackProf.
|
|
160
|
+
def profile(name, n)
|
|
161
|
+
return unless @profiler
|
|
162
|
+
|
|
163
|
+
# @type var sides: Hash[String, WorkerProcess]?
|
|
164
|
+
sides = {"baseline" => @bases.first, "candidate" => @candidates.first}
|
|
165
|
+
sides.each_value { |worker| worker.request("setup", name) }
|
|
166
|
+
sides.transform_values { |worker| worker.request("profile", name, n: n, runs: PROFILE_SAMPLES) }
|
|
167
|
+
ensure
|
|
168
|
+
sides&.each_value { |worker| worker.request("teardown", name) }
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
private
|
|
172
|
+
|
|
173
|
+
# A worker of +side+, with its own resources, and its version's Ruby
|
|
174
|
+
# and environment.
|
|
175
|
+
def spawn(side, index)
|
|
176
|
+
version = @sides.fetch(side)
|
|
177
|
+
label = "#{version.label}-#{index + 1}"
|
|
178
|
+
env = Resource.prepare(@config.prepares.fetch(:worker), File.join(@dir, label), label, version.root, env: version.env,
|
|
179
|
+
ruby: version.ruby, timeout: @settings.timeout, cache: @config.cache) { |resource| keep(@resources, resource) }
|
|
180
|
+
worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
|
|
181
|
+
config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
|
|
182
|
+
keep(@workers, worker)
|
|
183
|
+
worker
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Fails if a version's workers loaded a native extension that isn't a
|
|
187
|
+
# build of its own: one in the other version's tree, or one where the
|
|
188
|
+
# other version loaded the same from its own tree (an installed gem's,
|
|
189
|
+
# say). A checkout without a build of its own (see Config#build)
|
|
190
|
+
# quietly loads whichever build it finds, and the comparison wouldn't
|
|
191
|
+
# be of its code. An installed gem's extension (in the directories
|
|
192
|
+
# gems are installed in, by the worker that loaded it) is in no
|
|
193
|
+
# version's tree, even in a bundle there (vendor/bundle, say, which
|
|
194
|
+
# both versions may load from). And fails if a version's workers
|
|
195
|
+
# loaded one twice (see Worker.twice): the candidate's, that something
|
|
196
|
+
# required before Duel loaded the version, and then its own over it.
|
|
197
|
+
def check_builds
|
|
198
|
+
workers = {"baseline" => @bases.first, "candidate" => @candidates.first}
|
|
199
|
+
workers.each do |side, worker|
|
|
200
|
+
worker.twice.each do |feature, paths|
|
|
201
|
+
raise Failed, "the #{side} loaded #{feature} twice, from #{paths.join(" and ")}: something required it before " \
|
|
202
|
+
"Duel loaded the #{side}, and both stay loaded. Require what loads it in a block that runs once the version " \
|
|
203
|
+
"has loaded (c.in_worker, in .duel.rb), not as .duel.rb or a plugin loads"
|
|
204
|
+
end
|
|
205
|
+
other = (side == "baseline") ? "candidate" : "baseline"
|
|
206
|
+
theirs = workers.fetch(other).then { |w| w.extensions.select { |_, path| inside?(path, other, w) } }
|
|
207
|
+
worker.extensions.each do |feature, path|
|
|
208
|
+
next if inside?(path, side, worker) || !(inside?(path, other, worker) || theirs.key?(feature))
|
|
209
|
+
|
|
210
|
+
raise Failed, "the #{side} loaded #{path}, not a build of its own: #{feature} isn't built in " \
|
|
211
|
+
"#{@sides.fetch(side).root}. Set c.build in .duel.rb to build each version (by default, it's " \
|
|
212
|
+
"`bundle exec rake compile` for a project with ext/ and a Rakefile)"
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# Whether +path+ (a real path, that +worker+ loaded) is in +side+'s
|
|
218
|
+
# tree, and not in an installed gem (in +worker+'s gem directories).
|
|
219
|
+
def inside?(path, side, worker)
|
|
220
|
+
root = @sides.fetch(side).root
|
|
221
|
+
path.start_with?(File.join(File.exist?(root) ? File.realpath(root) : root, "")) &&
|
|
222
|
+
worker.gem_dirs.none? { |dir| path.start_with?(dir) }
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# Adds +item+ to +list+, from a thread of its own (see #in_parallel).
|
|
226
|
+
# mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
|
|
227
|
+
def keep(list, item) = @lock.synchronize { list << item }
|
|
228
|
+
|
|
229
|
+
def each_worker(&) = (@bases + @candidates).each(&)
|
|
230
|
+
|
|
231
|
+
# Sets the benchmark up in every worker, the first candidate first (a
|
|
232
|
+
# benchmark that fails there is broken) and the first baseline next
|
|
233
|
+
# (one that fails there is new: skipped), then the rest at once.
|
|
234
|
+
def set_up(name, result)
|
|
235
|
+
@candidates.first.request("setup", name)
|
|
236
|
+
begin
|
|
237
|
+
@bases.first.request("setup", name)
|
|
238
|
+
rescue Failed => e
|
|
239
|
+
@candidates.first.request("teardown", name)
|
|
240
|
+
result.skipped = "not on the baseline (#{e.message.lines.first.chomp})"
|
|
241
|
+
return
|
|
242
|
+
end
|
|
243
|
+
in_parallel(@bases.drop(1) + @candidates.drop(1)) { |worker| worker.request("setup", name) }
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# What's measured: ratio metrics' measures are in a sample's "timed",
|
|
247
|
+
# counted ones' in its "counted".
|
|
248
|
+
def metrics = @candidates.first.metrics
|
|
249
|
+
|
|
250
|
+
# Whether workers start (and stop) at once: not on TruffleRuby, whose
|
|
251
|
+
# Process.spawn, closing other descriptors (see WorkerProcess), fails
|
|
252
|
+
# when another thread closes one meanwhile.
|
|
253
|
+
PARALLEL = RUBY_ENGINE != "truffleruby"
|
|
254
|
+
|
|
255
|
+
# What the block returns for each item, each on a thread of its own
|
|
256
|
+
# (one after another, unless PARALLEL). Every one finishes before the
|
|
257
|
+
# first that failed, if any, raises.
|
|
258
|
+
def in_parallel(items, &)
|
|
259
|
+
return items.map(&) unless PARALLEL
|
|
260
|
+
|
|
261
|
+
threads = items.map do |item|
|
|
262
|
+
Thread.new do
|
|
263
|
+
Thread.current.report_on_exception = false
|
|
264
|
+
yield item
|
|
265
|
+
end
|
|
266
|
+
end
|
|
267
|
+
threads.each do |thread|
|
|
268
|
+
thread.join
|
|
269
|
+
rescue
|
|
270
|
+
nil # (raised by #value, once they're all done)
|
|
271
|
+
end
|
|
272
|
+
threads.map(&:value)
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def measure(name, result)
|
|
276
|
+
benchmark = benchmarks.fetch(name)
|
|
277
|
+
only = benchmark.fetch("only")
|
|
278
|
+
if only == "count"
|
|
279
|
+
count(name, result.n = COUNTS_ONLY, result)
|
|
280
|
+
return
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
timings_only = !only.nil? # (the only other choice)
|
|
284
|
+
target = timings_only ? TARGET_NS * 10 : TARGET_NS
|
|
285
|
+
n = calibrate(name, target, 1)
|
|
286
|
+
warm(name, n)
|
|
287
|
+
n = calibrate(name, target, n) if @warm.positive? # it may have got faster
|
|
288
|
+
floors = Array.new(WARMUP) { |i| pair(name, n, i) }
|
|
289
|
+
# (It got faster still, a JIT got to it, say: sized again, unless
|
|
290
|
+
# what it allocates holds it there.)
|
|
291
|
+
if below_cpu_floor?(floors) && (sized = calibrate(name, target, n)) > n
|
|
292
|
+
n = sized
|
|
293
|
+
floors = Array.new(WARMUP) { |i| pair(name, n, i) }
|
|
294
|
+
end
|
|
295
|
+
result.n = n
|
|
296
|
+
peak_memory(name, n, result) if benchmark.fetch("peak_memory")
|
|
297
|
+
judge(name, n, floors, result)
|
|
298
|
+
# (Counted once it's timed: on JRuby, the bytes a benchmark allocates
|
|
299
|
+
# can fall as its JIT compiles it, so a count right after a short
|
|
300
|
+
# warm-up could find one version's workers further along than the
|
|
301
|
+
# other's.)
|
|
302
|
+
count(name, n, result) unless timings_only
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# A process benchmark's samples (see DSL#duel_process): each the
|
|
306
|
+
# fastest of a few fresh processes (see #pair and #process_runs), as
|
|
307
|
+
# its one iteration, paired as any other's, after PROCESS_WARMUP pairs
|
|
308
|
+
# (the first runs read files the disk's cache hasn't yet), and each
|
|
309
|
+
# version's least peak memory, where it can be told.
|
|
310
|
+
def measure_process(name, result)
|
|
311
|
+
@peaks = {"baseline" => [], "candidate" => []}
|
|
312
|
+
result.n = 1
|
|
313
|
+
floors = Array.new(PROCESS_WARMUP) { |i| pair(name, 1, i) }
|
|
314
|
+
# (A process benchmark's.)
|
|
315
|
+
process = benchmarks.fetch(name).fetch("process") #: Definition::process
|
|
316
|
+
judge(name, process.fetch("runs") || process_runs(floors), floors, result)
|
|
317
|
+
return if @peaks.values.flatten.include?(nil)
|
|
318
|
+
|
|
319
|
+
# (Each told, so each least is.)
|
|
320
|
+
least = @peaks.transform_values(&:min) #: Hash[String, Integer]
|
|
321
|
+
result.memory = least
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# The runs a process benchmark's sample takes, by its warm-up's
|
|
325
|
+
# samples (+floors+, of a run each): as many as take
|
|
326
|
+
# PROCESS_SAMPLE_NS of CPU time on the slower version, up to
|
|
327
|
+
# PROCESS_RUNS, and one at least. A short process varies the most, and
|
|
328
|
+
# the fastest of a few is steadier, but a long one's pairs take long
|
|
329
|
+
# enough as they are.
|
|
330
|
+
def process_runs(floors)
|
|
331
|
+
longest = floors.flatten.map { |sample| sample.fetch("timed").fetch("cpu") }.max #: Integer
|
|
332
|
+
# (Any number, for one that took none at all.)
|
|
333
|
+
PROCESS_SAMPLE_NS.fdiv(longest).clamp(1, PROCESS_RUNS).floor
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
# Every worker runs the benchmark a few times, and then, all at once
|
|
337
|
+
# (none of it is measured, so they needn't take turns), for the
|
|
338
|
+
# seconds --warm says (Settings#warm), or else until it's steady (see
|
|
339
|
+
# #steady), for JIT_WARM's or YJIT_WARM's seconds at most.
|
|
340
|
+
def warm(name, n)
|
|
341
|
+
warm_until = monotonic + @warm
|
|
342
|
+
in_parallel(@bases + @candidates) do |worker|
|
|
343
|
+
2.times { worker.sample(name, n) }
|
|
344
|
+
if @settings.warm
|
|
345
|
+
worker.sample(name, n) until monotonic > warm_until
|
|
346
|
+
else
|
|
347
|
+
steady(worker, name, n, warm_until)
|
|
348
|
+
end
|
|
349
|
+
end
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
# A worker's warm-up ends once STEADY_WINDOWS windows of STEADY
|
|
353
|
+
# seconds in a row each have a median of its samples no faster, by
|
|
354
|
+
# STEADY_GAIN, than the last window that was (so code that creeps
|
|
355
|
+
# faster by less a window isn't steady): a JIT has compiled what it
|
|
356
|
+
# will. (Not once its fastest sample stops getting
|
|
357
|
+
# faster: the more samples, the faster the fastest of them, by
|
|
358
|
+
# chance. And not once one window is no faster: a pause, a collection
|
|
359
|
+
# say, can slow one.) A small library's code is steady in a few
|
|
360
|
+
# seconds, where a big app's JIT keeps compiling, for up to the
|
|
361
|
+
# warm-up's seconds.
|
|
362
|
+
STEADY = 0.5
|
|
363
|
+
STEADY_GAIN = 0.01
|
|
364
|
+
STEADY_WINDOWS = 3
|
|
365
|
+
|
|
366
|
+
# Samples +worker+ (+n+ iterations each), STEADY seconds at a time,
|
|
367
|
+
# until it's steady, or it's +warm_until+.
|
|
368
|
+
def steady(worker, name, n, warm_until)
|
|
369
|
+
window, now = window(worker, name, n, warm_until)
|
|
370
|
+
return if now > warm_until
|
|
371
|
+
|
|
372
|
+
fastest = Stats.median(window)
|
|
373
|
+
calm = 0
|
|
374
|
+
loop do
|
|
375
|
+
window, now = window(worker, name, n, warm_until)
|
|
376
|
+
return if now > warm_until
|
|
377
|
+
|
|
378
|
+
median = Stats.median(window)
|
|
379
|
+
if median > fastest * (1 - STEADY_GAIN)
|
|
380
|
+
calm += 1
|
|
381
|
+
return if calm == STEADY_WINDOWS
|
|
382
|
+
else
|
|
383
|
+
calm = 0
|
|
384
|
+
fastest = median
|
|
385
|
+
end
|
|
386
|
+
end
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
# STEADY seconds of +worker+'s samples' CPU time (a sample at least),
|
|
390
|
+
# or until +warm_until+, and when they ended.
|
|
391
|
+
def window(worker, name, n, warm_until)
|
|
392
|
+
window_until = monotonic + STEADY
|
|
393
|
+
samples = [] #: Array[Metric::value]
|
|
394
|
+
loop do
|
|
395
|
+
samples << worker.sample(name, n).fetch("timed").fetch("cpu")
|
|
396
|
+
now = monotonic
|
|
397
|
+
return [samples, now] if now > window_until || now > warm_until
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
# Takes pairs until every ratio metric worth judging (measured on
|
|
402
|
+
# both versions in the warm-up, and above its floors on either: one
|
|
403
|
+
# that rises from next to nothing is judged too) is decided, or
|
|
404
|
+
# MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
|
|
405
|
+
# and each interval over the pairs. That's for a process benchmark, or
|
|
406
|
+
# too few workers for an interval over them: otherwise, see
|
|
407
|
+
# #judge_processes. A process benchmark's pairs take a process each,
|
|
408
|
+
# so it's decided not slower (Result::CLEARED) after any pair too.
|
|
409
|
+
# That can only stop it before it would have been found slower, never
|
|
410
|
+
# make it slower: it's found slower only at the end of a round, as any
|
|
411
|
+
# benchmark is.
|
|
412
|
+
def judge(name, n, floors, result)
|
|
413
|
+
judged = metrics.select do |spec|
|
|
414
|
+
floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
|
|
415
|
+
floors.transpose.any? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, spec) }
|
|
416
|
+
end
|
|
417
|
+
return judge_processes(name, n, judged, result) if by_process?(name)
|
|
418
|
+
|
|
419
|
+
ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
420
|
+
# @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
|
|
421
|
+
raw = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
422
|
+
pending = judged.dup
|
|
423
|
+
every = benchmarks.fetch(name).fetch("process")
|
|
424
|
+
(0...MAX_PAIRS).each do |i|
|
|
425
|
+
break if pending.empty?
|
|
426
|
+
|
|
427
|
+
base, candidate = pair(name, n, i)
|
|
428
|
+
result.note(base, candidate)
|
|
429
|
+
ratios.each do |metric, list|
|
|
430
|
+
values = [base.fetch("timed").fetch(metric), candidate.fetch("timed").fetch(metric)] #: [Metric::value, Metric::value]
|
|
431
|
+
raw.fetch(metric) << values
|
|
432
|
+
list << values.last.fdiv([values.first, 1].max)
|
|
433
|
+
end
|
|
434
|
+
round = ((i + 1) % STEP).zero?
|
|
435
|
+
next unless round || every
|
|
436
|
+
|
|
437
|
+
pending.reject! { |spec| decided?(verdict(ratios.fetch(spec.fetch("name")), spec), round) }
|
|
438
|
+
end
|
|
439
|
+
judged.each do |spec|
|
|
440
|
+
list = ratios.fetch(spec.fetch("name"))
|
|
441
|
+
medians = raw.fetch(spec.fetch("name")).transpose.map { |values| Stats.median(values) }
|
|
442
|
+
result.timed[spec.fetch("name")] = {"median" => Stats.median(list), "limit" => limit(spec), "across" => spec.fetch("across"),
|
|
443
|
+
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
|
|
444
|
+
"interval" => Stats.median_interval(list, @settings.confidence),
|
|
445
|
+
"pairs" => list.size, "verdict" => verdict(list, spec).to_s,
|
|
446
|
+
"baseline" => medians.fetch(0), "candidate" => medians.fetch(1)}
|
|
447
|
+
end
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
# Whether a benchmark's timings are judged by process (see
|
|
451
|
+
# #judge_processes): unless it's a process benchmark (each of its
|
|
452
|
+
# samples is a fresh process already, so its pairs are independent),
|
|
453
|
+
# or there are too few workers of each version for an interval over
|
|
454
|
+
# them (Lane.by_process?).
|
|
455
|
+
def by_process?(name) = !benchmarks.fetch(name).fetch("process") && Lane.by_process?(@bases.size, @settings.confidence)
|
|
456
|
+
|
|
457
|
+
# Takes rounds of pairs, each a sample from every worker of each
|
|
458
|
+
# version (the baseline's in order, paired with the candidate's in
|
|
459
|
+
# turn: each round pairs each with another), until every ratio metric
|
|
460
|
+
# judged (see #judge) is decided, from the second round on, or
|
|
461
|
+
# #rounds run out. Each worker's median is a sample, and the interval
|
|
462
|
+
# is over workers (Stats.shift_interval): where a process's luck
|
|
463
|
+
# makes it faster or slower than another for its whole life (where
|
|
464
|
+
# things landed in its memory, or what a JIT compiled), the interval
|
|
465
|
+
# is as wide as that makes it. An interval over pairs would be as
|
|
466
|
+
# narrow as if every pair had processes of its own. Each worker's
|
|
467
|
+
# medians are kept, by metric, for across the board (Result#processes).
|
|
468
|
+
def judge_processes(name, n, judged, result)
|
|
469
|
+
k = @bases.size
|
|
470
|
+
# @type var values: Hash[String, Hash[String, Array[Array[Metric::value]]]]
|
|
471
|
+
values = judged.to_h { |spec| [spec.fetch("name"), {"baseline" => Array.new(k) { [] }, "candidate" => Array.new(k) { [] }}] }
|
|
472
|
+
pending = judged.dup
|
|
473
|
+
pairs = 0
|
|
474
|
+
rounds(k).times do |round|
|
|
475
|
+
break if pending.empty?
|
|
476
|
+
|
|
477
|
+
k.times do |i|
|
|
478
|
+
j = (i + round) % k
|
|
479
|
+
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
|
|
480
|
+
result.note(base, candidate)
|
|
481
|
+
values.each do |metric, sides|
|
|
482
|
+
sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
|
|
483
|
+
sides.fetch("candidate").fetch(j) << candidate.fetch("timed").fetch(metric)
|
|
484
|
+
end
|
|
485
|
+
pairs += 1
|
|
486
|
+
end
|
|
487
|
+
next if round + 1 < MIN_ROUNDS
|
|
488
|
+
|
|
489
|
+
pending.reject! { |spec| decided?(process_verdict(medians(values.fetch(spec.fetch("name"))), spec), true) }
|
|
490
|
+
end
|
|
491
|
+
judged.each do |spec|
|
|
492
|
+
metric = spec.fetch("name")
|
|
493
|
+
each = medians(values.fetch(metric))
|
|
494
|
+
bases, candidates = clamped(each)
|
|
495
|
+
result.timed[metric] = {"median" => Stats.shift(bases, candidates), "limit" => limit(spec), "across" => spec.fetch("across"),
|
|
496
|
+
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
|
|
497
|
+
"interval" => Stats.shift_interval(bases, candidates, @settings.confidence),
|
|
498
|
+
"pairs" => pairs, "processes" => k, "verdict" => process_verdict(each, spec).to_s,
|
|
499
|
+
"baseline" => Stats.median(each.fetch("baseline")), "candidate" => Stats.median(each.fetch("candidate"))}
|
|
500
|
+
result.processes[metric] = {"lane" => @id, "baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}
|
|
501
|
+
end
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
# The most rounds of +k+ workers' pairs: as many as make
|
|
505
|
+
# PROCESS_PAIRS pairs, and MIN_ROUNDS at least.
|
|
506
|
+
def rounds(k) = [PROCESS_PAIRS / k, MIN_ROUNDS].max
|
|
507
|
+
|
|
508
|
+
# Each worker's median, by version, of +values+ (each worker's
|
|
509
|
+
# measures, by version).
|
|
510
|
+
def medians(values) = values.transform_values { |lists| lists.map { |list| Stats.median(list) } }
|
|
511
|
+
|
|
512
|
+
# The baseline's medians (of +each+) at least 1, as the ratio's
|
|
513
|
+
# denominator (so a ratio to nothing is one to 1, as a pair's is),
|
|
514
|
+
# and the candidate's.
|
|
515
|
+
def clamped(each) = [each.fetch("baseline").map { |value| [value, 1].max }, each.fetch("candidate")]
|
|
516
|
+
|
|
517
|
+
def process_verdict(each, spec)
|
|
518
|
+
bases, candidates = clamped(each)
|
|
519
|
+
Stats.decide(Stats.shift_interval(bases, candidates, @settings.confidence), limit(spec))
|
|
520
|
+
end
|
|
521
|
+
|
|
522
|
+
# Whether both versions' CPU time in +floors+ is below its floor: then
|
|
523
|
+
# it wouldn't be judged at all.
|
|
524
|
+
def below_cpu_floor?(floors)
|
|
525
|
+
cpu = metrics.find { |spec| spec.fetch("name") == "cpu" } #: Metric::spec
|
|
526
|
+
floors.transpose.none? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, cpu) }
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
# Whether +samples+ (one version's timed measures) are above +spec+'s
|
|
530
|
+
# floors (the median): its own, and its share of CPU time.
|
|
531
|
+
def above_floors?(samples, spec)
|
|
532
|
+
values = samples.map { |sample| sample.fetch(spec.fetch("name")) }
|
|
533
|
+
cpu = samples.map { |sample| sample.fetch("cpu") }
|
|
534
|
+
median = Stats.median(values)
|
|
535
|
+
median >= spec.fetch("floor") && median * spec.fetch("unit_ns") >= spec.fetch("cpu_floor") * Stats.median(cpu)
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
def limit(spec) = spec.fetch("limit") || @settings.limit
|
|
539
|
+
|
|
540
|
+
# The verdicts that decide a measure at the end of a round (see
|
|
541
|
+
# #judge), and after any other pair: only those that clear it.
|
|
542
|
+
DECISIVE = {true => [:slower, :faster, :same], false => [:faster, :same]}.freeze #: Hash[bool, Array[Stats::verdict]]
|
|
543
|
+
|
|
544
|
+
# Whether +verdict+ decides a measure, at the end of a +round+ or not.
|
|
545
|
+
def decided?(verdict, round) = DECISIVE.fetch(round).include?(verdict)
|
|
546
|
+
|
|
547
|
+
def verdict(ratios, spec) = Stats.verdict(ratios, limit: limit(spec), confidence: @settings.confidence)
|
|
548
|
+
|
|
549
|
+
# The iterations per sample (from +n+ up) whose fastest run takes
|
|
550
|
+
# about +target+ on the first candidate, by its largest timed metric
|
|
551
|
+
# that sizes samples (CPU time does), found while warming up (twice in
|
|
552
|
+
# a row: a cold run may look like enough), and that allocate no more
|
|
553
|
+
# than HEAP_CAP a run.
|
|
554
|
+
def calibrate(name, target, n)
|
|
555
|
+
units = metrics.select { |spec| spec.fetch("sizes_samples") }.to_h { |spec| [spec.fetch("name"), spec.fetch("unit_ns")] }
|
|
556
|
+
reached = 0
|
|
557
|
+
until reached == 2
|
|
558
|
+
sample = pair(name, n, 0, runs: 3).last
|
|
559
|
+
# (CPU time sizes samples, and is always measured.)
|
|
560
|
+
timed = sample.fetch("timed")
|
|
561
|
+
work = units.filter_map { |metric, unit| timed[metric]&.*(unit) }.max #: Metric::value
|
|
562
|
+
most = most_iterations(n, sample.fetch("heap"))
|
|
563
|
+
if most <= n
|
|
564
|
+
# (Finite, when it's at most n.)
|
|
565
|
+
return most #: Integer
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
reached = (work >= target) ? reached + 1 : 0
|
|
569
|
+
next unless reached.zero?
|
|
570
|
+
|
|
571
|
+
# (An Integer: most is finite, or more.)
|
|
572
|
+
n = [(n * (target * 1.2 / work).clamp(2, 100)).ceil, most].min #: Integer
|
|
573
|
+
end
|
|
574
|
+
n
|
|
575
|
+
end
|
|
576
|
+
|
|
577
|
+
# The most iterations whose allocations fit HEAP_CAP, by +heap+ bytes
|
|
578
|
+
# over +n+ (at least one, and any number when it allocated nothing).
|
|
579
|
+
def most_iterations(n, heap)
|
|
580
|
+
return Float::INFINITY unless heap.positive?
|
|
581
|
+
|
|
582
|
+
[n * HEAP_CAP / heap, 1].max
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
# The +index+th pair: a sample (of +n+ iterations) from a baseline
|
|
586
|
+
# worker and from a candidate worker, each pair of workers in turn,
|
|
587
|
+
# baseline first on even pairs. A process benchmark's sample is the
|
|
588
|
+
# fastest of +n+ runs (see #measure_process), the two versions' taking
|
|
589
|
+
# turns: so interference, which only ever adds time, is mostly left
|
|
590
|
+
# out, and weighs on both alike.
|
|
591
|
+
def pair(name, n, index, runs: RUNS, count: false)
|
|
592
|
+
k = @bases.size
|
|
593
|
+
sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
|
|
594
|
+
end
|
|
595
|
+
|
|
596
|
+
# A sample (of +n+ iterations) from +base+, a baseline worker, and
|
|
597
|
+
# from +candidate+, a candidate worker, the baseline's first if
|
|
598
|
+
# +base_first+ (see #pair).
|
|
599
|
+
def sampled(name, n, base, candidate, base_first:, runs: RUNS, count: false)
|
|
600
|
+
order = base_first ? [base, candidate] : [candidate, base]
|
|
601
|
+
process = benchmarks.fetch(name).fetch("process")
|
|
602
|
+
replies = if process
|
|
603
|
+
# (Each run of one version's next to one of the other's.)
|
|
604
|
+
each_run = Array.new(n) { order.map { |worker| worker.run(process) } }
|
|
605
|
+
order.zip(each_run.transpose).to_h do |worker, samples|
|
|
606
|
+
# (Never nil: each worker in order has its runs.)
|
|
607
|
+
ran = samples #: Array[ProcessSample::sampled]
|
|
608
|
+
[worker, ProcessSample.fastest(ran)]
|
|
609
|
+
end
|
|
610
|
+
else
|
|
611
|
+
order.to_h { |worker| [worker, worker.sample(name, n, runs: runs, count: count)] }
|
|
612
|
+
end
|
|
613
|
+
if process
|
|
614
|
+
@peaks.fetch("baseline") << replies.fetch(base).fetch("peak")
|
|
615
|
+
@peaks.fetch("candidate") << replies.fetch(candidate).fetch("peak")
|
|
616
|
+
end
|
|
617
|
+
[replies.fetch(base), replies.fetch(candidate)]
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
# Each counted measure's fewest, on each version, over COUNTS pairs of
|
|
621
|
+
# samples, and what its metric said of those samples.
|
|
622
|
+
def count(name, n, result)
|
|
623
|
+
return count_by_process(name, n, result) if Lane.by_process?(@bases.size, @settings.confidence)
|
|
624
|
+
|
|
625
|
+
samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
|
|
626
|
+
metrics.each do |spec|
|
|
627
|
+
metric = spec.fetch("name")
|
|
628
|
+
base, candidate = samples.transpose.map do |side|
|
|
629
|
+
side.select { |sample| sample.fetch("counted").key?(metric) }.min_by { |sample| sample.fetch("counted").fetch(metric) }
|
|
630
|
+
end
|
|
631
|
+
next unless base && candidate
|
|
632
|
+
|
|
633
|
+
result.counted[metric] = {"baseline" => base.fetch("counted").fetch(metric), "candidate" => candidate.fetch("counted").fetch(metric),
|
|
634
|
+
"n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
|
|
635
|
+
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
|
|
636
|
+
result.note(base, candidate, metric)
|
|
637
|
+
end
|
|
638
|
+
end
|
|
639
|
+
|
|
640
|
+
# Each counted measure, with workers enough for an interval over them
|
|
641
|
+
# (see #judge_processes): each worker's fewest in COUNT_ROUNDS rounds
|
|
642
|
+
# of a sample from every worker, and how much more the candidate's
|
|
643
|
+
# are than the baseline's, at least (the low end of an interval for
|
|
644
|
+
# it, Stats.difference_interval), which is what's judged (see
|
|
645
|
+
# Result#worse). Where every worker's is the same, as CRuby's counts
|
|
646
|
+
# of objects are, that's exactly how many more. Where they differ, as
|
|
647
|
+
# JRuby's bytes do where one JVM's JIT does away with an allocation
|
|
648
|
+
# and another's doesn't, it's as little as the workers can tell.
|
|
649
|
+
# Each version's own is the median of its workers'.
|
|
650
|
+
def count_by_process(name, n, result)
|
|
651
|
+
k = @bases.size
|
|
652
|
+
# @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
|
|
653
|
+
fewest = Hash.new do |h, metric|
|
|
654
|
+
none = {} #: Hash[Integer, Metric::value]
|
|
655
|
+
h[metric] = {"baseline" => none, "candidate" => none.dup}
|
|
656
|
+
end
|
|
657
|
+
COUNT_ROUNDS.times do |round|
|
|
658
|
+
k.times do |i|
|
|
659
|
+
j = (i + round) % k
|
|
660
|
+
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
|
|
661
|
+
fewest!(fewest, "baseline", i, base)
|
|
662
|
+
fewest!(fewest, "candidate", j, candidate)
|
|
663
|
+
# (What metrics said of its counts alone: a timing's is the timed
|
|
664
|
+
# pairs'.)
|
|
665
|
+
base.fetch("counted").each_key { |metric| result.note(base, candidate, metric) }
|
|
666
|
+
end
|
|
667
|
+
end
|
|
668
|
+
metrics.each do |spec|
|
|
669
|
+
metric = spec.fetch("name")
|
|
670
|
+
next unless fewest.key?(metric)
|
|
671
|
+
|
|
672
|
+
found = fewest.fetch(metric)
|
|
673
|
+
bases = found.fetch("baseline").values
|
|
674
|
+
candidates = found.fetch("candidate").values
|
|
675
|
+
next if bases.empty? || candidates.empty?
|
|
676
|
+
|
|
677
|
+
result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
|
|
678
|
+
"more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
|
|
679
|
+
"n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
|
|
680
|
+
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
|
|
681
|
+
end
|
|
682
|
+
end
|
|
683
|
+
|
|
684
|
+
# Keeps each count of +sample+ (from +side+'s +worker+th worker) in
|
|
685
|
+
# +fewest+, where it's that worker's fewest so far.
|
|
686
|
+
def fewest!(fewest, side, worker, sample)
|
|
687
|
+
sample.fetch("counted").each do |metric, value|
|
|
688
|
+
least = fewest[metric].fetch(side)
|
|
689
|
+
least[worker] = [least[worker], value].compact.min
|
|
690
|
+
end
|
|
691
|
+
end
|
|
692
|
+
|
|
693
|
+
# The samples a count takes: JRUBY_COUNTS where a version is on JRuby,
|
|
694
|
+
# or else COUNTS.
|
|
695
|
+
def counts = (@sides.each_value.any? { |side| side.engine == "jruby" }) ? JRUBY_COUNTS : COUNTS
|
|
696
|
+
|
|
697
|
+
# Each version's least peak memory over MEMORY_SAMPLES fresh workers
|
|
698
|
+
# (a long-lived one has room left over from earlier work), turn about.
|
|
699
|
+
# Fresh means no trial run either: the allocator keeps what that freed,
|
|
700
|
+
# and the measured run would reuse it rather than grow.
|
|
701
|
+
def peak_memory(name, n, result)
|
|
702
|
+
return unless @candidates.first.memory
|
|
703
|
+
|
|
704
|
+
# @type var peaks: Hash[String, Array[Integer]]
|
|
705
|
+
peaks = {"baseline" => [], "candidate" => []}
|
|
706
|
+
MEMORY_SAMPLES.times do |i|
|
|
707
|
+
sides = i.even? ? %w[baseline candidate] : %w[candidate baseline]
|
|
708
|
+
sides.each do |side|
|
|
709
|
+
peaks.fetch(side) << fresh(side) do |worker|
|
|
710
|
+
worker.request("setup", name, trial: false)
|
|
711
|
+
worker.request("memory", name, n: n).fetch("peak")
|
|
712
|
+
end
|
|
713
|
+
end
|
|
714
|
+
end
|
|
715
|
+
result.memory = peaks.transform_values(&:min)
|
|
716
|
+
end
|
|
717
|
+
|
|
718
|
+
# Yields a worker of +side+ of its own, set up for nothing yet, then
|
|
719
|
+
# stops it and its resources.
|
|
720
|
+
def fresh(side)
|
|
721
|
+
# @type var workers: Integer
|
|
722
|
+
# @type var resources: Integer
|
|
723
|
+
workers = @workers.size
|
|
724
|
+
resources = @resources.size
|
|
725
|
+
worker = spawn(side, @workers.size)
|
|
726
|
+
yield worker
|
|
727
|
+
ensure
|
|
728
|
+
@workers.pop(@workers.size - workers).each(&:stop)
|
|
729
|
+
@resources.pop(@resources.size - resources).reverse_each(&:stop)
|
|
730
|
+
end
|
|
731
|
+
|
|
732
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
733
|
+
end
|
|
734
|
+
end
|