duel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +51 -27
- data/CHANGELOG.md +247 -1
- data/README.md +30 -14
- data/docs/benchmarks.md +28 -24
- data/docs/ci.md +328 -362
- data/docs/configuration.md +40 -37
- data/docs/how-it-works.md +23 -10
- data/docs/json.md +21 -25
- data/docs/plugins.md +40 -22
- data/docs/troubleshooting.md +3 -2
- data/lib/duel/across.rb +18 -18
- data/lib/duel/badge.rb +174 -0
- data/lib/duel/baseline.rb +38 -15
- data/lib/duel/cli.rb +175 -107
- data/lib/duel/comment.rb +80 -71
- data/lib/duel/config.rb +145 -63
- data/lib/duel/context.rb +29 -24
- data/lib/duel/forge.rb +87 -0
- data/lib/duel/init.rb +44 -87
- data/lib/duel/json_report.rb +34 -25
- data/lib/duel/lane.rb +107 -61
- data/lib/duel/lock.rb +1 -1
- data/lib/duel/mark_passed.rb +132 -0
- data/lib/duel/metric.rb +28 -27
- data/lib/duel/native.rb +7 -5
- data/lib/duel/platform.rb +4 -0
- data/lib/duel/process_sample.rb +9 -6
- data/lib/duel/rake_task.rb +4 -5
- data/lib/duel/resource.rb +10 -11
- data/lib/duel/result.rb +22 -14
- data/lib/duel/rubies.rb +6 -2
- data/lib/duel/runner.rb +27 -15
- data/lib/duel/spawn.rb +29 -4
- data/lib/duel/stats.rb +14 -12
- data/lib/duel/summary.rb +31 -29
- data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
- data/lib/duel/templates/duel.rb.erb +7 -7
- data/lib/duel/templates/perf-caller.yml.erb +22 -5
- data/lib/duel/templates/perf-comment.yml.erb +11 -11
- data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
- data/lib/duel/templates/perf-push.yml.erb +50 -0
- data/lib/duel/templates/perf.yml.erb +113 -335
- data/lib/duel/testing.rb +20 -25
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +16 -9
- data/lib/duel/worker_process.rb +6 -6
- data/lib/duel.rb +88 -33
- data/sig/duel/across.rbs +4 -0
- data/sig/duel/badge.rbs +51 -0
- data/sig/duel/baseline.rbs +7 -2
- data/sig/duel/cli.rbs +31 -20
- data/sig/duel/comment.rbs +14 -15
- data/sig/duel/config.rbs +71 -29
- data/sig/duel/context.rbs +4 -4
- data/sig/duel/forge.rbs +42 -0
- data/sig/duel/init.rbs +8 -26
- data/sig/duel/json_report.rbs +7 -6
- data/sig/duel/lane.rbs +29 -6
- data/sig/duel/mark_passed.rbs +39 -0
- data/sig/duel/metric.rbs +10 -10
- data/sig/duel/native.rbs +3 -0
- data/sig/duel/platform.rbs +1 -0
- data/sig/duel/process_sample.rbs +3 -0
- data/sig/duel/resource.rbs +3 -3
- data/sig/duel/result.rbs +12 -5
- data/sig/duel/rubies.rbs +3 -0
- data/sig/duel/runner.rbs +23 -16
- data/sig/duel/spawn.rbs +7 -0
- data/sig/duel/stats.rbs +4 -1
- data/sig/duel/summary.rbs +8 -0
- data/sig/duel/testing.rbs +2 -2
- data/sig/duel/worker.rbs +6 -5
- data/sig/duel/worker_process.rbs +3 -3
- data/sig/duel.rbs +50 -31
- metadata +9 -1
data/lib/duel/lane.rb
CHANGED
|
@@ -20,7 +20,7 @@ module Duel
|
|
|
20
20
|
# (Config#prepare(:worker)).
|
|
21
21
|
# @api private
|
|
22
22
|
class Lane
|
|
23
|
-
# CPU time (or a plugin's
|
|
23
|
+
# CPU time (or a plugin's timed metric's equivalent) a run aims for:
|
|
24
24
|
# long enough that a clock's resolution is lost in it. A sample is the
|
|
25
25
|
# fastest of RUNS runs.
|
|
26
26
|
TARGET_NS = 1_000_000
|
|
@@ -60,8 +60,6 @@ module Duel
|
|
|
60
60
|
# StackProf's a version: enough to tell a few percent apart).
|
|
61
61
|
PROFILE_SAMPLES = 500
|
|
62
62
|
|
|
63
|
-
attr_reader :profiler
|
|
64
|
-
|
|
65
63
|
# Its workers start at once, each with its resources (a database
|
|
66
64
|
# server takes seconds), and stop at once. +sides+ are the versions as
|
|
67
65
|
# the run prepared them (Runner#prepare), by name.
|
|
@@ -79,7 +77,7 @@ module Duel
|
|
|
79
77
|
@bases = started.first(processes)
|
|
80
78
|
@candidates = started.drop(processes)
|
|
81
79
|
@profiler = @candidates.first.profiler
|
|
82
|
-
@warm = settings.warm || (started.any?(&:
|
|
80
|
+
@warm = settings.warm || (started.any?(&:jit) ? CRUBY_JIT_WARM : JIT_WARM)
|
|
83
81
|
check_builds
|
|
84
82
|
rescue Exception # rubocop:disable Lint/RescueException
|
|
85
83
|
stop
|
|
@@ -123,7 +121,7 @@ module Duel
|
|
|
123
121
|
def benchmarks = @candidates.first.benchmarks
|
|
124
122
|
|
|
125
123
|
# The Ruby each version runs on (RUBY_DESCRIPTION), by version.
|
|
126
|
-
def rubies =
|
|
124
|
+
def rubies = firsts.transform_values(&:description)
|
|
127
125
|
|
|
128
126
|
# Stops its workers, then their resources: each worker's last first
|
|
129
127
|
# (one may need another), the workers' at once.
|
|
@@ -139,7 +137,7 @@ module Duel
|
|
|
139
137
|
def compare(name)
|
|
140
138
|
result = Result.new(name)
|
|
141
139
|
started = monotonic
|
|
142
|
-
if
|
|
140
|
+
if process_of(name)
|
|
143
141
|
measure_process(name, result)
|
|
144
142
|
else
|
|
145
143
|
set_up(name, result)
|
|
@@ -148,7 +146,7 @@ module Duel
|
|
|
148
146
|
begin
|
|
149
147
|
measure(name, result)
|
|
150
148
|
ensure
|
|
151
|
-
|
|
149
|
+
every_worker.each { |worker| worker.request("teardown", name) }
|
|
152
150
|
end
|
|
153
151
|
end
|
|
154
152
|
result.seconds = monotonic - started
|
|
@@ -161,7 +159,7 @@ module Duel
|
|
|
161
159
|
return unless @profiler
|
|
162
160
|
|
|
163
161
|
# @type var sides: Hash[String, WorkerProcess]?
|
|
164
|
-
sides =
|
|
162
|
+
sides = firsts
|
|
165
163
|
sides.each_value { |worker| worker.request("setup", name) }
|
|
166
164
|
sides.transform_values { |worker| worker.request("profile", name, n: n, runs: PROFILE_SAMPLES) }
|
|
167
165
|
ensure
|
|
@@ -170,6 +168,13 @@ module Duel
|
|
|
170
168
|
|
|
171
169
|
private
|
|
172
170
|
|
|
171
|
+
# How a benchmark runs as a process of its own, or nil where it's no
|
|
172
|
+
# process benchmark.
|
|
173
|
+
def process_of(name) = benchmarks.fetch(name).fetch("process")
|
|
174
|
+
|
|
175
|
+
# Each version's first worker, by version.
|
|
176
|
+
def firsts = {"baseline" => @bases.first, "candidate" => @candidates.first}
|
|
177
|
+
|
|
173
178
|
# A worker of +side+, with its own resources, and its version's Ruby
|
|
174
179
|
# and environment.
|
|
175
180
|
def spawn(side, index)
|
|
@@ -180,9 +185,30 @@ module Duel
|
|
|
180
185
|
worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
|
|
181
186
|
config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
|
|
182
187
|
keep(@workers, worker)
|
|
188
|
+
check_jit(side, worker, env, version.engine)
|
|
183
189
|
worker
|
|
184
190
|
end
|
|
185
191
|
|
|
192
|
+
# Fails where +env+, +worker+'s (over the runner's own), turns one of
|
|
193
|
+
# CRuby's JITs on (see JIT_VARIABLES) and +worker+ doesn't run it: its
|
|
194
|
+
# benchmarks would be measured without it, and nothing would say so.
|
|
195
|
+
# A CRuby without the JIT ignores its variable, as one before 4.0, or
|
|
196
|
+
# built without ZJIT, does RUBY_ZJIT_ENABLE, and so do JRuby and
|
|
197
|
+
# TruffleRuby, whose JIT is their own.
|
|
198
|
+
def check_jit(side, worker, env, engine)
|
|
199
|
+
jit, variable = JIT_VARIABLES.find { |_, name| JIT_ON.include?(env.fetch(name) { ENV[name] }) }
|
|
200
|
+
return if jit.nil? || worker.jit == jit
|
|
201
|
+
|
|
202
|
+
asked = jit.upcase
|
|
203
|
+
why = if engine == "ruby"
|
|
204
|
+
(jit == "zjit") ? "a Ruby before 4.0, or one built without ZJIT, ignores RUBY_ZJIT_ENABLE" : "a Ruby built without YJIT ignores RUBY_YJIT_ENABLE"
|
|
205
|
+
else
|
|
206
|
+
"#{asked} is CRuby's, and #{engine} has a JIT of its own: unset #{variable} for it"
|
|
207
|
+
end
|
|
208
|
+
raise Failed, "the #{side}'s workers run #{worker.jit&.upcase || "no JIT"}, not #{asked} as #{variable} asks, on " \
|
|
209
|
+
"#{worker.description}: #{why}"
|
|
210
|
+
end
|
|
211
|
+
|
|
186
212
|
# Fails if a version's workers loaded a native extension that isn't a
|
|
187
213
|
# build of its own: one in the other version's tree, or one where the
|
|
188
214
|
# other version loaded the same from its own tree (an installed gem's,
|
|
@@ -195,7 +221,7 @@ module Duel
|
|
|
195
221
|
# loaded one twice (see Worker.twice): the candidate's, that something
|
|
196
222
|
# required before Duel loaded the version, and then its own over it.
|
|
197
223
|
def check_builds
|
|
198
|
-
workers =
|
|
224
|
+
workers = firsts
|
|
199
225
|
workers.each do |side, worker|
|
|
200
226
|
worker.twice.each do |feature, paths|
|
|
201
227
|
raise Failed, "the #{side} loaded #{feature} twice, from #{paths.join(" and ")}: something required it before " \
|
|
@@ -226,7 +252,8 @@ module Duel
|
|
|
226
252
|
# mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
|
|
227
253
|
def keep(list, item) = @lock.synchronize { list << item }
|
|
228
254
|
|
|
229
|
-
|
|
255
|
+
# Each version's workers, the baseline's first.
|
|
256
|
+
def every_worker = @bases + @candidates
|
|
230
257
|
|
|
231
258
|
# Sets the benchmark up in every worker, the first candidate first (a
|
|
232
259
|
# benchmark that fails there is broken) and the first baseline next
|
|
@@ -243,7 +270,7 @@ module Duel
|
|
|
243
270
|
in_parallel(@bases.drop(1) + @candidates.drop(1)) { |worker| worker.request("setup", name) }
|
|
244
271
|
end
|
|
245
272
|
|
|
246
|
-
# What's measured:
|
|
273
|
+
# What's measured: timed metrics' measures are in a sample's "timed",
|
|
247
274
|
# counted ones' in its "counted".
|
|
248
275
|
def metrics = @candidates.first.metrics
|
|
249
276
|
|
|
@@ -273,27 +300,26 @@ module Duel
|
|
|
273
300
|
end
|
|
274
301
|
|
|
275
302
|
def measure(name, result)
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
if only == "count"
|
|
303
|
+
judge = benchmarks.fetch(name).fetch("judge")
|
|
304
|
+
unless judge.include?("timed")
|
|
279
305
|
count(name, result.n = COUNTS_ONLY, result)
|
|
280
306
|
return
|
|
281
307
|
end
|
|
282
308
|
|
|
283
|
-
timings_only = !
|
|
309
|
+
timings_only = !judge.include?("counted")
|
|
284
310
|
target = timings_only ? TARGET_NS * 10 : TARGET_NS
|
|
285
311
|
n = calibrate(name, target, 1)
|
|
286
312
|
warm(name, n)
|
|
287
313
|
n = calibrate(name, target, n) if @warm.positive? # it may have got faster
|
|
288
|
-
floors =
|
|
314
|
+
floors = warm_up_pairs(name, n)
|
|
289
315
|
# (It got faster still, a JIT got to it, say: sized again, unless
|
|
290
316
|
# what it allocates holds it there.)
|
|
291
317
|
if below_cpu_floor?(floors) && (sized = calibrate(name, target, n)) > n
|
|
292
318
|
n = sized
|
|
293
|
-
floors =
|
|
319
|
+
floors = warm_up_pairs(name, n)
|
|
294
320
|
end
|
|
295
321
|
result.n = n
|
|
296
|
-
peak_memory(name, n, result) if
|
|
322
|
+
peak_memory(name, n, result) if judge.include?("peak_memory")
|
|
297
323
|
judge(name, n, floors, result)
|
|
298
324
|
# (Counted once it's timed: on JRuby, the bytes a benchmark allocates
|
|
299
325
|
# can fall as its JIT compiles it, so a count right after a short
|
|
@@ -312,7 +338,7 @@ module Duel
|
|
|
312
338
|
result.n = 1
|
|
313
339
|
floors = Array.new(PROCESS_WARMUP) { |i| pair(name, 1, i) }
|
|
314
340
|
# (A process benchmark's.)
|
|
315
|
-
process =
|
|
341
|
+
process = process_of(name) #: Definition::process
|
|
316
342
|
judge(name, process.fetch("runs") || process_runs(floors), floors, result)
|
|
317
343
|
return if @peaks.values.flatten.include?(nil)
|
|
318
344
|
|
|
@@ -336,10 +362,10 @@ module Duel
|
|
|
336
362
|
# Every worker runs the benchmark a few times, and then, all at once
|
|
337
363
|
# (none of it is measured, so they needn't take turns), for the
|
|
338
364
|
# seconds --warm says (Settings#warm), or else until it's steady (see
|
|
339
|
-
# #steady), for JIT_WARM's or
|
|
365
|
+
# #steady), for JIT_WARM's or CRUBY_JIT_WARM's seconds at most.
|
|
340
366
|
def warm(name, n)
|
|
341
367
|
warm_until = monotonic + @warm
|
|
342
|
-
in_parallel(
|
|
368
|
+
in_parallel(every_worker) do |worker|
|
|
343
369
|
2.times { worker.sample(name, n) }
|
|
344
370
|
if @settings.warm
|
|
345
371
|
worker.sample(name, n) until monotonic > warm_until
|
|
@@ -398,7 +424,7 @@ module Duel
|
|
|
398
424
|
end
|
|
399
425
|
end
|
|
400
426
|
|
|
401
|
-
# Takes pairs until every
|
|
427
|
+
# Takes pairs until every timed metric worth judging (measured on
|
|
402
428
|
# both versions in the warm-up, and above its floors on either: one
|
|
403
429
|
# that rises from next to nothing is judged too) is decided, or
|
|
404
430
|
# MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
|
|
@@ -412,32 +438,27 @@ module Duel
|
|
|
412
438
|
def judge(name, n, floors, result)
|
|
413
439
|
judged = metrics.select do |spec|
|
|
414
440
|
floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
|
|
415
|
-
|
|
441
|
+
above_floors?(floors, spec)
|
|
416
442
|
end
|
|
417
443
|
return judge_processes(name, n, judged, result) if by_process?(name)
|
|
418
444
|
|
|
419
|
-
ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
420
445
|
# @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
|
|
421
446
|
raw = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
422
447
|
pending = judged.dup
|
|
423
|
-
every =
|
|
448
|
+
every = process_of(name)
|
|
424
449
|
(0...MAX_PAIRS).each do |i|
|
|
425
450
|
break if pending.empty?
|
|
426
451
|
|
|
427
452
|
base, candidate = pair(name, n, i)
|
|
428
453
|
result.note(base, candidate)
|
|
429
|
-
|
|
430
|
-
values = [base.fetch("timed").fetch(metric), candidate.fetch("timed").fetch(metric)] #: [Metric::value, Metric::value]
|
|
431
|
-
raw.fetch(metric) << values
|
|
432
|
-
list << values.last.fdiv([values.first, 1].max)
|
|
433
|
-
end
|
|
454
|
+
raw.each { |metric, pairs| pairs << [base.fetch("timed").fetch(metric), candidate.fetch("timed").fetch(metric)] }
|
|
434
455
|
round = ((i + 1) % STEP).zero?
|
|
435
456
|
next unless round || every
|
|
436
457
|
|
|
437
|
-
pending.reject! { |spec| decided?(verdict(ratios.fetch(spec.fetch("name")), spec), round) }
|
|
458
|
+
pending.reject! { |spec| decided?(verdict(ratios(raw.fetch(spec.fetch("name"))), spec), round) }
|
|
438
459
|
end
|
|
439
460
|
judged.each do |spec|
|
|
440
|
-
list = ratios.fetch(spec.fetch("name"))
|
|
461
|
+
list = ratios(raw.fetch(spec.fetch("name")))
|
|
441
462
|
medians = raw.fetch(spec.fetch("name")).transpose.map { |values| Stats.median(values) }
|
|
442
463
|
result.timed[spec.fetch("name")] = {"median" => Stats.median(list), "limit" => limit(spec), "across" => spec.fetch("across"),
|
|
443
464
|
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
|
|
@@ -451,12 +472,17 @@ module Duel
|
|
|
451
472
|
# #judge_processes): unless it's a process benchmark (each of its
|
|
452
473
|
# samples is a fresh process already, so its pairs are independent),
|
|
453
474
|
# or there are too few workers of each version for an interval over
|
|
454
|
-
# them (
|
|
455
|
-
def by_process?(name) = !
|
|
475
|
+
# them (see #enough_workers?).
|
|
476
|
+
def by_process?(name) = !process_of(name) && enough_workers?
|
|
477
|
+
|
|
478
|
+
# Whether there are workers enough of each version for an interval
|
|
479
|
+
# over them (Lane.by_process?): then timings and counts alike are
|
|
480
|
+
# judged by process.
|
|
481
|
+
def enough_workers? = Lane.by_process?(@bases.size, @settings.confidence)
|
|
456
482
|
|
|
457
483
|
# Takes rounds of pairs, each a sample from every worker of each
|
|
458
484
|
# version (the baseline's in order, paired with the candidate's in
|
|
459
|
-
# turn: each round pairs each with another), until every
|
|
485
|
+
# turn: each round pairs each with another), until every timed metric
|
|
460
486
|
# judged (see #judge) is decided, from the second round on, or
|
|
461
487
|
# #rounds run out. Each worker's median is a sample, and the interval
|
|
462
488
|
# is over workers (Stats.shift_interval): where a process's luck
|
|
@@ -474,9 +500,7 @@ module Duel
|
|
|
474
500
|
rounds(k).times do |round|
|
|
475
501
|
break if pending.empty?
|
|
476
502
|
|
|
477
|
-
|
|
478
|
-
j = (i + round) % k
|
|
479
|
-
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
|
|
503
|
+
in_round(name, n, round) do |i, j, base, candidate|
|
|
480
504
|
result.note(base, candidate)
|
|
481
505
|
values.each do |metric, sides|
|
|
482
506
|
sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
|
|
@@ -523,18 +547,26 @@ module Duel
|
|
|
523
547
|
# it wouldn't be judged at all.
|
|
524
548
|
def below_cpu_floor?(floors)
|
|
525
549
|
cpu = metrics.find { |spec| spec.fetch("name") == "cpu" } #: Metric::spec
|
|
526
|
-
|
|
550
|
+
!above_floors?(floors, cpu)
|
|
527
551
|
end
|
|
528
552
|
|
|
529
|
-
# Whether
|
|
530
|
-
# floors (the median): its own, and its share of
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
553
|
+
# Whether either version's samples in +floors+ (its timed measures)
|
|
554
|
+
# are above +spec+'s floors (the median): its own, and its share of
|
|
555
|
+
# CPU time.
|
|
556
|
+
def above_floors?(floors, spec)
|
|
557
|
+
floors.transpose.any? do |side|
|
|
558
|
+
samples = side.map { |sample| sample.fetch("timed") }
|
|
559
|
+
values = samples.map { |sample| sample.fetch(spec.fetch("name")) }
|
|
560
|
+
cpu = samples.map { |sample| sample.fetch("cpu") }
|
|
561
|
+
median = Stats.median(values)
|
|
562
|
+
median >= spec.fetch("floor") && median * spec.fetch("unit_ns") >= spec.fetch("cpu_floor") * Stats.median(cpu)
|
|
563
|
+
end
|
|
536
564
|
end
|
|
537
565
|
|
|
566
|
+
# Each of +pairs+' ratio: the candidate's value over the baseline's
|
|
567
|
+
# (at least 1).
|
|
568
|
+
def ratios(pairs) = pairs.map { |base, candidate| candidate.fdiv([base, 1].max) }
|
|
569
|
+
|
|
538
570
|
def limit(spec) = spec.fetch("limit") || @settings.limit
|
|
539
571
|
|
|
540
572
|
# The verdicts that decide a measure at the end of a round (see
|
|
@@ -552,7 +584,7 @@ module Duel
|
|
|
552
584
|
# a row: a cold run may look like enough), and that allocate no more
|
|
553
585
|
# than HEAP_CAP a run.
|
|
554
586
|
def calibrate(name, target, n)
|
|
555
|
-
units = metrics.select { |spec| spec.fetch("
|
|
587
|
+
units = metrics.select { |spec| spec.fetch("size_samples") }.to_h { |spec| [spec.fetch("name"), spec.fetch("unit_ns")] }
|
|
556
588
|
reached = 0
|
|
557
589
|
until reached == 2
|
|
558
590
|
sample = pair(name, n, 0, runs: 3).last
|
|
@@ -582,6 +614,10 @@ module Duel
|
|
|
582
614
|
[n * HEAP_CAP / heap, 1].max
|
|
583
615
|
end
|
|
584
616
|
|
|
617
|
+
# The warm-up's WARMUP pairs, of +n+ iterations each, the floors a
|
|
618
|
+
# benchmark's measures are judged above (see #judge).
|
|
619
|
+
def warm_up_pairs(name, n) = Array.new(WARMUP) { |i| pair(name, n, i) }
|
|
620
|
+
|
|
585
621
|
# The +index+th pair: a sample (of +n+ iterations) from a baseline
|
|
586
622
|
# worker and from a candidate worker, each pair of workers in turn,
|
|
587
623
|
# baseline first on even pairs. A process benchmark's sample is the
|
|
@@ -593,34 +629,47 @@ module Duel
|
|
|
593
629
|
sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
|
|
594
630
|
end
|
|
595
631
|
|
|
632
|
+
# Round +round+ of pairs taken by process (see #judge_processes and
|
|
633
|
+
# #count_by_process): a sample from every baseline worker, the +i+th,
|
|
634
|
+
# paired with the candidate's +j+th in turn (each round pairs each with
|
|
635
|
+
# another), baseline first alternating by worker and by round. Yields
|
|
636
|
+
# each pair's workers' indexes and samples.
|
|
637
|
+
def in_round(name, n, round, runs: RUNS, count: false)
|
|
638
|
+
k = @bases.size
|
|
639
|
+
k.times do |i|
|
|
640
|
+
j = (i + round) % k
|
|
641
|
+
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs:, count:)
|
|
642
|
+
yield i, j, base, candidate
|
|
643
|
+
end
|
|
644
|
+
end
|
|
645
|
+
|
|
596
646
|
# A sample (of +n+ iterations) from +base+, a baseline worker, and
|
|
597
647
|
# from +candidate+, a candidate worker, the baseline's first if
|
|
598
648
|
# +base_first+ (see #pair).
|
|
599
|
-
def sampled(name, n, base, candidate, base_first:, runs
|
|
649
|
+
def sampled(name, n, base, candidate, base_first:, runs:, count:)
|
|
600
650
|
order = base_first ? [base, candidate] : [candidate, base]
|
|
601
|
-
process =
|
|
651
|
+
process = process_of(name)
|
|
602
652
|
replies = if process
|
|
603
653
|
# (Each run of one version's next to one of the other's.)
|
|
604
654
|
each_run = Array.new(n) { order.map { |worker| worker.run(process) } }
|
|
605
|
-
order.zip(each_run.transpose).to_h do |worker, samples|
|
|
655
|
+
fastest = order.zip(each_run.transpose).to_h do |worker, samples|
|
|
606
656
|
# (Never nil: each worker in order has its runs.)
|
|
607
657
|
ran = samples #: Array[ProcessSample::sampled]
|
|
608
658
|
[worker, ProcessSample.fastest(ran)]
|
|
609
659
|
end
|
|
660
|
+
@peaks.fetch("baseline") << fastest.fetch(base).fetch("peak")
|
|
661
|
+
@peaks.fetch("candidate") << fastest.fetch(candidate).fetch("peak")
|
|
662
|
+
fastest
|
|
610
663
|
else
|
|
611
664
|
order.to_h { |worker| [worker, worker.sample(name, n, runs: runs, count: count)] }
|
|
612
665
|
end
|
|
613
|
-
if process
|
|
614
|
-
@peaks.fetch("baseline") << replies.fetch(base).fetch("peak")
|
|
615
|
-
@peaks.fetch("candidate") << replies.fetch(candidate).fetch("peak")
|
|
616
|
-
end
|
|
617
666
|
[replies.fetch(base), replies.fetch(candidate)]
|
|
618
667
|
end
|
|
619
668
|
|
|
620
669
|
# Each counted measure's fewest, on each version, over COUNTS pairs of
|
|
621
670
|
# samples, and what its metric said of those samples.
|
|
622
671
|
def count(name, n, result)
|
|
623
|
-
return count_by_process(name, n, result) if
|
|
672
|
+
return count_by_process(name, n, result) if enough_workers?
|
|
624
673
|
|
|
625
674
|
samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
|
|
626
675
|
metrics.each do |spec|
|
|
@@ -631,7 +680,7 @@ module Duel
|
|
|
631
680
|
next unless base && candidate
|
|
632
681
|
|
|
633
682
|
result.counted[metric] = {"baseline" => base.fetch("counted").fetch(metric), "candidate" => candidate.fetch("counted").fetch(metric),
|
|
634
|
-
"n" => n, "
|
|
683
|
+
"n" => n, "per_iteration" => spec.fetch("per_iteration"), "per_sample" => spec.fetch("per_sample"),
|
|
635
684
|
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
|
|
636
685
|
result.note(base, candidate, metric)
|
|
637
686
|
end
|
|
@@ -648,16 +697,13 @@ module Duel
|
|
|
648
697
|
# and another's doesn't, it's as little as the workers can tell.
|
|
649
698
|
# Each version's own is the median of its workers'.
|
|
650
699
|
def count_by_process(name, n, result)
|
|
651
|
-
k = @bases.size
|
|
652
700
|
# @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
|
|
653
701
|
fewest = Hash.new do |h, metric|
|
|
654
702
|
none = {} #: Hash[Integer, Metric::value]
|
|
655
703
|
h[metric] = {"baseline" => none, "candidate" => none.dup}
|
|
656
704
|
end
|
|
657
705
|
COUNT_ROUNDS.times do |round|
|
|
658
|
-
|
|
659
|
-
j = (i + round) % k
|
|
660
|
-
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
|
|
706
|
+
in_round(name, n, round, runs: 1, count: true) do |i, j, base, candidate|
|
|
661
707
|
fewest!(fewest, "baseline", i, base)
|
|
662
708
|
fewest!(fewest, "candidate", j, candidate)
|
|
663
709
|
# (What metrics said of its counts alone: a timing's is the timed
|
|
@@ -676,7 +722,7 @@ module Duel
|
|
|
676
722
|
|
|
677
723
|
result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
|
|
678
724
|
"more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
|
|
679
|
-
"n" => n, "
|
|
725
|
+
"n" => n, "per_iteration" => spec.fetch("per_iteration"), "per_sample" => spec.fetch("per_sample"),
|
|
680
726
|
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
|
|
681
727
|
end
|
|
682
728
|
end
|
data/lib/duel/lock.rb
CHANGED
|
@@ -38,7 +38,7 @@ module Duel
|
|
|
38
38
|
yield
|
|
39
39
|
ensure
|
|
40
40
|
# (Removed while it's held, so a run that opens it after makes a new
|
|
41
|
-
# one. `duel
|
|
41
|
+
# one. `duel clean` may have removed it already.)
|
|
42
42
|
FileUtils.rm_f(path)
|
|
43
43
|
file.close
|
|
44
44
|
FileUtils.rm_f(path) if windows
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
require_relative "forge"
|
|
5
|
+
|
|
6
|
+
module Duel
|
|
7
|
+
# `duel mark-passed BRANCH`: what the Mark the commit passed job runs,
|
|
8
|
+
# where Duel said a push passed, with the branch that marks it (see
|
|
9
|
+
# Baseline#passed_branch: the Duel job's passed output), where
|
|
10
|
+
# Config#push_base compares a push with the last commit Duel passed.
|
|
11
|
+
# It moves the branch to the push's commit (or makes it), but never
|
|
12
|
+
# back: where a newer push's run moved it first, it stays. On GitHub
|
|
13
|
+
# Actions, through gh (to GITHUB_SHA, in GITHUB_REPOSITORY), and on
|
|
14
|
+
# GitLab CI, through its API, signed in with DUEL_TOKEN (to
|
|
15
|
+
# CI_COMMIT_SHA). It checks nothing out, and runs none of the code.
|
|
16
|
+
# @api private
|
|
17
|
+
class MarkPassed
|
|
18
|
+
include Forge
|
|
19
|
+
|
|
20
|
+
USAGE = "Usage: duel mark-passed BRANCH\n\n" \
|
|
21
|
+
"Moves BRANCH, which marks the last commit Duel passed, forward to the pushed commit, or makes it: what the Mark\n" \
|
|
22
|
+
"the commit passed job runs.\n\n"
|
|
23
|
+
|
|
24
|
+
def initialize(argv, env:, out:, err:)
|
|
25
|
+
@argv = argv
|
|
26
|
+
@env = env
|
|
27
|
+
@out = out
|
|
28
|
+
@err = err
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Returns the exit status: 0 moved (or left where a newer push's run
|
|
32
|
+
# moved it), 1 it couldn't, 2 a bad option.
|
|
33
|
+
def run
|
|
34
|
+
branches = catch(:help) { parser.parse(@argv) } or return 0
|
|
35
|
+
raise OptionParser::MissingArgument, "BRANCH" if branches.empty?
|
|
36
|
+
raise OptionParser::NeedlessArgument, branches.drop(1).first if branches.size > 1
|
|
37
|
+
|
|
38
|
+
branch = branches.first #: String
|
|
39
|
+
gitlab? ? on_gitlab(branch, @env.fetch("CI_COMMIT_SHA")) : on_github(branch, @env.fetch("GITHUB_SHA"))
|
|
40
|
+
rescue OptionParser::ParseError => e
|
|
41
|
+
@err.puts "#{command}: #{e}"
|
|
42
|
+
2
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def parser
|
|
48
|
+
OptionParser.new do |o|
|
|
49
|
+
o.banner = USAGE
|
|
50
|
+
o.on("-h", "--help", "Print this help") do
|
|
51
|
+
@out.puts o
|
|
52
|
+
throw :help
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Moves +branch+ to +sha+ on GitHub: forward, where GitHub moves it
|
|
58
|
+
# only where it's behind the push. Where there's no branch, and making
|
|
59
|
+
# it fails, it looks once more, as something else may have made it (a
|
|
60
|
+
# push of it, say).
|
|
61
|
+
def on_github(branch, sha)
|
|
62
|
+
ref = "repos/#{repository}/git/refs/heads/#{branch}"
|
|
63
|
+
at = nil #: String?
|
|
64
|
+
2.times do
|
|
65
|
+
return moved(branch, sha) if gh(ref, "--method", "PATCH", "-f", "sha=#{sha}", quiet: true)
|
|
66
|
+
|
|
67
|
+
at = gh("repos/#{repository}/git/ref/heads/#{branch}", "--jq", ".object.sha", quiet: true)&.chomp
|
|
68
|
+
break if at
|
|
69
|
+
return moved(branch, sha) if gh("repos/#{repository}/git/refs", "--method", "POST", "-f", "ref=refs/heads/#{branch}", "-f", "sha=#{sha}")
|
|
70
|
+
end
|
|
71
|
+
found = at or return blocked(branch) { |prefix| gh("repos/#{repository}/git/ref/heads/#{prefix}", quiet: true) }
|
|
72
|
+
|
|
73
|
+
case (compared = compared(sha, found))
|
|
74
|
+
# (A newer push's run moved it, or this one's did, run again.)
|
|
75
|
+
when "ahead", "identical" then left(branch)
|
|
76
|
+
# (History rewritten, a force push's, say, or replaced: moved anyway.)
|
|
77
|
+
when "diverged", "behind", "unrelated"
|
|
78
|
+
gh(ref, "--method", "PATCH", "-f", "sha=#{sha}", "-F", "force=true") ? moved(branch, sha) : 1
|
|
79
|
+
else
|
|
80
|
+
@out.puts "::error::Couldn't compare #{branch} with this push: #{compared}"
|
|
81
|
+
1
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# How +at+ (the branch's commit) compares with the push's commit
|
|
86
|
+
# +sha+, as GitHub says: ahead, identical, behind or diverged, or
|
|
87
|
+
# unrelated, which GitHub can't compare (commits with no history in
|
|
88
|
+
# common), or what went wrong.
|
|
89
|
+
def compared(sha, at)
|
|
90
|
+
out, err, status = Spawn.open3(:capture3, "gh", "api", "repos/#{repository}/compare/#{sha}...#{at}", "--jq", ".status")
|
|
91
|
+
return out.chomp if status.success?
|
|
92
|
+
|
|
93
|
+
err.include?("No common ancestor") ? "unrelated" : err.strip
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Moves +branch+ to +sha+ on GitLab, which moves no branch, so it's
|
|
97
|
+
# deleted, and made again where it is: unless it's at a newer push's
|
|
98
|
+
# commit (one that has +sha+).
|
|
99
|
+
def on_gitlab(branch, sha)
|
|
100
|
+
path = "#{project}/repository/branches/#{encoded(branch)}"
|
|
101
|
+
at = gitlab(:Get, path, quiet: true)&.dig("commit", "id")
|
|
102
|
+
if at
|
|
103
|
+
base = gitlab(:Get, "#{project}/repository/merge_base?refs%5B%5D=#{sha}&refs%5B%5D=#{at}")&.fetch("id", nil)
|
|
104
|
+
unless base
|
|
105
|
+
@out.puts "Couldn't compare #{branch} with this push"
|
|
106
|
+
return 1
|
|
107
|
+
end
|
|
108
|
+
# (A newer push's job moved it, or this one's did, run again.)
|
|
109
|
+
return left(branch) if base == sha
|
|
110
|
+
end
|
|
111
|
+
gitlab(:Delete, path, quiet: true)
|
|
112
|
+
return moved(branch, sha) if gitlab(:Post, "#{project}/repository/branches", {"branch" => branch, "ref" => sha}, quiet: true)
|
|
113
|
+
|
|
114
|
+
blocked(branch) { |prefix| gitlab(:Get, "#{project}/repository/branches/#{encoded(prefix)}", quiet: true) }
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def moved(branch, sha)
|
|
118
|
+
@out.puts "#{command}: #{branch} is at #{sha}"
|
|
119
|
+
0
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def left(branch)
|
|
123
|
+
@out.puts "#{command}: #{branch} is at a newer push's commit, which it's left at"
|
|
124
|
+
0
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# (For Forge.)
|
|
128
|
+
def command = "duel mark-passed"
|
|
129
|
+
|
|
130
|
+
def gitlab_token = "DUEL_TOKEN"
|
|
131
|
+
end
|
|
132
|
+
end
|