duel 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +51 -27
  3. data/CHANGELOG.md +247 -1
  4. data/README.md +30 -14
  5. data/docs/benchmarks.md +28 -24
  6. data/docs/ci.md +328 -362
  7. data/docs/configuration.md +40 -37
  8. data/docs/how-it-works.md +23 -10
  9. data/docs/json.md +21 -25
  10. data/docs/plugins.md +40 -22
  11. data/docs/troubleshooting.md +3 -2
  12. data/lib/duel/across.rb +18 -18
  13. data/lib/duel/badge.rb +174 -0
  14. data/lib/duel/baseline.rb +38 -15
  15. data/lib/duel/cli.rb +175 -107
  16. data/lib/duel/comment.rb +80 -71
  17. data/lib/duel/config.rb +145 -63
  18. data/lib/duel/context.rb +29 -24
  19. data/lib/duel/forge.rb +87 -0
  20. data/lib/duel/init.rb +44 -87
  21. data/lib/duel/json_report.rb +34 -25
  22. data/lib/duel/lane.rb +107 -61
  23. data/lib/duel/lock.rb +1 -1
  24. data/lib/duel/mark_passed.rb +132 -0
  25. data/lib/duel/metric.rb +28 -27
  26. data/lib/duel/native.rb +7 -5
  27. data/lib/duel/platform.rb +4 -0
  28. data/lib/duel/process_sample.rb +9 -6
  29. data/lib/duel/rake_task.rb +4 -5
  30. data/lib/duel/resource.rb +10 -11
  31. data/lib/duel/result.rb +22 -14
  32. data/lib/duel/rubies.rb +6 -2
  33. data/lib/duel/runner.rb +27 -15
  34. data/lib/duel/spawn.rb +29 -4
  35. data/lib/duel/stats.rb +14 -12
  36. data/lib/duel/summary.rb +31 -29
  37. data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
  38. data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
  39. data/lib/duel/templates/duel.rb.erb +7 -7
  40. data/lib/duel/templates/perf-caller.yml.erb +22 -5
  41. data/lib/duel/templates/perf-comment.yml.erb +11 -11
  42. data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
  43. data/lib/duel/templates/perf-push.yml.erb +50 -0
  44. data/lib/duel/templates/perf.yml.erb +113 -335
  45. data/lib/duel/testing.rb +20 -25
  46. data/lib/duel/version.rb +1 -1
  47. data/lib/duel/worker.rb +16 -9
  48. data/lib/duel/worker_process.rb +6 -6
  49. data/lib/duel.rb +88 -33
  50. data/sig/duel/across.rbs +4 -0
  51. data/sig/duel/badge.rbs +51 -0
  52. data/sig/duel/baseline.rbs +7 -2
  53. data/sig/duel/cli.rbs +31 -20
  54. data/sig/duel/comment.rbs +14 -15
  55. data/sig/duel/config.rbs +71 -29
  56. data/sig/duel/context.rbs +4 -4
  57. data/sig/duel/forge.rbs +42 -0
  58. data/sig/duel/init.rbs +8 -26
  59. data/sig/duel/json_report.rbs +7 -6
  60. data/sig/duel/lane.rbs +29 -6
  61. data/sig/duel/mark_passed.rbs +39 -0
  62. data/sig/duel/metric.rbs +10 -10
  63. data/sig/duel/native.rbs +3 -0
  64. data/sig/duel/platform.rbs +1 -0
  65. data/sig/duel/process_sample.rbs +3 -0
  66. data/sig/duel/resource.rbs +3 -3
  67. data/sig/duel/result.rbs +12 -5
  68. data/sig/duel/rubies.rbs +3 -0
  69. data/sig/duel/runner.rbs +23 -16
  70. data/sig/duel/spawn.rbs +7 -0
  71. data/sig/duel/stats.rbs +4 -1
  72. data/sig/duel/summary.rbs +8 -0
  73. data/sig/duel/testing.rbs +2 -2
  74. data/sig/duel/worker.rbs +6 -5
  75. data/sig/duel/worker_process.rbs +3 -3
  76. data/sig/duel.rbs +50 -31
  77. metadata +9 -1
data/lib/duel/lane.rb CHANGED
@@ -20,7 +20,7 @@ module Duel
20
20
  # (Config#prepare(:worker)).
21
21
  # @api private
22
22
  class Lane
23
- # CPU time (or a plugin's ratio metric's equivalent) a run aims for:
23
+ # CPU time (or a plugin's timed metric's equivalent) a run aims for:
24
24
  # long enough that a clock's resolution is lost in it. A sample is the
25
25
  # fastest of RUNS runs.
26
26
  TARGET_NS = 1_000_000
@@ -60,8 +60,6 @@ module Duel
60
60
  # StackProf's a version: enough to tell a few percent apart).
61
61
  PROFILE_SAMPLES = 500
62
62
 
63
- attr_reader :profiler
64
-
65
63
  # Its workers start at once, each with its resources (a database
66
64
  # server takes seconds), and stop at once. +sides+ are the versions as
67
65
  # the run prepared them (Runner#prepare), by name.
@@ -79,7 +77,7 @@ module Duel
79
77
  @bases = started.first(processes)
80
78
  @candidates = started.drop(processes)
81
79
  @profiler = @candidates.first.profiler
82
- @warm = settings.warm || (started.any?(&:yjit) ? YJIT_WARM : JIT_WARM)
80
+ @warm = settings.warm || (started.any?(&:jit) ? CRUBY_JIT_WARM : JIT_WARM)
83
81
  check_builds
84
82
  rescue Exception # rubocop:disable Lint/RescueException
85
83
  stop
@@ -123,7 +121,7 @@ module Duel
123
121
  def benchmarks = @candidates.first.benchmarks
124
122
 
125
123
  # The Ruby each version runs on (RUBY_DESCRIPTION), by version.
126
- def rubies = {"baseline" => @bases.first.description, "candidate" => @candidates.first.description}
124
+ def rubies = firsts.transform_values(&:description)
127
125
 
128
126
  # Stops its workers, then their resources: each worker's last first
129
127
  # (one may need another), the workers' at once.
@@ -139,7 +137,7 @@ module Duel
139
137
  def compare(name)
140
138
  result = Result.new(name)
141
139
  started = monotonic
142
- if benchmarks.fetch(name).fetch("process")
140
+ if process_of(name)
143
141
  measure_process(name, result)
144
142
  else
145
143
  set_up(name, result)
@@ -148,7 +146,7 @@ module Duel
148
146
  begin
149
147
  measure(name, result)
150
148
  ensure
151
- each_worker { |worker| worker.request("teardown", name) }
149
+ every_worker.each { |worker| worker.request("teardown", name) }
152
150
  end
153
151
  end
154
152
  result.seconds = monotonic - started
@@ -161,7 +159,7 @@ module Duel
161
159
  return unless @profiler
162
160
 
163
161
  # @type var sides: Hash[String, WorkerProcess]?
164
- sides = {"baseline" => @bases.first, "candidate" => @candidates.first}
162
+ sides = firsts
165
163
  sides.each_value { |worker| worker.request("setup", name) }
166
164
  sides.transform_values { |worker| worker.request("profile", name, n: n, runs: PROFILE_SAMPLES) }
167
165
  ensure
@@ -170,6 +168,13 @@ module Duel
170
168
 
171
169
  private
172
170
 
171
+ # How a benchmark runs as a process of its own, or nil where it's no
172
+ # process benchmark.
173
+ def process_of(name) = benchmarks.fetch(name).fetch("process")
174
+
175
+ # Each version's first worker, by version.
176
+ def firsts = {"baseline" => @bases.first, "candidate" => @candidates.first}
177
+
173
178
  # A worker of +side+, with its own resources, and its version's Ruby
174
179
  # and environment.
175
180
  def spawn(side, index)
@@ -180,9 +185,30 @@ module Duel
180
185
  worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
181
186
  config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
182
187
  keep(@workers, worker)
188
+ check_jit(side, worker, env, version.engine)
183
189
  worker
184
190
  end
185
191
 
192
+ # Fails where +env+, +worker+'s (over the runner's own), turns one of
193
+ # CRuby's JITs on (see JIT_VARIABLES) and +worker+ doesn't run it: its
194
+ # benchmarks would be measured without it, and nothing would say so.
195
+ # A CRuby without the JIT ignores its variable, as one before 4.0, or
196
+ # built without ZJIT, does RUBY_ZJIT_ENABLE, and so do JRuby and
197
+ # TruffleRuby, whose JIT is their own.
198
+ def check_jit(side, worker, env, engine)
199
+ jit, variable = JIT_VARIABLES.find { |_, name| JIT_ON.include?(env.fetch(name) { ENV[name] }) }
200
+ return if jit.nil? || worker.jit == jit
201
+
202
+ asked = jit.upcase
203
+ why = if engine == "ruby"
204
+ (jit == "zjit") ? "a Ruby before 4.0, or one built without ZJIT, ignores RUBY_ZJIT_ENABLE" : "a Ruby built without YJIT ignores RUBY_YJIT_ENABLE"
205
+ else
206
+ "#{asked} is CRuby's, and #{engine} has a JIT of its own: unset #{variable} for it"
207
+ end
208
+ raise Failed, "the #{side}'s workers run #{worker.jit&.upcase || "no JIT"}, not #{asked} as #{variable} asks, on " \
209
+ "#{worker.description}: #{why}"
210
+ end
211
+
186
212
  # Fails if a version's workers loaded a native extension that isn't a
187
213
  # build of its own: one in the other version's tree, or one where the
188
214
  # other version loaded the same from its own tree (an installed gem's,
@@ -195,7 +221,7 @@ module Duel
195
221
  # loaded one twice (see Worker.twice): the candidate's, that something
196
222
  # required before Duel loaded the version, and then its own over it.
197
223
  def check_builds
198
- workers = {"baseline" => @bases.first, "candidate" => @candidates.first}
224
+ workers = firsts
199
225
  workers.each do |side, worker|
200
226
  worker.twice.each do |feature, paths|
201
227
  raise Failed, "the #{side} loaded #{feature} twice, from #{paths.join(" and ")}: something required it before " \
@@ -226,7 +252,8 @@ module Duel
226
252
  # mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
227
253
  def keep(list, item) = @lock.synchronize { list << item }
228
254
 
229
- def each_worker(&) = (@bases + @candidates).each(&)
255
+ # Each version's workers, the baseline's first.
256
+ def every_worker = @bases + @candidates
230
257
 
231
258
  # Sets the benchmark up in every worker, the first candidate first (a
232
259
  # benchmark that fails there is broken) and the first baseline next
@@ -243,7 +270,7 @@ module Duel
243
270
  in_parallel(@bases.drop(1) + @candidates.drop(1)) { |worker| worker.request("setup", name) }
244
271
  end
245
272
 
246
- # What's measured: ratio metrics' measures are in a sample's "timed",
273
+ # What's measured: timed metrics' measures are in a sample's "timed",
247
274
  # counted ones' in its "counted".
248
275
  def metrics = @candidates.first.metrics
249
276
 
@@ -273,27 +300,26 @@ module Duel
273
300
  end
274
301
 
275
302
  def measure(name, result)
276
- benchmark = benchmarks.fetch(name)
277
- only = benchmark.fetch("only")
278
- if only == "count"
303
+ judge = benchmarks.fetch(name).fetch("judge")
304
+ unless judge.include?("timed")
279
305
  count(name, result.n = COUNTS_ONLY, result)
280
306
  return
281
307
  end
282
308
 
283
- timings_only = !only.nil? # (the only other choice)
309
+ timings_only = !judge.include?("counted")
284
310
  target = timings_only ? TARGET_NS * 10 : TARGET_NS
285
311
  n = calibrate(name, target, 1)
286
312
  warm(name, n)
287
313
  n = calibrate(name, target, n) if @warm.positive? # it may have got faster
288
- floors = Array.new(WARMUP) { |i| pair(name, n, i) }
314
+ floors = warm_up_pairs(name, n)
289
315
  # (It got faster still, a JIT got to it, say: sized again, unless
290
316
  # what it allocates holds it there.)
291
317
  if below_cpu_floor?(floors) && (sized = calibrate(name, target, n)) > n
292
318
  n = sized
293
- floors = Array.new(WARMUP) { |i| pair(name, n, i) }
319
+ floors = warm_up_pairs(name, n)
294
320
  end
295
321
  result.n = n
296
- peak_memory(name, n, result) if benchmark.fetch("peak_memory")
322
+ peak_memory(name, n, result) if judge.include?("peak_memory")
297
323
  judge(name, n, floors, result)
298
324
  # (Counted once it's timed: on JRuby, the bytes a benchmark allocates
299
325
  # can fall as its JIT compiles it, so a count right after a short
@@ -312,7 +338,7 @@ module Duel
312
338
  result.n = 1
313
339
  floors = Array.new(PROCESS_WARMUP) { |i| pair(name, 1, i) }
314
340
  # (A process benchmark's.)
315
- process = benchmarks.fetch(name).fetch("process") #: Definition::process
341
+ process = process_of(name) #: Definition::process
316
342
  judge(name, process.fetch("runs") || process_runs(floors), floors, result)
317
343
  return if @peaks.values.flatten.include?(nil)
318
344
 
@@ -336,10 +362,10 @@ module Duel
336
362
  # Every worker runs the benchmark a few times, and then, all at once
337
363
  # (none of it is measured, so they needn't take turns), for the
338
364
  # seconds --warm says (Settings#warm), or else until it's steady (see
339
- # #steady), for JIT_WARM's or YJIT_WARM's seconds at most.
365
+ # #steady), for JIT_WARM's or CRUBY_JIT_WARM's seconds at most.
340
366
  def warm(name, n)
341
367
  warm_until = monotonic + @warm
342
- in_parallel(@bases + @candidates) do |worker|
368
+ in_parallel(every_worker) do |worker|
343
369
  2.times { worker.sample(name, n) }
344
370
  if @settings.warm
345
371
  worker.sample(name, n) until monotonic > warm_until
@@ -398,7 +424,7 @@ module Duel
398
424
  end
399
425
  end
400
426
 
401
- # Takes pairs until every ratio metric worth judging (measured on
427
+ # Takes pairs until every timed metric worth judging (measured on
402
428
  # both versions in the warm-up, and above its floors on either: one
403
429
  # that rises from next to nothing is judged too) is decided, or
404
430
  # MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
@@ -412,32 +438,27 @@ module Duel
412
438
  def judge(name, n, floors, result)
413
439
  judged = metrics.select do |spec|
414
440
  floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
415
- floors.transpose.any? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, spec) }
441
+ above_floors?(floors, spec)
416
442
  end
417
443
  return judge_processes(name, n, judged, result) if by_process?(name)
418
444
 
419
- ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
420
445
  # @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
421
446
  raw = judged.to_h { |spec| [spec.fetch("name"), []] }
422
447
  pending = judged.dup
423
- every = benchmarks.fetch(name).fetch("process")
448
+ every = process_of(name)
424
449
  (0...MAX_PAIRS).each do |i|
425
450
  break if pending.empty?
426
451
 
427
452
  base, candidate = pair(name, n, i)
428
453
  result.note(base, candidate)
429
- ratios.each do |metric, list|
430
- values = [base.fetch("timed").fetch(metric), candidate.fetch("timed").fetch(metric)] #: [Metric::value, Metric::value]
431
- raw.fetch(metric) << values
432
- list << values.last.fdiv([values.first, 1].max)
433
- end
454
+ raw.each { |metric, pairs| pairs << [base.fetch("timed").fetch(metric), candidate.fetch("timed").fetch(metric)] }
434
455
  round = ((i + 1) % STEP).zero?
435
456
  next unless round || every
436
457
 
437
- pending.reject! { |spec| decided?(verdict(ratios.fetch(spec.fetch("name")), spec), round) }
458
+ pending.reject! { |spec| decided?(verdict(ratios(raw.fetch(spec.fetch("name"))), spec), round) }
438
459
  end
439
460
  judged.each do |spec|
440
- list = ratios.fetch(spec.fetch("name"))
461
+ list = ratios(raw.fetch(spec.fetch("name")))
441
462
  medians = raw.fetch(spec.fetch("name")).transpose.map { |values| Stats.median(values) }
442
463
  result.timed[spec.fetch("name")] = {"median" => Stats.median(list), "limit" => limit(spec), "across" => spec.fetch("across"),
443
464
  "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
@@ -451,12 +472,17 @@ module Duel
451
472
  # #judge_processes): unless it's a process benchmark (each of its
452
473
  # samples is a fresh process already, so its pairs are independent),
453
474
  # or there are too few workers of each version for an interval over
454
- # them (Lane.by_process?).
455
- def by_process?(name) = !benchmarks.fetch(name).fetch("process") && Lane.by_process?(@bases.size, @settings.confidence)
475
+ # them (see #enough_workers?).
476
+ def by_process?(name) = !process_of(name) && enough_workers?
477
+
478
+ # Whether there are workers enough of each version for an interval
479
+ # over them (Lane.by_process?): then timings and counts alike are
480
+ # judged by process.
481
+ def enough_workers? = Lane.by_process?(@bases.size, @settings.confidence)
456
482
 
457
483
  # Takes rounds of pairs, each a sample from every worker of each
458
484
  # version (the baseline's in order, paired with the candidate's in
459
- # turn: each round pairs each with another), until every ratio metric
485
+ # turn: each round pairs each with another), until every timed metric
460
486
  # judged (see #judge) is decided, from the second round on, or
461
487
  # #rounds run out. Each worker's median is a sample, and the interval
462
488
  # is over workers (Stats.shift_interval): where a process's luck
@@ -474,9 +500,7 @@ module Duel
474
500
  rounds(k).times do |round|
475
501
  break if pending.empty?
476
502
 
477
- k.times do |i|
478
- j = (i + round) % k
479
- base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
503
+ in_round(name, n, round) do |i, j, base, candidate|
480
504
  result.note(base, candidate)
481
505
  values.each do |metric, sides|
482
506
  sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
@@ -523,18 +547,26 @@ module Duel
523
547
  # it wouldn't be judged at all.
524
548
  def below_cpu_floor?(floors)
525
549
  cpu = metrics.find { |spec| spec.fetch("name") == "cpu" } #: Metric::spec
526
- floors.transpose.none? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, cpu) }
550
+ !above_floors?(floors, cpu)
527
551
  end
528
552
 
529
- # Whether +samples+ (one version's timed measures) are above +spec+'s
530
- # floors (the median): its own, and its share of CPU time.
531
- def above_floors?(samples, spec)
532
- values = samples.map { |sample| sample.fetch(spec.fetch("name")) }
533
- cpu = samples.map { |sample| sample.fetch("cpu") }
534
- median = Stats.median(values)
535
- median >= spec.fetch("floor") && median * spec.fetch("unit_ns") >= spec.fetch("cpu_floor") * Stats.median(cpu)
553
+ # Whether either version's samples in +floors+ (its timed measures)
554
+ # are above +spec+'s floors (the median): its own, and its share of
555
+ # CPU time.
556
+ def above_floors?(floors, spec)
557
+ floors.transpose.any? do |side|
558
+ samples = side.map { |sample| sample.fetch("timed") }
559
+ values = samples.map { |sample| sample.fetch(spec.fetch("name")) }
560
+ cpu = samples.map { |sample| sample.fetch("cpu") }
561
+ median = Stats.median(values)
562
+ median >= spec.fetch("floor") && median * spec.fetch("unit_ns") >= spec.fetch("cpu_floor") * Stats.median(cpu)
563
+ end
536
564
  end
537
565
 
566
+ # Each of +pairs+' ratio: the candidate's value over the baseline's
567
+ # (at least 1).
568
+ def ratios(pairs) = pairs.map { |base, candidate| candidate.fdiv([base, 1].max) }
569
+
538
570
  def limit(spec) = spec.fetch("limit") || @settings.limit
539
571
 
540
572
  # The verdicts that decide a measure at the end of a round (see
@@ -552,7 +584,7 @@ module Duel
552
584
  # a row: a cold run may look like enough), and that allocate no more
553
585
  # than HEAP_CAP a run.
554
586
  def calibrate(name, target, n)
555
- units = metrics.select { |spec| spec.fetch("sizes_samples") }.to_h { |spec| [spec.fetch("name"), spec.fetch("unit_ns")] }
587
+ units = metrics.select { |spec| spec.fetch("size_samples") }.to_h { |spec| [spec.fetch("name"), spec.fetch("unit_ns")] }
556
588
  reached = 0
557
589
  until reached == 2
558
590
  sample = pair(name, n, 0, runs: 3).last
@@ -582,6 +614,10 @@ module Duel
582
614
  [n * HEAP_CAP / heap, 1].max
583
615
  end
584
616
 
617
+ # The warm-up's WARMUP pairs, of +n+ iterations each, the floors a
618
+ # benchmark's measures are judged above (see #judge).
619
+ def warm_up_pairs(name, n) = Array.new(WARMUP) { |i| pair(name, n, i) }
620
+
585
621
  # The +index+th pair: a sample (of +n+ iterations) from a baseline
586
622
  # worker and from a candidate worker, each pair of workers in turn,
587
623
  # baseline first on even pairs. A process benchmark's sample is the
@@ -593,34 +629,47 @@ module Duel
593
629
  sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
594
630
  end
595
631
 
632
+ # Round +round+ of pairs taken by process (see #judge_processes and
633
+ # #count_by_process): a sample from every baseline worker, the +i+th,
634
+ # paired with the candidate's +j+th in turn (each round pairs each with
635
+ # another), baseline first alternating by worker and by round. Yields
636
+ # each pair's workers' indexes and samples.
637
+ def in_round(name, n, round, runs: RUNS, count: false)
638
+ k = @bases.size
639
+ k.times do |i|
640
+ j = (i + round) % k
641
+ base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs:, count:)
642
+ yield i, j, base, candidate
643
+ end
644
+ end
645
+
596
646
  # A sample (of +n+ iterations) from +base+, a baseline worker, and
597
647
  # from +candidate+, a candidate worker, the baseline's first if
598
648
  # +base_first+ (see #pair).
599
- def sampled(name, n, base, candidate, base_first:, runs: RUNS, count: false)
649
+ def sampled(name, n, base, candidate, base_first:, runs:, count:)
600
650
  order = base_first ? [base, candidate] : [candidate, base]
601
- process = benchmarks.fetch(name).fetch("process")
651
+ process = process_of(name)
602
652
  replies = if process
603
653
  # (Each run of one version's next to one of the other's.)
604
654
  each_run = Array.new(n) { order.map { |worker| worker.run(process) } }
605
- order.zip(each_run.transpose).to_h do |worker, samples|
655
+ fastest = order.zip(each_run.transpose).to_h do |worker, samples|
606
656
  # (Never nil: each worker in order has its runs.)
607
657
  ran = samples #: Array[ProcessSample::sampled]
608
658
  [worker, ProcessSample.fastest(ran)]
609
659
  end
660
+ @peaks.fetch("baseline") << fastest.fetch(base).fetch("peak")
661
+ @peaks.fetch("candidate") << fastest.fetch(candidate).fetch("peak")
662
+ fastest
610
663
  else
611
664
  order.to_h { |worker| [worker, worker.sample(name, n, runs: runs, count: count)] }
612
665
  end
613
- if process
614
- @peaks.fetch("baseline") << replies.fetch(base).fetch("peak")
615
- @peaks.fetch("candidate") << replies.fetch(candidate).fetch("peak")
616
- end
617
666
  [replies.fetch(base), replies.fetch(candidate)]
618
667
  end
619
668
 
620
669
  # Each counted measure's fewest, on each version, over COUNTS pairs of
621
670
  # samples, and what its metric said of those samples.
622
671
  def count(name, n, result)
623
- return count_by_process(name, n, result) if Lane.by_process?(@bases.size, @settings.confidence)
672
+ return count_by_process(name, n, result) if enough_workers?
624
673
 
625
674
  samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
626
675
  metrics.each do |spec|
@@ -631,7 +680,7 @@ module Duel
631
680
  next unless base && candidate
632
681
 
633
682
  result.counted[metric] = {"baseline" => base.fetch("counted").fetch(metric), "candidate" => candidate.fetch("counted").fetch(metric),
634
- "n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
683
+ "n" => n, "per_iteration" => spec.fetch("per_iteration"), "per_sample" => spec.fetch("per_sample"),
635
684
  "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
636
685
  result.note(base, candidate, metric)
637
686
  end
@@ -648,16 +697,13 @@ module Duel
648
697
  # and another's doesn't, it's as little as the workers can tell.
649
698
  # Each version's own is the median of its workers'.
650
699
  def count_by_process(name, n, result)
651
- k = @bases.size
652
700
  # @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
653
701
  fewest = Hash.new do |h, metric|
654
702
  none = {} #: Hash[Integer, Metric::value]
655
703
  h[metric] = {"baseline" => none, "candidate" => none.dup}
656
704
  end
657
705
  COUNT_ROUNDS.times do |round|
658
- k.times do |i|
659
- j = (i + round) % k
660
- base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
706
+ in_round(name, n, round, runs: 1, count: true) do |i, j, base, candidate|
661
707
  fewest!(fewest, "baseline", i, base)
662
708
  fewest!(fewest, "candidate", j, candidate)
663
709
  # (What metrics said of its counts alone: a timing's is the timed
@@ -676,7 +722,7 @@ module Duel
676
722
 
677
723
  result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
678
724
  "more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
679
- "n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
725
+ "n" => n, "per_iteration" => spec.fetch("per_iteration"), "per_sample" => spec.fetch("per_sample"),
680
726
  "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
681
727
  end
682
728
  end
data/lib/duel/lock.rb CHANGED
@@ -38,7 +38,7 @@ module Duel
38
38
  yield
39
39
  ensure
40
40
  # (Removed while it's held, so a run that opens it after makes a new
41
- # one. `duel --clean` may have removed it already.)
41
+ # one. `duel clean` may have removed it already.)
42
42
  FileUtils.rm_f(path)
43
43
  file.close
44
44
  FileUtils.rm_f(path) if windows
@@ -0,0 +1,132 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "optparse"
4
+ require_relative "forge"
5
+
6
+ module Duel
7
+ # `duel mark-passed BRANCH`: what the Mark the commit passed job runs,
8
+ # where Duel said a push passed, with the branch that marks it (see
9
+ # Baseline#passed_branch: the Duel job's passed output), where
10
+ # Config#push_base compares a push with the last commit Duel passed.
11
+ # It moves the branch to the push's commit (or makes it), but never
12
+ # back: where a newer push's run moved it first, it stays. On GitHub
13
+ # Actions, through gh (to GITHUB_SHA, in GITHUB_REPOSITORY), and on
14
+ # GitLab CI, through its API, signed in with DUEL_TOKEN (to
15
+ # CI_COMMIT_SHA). It checks nothing out, and runs none of the code.
16
+ # @api private
17
+ class MarkPassed
18
+ include Forge
19
+
20
+ USAGE = "Usage: duel mark-passed BRANCH\n\n" \
21
+ "Moves BRANCH, which marks the last commit Duel passed, forward to the pushed commit, or makes it: what the Mark\n" \
22
+ "the commit passed job runs.\n\n"
23
+
24
+ def initialize(argv, env:, out:, err:)
25
+ @argv = argv
26
+ @env = env
27
+ @out = out
28
+ @err = err
29
+ end
30
+
31
+ # Returns the exit status: 0 moved (or left where a newer push's run
32
+ # moved it), 1 it couldn't, 2 a bad option.
33
+ def run
34
+ branches = catch(:help) { parser.parse(@argv) } or return 0
35
+ raise OptionParser::MissingArgument, "BRANCH" if branches.empty?
36
+ raise OptionParser::NeedlessArgument, branches.drop(1).first if branches.size > 1
37
+
38
+ branch = branches.first #: String
39
+ gitlab? ? on_gitlab(branch, @env.fetch("CI_COMMIT_SHA")) : on_github(branch, @env.fetch("GITHUB_SHA"))
40
+ rescue OptionParser::ParseError => e
41
+ @err.puts "#{command}: #{e}"
42
+ 2
43
+ end
44
+
45
+ private
46
+
47
+ def parser
48
+ OptionParser.new do |o|
49
+ o.banner = USAGE
50
+ o.on("-h", "--help", "Print this help") do
51
+ @out.puts o
52
+ throw :help
53
+ end
54
+ end
55
+ end
56
+
57
+ # Moves +branch+ to +sha+ on GitHub: forward, where GitHub moves it
58
+ # only where it's behind the push. Where there's no branch, and making
59
+ # it fails, it looks once more, as something else may have made it (a
60
+ # push of it, say).
61
+ def on_github(branch, sha)
62
+ ref = "repos/#{repository}/git/refs/heads/#{branch}"
63
+ at = nil #: String?
64
+ 2.times do
65
+ return moved(branch, sha) if gh(ref, "--method", "PATCH", "-f", "sha=#{sha}", quiet: true)
66
+
67
+ at = gh("repos/#{repository}/git/ref/heads/#{branch}", "--jq", ".object.sha", quiet: true)&.chomp
68
+ break if at
69
+ return moved(branch, sha) if gh("repos/#{repository}/git/refs", "--method", "POST", "-f", "ref=refs/heads/#{branch}", "-f", "sha=#{sha}")
70
+ end
71
+ found = at or return blocked(branch) { |prefix| gh("repos/#{repository}/git/ref/heads/#{prefix}", quiet: true) }
72
+
73
+ case (compared = compared(sha, found))
74
+ # (A newer push's run moved it, or this one's did, run again.)
75
+ when "ahead", "identical" then left(branch)
76
+ # (History rewritten, a force push's, say, or replaced: moved anyway.)
77
+ when "diverged", "behind", "unrelated"
78
+ gh(ref, "--method", "PATCH", "-f", "sha=#{sha}", "-F", "force=true") ? moved(branch, sha) : 1
79
+ else
80
+ @out.puts "::error::Couldn't compare #{branch} with this push: #{compared}"
81
+ 1
82
+ end
83
+ end
84
+
85
+ # How +at+ (the branch's commit) compares with the push's commit
86
+ # +sha+, as GitHub says: ahead, identical, behind or diverged, or
87
+ # unrelated, which GitHub can't compare (commits with no history in
88
+ # common), or what went wrong.
89
+ def compared(sha, at)
90
+ out, err, status = Spawn.open3(:capture3, "gh", "api", "repos/#{repository}/compare/#{sha}...#{at}", "--jq", ".status")
91
+ return out.chomp if status.success?
92
+
93
+ err.include?("No common ancestor") ? "unrelated" : err.strip
94
+ end
95
+
96
+ # Moves +branch+ to +sha+ on GitLab, which moves no branch, so it's
97
+ # deleted, and made again where it is: unless it's at a newer push's
98
+ # commit (one that has +sha+).
99
+ def on_gitlab(branch, sha)
100
+ path = "#{project}/repository/branches/#{encoded(branch)}"
101
+ at = gitlab(:Get, path, quiet: true)&.dig("commit", "id")
102
+ if at
103
+ base = gitlab(:Get, "#{project}/repository/merge_base?refs%5B%5D=#{sha}&refs%5B%5D=#{at}")&.fetch("id", nil)
104
+ unless base
105
+ @out.puts "Couldn't compare #{branch} with this push"
106
+ return 1
107
+ end
108
+ # (A newer push's job moved it, or this one's did, run again.)
109
+ return left(branch) if base == sha
110
+ end
111
+ gitlab(:Delete, path, quiet: true)
112
+ return moved(branch, sha) if gitlab(:Post, "#{project}/repository/branches", {"branch" => branch, "ref" => sha}, quiet: true)
113
+
114
+ blocked(branch) { |prefix| gitlab(:Get, "#{project}/repository/branches/#{encoded(prefix)}", quiet: true) }
115
+ end
116
+
117
+ def moved(branch, sha)
118
+ @out.puts "#{command}: #{branch} is at #{sha}"
119
+ 0
120
+ end
121
+
122
+ def left(branch)
123
+ @out.puts "#{command}: #{branch} is at a newer push's commit, which it's left at"
124
+ 0
125
+ end
126
+
127
+ # (For Forge.)
128
+ def command = "duel mark-passed"
129
+
130
+ def gitlab_token = "DUEL_TOKEN"
131
+ end
132
+ end