duel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +51 -27
- data/CHANGELOG.md +247 -1
- data/README.md +30 -14
- data/docs/benchmarks.md +28 -24
- data/docs/ci.md +328 -362
- data/docs/configuration.md +40 -37
- data/docs/how-it-works.md +23 -10
- data/docs/json.md +21 -25
- data/docs/plugins.md +40 -22
- data/docs/troubleshooting.md +3 -2
- data/lib/duel/across.rb +18 -18
- data/lib/duel/badge.rb +174 -0
- data/lib/duel/baseline.rb +38 -15
- data/lib/duel/cli.rb +175 -107
- data/lib/duel/comment.rb +80 -71
- data/lib/duel/config.rb +145 -63
- data/lib/duel/context.rb +29 -24
- data/lib/duel/forge.rb +87 -0
- data/lib/duel/init.rb +44 -87
- data/lib/duel/json_report.rb +34 -25
- data/lib/duel/lane.rb +107 -61
- data/lib/duel/lock.rb +1 -1
- data/lib/duel/mark_passed.rb +132 -0
- data/lib/duel/metric.rb +28 -27
- data/lib/duel/native.rb +7 -5
- data/lib/duel/platform.rb +4 -0
- data/lib/duel/process_sample.rb +9 -6
- data/lib/duel/rake_task.rb +4 -5
- data/lib/duel/resource.rb +10 -11
- data/lib/duel/result.rb +22 -14
- data/lib/duel/rubies.rb +6 -2
- data/lib/duel/runner.rb +27 -15
- data/lib/duel/spawn.rb +29 -4
- data/lib/duel/stats.rb +14 -12
- data/lib/duel/summary.rb +31 -29
- data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
- data/lib/duel/templates/duel.rb.erb +7 -7
- data/lib/duel/templates/perf-caller.yml.erb +22 -5
- data/lib/duel/templates/perf-comment.yml.erb +11 -11
- data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
- data/lib/duel/templates/perf-push.yml.erb +50 -0
- data/lib/duel/templates/perf.yml.erb +113 -335
- data/lib/duel/testing.rb +20 -25
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +16 -9
- data/lib/duel/worker_process.rb +6 -6
- data/lib/duel.rb +88 -33
- data/sig/duel/across.rbs +4 -0
- data/sig/duel/badge.rbs +51 -0
- data/sig/duel/baseline.rbs +7 -2
- data/sig/duel/cli.rbs +31 -20
- data/sig/duel/comment.rbs +14 -15
- data/sig/duel/config.rbs +71 -29
- data/sig/duel/context.rbs +4 -4
- data/sig/duel/forge.rbs +42 -0
- data/sig/duel/init.rbs +8 -26
- data/sig/duel/json_report.rbs +7 -6
- data/sig/duel/lane.rbs +29 -6
- data/sig/duel/mark_passed.rbs +39 -0
- data/sig/duel/metric.rbs +10 -10
- data/sig/duel/native.rbs +3 -0
- data/sig/duel/platform.rbs +1 -0
- data/sig/duel/process_sample.rbs +3 -0
- data/sig/duel/resource.rbs +3 -3
- data/sig/duel/result.rbs +12 -5
- data/sig/duel/rubies.rbs +3 -0
- data/sig/duel/runner.rbs +23 -16
- data/sig/duel/spawn.rbs +7 -0
- data/sig/duel/stats.rbs +4 -1
- data/sig/duel/summary.rbs +8 -0
- data/sig/duel/testing.rbs +2 -2
- data/sig/duel/worker.rbs +6 -5
- data/sig/duel/worker_process.rbs +3 -3
- data/sig/duel.rbs +50 -31
- metadata +9 -1
data/lib/duel/metric.rb
CHANGED
|
@@ -9,7 +9,7 @@ module Duel
|
|
|
9
9
|
# judged, in +measures+: {"name" => settings}, each of these, any left
|
|
10
10
|
# out as by default:
|
|
11
11
|
#
|
|
12
|
-
# - +kind+: :
|
|
12
|
+
# - +kind+: :timed (noisy: judged on paired ratios with a confidence
|
|
13
13
|
# interval) or :counted (exact: any increase beyond what's allowed is
|
|
14
14
|
# a regression, counted in separate samples).
|
|
15
15
|
# - +unit+: what it's in, as reports show it: a time unit ("ns", "µs",
|
|
@@ -17,24 +17,24 @@ module Duel
|
|
|
17
17
|
# shown after the number, or nil for none. A word comes with its
|
|
18
18
|
# plural, [singular, plural] (%w[row rows]: 1 row, 2 rows), and a symbol ("B",
|
|
19
19
|
# "KB") is one String, the same for any number.
|
|
20
|
-
# - +floor+: for a
|
|
20
|
+
# - +floor+: for a timed metric, the least a sample must measure (in
|
|
21
21
|
# its own unit) on either version for it to be judged. Below it, it's
|
|
22
22
|
# mostly noise.
|
|
23
|
-
# - +cpu_floor+: for a
|
|
23
|
+
# - +cpu_floor+: for a timed metric measured in time (see +unit_ns+),
|
|
24
24
|
# the least share of the sample's CPU time it must be on either
|
|
25
25
|
# version to be judged (0.05: 5%): a floor that scales with the work.
|
|
26
|
-
# - +limit+: for a
|
|
26
|
+
# - +limit+: for a timed metric, its own ratio above which it's slower
|
|
27
27
|
# (nil: the run's, 1.02 by default).
|
|
28
|
-
# - +unit_ns+: for a
|
|
28
|
+
# - +unit_ns+: for a timed metric, nanoseconds per unit, for comparing
|
|
29
29
|
# it with CPU time when sizing samples.
|
|
30
|
-
# - +
|
|
30
|
+
# - +size_samples+: for a timed metric, whether samples are made long
|
|
31
31
|
# enough for it to settle, as they are for CPU time (false for one
|
|
32
32
|
# that's noisy whatever the length, or not in step with the work).
|
|
33
|
-
# - +across+: for a
|
|
33
|
+
# - +across+: for a timed metric, whether it's judged across the board
|
|
34
34
|
# too: false for one too noisy for so small a margin.
|
|
35
|
-
# - +
|
|
36
|
-
# still isn't one (0: one more is).
|
|
37
|
-
# - +
|
|
35
|
+
# - +per_iteration+: for a counted metric, the increase per iteration
|
|
36
|
+
# that still isn't one (0: one more is).
|
|
37
|
+
# - +per_sample+: for a counted metric, the increase per sample that
|
|
38
38
|
# still isn't one, whatever its length (noise that doesn't grow with
|
|
39
39
|
# the iterations).
|
|
40
40
|
#
|
|
@@ -65,7 +65,7 @@ module Duel
|
|
|
65
65
|
# Every setting there is (see Metric), as #settings resolves them, and
|
|
66
66
|
# running totals' calibrate and Config#measure's skip.
|
|
67
67
|
# @api private
|
|
68
|
-
SETTINGS = %i[kind unit floor cpu_floor limit unit_ns
|
|
68
|
+
SETTINGS = %i[kind unit floor cpu_floor limit unit_ns size_samples across per_iteration per_sample calibrate skip].freeze
|
|
69
69
|
|
|
70
70
|
# A unit that's a word (or words), which needs its plural given: one
|
|
71
71
|
# that isn't (a symbol: "B", "KB", "op/s") is the same for any number.
|
|
@@ -111,8 +111,8 @@ module Duel
|
|
|
111
111
|
def check(name, settings)
|
|
112
112
|
unknown = settings.keys - SETTINGS
|
|
113
113
|
raise Error, "#{name}: no setting #{unknown.join(", ")} (settings: #{SETTINGS.join(", ")})" unless unknown.empty?
|
|
114
|
-
unless %i[
|
|
115
|
-
raise Error, "#{name}: kind is :
|
|
114
|
+
unless %i[timed counted].include?(settings.fetch(:kind, :counted))
|
|
115
|
+
raise Error, "#{name}: kind is :timed or :counted, not #{settings.fetch(:kind).inspect}"
|
|
116
116
|
end
|
|
117
117
|
unit = settings.fetch(:unit, nil)
|
|
118
118
|
if unit.instance_of?(String) && unit.match?(WORD) && !TIME_UNITS.key?(unit)
|
|
@@ -124,10 +124,10 @@ module Duel
|
|
|
124
124
|
end
|
|
125
125
|
|
|
126
126
|
# Whether +more+ (what a count grew by, over +n+ iterations) is
|
|
127
|
-
# still within what's allowed: +
|
|
128
|
-
# +
|
|
127
|
+
# still within what's allowed: +per_iteration+ an iteration and
|
|
128
|
+
# +per_sample+ a sample. Duel judges counts so, and so can a metric, to
|
|
129
129
|
# tell whether it's worth explaining one (see #within?).
|
|
130
|
-
def within?(more, n,
|
|
130
|
+
def within?(more, n, per_iteration:, per_sample:) = more <= per_sample + (per_iteration * n)
|
|
131
131
|
|
|
132
132
|
# Each measure's settings, by name, every one of them filled in (as
|
|
133
133
|
# declared, and the rest as by default): what a plugin's tests can
|
|
@@ -148,8 +148,8 @@ module Duel
|
|
|
148
148
|
{kind: given.fetch(:kind, :counted), unit: unit, plural: plural || unit, floor: given.fetch(:floor, 0),
|
|
149
149
|
cpu_floor: given.fetch(:cpu_floor, 0),
|
|
150
150
|
limit: given.fetch(:limit, nil), unit_ns: given.fetch(:unit_ns) { TIME_UNITS.fetch(unit, 1) },
|
|
151
|
-
|
|
152
|
-
|
|
151
|
+
size_samples: given.fetch(:size_samples, true), across: given.fetch(:across, true),
|
|
152
|
+
per_iteration: given.fetch(:per_iteration, 0), per_sample: given.fetch(:per_sample, 0)}
|
|
153
153
|
end
|
|
154
154
|
|
|
155
155
|
# Its measures as the runner learns them, with +overrides+ (see
|
|
@@ -183,15 +183,16 @@ module Duel
|
|
|
183
183
|
# @api private
|
|
184
184
|
def first_measure = @first_measure ||= settings.keys.first
|
|
185
185
|
|
|
186
|
-
# The setting +key+ (:
|
|
186
|
+
# The setting +key+ (:per_sample, say) of +measure+: by default, of the
|
|
187
187
|
# one it measures (one that measures several names it).
|
|
188
188
|
def setting(key, measure = only_measure) = settings.fetch(measure).fetch(key)
|
|
189
189
|
|
|
190
190
|
# Whether +more+ (what +measure+'s count grew by, over +n+
|
|
191
|
-
# iterations) is still within its
|
|
191
|
+
# iterations) is still within what its per_iteration and per_sample
|
|
192
|
+
# allow, as Duel
|
|
192
193
|
# judges it (see Metric.within?): whether it's worth explaining.
|
|
193
194
|
def within?(more, n, measure = only_measure)
|
|
194
|
-
Metric.within?(more, n,
|
|
195
|
+
Metric.within?(more, n, per_iteration: setting(:per_iteration, measure), per_sample: setting(:per_sample, measure))
|
|
195
196
|
end
|
|
196
197
|
|
|
197
198
|
# Sets up what it reads (a subscription, say), once in each worker,
|
|
@@ -249,7 +250,7 @@ module Duel
|
|
|
249
250
|
# twice in a row, CALIBRATION times, and the least each counted total
|
|
250
251
|
# grew by is taken from every sample's: unless a measure says
|
|
251
252
|
# calibrate: false (its reading costs nothing, or a varying amount its
|
|
252
|
-
# block takes away itself). A
|
|
253
|
+
# block takes away itself). A timed one is left alone: its reading's
|
|
253
254
|
# cost isn't steady enough to take away.
|
|
254
255
|
#
|
|
255
256
|
# Config#metric makes one, and returns it (to take .open, .close,
|
|
@@ -312,7 +313,7 @@ module Duel
|
|
|
312
313
|
# around_iteration hooks, and what they add to it is taken away
|
|
313
314
|
# instead (see Metric). Returns the metric.
|
|
314
315
|
#
|
|
315
|
-
# c.metric("
|
|
316
|
+
# c.metric("postgresql_queries") { queries }.costly
|
|
316
317
|
def costly
|
|
317
318
|
@costly = true
|
|
318
319
|
self
|
|
@@ -513,7 +514,7 @@ module Duel
|
|
|
513
514
|
# processes on the machine don't inflate. Always measured last, inside
|
|
514
515
|
# every other metric.
|
|
515
516
|
class CPU < Metric
|
|
516
|
-
def self.measures = {"cpu" => {kind: :
|
|
517
|
+
def self.measures = {"cpu" => {kind: :timed, unit: "ns", floor: 300_000}}
|
|
517
518
|
|
|
518
519
|
def start_sample(sample)
|
|
519
520
|
@clock = sample.benchmark.cpu
|
|
@@ -534,7 +535,7 @@ module Duel
|
|
|
534
535
|
# whole process's, if it can. It misses what instructions don't show,
|
|
535
536
|
# like cache misses: CPU time is still judged.
|
|
536
537
|
class Instructions < Metric
|
|
537
|
-
def self.measures = {"instructions" => {kind: :
|
|
538
|
+
def self.measures = {"instructions" => {kind: :timed, floor: 100_000, size_samples: false, limit: 1.01}}
|
|
538
539
|
|
|
539
540
|
def self.available? = !counter.nil?
|
|
540
541
|
|
|
@@ -566,7 +567,7 @@ module Duel
|
|
|
566
567
|
# sample here and there some 31 bytes an iteration apart: so more by
|
|
567
568
|
# 32 bytes an iteration, about a small object's, isn't more yet.
|
|
568
569
|
class AllocatedBytes < Metric
|
|
569
|
-
def self.measures = {"allocated_bytes" => {
|
|
570
|
+
def self.measures = {"allocated_bytes" => {per_iteration: 32, unit: %w[byte bytes]}}
|
|
570
571
|
|
|
571
572
|
# mutant:disable -- mutant runs on CRuby, where this is always false
|
|
572
573
|
def self.available? = Platform::JRUBY
|
|
@@ -583,7 +584,7 @@ module Duel
|
|
|
583
584
|
# a few from run to run (ids of random lengths, say), so it's one
|
|
584
585
|
# more per two iterations, not one more, that's a regression.
|
|
585
586
|
class Allocations < Metric
|
|
586
|
-
def self.measures = {"allocations" => {
|
|
587
|
+
def self.measures = {"allocations" => {per_iteration: 0.5}}
|
|
587
588
|
|
|
588
589
|
# mutant:disable -- mutant runs on CRuby, where this is always true
|
|
589
590
|
def self.available? = Platform::CRUBY
|
data/lib/duel/native.rb
CHANGED
|
@@ -167,11 +167,14 @@ module Duel
|
|
|
167
167
|
STATUS = 64 # sizeof(MEMORYSTATUSEX)
|
|
168
168
|
AVAILABLE = 16 # offsetof(ullAvailPhys)
|
|
169
169
|
|
|
170
|
+
# K32GetProcessMemoryInfo, a process's memory (raising
|
|
171
|
+
# Fiddle::DLError where there's no such function).
|
|
172
|
+
def self.process_memory_info = Native.function("K32GetProcessMemoryInfo", %i[VOIDP VOIDP INT], :INT, library: KERNEL32)
|
|
173
|
+
|
|
170
174
|
def self.open(platform: Platform::OS, memory_info: nil)
|
|
171
175
|
return unless platform.match?(Platform::WINDOWS_OS)
|
|
172
176
|
|
|
173
|
-
|
|
174
|
-
windows = new(memory_info)
|
|
177
|
+
windows = new(memory_info || process_memory_info)
|
|
175
178
|
windows if windows.resident
|
|
176
179
|
rescue Fiddle::DLError # no such function
|
|
177
180
|
nil
|
|
@@ -262,8 +265,7 @@ module Duel
|
|
|
262
265
|
def initialize(handle, cycles: nil, memory_info: nil, close: nil)
|
|
263
266
|
@handle = handle
|
|
264
267
|
@cycles = cycles || Cycles.new(:process, handle: handle)
|
|
265
|
-
@memory = Windows.new(memory_info ||
|
|
266
|
-
handle: handle)
|
|
268
|
+
@memory = Windows.new(memory_info || Windows.process_memory_info, handle: handle)
|
|
267
269
|
@close = close || Native.function("CloseHandle", %i[INTPTR_T], :INT, library: KERNEL32)
|
|
268
270
|
end
|
|
269
271
|
|
|
@@ -329,7 +331,7 @@ module Duel
|
|
|
329
331
|
# mutant:disable -- kept only so as not to open it again: it's the
|
|
330
332
|
# same reader however often it's opened
|
|
331
333
|
def self.opened_reader
|
|
332
|
-
return @reader if
|
|
334
|
+
return @reader if instance_variable_defined?(:@reader)
|
|
333
335
|
|
|
334
336
|
@reader = (MacOS.memory || Windows.open if FIDDLE)
|
|
335
337
|
end
|
data/lib/duel/platform.rb
CHANGED
|
@@ -19,6 +19,10 @@ module Duel
|
|
|
19
19
|
WINDOWS_OS = /mingw|mswin/
|
|
20
20
|
WINDOWS = OS.match?(WINDOWS_OS)
|
|
21
21
|
|
|
22
|
+
# What a program's file ends with: on Windows, .exe (see
|
|
23
|
+
# Duel.find_program).
|
|
24
|
+
EXE = RbConfig::CONFIG["EXEEXT"]
|
|
25
|
+
|
|
22
26
|
CRUBY = RUBY_ENGINE == "ruby"
|
|
23
27
|
JRUBY = RUBY_ENGINE == "jruby"
|
|
24
28
|
|
data/lib/duel/process_sample.rb
CHANGED
|
@@ -41,9 +41,14 @@ module Duel
|
|
|
41
41
|
def self.fastest(samples)
|
|
42
42
|
peaks = samples.map { |sample| sample.fetch("peak") }
|
|
43
43
|
cpu = samples.map { |sample| sample.fetch("timed").fetch("cpu") }.min #: Integer
|
|
44
|
-
|
|
44
|
+
sampled(cpu, (peaks.min unless peaks.include?(nil)))
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
+
# A process benchmark's sample, as a worker's is: its CPU time
|
|
48
|
+
# (nanoseconds) and its peak memory (bytes, or nil), counting nothing
|
|
49
|
+
# and leaving no heap.
|
|
50
|
+
def self.sampled(cpu, peak) = {"timed" => {"cpu" => cpu}, "counted" => {}, "heap" => 0, "peak" => peak}
|
|
51
|
+
|
|
47
52
|
# Runs +command+ (as Process.spawn takes it) once, and returns its
|
|
48
53
|
# sample ({"timed" => {"cpu" => nanoseconds}, "peak" => bytes or nil},
|
|
49
54
|
# as a worker's has it). One that fails, or takes more than the
|
|
@@ -71,11 +76,9 @@ module Duel
|
|
|
71
76
|
|
|
72
77
|
ended = JSON.parse(reply)
|
|
73
78
|
failed = ended.fetch("failed")
|
|
74
|
-
if failed
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
end
|
|
78
|
-
{"timed" => {"cpu" => ended.fetch("cpu")}, "counted" => {}, "heap" => 0, "peak" => ended.fetch("peak")}
|
|
79
|
+
raise Failed, Spawn.with_tail("#{@label}: #{command.join(" ")} #{failed}", @log) if failed
|
|
80
|
+
|
|
81
|
+
ProcessSample.sampled(ended.fetch("cpu"), ended.fetch("peak"))
|
|
79
82
|
end
|
|
80
83
|
|
|
81
84
|
# Ends the Ruby process of its own, if it started: closing its input
|
data/lib/duel/rake_task.rb
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
require "rake"
|
|
4
4
|
|
|
5
5
|
module Duel
|
|
6
|
-
# A rake task running `duel` with +args
|
|
6
|
+
# A rake task running `duel` with +args+, which exits as `duel` does:
|
|
7
|
+
# with its status, where it isn't 0 (see CLI).
|
|
7
8
|
#
|
|
8
9
|
# require "duel/rake_task"
|
|
9
10
|
# Duel::RakeTask.new(:perf)
|
|
@@ -24,10 +25,8 @@ module Duel
|
|
|
24
25
|
desc "Fail if this tree is slower than its baseline at any benchmark (duel)"
|
|
25
26
|
task(name) do
|
|
26
27
|
RakeTask.load_duel
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
when 2 then abort "duel: couldn't compare"
|
|
30
|
-
end
|
|
28
|
+
status = CLI.new(args).run
|
|
29
|
+
exit(status) unless status.zero?
|
|
31
30
|
end
|
|
32
31
|
end
|
|
33
32
|
end
|
data/lib/duel/resource.rb
CHANGED
|
@@ -134,8 +134,9 @@ module Duel
|
|
|
134
134
|
|
|
135
135
|
# A port of 127.0.0.1's that's free (one the system chose, let go of
|
|
136
136
|
# again), for a server of its own to listen on, rather than on a Unix
|
|
137
|
-
# socket in its directory (see #path)
|
|
138
|
-
|
|
137
|
+
# socket in its directory (see #path): another each call, so keep the
|
|
138
|
+
# one a server's given.
|
|
139
|
+
def free_port
|
|
139
140
|
server = Control.tcp_server
|
|
140
141
|
begin
|
|
141
142
|
server.addr.fetch(1)
|
|
@@ -149,7 +150,7 @@ module Duel
|
|
|
149
150
|
# copy): the first that asks for +key+ builds it, in a directory the
|
|
150
151
|
# block is given, and the rest get it built. A key names what's built
|
|
151
152
|
# (put a version in it), as a file name: tmp/duel/cache/+key+, which
|
|
152
|
-
# `duel
|
|
153
|
+
# `duel clean` removes. A build that fails leaves nothing behind.
|
|
153
154
|
# A key that ends with a hash (and a version before it, if you like:
|
|
154
155
|
# mysql-0.1.2-3f9c2a1b) replaces the builds of its name when the run
|
|
155
156
|
# ends (see .evict): the one a run used is the one that's kept.
|
|
@@ -188,7 +189,7 @@ module Duel
|
|
|
188
189
|
code, signal = start(command, log, options).wait
|
|
189
190
|
return if code == 0
|
|
190
191
|
|
|
191
|
-
|
|
192
|
+
fail!("#{File.basename(log, ".log")} #{Spawn.why(code, signal)}", log: log)
|
|
192
193
|
end
|
|
193
194
|
|
|
194
195
|
# Waits until the block is true: fails if a process spawned dies
|
|
@@ -201,9 +202,9 @@ module Duel
|
|
|
201
202
|
# (It may have got ready, and ended, since the last look.)
|
|
202
203
|
break if yield
|
|
203
204
|
|
|
204
|
-
|
|
205
|
+
fail!("#{ended.program} exited before it was ready", log: ended.log)
|
|
205
206
|
end
|
|
206
|
-
|
|
207
|
+
fail!("not ready within #{@timeout}s", log: @processes.last&.log) if monotonic > deadline
|
|
207
208
|
sleep POLL
|
|
208
209
|
end
|
|
209
210
|
end
|
|
@@ -213,9 +214,8 @@ module Duel
|
|
|
213
214
|
|
|
214
215
|
# Fails, saying it's this resource that did, with the end of +log+ (a
|
|
215
216
|
# path) to say why.
|
|
216
|
-
def
|
|
217
|
-
|
|
218
|
-
raise Failed, "#{@name}'s resource: #{message}#{"\n#{tail}" unless tail.empty?}"
|
|
217
|
+
def fail!(message, log: nil)
|
|
218
|
+
raise Failed, Spawn.with_tail("#{@name}'s resource: #{message}", log)
|
|
219
219
|
end
|
|
220
220
|
|
|
221
221
|
# Stops it: the at_stop blocks, last first, then each process.
|
|
@@ -263,8 +263,7 @@ module Duel
|
|
|
263
263
|
# can't interrupt a wait on JRuby.)
|
|
264
264
|
def start(command, log, options)
|
|
265
265
|
# (Its own environment, if it's given one, over its version's.)
|
|
266
|
-
|
|
267
|
-
env, *program = command.first.is_a?(Hash) ? command : [none, *command]
|
|
266
|
+
env, program = Spawn.environment_and(command)
|
|
268
267
|
@starter.logged(@env.merge(env), program, log: log, mode: "a", close_others: true, **options).detach
|
|
269
268
|
end
|
|
270
269
|
|
data/lib/duel/result.rb
CHANGED
|
@@ -45,11 +45,16 @@ module Duel
|
|
|
45
45
|
|
|
46
46
|
# The measurements again that were too close to call (see #confirm).
|
|
47
47
|
# @api private
|
|
48
|
-
attr_reader :
|
|
48
|
+
attr_reader :close_calls
|
|
49
49
|
|
|
50
50
|
# @api private
|
|
51
51
|
attr_writer :n, :seconds, :skipped, :error, :memory, :profile
|
|
52
52
|
|
|
53
|
+
# A counted measure's (+c+'s) count an iteration (of its "n", one at
|
|
54
|
+
# least): [baseline's, candidate's].
|
|
55
|
+
# @api private
|
|
56
|
+
def self.per_iteration(c) = [c.fetch("baseline").fdiv(c.fetch("n")), c.fetch("candidate").fdiv(c.fetch("n"))]
|
|
57
|
+
|
|
53
58
|
# @api private
|
|
54
59
|
def initialize(name)
|
|
55
60
|
@name = name
|
|
@@ -57,7 +62,7 @@ module Duel
|
|
|
57
62
|
@counted = {}
|
|
58
63
|
@details = {}
|
|
59
64
|
@confirmations = []
|
|
60
|
-
@
|
|
65
|
+
@close_calls = []
|
|
61
66
|
@processes = {}
|
|
62
67
|
end
|
|
63
68
|
|
|
@@ -73,14 +78,14 @@ module Duel
|
|
|
73
78
|
end
|
|
74
79
|
|
|
75
80
|
# The metrics on which the candidate did worse: a timed one slower, or
|
|
76
|
-
# a counted one up by more than its
|
|
77
|
-
#
|
|
81
|
+
# a counted one up by more than its per_iteration an iteration and its
|
|
82
|
+
# per_sample a sample (counted by process, up by that much at least:
|
|
78
83
|
# see Lane#count_by_process).
|
|
79
84
|
# @api private
|
|
80
85
|
def worse
|
|
81
86
|
more = @counted.select do |_, c|
|
|
82
|
-
!Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"),
|
|
83
|
-
|
|
87
|
+
!Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), per_iteration: c.fetch("per_iteration"),
|
|
88
|
+
per_sample: c.fetch("per_sample"))
|
|
84
89
|
end
|
|
85
90
|
slower + more.keys + (more_memory? ? ["peak_memory"] : [])
|
|
86
91
|
end
|
|
@@ -108,16 +113,16 @@ module Duel
|
|
|
108
113
|
# Keeps +again+, a measurement of it again from scratch (see
|
|
109
114
|
# Runner#confirm), as a confirmation, or sets it aside where it's too
|
|
110
115
|
# close to call, up to Runner::CONFIRM times: every timing is still
|
|
111
|
-
#
|
|
116
|
+
# unsettled there (none slower, none the same or faster), and each
|
|
112
117
|
# measure it regressed on so far is one of them. One that can't tell
|
|
113
118
|
# neither confirms it nor clears it, so it's measured again instead.
|
|
114
119
|
# Returns whether +again+ was set aside.
|
|
115
120
|
# @api private
|
|
116
121
|
def confirm(again)
|
|
117
122
|
timed = again.timed
|
|
118
|
-
close = timed.all? { |_, t| t.fetch("verdict") == "
|
|
119
|
-
@
|
|
120
|
-
(close ? @
|
|
123
|
+
close = timed.all? { |_, t| t.fetch("verdict") == "unsettled" } && regressed.all? { |metric| timed.key?(metric) } &&
|
|
124
|
+
@close_calls.size < Runner::CONFIRM
|
|
125
|
+
(close ? @close_calls : @confirmations) << again
|
|
121
126
|
close
|
|
122
127
|
end
|
|
123
128
|
|
|
@@ -137,8 +142,8 @@ module Duel
|
|
|
137
142
|
def regressed
|
|
138
143
|
return [] if @error || @skipped
|
|
139
144
|
|
|
140
|
-
|
|
141
|
-
return
|
|
145
|
+
worse = agreed
|
|
146
|
+
return worse unless worse.empty? && measurements.all?(&:slower_in_time?)
|
|
142
147
|
|
|
143
148
|
measurements.flat_map(&:slower).uniq
|
|
144
149
|
end
|
|
@@ -148,18 +153,21 @@ module Duel
|
|
|
148
153
|
# instructions in the next, say), slower in every one with none saying
|
|
149
154
|
# it isn't: that takes one measurement more than one metric does.
|
|
150
155
|
# @api private
|
|
151
|
-
def mixed? =
|
|
156
|
+
def mixed? = agreed.empty? && !regressed.empty?
|
|
152
157
|
|
|
153
158
|
# Whether it regressed on nothing, and didn't fail.
|
|
154
159
|
def passed? = regressed.empty? && !@error
|
|
155
160
|
|
|
156
161
|
# Whether a timed measure's verdict was too close to the limit to settle.
|
|
157
162
|
# @api private
|
|
158
|
-
def unsettled? = @timed.any? { |_, t| t.fetch("verdict") == "
|
|
163
|
+
def unsettled? = @timed.any? { |_, t| t.fetch("verdict") == "unsettled" }
|
|
159
164
|
|
|
160
165
|
private
|
|
161
166
|
|
|
162
167
|
# The first measurement and its confirmations.
|
|
163
168
|
def measurements = [self, *@confirmations]
|
|
169
|
+
|
|
170
|
+
# The metrics it was worse on in every measurement.
|
|
171
|
+
def agreed = measurements.map(&:worse).reduce(:&)
|
|
164
172
|
end
|
|
165
173
|
end
|
data/lib/duel/rubies.rb
CHANGED
|
@@ -19,6 +19,10 @@ module Duel
|
|
|
19
19
|
# directory.
|
|
20
20
|
VERSION_FILES = %w[.ruby-version .tool-versions mise.toml .mise.toml].freeze
|
|
21
21
|
|
|
22
|
+
# What a Ruby's bin/ may name its executable: CRuby's, or else JRuby's,
|
|
23
|
+
# each on Windows with .exe.
|
|
24
|
+
PROGRAMS = %w[ruby ruby.exe jruby jruby.exe].freeze
|
|
25
|
+
|
|
22
26
|
# Where chruby finds Rubies.
|
|
23
27
|
CHRUBY = ["/opt/rubies", File.join(Dir.home, ".rubies")].freeze
|
|
24
28
|
|
|
@@ -194,8 +198,8 @@ module Duel
|
|
|
194
198
|
|
|
195
199
|
MANAGERS.each_value do |manager|
|
|
196
200
|
prefix = manager.call(version)
|
|
197
|
-
# (JRuby's may have no bin/ruby.)
|
|
198
|
-
ruby = prefix &&
|
|
201
|
+
# (JRuby's may have no bin/ruby, and on Windows, each is a .exe.)
|
|
202
|
+
ruby = prefix && PROGRAMS.map { |program| File.join(prefix, "bin", program) }.find { |path| File.executable?(path) }
|
|
199
203
|
return ruby if ruby
|
|
200
204
|
end
|
|
201
205
|
raise Error, "the #{name}'s #{file} asks for Ruby #{version}, which isn't installed with rbenv, chruby, mise or " \
|
data/lib/duel/runner.rb
CHANGED
|
@@ -21,12 +21,24 @@ module Duel
|
|
|
21
21
|
JIT_WARM = {"jruby" => 60.0, "truffleruby" => 60.0}.fetch(RUBY_ENGINE, 0.0)
|
|
22
22
|
|
|
23
23
|
# The most seconds each benchmark runs before it's measured where a
|
|
24
|
-
# worker has
|
|
25
|
-
# does), unless --warm says.
|
|
26
|
-
#
|
|
27
|
-
#
|
|
24
|
+
# worker has one of CRuby's JITs on, YJIT or ZJIT (an app may turn one
|
|
25
|
+
# on as it boots, as Rails does YJIT), unless --warm says. Each compiles
|
|
26
|
+
# a method after 30 calls (YJIT, in a big app, after 120), so a small
|
|
27
|
+
# library's methods are compiled within the warm-up samples, but a big
|
|
28
|
+
# app's take more calls than a slow benchmark's warm-up samples make.
|
|
28
29
|
# @api private
|
|
29
|
-
|
|
30
|
+
CRUBY_JIT_WARM = 10.0
|
|
31
|
+
|
|
32
|
+
# CRuby's JITs, by the variable that turns each on in a worker, which
|
|
33
|
+
# Ruby takes as on where it's one of JIT_ON. A worker whose environment
|
|
34
|
+
# turns one on must run it (see Lane#check_jit).
|
|
35
|
+
# @api private
|
|
36
|
+
JIT_VARIABLES = {"yjit" => "RUBY_YJIT_ENABLE", "zjit" => "RUBY_ZJIT_ENABLE"}.freeze
|
|
37
|
+
|
|
38
|
+
# The values of a JIT's variable that turn it on, as Ruby reads them
|
|
39
|
+
# (exactly: TRUE or on doesn't).
|
|
40
|
+
# @api private
|
|
41
|
+
JIT_ON = %w[1 true yes].freeze
|
|
30
42
|
|
|
31
43
|
# Worker processes of each version, by default: enough for an interval
|
|
32
44
|
# over processes (see Lane#judge_processes), which takes 5 at 99%, and
|
|
@@ -62,7 +74,7 @@ module Duel
|
|
|
62
74
|
# the memory available (see Runner#fit): the defaults, but not what
|
|
63
75
|
# an option set.
|
|
64
76
|
Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
|
|
65
|
-
:config_path, :timeout, :ruby, :
|
|
77
|
+
:config_path, :timeout, :ruby, :pushed, :target, :time_limit, :accepted, :repository, :fit) do
|
|
66
78
|
# @type self: singleton(Settings)
|
|
67
79
|
def self.defaults = new(filter: //, jobs: 1, processes: PROCESSES, limit: 1.02, confidence: 0.99, warm: nil, force: false,
|
|
68
80
|
timeout: TIMEOUT, time_limit: TIME_LIMIT, fit: [])
|
|
@@ -145,7 +157,7 @@ module Duel
|
|
|
145
157
|
@base_root = base_root
|
|
146
158
|
@out = out
|
|
147
159
|
@prepared = []
|
|
148
|
-
@started =
|
|
160
|
+
@started = monotonic
|
|
149
161
|
@lock = Mutex.new
|
|
150
162
|
@unmeasured = [] #: Array[String]
|
|
151
163
|
@unconfirmed = [] #: Array[String]
|
|
@@ -155,10 +167,10 @@ module Duel
|
|
|
155
167
|
# Returns the Results.
|
|
156
168
|
def run
|
|
157
169
|
CycleClock.prepare
|
|
158
|
-
|
|
170
|
+
@started = monotonic
|
|
159
171
|
@dir = Lane.tmpdir
|
|
160
172
|
begin
|
|
161
|
-
compare_all
|
|
173
|
+
compare_all
|
|
162
174
|
ensure
|
|
163
175
|
@prepared.reverse_each(&:stop)
|
|
164
176
|
FileUtils.rm_rf(@dir)
|
|
@@ -191,7 +203,7 @@ module Duel
|
|
|
191
203
|
# fails what still looks worse in all of them, so a run cut short
|
|
192
204
|
# never passes a regression it would have failed. Each measurement it
|
|
193
205
|
# started runs to its end.
|
|
194
|
-
def out_of_time? =
|
|
206
|
+
def out_of_time? = monotonic - @started > @settings.time_limit
|
|
195
207
|
|
|
196
208
|
# The plugins the configuration used (Config#plugins), by name.
|
|
197
209
|
def plugins = @config.plugins
|
|
@@ -203,8 +215,10 @@ module Duel
|
|
|
203
215
|
|
|
204
216
|
private
|
|
205
217
|
|
|
218
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
219
|
+
|
|
206
220
|
# What #run does, with its versions prepared in its directory.
|
|
207
|
-
def compare_all
|
|
221
|
+
def compare_all
|
|
208
222
|
@sides = prepare
|
|
209
223
|
fit
|
|
210
224
|
names, results = with_lanes do |lanes|
|
|
@@ -221,7 +235,7 @@ module Duel
|
|
|
221
235
|
confirm(results.select { |r| r.worse.any? })
|
|
222
236
|
@across = judge_across(results, names)
|
|
223
237
|
profile(results.select { |r| (r.regressed & r.timed.keys).any? && !@benchmarks.fetch(r.name).fetch("process") })
|
|
224
|
-
report(results, @seconds =
|
|
238
|
+
report(results, @seconds = monotonic - @started)
|
|
225
239
|
results
|
|
226
240
|
end
|
|
227
241
|
|
|
@@ -465,9 +479,7 @@ module Duel
|
|
|
465
479
|
end
|
|
466
480
|
# (An iteration's, as the explanation's are.)
|
|
467
481
|
counted = result.counted.map do |metric, c|
|
|
468
|
-
|
|
469
|
-
base = c.fetch("baseline").fdiv(n)
|
|
470
|
-
candidate = c.fetch("candidate").fdiv(n)
|
|
482
|
+
base, candidate = Result.per_iteration(c)
|
|
471
483
|
# (No unit its name ends in: "retained_objects 2", not "2 objects".)
|
|
472
484
|
named = c.fetch("plural")&.then { |plural| metric.end_with?(plural.tr(" ", "_")) }
|
|
473
485
|
[format("%s %s%s", metric, Runner.number(base), ("→#{Runner.number(candidate)}" unless base == candidate)),
|
data/lib/duel/spawn.rb
CHANGED
|
@@ -100,8 +100,20 @@ module Duel
|
|
|
100
100
|
nil
|
|
101
101
|
end
|
|
102
102
|
|
|
103
|
-
# Waits up to +seconds+ for it to end, then kills it (see #kill).
|
|
104
|
-
|
|
103
|
+
# Waits up to +seconds+ for it to end, then kills it (see #kill). One
|
|
104
|
+
# in a group of its own that ended leaves none of its group running:
|
|
105
|
+
# what it started, orphaned, would go on (a command it ran when it
|
|
106
|
+
# was killed, say), so what's left of the group is killed. (Not on
|
|
107
|
+
# Windows, where a process that's ended has no tree left to find.)
|
|
108
|
+
def stop(seconds)
|
|
109
|
+
if waiter.join(seconds)
|
|
110
|
+
Process.kill("KILL", -@pid) if @group && !@windows
|
|
111
|
+
else
|
|
112
|
+
kill
|
|
113
|
+
end
|
|
114
|
+
rescue *GONE
|
|
115
|
+
nil
|
|
116
|
+
end
|
|
105
117
|
end
|
|
106
118
|
|
|
107
119
|
# Starts a process with Process.spawn, given its redirects and pipes.
|
|
@@ -210,6 +222,13 @@ module Duel
|
|
|
210
222
|
# LOG_LINES lines, without CRLF's CR (a Windows program's).
|
|
211
223
|
def self.tail(path) = File.readlines(path, chomp: true).last(LOG_LINES).map { |line| "#{line}\n" }.join
|
|
212
224
|
|
|
225
|
+
# +message+, then the end of the log at +path+ (see .tail), where
|
|
226
|
+
# there's a log with anything in it.
|
|
227
|
+
def self.with_tail(message, path)
|
|
228
|
+
tail = path ? tail(path) : ""
|
|
229
|
+
tail.empty? ? message : "#{message}\n#{tail}"
|
|
230
|
+
end
|
|
231
|
+
|
|
213
232
|
# The log at +path+, opened with +mode+, for a process's output: here,
|
|
214
233
|
# so it's there for a failure to show, and binary, so what's copied to
|
|
215
234
|
# it is as the process wrote it (on Windows, CRuby's text mode would
|
|
@@ -251,8 +270,7 @@ module Duel
|
|
|
251
270
|
def self.open3(name, *args, jruby_on_windows: Platform::JRUBY_ON_WINDOWS, **options, &)
|
|
252
271
|
return Open3.public_send(name, *args, **options, &) unless jruby_on_windows
|
|
253
272
|
|
|
254
|
-
|
|
255
|
-
env, *command = args.first.is_a?(Hash) ? args : [none, *args]
|
|
273
|
+
env, command = environment_and(args)
|
|
256
274
|
current = ENV.to_h
|
|
257
275
|
under = options[:unsetenv_others] ? current.keys.product([nil]).to_h : current #: Hash[String, String?]
|
|
258
276
|
# (Each given by ENV's name for it, whatever its case: Windows's
|
|
@@ -262,6 +280,13 @@ module Duel
|
|
|
262
280
|
Open3.public_send(name, under.merge(given), *program(command), **options, &)
|
|
263
281
|
end
|
|
264
282
|
|
|
283
|
+
# +command+ as Process.spawn takes it, split: its environment (none,
|
|
284
|
+
# if it doesn't start with one), and the rest.
|
|
285
|
+
def self.environment_and(command)
|
|
286
|
+
none = {} #: Hash[String, String?]
|
|
287
|
+
command.first.is_a?(Hash) ? [command.first, command.drop(1)] : [none, command]
|
|
288
|
+
end
|
|
289
|
+
|
|
265
290
|
# +command+ as Windows runs it: a lone String that a shell runs (with
|
|
266
291
|
# a space in it, say) as cmd.exe's, and otherwise with its program
|
|
267
292
|
# found (by name, in +path+'s directories), as the program it is, or
|