duel 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +203 -1
- data/README.md +41 -1
- data/docs/ci.md +170 -76
- data/docs/configuration.md +38 -2
- data/docs/how-it-works.md +18 -9
- data/docs/plugins.md +23 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +96 -28
- data/lib/duel/comment.rb +296 -0
- data/lib/duel/config.rb +124 -4
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +255 -20
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +33 -1044
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +21 -15
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +61 -44
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +35 -3
- data/sig/duel/cli.rbs +19 -0
- data/sig/duel/comment.rbs +97 -0
- data/sig/duel/config.rbs +57 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +95 -1
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +6 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +6 -3
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -105
data/sig/duel/result.rbs
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
module Duel
|
|
2
|
+
# What one comparison of a benchmark found (Testing::Comparison#results
|
|
3
|
+
# has them).
|
|
4
|
+
class Result
|
|
5
|
+
# A ratio metric's verdict, and each version's (the median of its
|
|
6
|
+
# samples).
|
|
7
|
+
type timed = { "median" => Float, "limit" => Float, "across" => bool, "unit" => String?, "plural" => String?, "interval" => Array[Float]?, "pairs" => Integer,
|
|
8
|
+
?"processes" => Integer, "verdict" => String, "baseline" => Metric::value, "candidate" => Metric::value }
|
|
9
|
+
|
|
10
|
+
# Each worker's median of a measure judged by process, by version,
|
|
11
|
+
# and its lane's number.
|
|
12
|
+
type processes = { "lane" => Integer, "baseline" => Array[Metric::value], "candidate" => Array[Metric::value] }
|
|
13
|
+
|
|
14
|
+
# A counted metric's fewest, on each version, in +n+ iterations.
|
|
15
|
+
type counted = { "baseline" => Metric::value, "candidate" => Metric::value, ?"more" => Metric::value?, "n" => Integer,
|
|
16
|
+
"tolerance" => Metric::value, "allowance" => Metric::value, "unit" => String?, "plural" => String? }
|
|
17
|
+
|
|
18
|
+
# Where a regression's time went (see Profile#to_h), or why there's no
|
|
19
|
+
# telling.
|
|
20
|
+
# @api private
|
|
21
|
+
type profile = Profile::report | { "unavailable" => String }
|
|
22
|
+
|
|
23
|
+
# Peak memory grows by more than MEMORY_RATIO, and by MEMORY_FLOOR
|
|
24
|
+
# bytes at least, before it's worse.
|
|
25
|
+
# @api private
|
|
26
|
+
MEMORY_RATIO: Float
|
|
27
|
+
|
|
28
|
+
# @api private
|
|
29
|
+
MEMORY_FLOOR: Integer
|
|
30
|
+
|
|
31
|
+
# What a sample says when no metric said anything.
|
|
32
|
+
# @api private
|
|
33
|
+
NOTHING_SAID: Hash[String, String]
|
|
34
|
+
|
|
35
|
+
# The verdicts that say a timing isn't slower.
|
|
36
|
+
# @api private
|
|
37
|
+
CLEARED: Array[String]
|
|
38
|
+
|
|
39
|
+
@name: String
|
|
40
|
+
@timed: Hash[String, timed]
|
|
41
|
+
@counted: Hash[String, counted]
|
|
42
|
+
@details: Hash[String, Hash[String, String]]
|
|
43
|
+
@confirmations: Array[Result]
|
|
44
|
+
@undecided: Array[Result]
|
|
45
|
+
@processes: Hash[String, processes]
|
|
46
|
+
@n: Integer?
|
|
47
|
+
@seconds: Float?
|
|
48
|
+
@skipped: String?
|
|
49
|
+
@error: String?
|
|
50
|
+
@memory: Hash[String, Integer]?
|
|
51
|
+
@profile: profile?
|
|
52
|
+
|
|
53
|
+
# The benchmark's name.
|
|
54
|
+
attr_reader name: String
|
|
55
|
+
|
|
56
|
+
# Each ratio measure's verdict and medians, by name.
|
|
57
|
+
attr_reader timed: Hash[String, timed]
|
|
58
|
+
|
|
59
|
+
# Each counted measure's fewest on each version, by name.
|
|
60
|
+
attr_reader counted: Hash[String, counted]
|
|
61
|
+
|
|
62
|
+
# What metrics said of the latest samples (Metric#details): by
|
|
63
|
+
# measure, by version.
|
|
64
|
+
attr_reader details: Hash[String, Hash[String, String]]
|
|
65
|
+
|
|
66
|
+
# The measurements that confirmed it was worse.
|
|
67
|
+
attr_reader confirmations: Array[Result]
|
|
68
|
+
|
|
69
|
+
# The measurements again that were too close to call.
|
|
70
|
+
# @api private
|
|
71
|
+
attr_reader undecided: Array[Result]
|
|
72
|
+
|
|
73
|
+
# Each worker's median of each measure judged by process, by metric.
|
|
74
|
+
# @api private
|
|
75
|
+
attr_reader processes: Hash[String, processes]
|
|
76
|
+
|
|
77
|
+
# Its iterations a sample.
|
|
78
|
+
attr_reader n: Integer?
|
|
79
|
+
|
|
80
|
+
# @api private
|
|
81
|
+
attr_writer n: Integer?
|
|
82
|
+
|
|
83
|
+
# The seconds it took.
|
|
84
|
+
attr_reader seconds: Float?
|
|
85
|
+
|
|
86
|
+
# @api private
|
|
87
|
+
attr_writer seconds: Float?
|
|
88
|
+
|
|
89
|
+
# Why it was skipped (the baseline couldn't run it), or nil.
|
|
90
|
+
attr_reader skipped: String?
|
|
91
|
+
|
|
92
|
+
# @api private
|
|
93
|
+
attr_writer skipped: String?
|
|
94
|
+
|
|
95
|
+
# How it failed, or nil.
|
|
96
|
+
attr_reader error: String?
|
|
97
|
+
|
|
98
|
+
# @api private
|
|
99
|
+
attr_writer error: String?
|
|
100
|
+
|
|
101
|
+
# Each version's peak memory, in bytes.
|
|
102
|
+
attr_reader memory: Hash[String, Integer]?
|
|
103
|
+
|
|
104
|
+
# @api private
|
|
105
|
+
attr_writer memory: Hash[String, Integer]?
|
|
106
|
+
|
|
107
|
+
# Where a regression's time went.
|
|
108
|
+
# @api private
|
|
109
|
+
attr_accessor profile: profile?
|
|
110
|
+
|
|
111
|
+
# @api private
|
|
112
|
+
def initialize: (String name) -> void
|
|
113
|
+
|
|
114
|
+
# Keeps what metrics said of a pair of samples.
|
|
115
|
+
# @api private
|
|
116
|
+
def note: (Worker::sampled base, Worker::sampled candidate, ?String? measure) -> void
|
|
117
|
+
|
|
118
|
+
# The metrics on which the candidate did worse.
|
|
119
|
+
# @api private
|
|
120
|
+
def worse: () -> Array[String]
|
|
121
|
+
|
|
122
|
+
# The timed metrics on which the candidate was slower.
|
|
123
|
+
# @api private
|
|
124
|
+
def slower: () -> Array[String]
|
|
125
|
+
|
|
126
|
+
# Whether its timings say slower: one is, and none says it isn't.
|
|
127
|
+
# @api private
|
|
128
|
+
def slower_in_time?: () -> bool
|
|
129
|
+
|
|
130
|
+
# Whether it's to be measured again.
|
|
131
|
+
# @api private
|
|
132
|
+
def confirming?: () -> bool
|
|
133
|
+
|
|
134
|
+
# Keeps a measurement again as a confirmation, or sets it aside
|
|
135
|
+
# where it's too close to call. Whether it was set aside.
|
|
136
|
+
# @api private
|
|
137
|
+
def confirm: (Result again) -> bool
|
|
138
|
+
|
|
139
|
+
# Whether its peak memory grew by too much.
|
|
140
|
+
# @api private
|
|
141
|
+
def more_memory?: () -> bool
|
|
142
|
+
|
|
143
|
+
# Worse on a metric in the first measurement and in every confirmation,
|
|
144
|
+
# or slower in time in all of them (see #mixed?).
|
|
145
|
+
def regressed: () -> Array[String]
|
|
146
|
+
|
|
147
|
+
# Whether it regressed in time on one timed metric in some
|
|
148
|
+
# measurements and on another in the rest.
|
|
149
|
+
# @api private
|
|
150
|
+
def mixed?: () -> bool
|
|
151
|
+
|
|
152
|
+
# Whether it regressed on nothing, and didn't fail.
|
|
153
|
+
def passed?: () -> bool
|
|
154
|
+
|
|
155
|
+
# Whether a timed measure's verdict was too close to the limit to settle.
|
|
156
|
+
# @api private
|
|
157
|
+
def unsettled?: () -> bool
|
|
158
|
+
|
|
159
|
+
private
|
|
160
|
+
|
|
161
|
+
# The first measurement and its confirmations.
|
|
162
|
+
def measurements: () -> Array[Result]
|
|
163
|
+
end
|
|
164
|
+
end
|
data/sig/duel/rubies.rbs
CHANGED
|
@@ -33,6 +33,10 @@ module Duel
|
|
|
33
33
|
# Whether +ruby+ is the runner's own.
|
|
34
34
|
def self.runner?: (String ruby) -> bool
|
|
35
35
|
|
|
36
|
+
# The command that starts +ruby+ with +env+: +ruby+, or on JRuby on
|
|
37
|
+
# Windows, the JVM's own command.
|
|
38
|
+
def self.launch: (String ruby, String engine, Hash[String, String?] env, ?windows: bool) -> Array[String]
|
|
39
|
+
|
|
36
40
|
# What a version's processes on +ruby+ get in their environment.
|
|
37
41
|
def self.environment: (String ruby, ?path: String?) -> Hash[String, String?]
|
|
38
42
|
|
data/sig/duel/runner.rbs
CHANGED
|
@@ -108,470 +108,15 @@ module Duel
|
|
|
108
108
|
def label: () -> String
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
-
# A worker process (Duel::Worker) running one version of the code.
|
|
112
|
-
# @api private
|
|
113
|
-
class WorkerProcess
|
|
114
|
-
# A worker's reply to a request (see Worker#handle), parsed from JSON.
|
|
115
|
-
type reply = untyped
|
|
116
|
-
|
|
117
|
-
LIB: String
|
|
118
|
-
POLL: Float
|
|
119
|
-
STOP_WAIT: Integer
|
|
120
|
-
# The gems that are Duel's: Duel, and its plugins.
|
|
121
|
-
DUEL_GEM: Regexp
|
|
122
|
-
|
|
123
|
-
@label: String
|
|
124
|
-
@timeout: Float
|
|
125
|
-
@root: String
|
|
126
|
-
@dir: String
|
|
127
|
-
@env: Hash[String, String?]
|
|
128
|
-
@ruby: String
|
|
129
|
-
@runs: ProcessSample?
|
|
130
|
-
@description: String
|
|
131
|
-
@extensions: Hash[String, String]
|
|
132
|
-
@twice: Hash[String, Array[String]]
|
|
133
|
-
@gem_dirs: Array[String]
|
|
134
|
-
@pid: Integer
|
|
135
|
-
@socket: UNIXSocket
|
|
136
|
-
@benchmarks: Hash[String, Definition::summary]
|
|
137
|
-
@metrics: Array[Metric::spec]
|
|
138
|
-
@memory: bool
|
|
139
|
-
@profiler: bool
|
|
140
|
-
@yjit: bool
|
|
141
|
-
|
|
142
|
-
attr_reader label: String
|
|
143
|
-
|
|
144
|
-
attr_reader benchmarks: Hash[String, Definition::summary]
|
|
145
|
-
|
|
146
|
-
attr_reader metrics: Array[Metric::spec]
|
|
147
|
-
|
|
148
|
-
attr_reader memory: bool
|
|
149
|
-
|
|
150
|
-
attr_reader profiler: bool
|
|
151
|
-
|
|
152
|
-
attr_reader yjit: bool
|
|
153
|
-
|
|
154
|
-
# The Ruby it runs on (RUBY_DESCRIPTION).
|
|
155
|
-
attr_reader description: String
|
|
156
|
-
|
|
157
|
-
# The native extensions it loaded, by feature.
|
|
158
|
-
attr_reader extensions: Hash[String, String]
|
|
159
|
-
|
|
160
|
-
# Those it loaded from more than one file, by feature.
|
|
161
|
-
attr_reader twice: Hash[String, Array[String]]
|
|
162
|
-
|
|
163
|
-
# The directories gems are installed in, as real paths ending in "/".
|
|
164
|
-
attr_reader gem_dirs: Array[String]
|
|
165
|
-
|
|
166
|
-
# Where a worker on +ruby+ loads Duel from, ahead of anything else.
|
|
167
|
-
def self.load_path: (String ruby, ?specs: Hash[String, Gem::BasicSpecification]) -> Array[String]
|
|
168
|
-
|
|
169
|
-
def initialize: (String label, String root, String candidate_root, String dir, config_path: String?,
|
|
170
|
-
env: Hash[String, String?], timeout: Float, ruby: String, engine: String) -> void
|
|
171
|
-
|
|
172
|
-
# Sends a request, and returns the reply, or raises Failed with its
|
|
173
|
-
# error.
|
|
174
|
-
def request: (String op, String name, **untyped args) -> reply
|
|
175
|
-
|
|
176
|
-
def sample: (String name, Integer n, ?runs: Integer, ?count: bool) -> reply
|
|
177
|
-
|
|
178
|
-
# A run of a process benchmark: its process, fresh, started as this
|
|
179
|
-
# worker was.
|
|
180
|
-
def run: (Definition::process process) -> ProcessSample::sampled
|
|
181
|
-
|
|
182
|
-
# Closing the connection ends the worker. One still running +wait+
|
|
183
|
-
# seconds later is killed.
|
|
184
|
-
def stop: (?wait: Numeric) -> void
|
|
185
|
-
|
|
186
|
-
private
|
|
187
|
-
|
|
188
|
-
def accept: (UNIXServer server, String path) -> UNIXSocket
|
|
189
|
-
|
|
190
|
-
def read: () -> reply
|
|
191
|
-
|
|
192
|
-
# Kills a worker that didn't +what+ within the timeout, and fails.
|
|
193
|
-
def hung: (String what) -> bot
|
|
194
|
-
|
|
195
|
-
def kill: () -> void
|
|
196
|
-
|
|
197
|
-
def monotonic: () -> Float
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
# What Duel raises when something a run needs fails: a resource that
|
|
201
|
-
# didn't start, a worker that died or hung.
|
|
202
|
-
class Failed < Error
|
|
203
|
-
end
|
|
204
|
-
|
|
205
|
-
# Workers of both versions, taking turns.
|
|
206
|
-
# @api private
|
|
207
|
-
class Lane
|
|
208
|
-
TARGET_NS: Integer
|
|
209
|
-
RUNS: Integer
|
|
210
|
-
WARMUP: Integer
|
|
211
|
-
HEAP_CAP: Integer
|
|
212
|
-
STEP: Integer
|
|
213
|
-
MAX_PAIRS: Integer
|
|
214
|
-
# The round of a sample from every worker from which a benchmark
|
|
215
|
-
# judged by process is decided.
|
|
216
|
-
MIN_ROUNDS: Integer
|
|
217
|
-
PROCESS_PAIRS: Integer
|
|
218
|
-
COUNTS: Integer
|
|
219
|
-
JRUBY_COUNTS: Integer
|
|
220
|
-
# The rounds of a sample from every worker a count by process takes.
|
|
221
|
-
COUNT_ROUNDS: Integer
|
|
222
|
-
MEMORY_SAMPLES: Integer
|
|
223
|
-
PROFILE_SAMPLES: Integer
|
|
224
|
-
# A process benchmark's pairs before any are measured, and its runs a
|
|
225
|
-
# sample (unless it says), each sample the fastest of them, as many
|
|
226
|
-
# as take PROCESS_SAMPLE_NS of CPU time.
|
|
227
|
-
PROCESS_WARMUP: Integer
|
|
228
|
-
PROCESS_RUNS: Integer
|
|
229
|
-
PROCESS_SAMPLE_NS: Integer
|
|
230
|
-
# The longest TMPDIR a lane's directory goes in.
|
|
231
|
-
SHORT: Integer
|
|
232
|
-
|
|
233
|
-
self.@ids: Integer
|
|
234
|
-
self.@ids_lock: Thread::Mutex
|
|
235
|
-
|
|
236
|
-
# Its number (see .next_id).
|
|
237
|
-
@id: Integer
|
|
238
|
-
@dir: String
|
|
239
|
-
@resources: Array[Resource]
|
|
240
|
-
@workers: Array[WorkerProcess]
|
|
241
|
-
@lock: Thread::Mutex
|
|
242
|
-
@config: Config
|
|
243
|
-
@settings: Settings
|
|
244
|
-
@sides: Hash[String, Side]
|
|
245
|
-
# A process benchmark's peak memory samples, by version.
|
|
246
|
-
@peaks: Hash[String, Array[Integer?]]
|
|
247
|
-
@bases: Array[WorkerProcess]
|
|
248
|
-
@candidates: Array[WorkerProcess]
|
|
249
|
-
@profiler: bool
|
|
250
|
-
# The warm-up's seconds (see #warm).
|
|
251
|
-
@warm: Float
|
|
252
|
-
|
|
253
|
-
attr_reader profiler: bool
|
|
254
|
-
|
|
255
|
-
def initialize: (Config config, Settings settings, Hash[String, Side] sides, ?processes: Integer) -> void
|
|
256
|
-
|
|
257
|
-
# A number of its own for each lane, from 1.
|
|
258
|
-
def self.next_id: () -> Integer
|
|
259
|
-
|
|
260
|
-
# Whether +processes+ workers of each version are enough for a
|
|
261
|
-
# +confidence+ interval over them.
|
|
262
|
-
def self.by_process?: (Integer processes, Float confidence) -> bool
|
|
263
|
-
|
|
264
|
-
# A directory for a lane's sockets.
|
|
265
|
-
def self.tmpdir: () -> String
|
|
266
|
-
|
|
267
|
-
# Where lanes' directories go: TMPDIR if it's short, else /tmp.
|
|
268
|
-
def self.short_tmp: (?String tmpdir) -> String
|
|
269
|
-
|
|
270
|
-
# Opens a lane, yields it and always closes it.
|
|
271
|
-
def self.open: [T] (Config config, Settings settings, Hash[String, Side] sides, ?processes: Integer) { (Lane lane) -> T } -> T
|
|
272
|
-
|
|
273
|
-
def benchmarks: () -> Hash[String, Definition::summary]
|
|
274
|
-
|
|
275
|
-
# The Ruby each version runs on, by version.
|
|
276
|
-
def rubies: () -> Hash[String, String]
|
|
277
|
-
|
|
278
|
-
def stop: () -> void
|
|
279
|
-
|
|
280
|
-
# Measures one benchmark on both versions.
|
|
281
|
-
def compare: (String name) -> Result
|
|
282
|
-
|
|
283
|
-
# Profiles of the benchmark on each version, or nil without StackProf.
|
|
284
|
-
def profile: (String name, Integer n) -> Hash[String, Profile::profile]?
|
|
285
|
-
|
|
286
|
-
private
|
|
287
|
-
|
|
288
|
-
def spawn: (String side, Integer index) -> WorkerProcess
|
|
289
|
-
|
|
290
|
-
# Fails if a version's workers loaded a native extension that isn't a
|
|
291
|
-
# build of its own.
|
|
292
|
-
def check_builds: () -> void
|
|
293
|
-
|
|
294
|
-
# Whether +path+ (that +worker+ loaded) is in +side+'s tree, and not
|
|
295
|
-
# in an installed gem.
|
|
296
|
-
def inside?: (String path, String side, WorkerProcess worker) -> bool
|
|
297
|
-
|
|
298
|
-
def each_worker: () { (WorkerProcess worker) -> void } -> void
|
|
299
|
-
|
|
300
|
-
def set_up: (String name, Result result) -> void
|
|
301
|
-
|
|
302
|
-
def metrics: () -> Array[Metric::spec]
|
|
303
|
-
|
|
304
|
-
# What the block returns for each item, each on a thread of its own.
|
|
305
|
-
# Adds an item to a list, from a thread of its own.
|
|
306
|
-
def keep: [T] (Array[T] list, T item) -> void
|
|
307
|
-
|
|
308
|
-
# Whether workers start (and stop) at once.
|
|
309
|
-
PARALLEL: bool
|
|
310
|
-
|
|
311
|
-
# (Untyped: Steep loses a generic in a thread's block.)
|
|
312
|
-
def in_parallel: (Array[untyped] items) { (untyped item) -> untyped } -> Array[untyped]
|
|
313
|
-
|
|
314
|
-
def measure: (String name, Result result) -> void
|
|
315
|
-
|
|
316
|
-
# A process benchmark's samples.
|
|
317
|
-
def measure_process: (String name, Result result) -> void
|
|
318
|
-
|
|
319
|
-
# The runs a process benchmark's sample takes, by its warm-up's.
|
|
320
|
-
def process_runs: (Array[[WorkerProcess::reply, WorkerProcess::reply]] floors) -> Integer
|
|
321
|
-
|
|
322
|
-
def warm: (String name, Integer n) -> void
|
|
323
|
-
|
|
324
|
-
# The seconds of samples whose median, no faster by STEADY_GAIN than
|
|
325
|
-
# the last's, end a worker's warm-up, without --warm.
|
|
326
|
-
STEADY: Float
|
|
327
|
-
STEADY_GAIN: Float
|
|
328
|
-
STEADY_WINDOWS: Integer
|
|
329
|
-
|
|
330
|
-
# Samples a worker until it's steady, or it's +warm_until+.
|
|
331
|
-
def steady: (WorkerProcess worker, String name, Integer n, Float warm_until) -> void
|
|
332
|
-
|
|
333
|
-
# STEADY seconds of a worker's samples' CPU time, and when they ended.
|
|
334
|
-
def window: (WorkerProcess worker, String name, Integer n, Float warm_until) -> [Array[Metric::value], Float]
|
|
335
|
-
|
|
336
|
-
def judge: (String name, Integer n, Array[[WorkerProcess::reply, WorkerProcess::reply]] floors, Result result) -> void
|
|
337
|
-
|
|
338
|
-
# Whether a benchmark's timings are judged by process.
|
|
339
|
-
def by_process?: (String name) -> bool
|
|
340
|
-
|
|
341
|
-
# Judges each of +judged+ over workers, in rounds of a sample from
|
|
342
|
-
# every worker.
|
|
343
|
-
def judge_processes: (String name, Integer n, Array[Metric::spec] judged, Result result) -> void
|
|
344
|
-
|
|
345
|
-
# The most rounds of +k+ workers' pairs.
|
|
346
|
-
def rounds: (Integer k) -> Integer
|
|
347
|
-
|
|
348
|
-
# Each worker's median, by version.
|
|
349
|
-
def medians: (Hash[String, Array[Array[Metric::value]]] values) -> Hash[String, Array[Metric::value]]
|
|
350
|
-
|
|
351
|
-
# The baseline's medians, at least 1, and the candidate's.
|
|
352
|
-
def clamped: (Hash[String, Array[Metric::value]] each) -> [Array[Metric::value], Array[Metric::value]]
|
|
353
|
-
|
|
354
|
-
# What each worker's medians say against the spec's limit.
|
|
355
|
-
def process_verdict: (Hash[String, Array[Metric::value]] each, Metric::spec spec) -> Stats::verdict
|
|
356
|
-
|
|
357
|
-
# Whether both versions' CPU time is below its floor.
|
|
358
|
-
def below_cpu_floor?: (Array[[Worker::sampled, Worker::sampled]] floors) -> bool
|
|
359
|
-
|
|
360
|
-
# Whether one version's samples are above a spec's floors.
|
|
361
|
-
def above_floors?: (Array[Hash[String, Metric::value]] samples, Metric::spec spec) -> bool
|
|
362
|
-
|
|
363
|
-
def limit: (Metric::spec spec) -> Float
|
|
364
|
-
|
|
365
|
-
def verdict: (Array[Float] ratios, Metric::spec spec) -> Stats::verdict
|
|
366
|
-
|
|
367
|
-
# The verdicts that decide a measure at the end of a round, and after
|
|
368
|
-
# any other pair.
|
|
369
|
-
DECISIVE: Hash[bool, Array[Stats::verdict]]
|
|
370
|
-
|
|
371
|
-
# Whether +verdict+ decides a measure, at the end of a round or not.
|
|
372
|
-
def decided?: (Stats::verdict verdict, bool round) -> bool
|
|
373
|
-
|
|
374
|
-
def calibrate: (String name, Integer target, Integer n) -> Integer
|
|
375
|
-
|
|
376
|
-
# The most iterations whose allocations fit HEAP_CAP (infinite when it
|
|
377
|
-
# allocated nothing).
|
|
378
|
-
def most_iterations: (Integer n, Integer heap) -> (Integer | Float)
|
|
379
|
-
|
|
380
|
-
# The +index+th pair: [baseline's reply, candidate's].
|
|
381
|
-
def pair: (String name, Integer n, Integer index, ?runs: Integer, ?count: bool) -> [WorkerProcess::reply, WorkerProcess::reply]
|
|
382
|
-
|
|
383
|
-
# A sample from a baseline worker and from a candidate worker: [baseline's reply, candidate's].
|
|
384
|
-
def sampled: (String name, Integer n, WorkerProcess base, WorkerProcess candidate, base_first: bool, ?runs: Integer, ?count: bool) -> [WorkerProcess::reply, WorkerProcess::reply]
|
|
385
|
-
|
|
386
|
-
def count: (String name, Integer n, Result result) -> void
|
|
387
|
-
|
|
388
|
-
# The samples a count takes.
|
|
389
|
-
def counts: () -> Integer
|
|
390
|
-
|
|
391
|
-
# Each counted measure, by each worker's fewest, and how much more
|
|
392
|
-
# the candidate's are at least.
|
|
393
|
-
def count_by_process: (String name, Integer n, Result result) -> void
|
|
394
|
-
|
|
395
|
-
# Keeps each count of a worker's sample where it's that worker's
|
|
396
|
-
# fewest so far.
|
|
397
|
-
def fewest!: (Hash[String, Hash[String, Hash[Integer, Metric::value]]] fewest, String side, Integer worker, WorkerProcess::reply sample) -> void
|
|
398
|
-
|
|
399
|
-
def peak_memory: (String name, Integer n, Result result) -> void
|
|
400
|
-
|
|
401
|
-
def fresh: [T] (String side) { (WorkerProcess worker) -> T } -> T
|
|
402
|
-
|
|
403
|
-
def monotonic: () -> Float
|
|
404
|
-
end
|
|
405
|
-
|
|
406
|
-
# What one comparison of a benchmark found (Testing::Comparison#results
|
|
407
|
-
# has them).
|
|
408
|
-
class Result
|
|
409
|
-
# A ratio metric's verdict, and each version's (the median of its
|
|
410
|
-
# samples).
|
|
411
|
-
type timed = { "median" => Float, "limit" => Float, "across" => bool, "unit" => String?, "plural" => String?, "interval" => Array[Float]?, "pairs" => Integer,
|
|
412
|
-
?"processes" => Integer, "verdict" => String, "baseline" => Metric::value, "candidate" => Metric::value }
|
|
413
|
-
|
|
414
|
-
# Each worker's median of a measure judged by process, by version,
|
|
415
|
-
# and its lane's number.
|
|
416
|
-
type processes = { "lane" => Integer, "baseline" => Array[Metric::value], "candidate" => Array[Metric::value] }
|
|
417
|
-
|
|
418
|
-
# A counted metric's fewest, on each version, in +n+ iterations.
|
|
419
|
-
type counted = { "baseline" => Metric::value, "candidate" => Metric::value, ?"more" => Metric::value?, "n" => Integer,
|
|
420
|
-
"tolerance" => Metric::value, "allowance" => Metric::value, "unit" => String?, "plural" => String? }
|
|
421
|
-
|
|
422
|
-
# Where a regression's time went (see Profile#to_h), or why there's no
|
|
423
|
-
# telling.
|
|
424
|
-
# @api private
|
|
425
|
-
type profile = Profile::report | { "unavailable" => String }
|
|
426
|
-
|
|
427
|
-
# Peak memory grows by more than MEMORY_RATIO, and by MEMORY_FLOOR
|
|
428
|
-
# bytes at least, before it's worse.
|
|
429
|
-
# @api private
|
|
430
|
-
MEMORY_RATIO: Float
|
|
431
|
-
|
|
432
|
-
# @api private
|
|
433
|
-
MEMORY_FLOOR: Integer
|
|
434
|
-
|
|
435
|
-
# What a sample says when no metric said anything.
|
|
436
|
-
# @api private
|
|
437
|
-
NOTHING_SAID: Hash[String, String]
|
|
438
|
-
|
|
439
|
-
# The verdicts that say a timing isn't slower.
|
|
440
|
-
# @api private
|
|
441
|
-
CLEARED: Array[String]
|
|
442
|
-
|
|
443
|
-
@name: String
|
|
444
|
-
@timed: Hash[String, timed]
|
|
445
|
-
@counted: Hash[String, counted]
|
|
446
|
-
@details: Hash[String, Hash[String, String]]
|
|
447
|
-
@confirmations: Array[Result]
|
|
448
|
-
@undecided: Array[Result]
|
|
449
|
-
@processes: Hash[String, processes]
|
|
450
|
-
@n: Integer?
|
|
451
|
-
@seconds: Float?
|
|
452
|
-
@skipped: String?
|
|
453
|
-
@error: String?
|
|
454
|
-
@memory: Hash[String, Integer]?
|
|
455
|
-
@profile: profile?
|
|
456
|
-
|
|
457
|
-
# The benchmark's name.
|
|
458
|
-
attr_reader name: String
|
|
459
|
-
|
|
460
|
-
# Each ratio measure's verdict and medians, by name.
|
|
461
|
-
attr_reader timed: Hash[String, timed]
|
|
462
|
-
|
|
463
|
-
# Each counted measure's fewest on each version, by name.
|
|
464
|
-
attr_reader counted: Hash[String, counted]
|
|
465
|
-
|
|
466
|
-
# What metrics said of the latest samples (Metric#details): by
|
|
467
|
-
# measure, by version.
|
|
468
|
-
attr_reader details: Hash[String, Hash[String, String]]
|
|
469
|
-
|
|
470
|
-
# The measurements that confirmed it was worse.
|
|
471
|
-
attr_reader confirmations: Array[Result]
|
|
472
|
-
|
|
473
|
-
# The measurements again that were too close to call.
|
|
474
|
-
# @api private
|
|
475
|
-
attr_reader undecided: Array[Result]
|
|
476
|
-
|
|
477
|
-
# Each worker's median of each measure judged by process, by metric.
|
|
478
|
-
# @api private
|
|
479
|
-
attr_reader processes: Hash[String, processes]
|
|
480
|
-
|
|
481
|
-
# Its iterations a sample.
|
|
482
|
-
attr_reader n: Integer?
|
|
483
|
-
|
|
484
|
-
# @api private
|
|
485
|
-
attr_writer n: Integer?
|
|
486
|
-
|
|
487
|
-
# The seconds it took.
|
|
488
|
-
attr_reader seconds: Float?
|
|
489
|
-
|
|
490
|
-
# @api private
|
|
491
|
-
attr_writer seconds: Float?
|
|
492
|
-
|
|
493
|
-
# Why it was skipped (the baseline couldn't run it), or nil.
|
|
494
|
-
attr_reader skipped: String?
|
|
495
|
-
|
|
496
|
-
# @api private
|
|
497
|
-
attr_writer skipped: String?
|
|
498
|
-
|
|
499
|
-
# How it failed, or nil.
|
|
500
|
-
attr_reader error: String?
|
|
501
|
-
|
|
502
|
-
# @api private
|
|
503
|
-
attr_writer error: String?
|
|
504
|
-
|
|
505
|
-
# Each version's peak memory, in bytes.
|
|
506
|
-
attr_reader memory: Hash[String, Integer]?
|
|
507
|
-
|
|
508
|
-
# @api private
|
|
509
|
-
attr_writer memory: Hash[String, Integer]?
|
|
510
|
-
|
|
511
|
-
# Where a regression's time went.
|
|
512
|
-
# @api private
|
|
513
|
-
attr_accessor profile: profile?
|
|
514
|
-
|
|
515
|
-
# @api private
|
|
516
|
-
def initialize: (String name) -> void
|
|
517
|
-
|
|
518
|
-
# Keeps what metrics said of a pair of samples.
|
|
519
|
-
# @api private
|
|
520
|
-
def note: (Worker::sampled base, Worker::sampled candidate, ?String? measure) -> void
|
|
521
|
-
|
|
522
|
-
# The metrics on which the candidate did worse.
|
|
523
|
-
# @api private
|
|
524
|
-
def worse: () -> Array[String]
|
|
525
|
-
|
|
526
|
-
# The timed metrics on which the candidate was slower.
|
|
527
|
-
# @api private
|
|
528
|
-
def slower: () -> Array[String]
|
|
529
|
-
|
|
530
|
-
# Whether its timings say slower: one is, and none says it isn't.
|
|
531
|
-
# @api private
|
|
532
|
-
def slower_in_time?: () -> bool
|
|
533
|
-
|
|
534
|
-
# Whether it's to be measured again.
|
|
535
|
-
# @api private
|
|
536
|
-
def confirming?: () -> bool
|
|
537
|
-
|
|
538
|
-
# Keeps a measurement again as a confirmation, or sets it aside
|
|
539
|
-
# where it's too close to call. Whether it was set aside.
|
|
540
|
-
# @api private
|
|
541
|
-
def confirm: (Result again) -> bool
|
|
542
|
-
|
|
543
|
-
# Whether its peak memory grew by too much.
|
|
544
|
-
# @api private
|
|
545
|
-
def more_memory?: () -> bool
|
|
546
|
-
|
|
547
|
-
# Worse on a metric in the first measurement and in every confirmation,
|
|
548
|
-
# or slower in time in all of them (see #mixed?).
|
|
549
|
-
def regressed: () -> Array[String]
|
|
550
|
-
|
|
551
|
-
# Whether it regressed in time on one timed metric in some
|
|
552
|
-
# measurements and on another in the rest.
|
|
553
|
-
# @api private
|
|
554
|
-
def mixed?: () -> bool
|
|
555
|
-
|
|
556
|
-
# Whether it regressed on nothing, and didn't fail.
|
|
557
|
-
def passed?: () -> bool
|
|
558
|
-
|
|
559
|
-
# Whether a timed measure's verdict was too close to the limit to settle.
|
|
560
|
-
# @api private
|
|
561
|
-
def unsettled?: () -> bool
|
|
562
|
-
|
|
563
|
-
private
|
|
564
|
-
|
|
565
|
-
# The first measurement and its confirmations.
|
|
566
|
-
def measurements: () -> Array[Result]
|
|
567
|
-
end
|
|
568
|
-
|
|
569
111
|
# Measures every benchmark, confirms what looks worse, and reports.
|
|
570
112
|
# @api private
|
|
571
113
|
class Runner
|
|
572
114
|
# A count to four figures, or a whole number from NUMBER.
|
|
573
115
|
def self.number: (Metric::value value) -> String
|
|
574
116
|
|
|
117
|
+
# Bytes of memory, in megabytes to a tenth.
|
|
118
|
+
def self.mb: (Integer bytes) -> String
|
|
119
|
+
|
|
575
120
|
# The least count #number shows as a whole number.
|
|
576
121
|
NUMBER: Float
|
|
577
122
|
|
|
@@ -657,6 +202,9 @@ module Duel
|
|
|
657
202
|
# reckoning.
|
|
658
203
|
HEADROOM: Float
|
|
659
204
|
|
|
205
|
+
# How many times a JIT engine's worker grows as it warms up.
|
|
206
|
+
JIT_GROWTH: Hash[String, Float]
|
|
207
|
+
|
|
660
208
|
# Lowers the jobs, then the processes, that Settings#fit lets it, to
|
|
661
209
|
# fit the memory available.
|
|
662
210
|
def fit: () -> void
|
|
@@ -690,8 +238,6 @@ module Duel
|
|
|
690
238
|
|
|
691
239
|
def line: (Result result) -> String
|
|
692
240
|
|
|
693
|
-
def mb: (Integer bytes) -> String
|
|
694
|
-
|
|
695
241
|
# The unit of +measured+, for +number+ of it: singular for one.
|
|
696
242
|
def unit: (Metric::value number, Result::timed | Result::counted measured) -> String?
|
|
697
243
|
|