duel 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +146 -1
- data/README.md +35 -0
- data/docs/ci.md +96 -3
- data/docs/configuration.md +35 -1
- data/docs/how-it-works.md +14 -8
- data/docs/plugins.md +16 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +45 -19
- data/lib/duel/comment.rb +71 -10
- data/lib/duel/config.rb +113 -2
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +142 -18
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +13 -1039
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +10 -10
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +45 -3
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +27 -3
- data/sig/duel/cli.rbs +8 -0
- data/sig/duel/comment.rbs +16 -1
- data/sig/duel/config.rbs +52 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +56 -0
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +3 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +3 -2
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -107
data/sig/duel/runner.rbs
CHANGED
|
@@ -108,470 +108,15 @@ module Duel
|
|
|
108
108
|
def label: () -> String
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
-
# A worker process (Duel::Worker) running one version of the code.
|
|
112
|
-
# @api private
|
|
113
|
-
class WorkerProcess
|
|
114
|
-
# A worker's reply to a request (see Worker#handle), parsed from JSON.
|
|
115
|
-
type reply = untyped
|
|
116
|
-
|
|
117
|
-
LIB: String
|
|
118
|
-
POLL: Float
|
|
119
|
-
STOP_WAIT: Integer
|
|
120
|
-
# The gems that are Duel's: Duel, and its plugins.
|
|
121
|
-
DUEL_GEM: Regexp
|
|
122
|
-
|
|
123
|
-
@label: String
|
|
124
|
-
@timeout: Float
|
|
125
|
-
@root: String
|
|
126
|
-
@dir: String
|
|
127
|
-
@env: Hash[String, String?]
|
|
128
|
-
@ruby: String
|
|
129
|
-
@runs: ProcessSample?
|
|
130
|
-
@description: String
|
|
131
|
-
@extensions: Hash[String, String]
|
|
132
|
-
@twice: Hash[String, Array[String]]
|
|
133
|
-
@gem_dirs: Array[String]
|
|
134
|
-
@pid: Integer
|
|
135
|
-
@socket: UNIXSocket
|
|
136
|
-
@benchmarks: Hash[String, Definition::summary]
|
|
137
|
-
@metrics: Array[Metric::spec]
|
|
138
|
-
@memory: bool
|
|
139
|
-
@profiler: bool
|
|
140
|
-
@yjit: bool
|
|
141
|
-
|
|
142
|
-
attr_reader label: String
|
|
143
|
-
|
|
144
|
-
attr_reader benchmarks: Hash[String, Definition::summary]
|
|
145
|
-
|
|
146
|
-
attr_reader metrics: Array[Metric::spec]
|
|
147
|
-
|
|
148
|
-
attr_reader memory: bool
|
|
149
|
-
|
|
150
|
-
attr_reader profiler: bool
|
|
151
|
-
|
|
152
|
-
attr_reader yjit: bool
|
|
153
|
-
|
|
154
|
-
# The Ruby it runs on (RUBY_DESCRIPTION).
|
|
155
|
-
attr_reader description: String
|
|
156
|
-
|
|
157
|
-
# The native extensions it loaded, by feature.
|
|
158
|
-
attr_reader extensions: Hash[String, String]
|
|
159
|
-
|
|
160
|
-
# Those it loaded from more than one file, by feature.
|
|
161
|
-
attr_reader twice: Hash[String, Array[String]]
|
|
162
|
-
|
|
163
|
-
# The directories gems are installed in, as real paths ending in "/".
|
|
164
|
-
attr_reader gem_dirs: Array[String]
|
|
165
|
-
|
|
166
|
-
# Where a worker on +ruby+ loads Duel from, ahead of anything else.
|
|
167
|
-
def self.load_path: (String ruby, ?specs: Hash[String, Gem::BasicSpecification]) -> Array[String]
|
|
168
|
-
|
|
169
|
-
def initialize: (String label, String root, String candidate_root, String dir, config_path: String?,
|
|
170
|
-
env: Hash[String, String?], timeout: Float, ruby: String, engine: String) -> void
|
|
171
|
-
|
|
172
|
-
# Sends a request, and returns the reply, or raises Failed with its
|
|
173
|
-
# error.
|
|
174
|
-
def request: (String op, String name, **untyped args) -> reply
|
|
175
|
-
|
|
176
|
-
def sample: (String name, Integer n, ?runs: Integer, ?count: bool) -> reply
|
|
177
|
-
|
|
178
|
-
# A run of a process benchmark: its process, fresh, started as this
|
|
179
|
-
# worker was.
|
|
180
|
-
def run: (Definition::process process) -> ProcessSample::sampled
|
|
181
|
-
|
|
182
|
-
# Closing the connection ends the worker. One still running +wait+
|
|
183
|
-
# seconds later is killed.
|
|
184
|
-
def stop: (?wait: Numeric) -> void
|
|
185
|
-
|
|
186
|
-
private
|
|
187
|
-
|
|
188
|
-
def accept: (UNIXServer server, String path) -> UNIXSocket
|
|
189
|
-
|
|
190
|
-
def read: () -> reply
|
|
191
|
-
|
|
192
|
-
# Kills a worker that didn't +what+ within the timeout, and fails.
|
|
193
|
-
def hung: (String what) -> bot
|
|
194
|
-
|
|
195
|
-
def kill: () -> void
|
|
196
|
-
|
|
197
|
-
def monotonic: () -> Float
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
# What Duel raises when something a run needs fails: a resource that
|
|
201
|
-
# didn't start, a worker that died or hung.
|
|
202
|
-
class Failed < Error
|
|
203
|
-
end
|
|
204
|
-
|
|
205
|
-
# Workers of both versions, taking turns.
|
|
206
|
-
# @api private
|
|
207
|
-
class Lane
|
|
208
|
-
TARGET_NS: Integer
|
|
209
|
-
RUNS: Integer
|
|
210
|
-
WARMUP: Integer
|
|
211
|
-
HEAP_CAP: Integer
|
|
212
|
-
STEP: Integer
|
|
213
|
-
MAX_PAIRS: Integer
|
|
214
|
-
# The round of a sample from every worker from which a benchmark
|
|
215
|
-
# judged by process is decided.
|
|
216
|
-
MIN_ROUNDS: Integer
|
|
217
|
-
PROCESS_PAIRS: Integer
|
|
218
|
-
COUNTS: Integer
|
|
219
|
-
JRUBY_COUNTS: Integer
|
|
220
|
-
# The rounds of a sample from every worker a count by process takes.
|
|
221
|
-
COUNT_ROUNDS: Integer
|
|
222
|
-
MEMORY_SAMPLES: Integer
|
|
223
|
-
PROFILE_SAMPLES: Integer
|
|
224
|
-
# A process benchmark's pairs before any are measured, and its runs a
|
|
225
|
-
# sample (unless it says), each sample the fastest of them, as many
|
|
226
|
-
# as take PROCESS_SAMPLE_NS of CPU time.
|
|
227
|
-
PROCESS_WARMUP: Integer
|
|
228
|
-
PROCESS_RUNS: Integer
|
|
229
|
-
PROCESS_SAMPLE_NS: Integer
|
|
230
|
-
# The longest TMPDIR a lane's directory goes in.
|
|
231
|
-
SHORT: Integer
|
|
232
|
-
|
|
233
|
-
self.@ids: Integer
|
|
234
|
-
self.@ids_lock: Thread::Mutex
|
|
235
|
-
|
|
236
|
-
# Its number (see .next_id).
|
|
237
|
-
@id: Integer
|
|
238
|
-
@dir: String
|
|
239
|
-
@resources: Array[Resource]
|
|
240
|
-
@workers: Array[WorkerProcess]
|
|
241
|
-
@lock: Thread::Mutex
|
|
242
|
-
@config: Config
|
|
243
|
-
@settings: Settings
|
|
244
|
-
@sides: Hash[String, Side]
|
|
245
|
-
# A process benchmark's peak memory samples, by version.
|
|
246
|
-
@peaks: Hash[String, Array[Integer?]]
|
|
247
|
-
@bases: Array[WorkerProcess]
|
|
248
|
-
@candidates: Array[WorkerProcess]
|
|
249
|
-
@profiler: bool
|
|
250
|
-
# The warm-up's seconds (see #warm).
|
|
251
|
-
@warm: Float
|
|
252
|
-
|
|
253
|
-
attr_reader profiler: bool
|
|
254
|
-
|
|
255
|
-
def initialize: (Config config, Settings settings, Hash[String, Side] sides, ?processes: Integer) -> void
|
|
256
|
-
|
|
257
|
-
# A number of its own for each lane, from 1.
|
|
258
|
-
def self.next_id: () -> Integer
|
|
259
|
-
|
|
260
|
-
# Whether +processes+ workers of each version are enough for a
|
|
261
|
-
# +confidence+ interval over them.
|
|
262
|
-
def self.by_process?: (Integer processes, Float confidence) -> bool
|
|
263
|
-
|
|
264
|
-
# A directory for a lane's sockets.
|
|
265
|
-
def self.tmpdir: () -> String
|
|
266
|
-
|
|
267
|
-
# Where lanes' directories go: TMPDIR if it's short, else /tmp.
|
|
268
|
-
def self.short_tmp: (?String tmpdir) -> String
|
|
269
|
-
|
|
270
|
-
# Opens a lane, yields it and always closes it.
|
|
271
|
-
def self.open: [T] (Config config, Settings settings, Hash[String, Side] sides, ?processes: Integer) { (Lane lane) -> T } -> T
|
|
272
|
-
|
|
273
|
-
def benchmarks: () -> Hash[String, Definition::summary]
|
|
274
|
-
|
|
275
|
-
# The Ruby each version runs on, by version.
|
|
276
|
-
def rubies: () -> Hash[String, String]
|
|
277
|
-
|
|
278
|
-
def stop: () -> void
|
|
279
|
-
|
|
280
|
-
# Measures one benchmark on both versions.
|
|
281
|
-
def compare: (String name) -> Result
|
|
282
|
-
|
|
283
|
-
# Profiles of the benchmark on each version, or nil without StackProf.
|
|
284
|
-
def profile: (String name, Integer n) -> Hash[String, Profile::profile]?
|
|
285
|
-
|
|
286
|
-
private
|
|
287
|
-
|
|
288
|
-
def spawn: (String side, Integer index) -> WorkerProcess
|
|
289
|
-
|
|
290
|
-
# Fails if a version's workers loaded a native extension that isn't a
|
|
291
|
-
# build of its own.
|
|
292
|
-
def check_builds: () -> void
|
|
293
|
-
|
|
294
|
-
# Whether +path+ (that +worker+ loaded) is in +side+'s tree, and not
|
|
295
|
-
# in an installed gem.
|
|
296
|
-
def inside?: (String path, String side, WorkerProcess worker) -> bool
|
|
297
|
-
|
|
298
|
-
def each_worker: () { (WorkerProcess worker) -> void } -> void
|
|
299
|
-
|
|
300
|
-
def set_up: (String name, Result result) -> void
|
|
301
|
-
|
|
302
|
-
def metrics: () -> Array[Metric::spec]
|
|
303
|
-
|
|
304
|
-
# What the block returns for each item, each on a thread of its own.
|
|
305
|
-
# Adds an item to a list, from a thread of its own.
|
|
306
|
-
def keep: [T] (Array[T] list, T item) -> void
|
|
307
|
-
|
|
308
|
-
# Whether workers start (and stop) at once.
|
|
309
|
-
PARALLEL: bool
|
|
310
|
-
|
|
311
|
-
# (Untyped: Steep loses a generic in a thread's block.)
|
|
312
|
-
def in_parallel: (Array[untyped] items) { (untyped item) -> untyped } -> Array[untyped]
|
|
313
|
-
|
|
314
|
-
def measure: (String name, Result result) -> void
|
|
315
|
-
|
|
316
|
-
# A process benchmark's samples.
|
|
317
|
-
def measure_process: (String name, Result result) -> void
|
|
318
|
-
|
|
319
|
-
# The runs a process benchmark's sample takes, by its warm-up's.
|
|
320
|
-
def process_runs: (Array[[WorkerProcess::reply, WorkerProcess::reply]] floors) -> Integer
|
|
321
|
-
|
|
322
|
-
def warm: (String name, Integer n) -> void
|
|
323
|
-
|
|
324
|
-
# The seconds of samples whose median, no faster by STEADY_GAIN than
|
|
325
|
-
# the last's, end a worker's warm-up, without --warm.
|
|
326
|
-
STEADY: Float
|
|
327
|
-
STEADY_GAIN: Float
|
|
328
|
-
STEADY_WINDOWS: Integer
|
|
329
|
-
|
|
330
|
-
# Samples a worker until it's steady, or it's +warm_until+.
|
|
331
|
-
def steady: (WorkerProcess worker, String name, Integer n, Float warm_until) -> void
|
|
332
|
-
|
|
333
|
-
# STEADY seconds of a worker's samples' CPU time, and when they ended.
|
|
334
|
-
def window: (WorkerProcess worker, String name, Integer n, Float warm_until) -> [Array[Metric::value], Float]
|
|
335
|
-
|
|
336
|
-
def judge: (String name, Integer n, Array[[WorkerProcess::reply, WorkerProcess::reply]] floors, Result result) -> void
|
|
337
|
-
|
|
338
|
-
# Whether a benchmark's timings are judged by process.
|
|
339
|
-
def by_process?: (String name) -> bool
|
|
340
|
-
|
|
341
|
-
# Judges each of +judged+ over workers, in rounds of a sample from
|
|
342
|
-
# every worker.
|
|
343
|
-
def judge_processes: (String name, Integer n, Array[Metric::spec] judged, Result result) -> void
|
|
344
|
-
|
|
345
|
-
# The most rounds of +k+ workers' pairs.
|
|
346
|
-
def rounds: (Integer k) -> Integer
|
|
347
|
-
|
|
348
|
-
# Each worker's median, by version.
|
|
349
|
-
def medians: (Hash[String, Array[Array[Metric::value]]] values) -> Hash[String, Array[Metric::value]]
|
|
350
|
-
|
|
351
|
-
# The baseline's medians, at least 1, and the candidate's.
|
|
352
|
-
def clamped: (Hash[String, Array[Metric::value]] each) -> [Array[Metric::value], Array[Metric::value]]
|
|
353
|
-
|
|
354
|
-
# What each worker's medians say against the spec's limit.
|
|
355
|
-
def process_verdict: (Hash[String, Array[Metric::value]] each, Metric::spec spec) -> Stats::verdict
|
|
356
|
-
|
|
357
|
-
# Whether both versions' CPU time is below its floor.
|
|
358
|
-
def below_cpu_floor?: (Array[[Worker::sampled, Worker::sampled]] floors) -> bool
|
|
359
|
-
|
|
360
|
-
# Whether one version's samples are above a spec's floors.
|
|
361
|
-
def above_floors?: (Array[Hash[String, Metric::value]] samples, Metric::spec spec) -> bool
|
|
362
|
-
|
|
363
|
-
def limit: (Metric::spec spec) -> Float
|
|
364
|
-
|
|
365
|
-
def verdict: (Array[Float] ratios, Metric::spec spec) -> Stats::verdict
|
|
366
|
-
|
|
367
|
-
# The verdicts that decide a measure at the end of a round, and after
|
|
368
|
-
# any other pair.
|
|
369
|
-
DECISIVE: Hash[bool, Array[Stats::verdict]]
|
|
370
|
-
|
|
371
|
-
# Whether +verdict+ decides a measure, at the end of a round or not.
|
|
372
|
-
def decided?: (Stats::verdict verdict, bool round) -> bool
|
|
373
|
-
|
|
374
|
-
def calibrate: (String name, Integer target, Integer n) -> Integer
|
|
375
|
-
|
|
376
|
-
# The most iterations whose allocations fit HEAP_CAP (infinite when it
|
|
377
|
-
# allocated nothing).
|
|
378
|
-
def most_iterations: (Integer n, Integer heap) -> (Integer | Float)
|
|
379
|
-
|
|
380
|
-
# The +index+th pair: [baseline's reply, candidate's].
|
|
381
|
-
def pair: (String name, Integer n, Integer index, ?runs: Integer, ?count: bool) -> [WorkerProcess::reply, WorkerProcess::reply]
|
|
382
|
-
|
|
383
|
-
# A sample from a baseline worker and from a candidate worker: [baseline's reply, candidate's].
|
|
384
|
-
def sampled: (String name, Integer n, WorkerProcess base, WorkerProcess candidate, base_first: bool, ?runs: Integer, ?count: bool) -> [WorkerProcess::reply, WorkerProcess::reply]
|
|
385
|
-
|
|
386
|
-
def count: (String name, Integer n, Result result) -> void
|
|
387
|
-
|
|
388
|
-
# The samples a count takes.
|
|
389
|
-
def counts: () -> Integer
|
|
390
|
-
|
|
391
|
-
# Each counted measure, by each worker's fewest, and how much more
|
|
392
|
-
# the candidate's are at least.
|
|
393
|
-
def count_by_process: (String name, Integer n, Result result) -> void
|
|
394
|
-
|
|
395
|
-
# Keeps each count of a worker's sample where it's that worker's
|
|
396
|
-
# fewest so far.
|
|
397
|
-
def fewest!: (Hash[String, Hash[String, Hash[Integer, Metric::value]]] fewest, String side, Integer worker, WorkerProcess::reply sample) -> void
|
|
398
|
-
|
|
399
|
-
def peak_memory: (String name, Integer n, Result result) -> void
|
|
400
|
-
|
|
401
|
-
def fresh: [T] (String side) { (WorkerProcess worker) -> T } -> T
|
|
402
|
-
|
|
403
|
-
def monotonic: () -> Float
|
|
404
|
-
end
|
|
405
|
-
|
|
406
|
-
# What one comparison of a benchmark found (Testing::Comparison#results
|
|
407
|
-
# has them).
|
|
408
|
-
class Result
|
|
409
|
-
# A ratio metric's verdict, and each version's (the median of its
|
|
410
|
-
# samples).
|
|
411
|
-
type timed = { "median" => Float, "limit" => Float, "across" => bool, "unit" => String?, "plural" => String?, "interval" => Array[Float]?, "pairs" => Integer,
|
|
412
|
-
?"processes" => Integer, "verdict" => String, "baseline" => Metric::value, "candidate" => Metric::value }
|
|
413
|
-
|
|
414
|
-
# Each worker's median of a measure judged by process, by version,
|
|
415
|
-
# and its lane's number.
|
|
416
|
-
type processes = { "lane" => Integer, "baseline" => Array[Metric::value], "candidate" => Array[Metric::value] }
|
|
417
|
-
|
|
418
|
-
# A counted metric's fewest, on each version, in +n+ iterations.
|
|
419
|
-
type counted = { "baseline" => Metric::value, "candidate" => Metric::value, ?"more" => Metric::value?, "n" => Integer,
|
|
420
|
-
"tolerance" => Metric::value, "allowance" => Metric::value, "unit" => String?, "plural" => String? }
|
|
421
|
-
|
|
422
|
-
# Where a regression's time went (see Profile#to_h), or why there's no
|
|
423
|
-
# telling.
|
|
424
|
-
# @api private
|
|
425
|
-
type profile = Profile::report | { "unavailable" => String }
|
|
426
|
-
|
|
427
|
-
# Peak memory grows by more than MEMORY_RATIO, and by MEMORY_FLOOR
|
|
428
|
-
# bytes at least, before it's worse.
|
|
429
|
-
# @api private
|
|
430
|
-
MEMORY_RATIO: Float
|
|
431
|
-
|
|
432
|
-
# @api private
|
|
433
|
-
MEMORY_FLOOR: Integer
|
|
434
|
-
|
|
435
|
-
# What a sample says when no metric said anything.
|
|
436
|
-
# @api private
|
|
437
|
-
NOTHING_SAID: Hash[String, String]
|
|
438
|
-
|
|
439
|
-
# The verdicts that say a timing isn't slower.
|
|
440
|
-
# @api private
|
|
441
|
-
CLEARED: Array[String]
|
|
442
|
-
|
|
443
|
-
@name: String
|
|
444
|
-
@timed: Hash[String, timed]
|
|
445
|
-
@counted: Hash[String, counted]
|
|
446
|
-
@details: Hash[String, Hash[String, String]]
|
|
447
|
-
@confirmations: Array[Result]
|
|
448
|
-
@undecided: Array[Result]
|
|
449
|
-
@processes: Hash[String, processes]
|
|
450
|
-
@n: Integer?
|
|
451
|
-
@seconds: Float?
|
|
452
|
-
@skipped: String?
|
|
453
|
-
@error: String?
|
|
454
|
-
@memory: Hash[String, Integer]?
|
|
455
|
-
@profile: profile?
|
|
456
|
-
|
|
457
|
-
# The benchmark's name.
|
|
458
|
-
attr_reader name: String
|
|
459
|
-
|
|
460
|
-
# Each ratio measure's verdict and medians, by name.
|
|
461
|
-
attr_reader timed: Hash[String, timed]
|
|
462
|
-
|
|
463
|
-
# Each counted measure's fewest on each version, by name.
|
|
464
|
-
attr_reader counted: Hash[String, counted]
|
|
465
|
-
|
|
466
|
-
# What metrics said of the latest samples (Metric#details): by
|
|
467
|
-
# measure, by version.
|
|
468
|
-
attr_reader details: Hash[String, Hash[String, String]]
|
|
469
|
-
|
|
470
|
-
# The measurements that confirmed it was worse.
|
|
471
|
-
attr_reader confirmations: Array[Result]
|
|
472
|
-
|
|
473
|
-
# The measurements again that were too close to call.
|
|
474
|
-
# @api private
|
|
475
|
-
attr_reader undecided: Array[Result]
|
|
476
|
-
|
|
477
|
-
# Each worker's median of each measure judged by process, by metric.
|
|
478
|
-
# @api private
|
|
479
|
-
attr_reader processes: Hash[String, processes]
|
|
480
|
-
|
|
481
|
-
# Its iterations a sample.
|
|
482
|
-
attr_reader n: Integer?
|
|
483
|
-
|
|
484
|
-
# @api private
|
|
485
|
-
attr_writer n: Integer?
|
|
486
|
-
|
|
487
|
-
# The seconds it took.
|
|
488
|
-
attr_reader seconds: Float?
|
|
489
|
-
|
|
490
|
-
# @api private
|
|
491
|
-
attr_writer seconds: Float?
|
|
492
|
-
|
|
493
|
-
# Why it was skipped (the baseline couldn't run it), or nil.
|
|
494
|
-
attr_reader skipped: String?
|
|
495
|
-
|
|
496
|
-
# @api private
|
|
497
|
-
attr_writer skipped: String?
|
|
498
|
-
|
|
499
|
-
# How it failed, or nil.
|
|
500
|
-
attr_reader error: String?
|
|
501
|
-
|
|
502
|
-
# @api private
|
|
503
|
-
attr_writer error: String?
|
|
504
|
-
|
|
505
|
-
# Each version's peak memory, in bytes.
|
|
506
|
-
attr_reader memory: Hash[String, Integer]?
|
|
507
|
-
|
|
508
|
-
# @api private
|
|
509
|
-
attr_writer memory: Hash[String, Integer]?
|
|
510
|
-
|
|
511
|
-
# Where a regression's time went.
|
|
512
|
-
# @api private
|
|
513
|
-
attr_accessor profile: profile?
|
|
514
|
-
|
|
515
|
-
# @api private
|
|
516
|
-
def initialize: (String name) -> void
|
|
517
|
-
|
|
518
|
-
# Keeps what metrics said of a pair of samples.
|
|
519
|
-
# @api private
|
|
520
|
-
def note: (Worker::sampled base, Worker::sampled candidate, ?String? measure) -> void
|
|
521
|
-
|
|
522
|
-
# The metrics on which the candidate did worse.
|
|
523
|
-
# @api private
|
|
524
|
-
def worse: () -> Array[String]
|
|
525
|
-
|
|
526
|
-
# The timed metrics on which the candidate was slower.
|
|
527
|
-
# @api private
|
|
528
|
-
def slower: () -> Array[String]
|
|
529
|
-
|
|
530
|
-
# Whether its timings say slower: one is, and none says it isn't.
|
|
531
|
-
# @api private
|
|
532
|
-
def slower_in_time?: () -> bool
|
|
533
|
-
|
|
534
|
-
# Whether it's to be measured again.
|
|
535
|
-
# @api private
|
|
536
|
-
def confirming?: () -> bool
|
|
537
|
-
|
|
538
|
-
# Keeps a measurement again as a confirmation, or sets it aside
|
|
539
|
-
# where it's too close to call. Whether it was set aside.
|
|
540
|
-
# @api private
|
|
541
|
-
def confirm: (Result again) -> bool
|
|
542
|
-
|
|
543
|
-
# Whether its peak memory grew by too much.
|
|
544
|
-
# @api private
|
|
545
|
-
def more_memory?: () -> bool
|
|
546
|
-
|
|
547
|
-
# Worse on a metric in the first measurement and in every confirmation,
|
|
548
|
-
# or slower in time in all of them (see #mixed?).
|
|
549
|
-
def regressed: () -> Array[String]
|
|
550
|
-
|
|
551
|
-
# Whether it regressed in time on one timed metric in some
|
|
552
|
-
# measurements and on another in the rest.
|
|
553
|
-
# @api private
|
|
554
|
-
def mixed?: () -> bool
|
|
555
|
-
|
|
556
|
-
# Whether it regressed on nothing, and didn't fail.
|
|
557
|
-
def passed?: () -> bool
|
|
558
|
-
|
|
559
|
-
# Whether a timed measure's verdict was too close to the limit to settle.
|
|
560
|
-
# @api private
|
|
561
|
-
def unsettled?: () -> bool
|
|
562
|
-
|
|
563
|
-
private
|
|
564
|
-
|
|
565
|
-
# The first measurement and its confirmations.
|
|
566
|
-
def measurements: () -> Array[Result]
|
|
567
|
-
end
|
|
568
|
-
|
|
569
111
|
# Measures every benchmark, confirms what looks worse, and reports.
|
|
570
112
|
# @api private
|
|
571
113
|
class Runner
|
|
572
114
|
# A count to four figures, or a whole number from NUMBER.
|
|
573
115
|
def self.number: (Metric::value value) -> String
|
|
574
116
|
|
|
117
|
+
# Bytes of memory, in megabytes to a tenth.
|
|
118
|
+
def self.mb: (Integer bytes) -> String
|
|
119
|
+
|
|
575
120
|
# The least count #number shows as a whole number.
|
|
576
121
|
NUMBER: Float
|
|
577
122
|
|
|
@@ -693,8 +238,6 @@ module Duel
|
|
|
693
238
|
|
|
694
239
|
def line: (Result result) -> String
|
|
695
240
|
|
|
696
|
-
def mb: (Integer bytes) -> String
|
|
697
|
-
|
|
698
241
|
# The unit of +measured+, for +number+ of it: singular for one.
|
|
699
242
|
def unit: (Metric::value number, Result::timed | Result::counted measured) -> String?
|
|
700
243
|
|
data/sig/duel/spawn.rbs
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
module Duel
|
|
2
|
+
# Starts a process with its output in a log, or pipes for its input and
|
|
3
|
+
# output, and gives the Child to wait for it and stop it by: with
|
|
4
|
+
# Process.spawn's redirects, or through Open3 where spawn can't redirect
|
|
5
|
+
# or wait (JRuby on Windows).
|
|
6
|
+
# @api private
|
|
7
|
+
module Spawn
|
|
8
|
+
# How a process starts: Directly, or ThroughOpen3.
|
|
9
|
+
type starter = singleton(Directly) | singleton(ThroughOpen3)
|
|
10
|
+
|
|
11
|
+
REDIRECTS: bool
|
|
12
|
+
WAITS: bool
|
|
13
|
+
SHELL: Regexp
|
|
14
|
+
COPY_WAIT: Integer
|
|
15
|
+
CHUNK: Integer
|
|
16
|
+
LOG_LINES: Integer
|
|
17
|
+
STARTER: starter
|
|
18
|
+
|
|
19
|
+
# A process started, to wait for and stop.
|
|
20
|
+
class Child
|
|
21
|
+
GONE: Array[singleton(SystemCallError)]
|
|
22
|
+
|
|
23
|
+
attr_reader pid: Integer
|
|
24
|
+
|
|
25
|
+
@pid: Integer
|
|
26
|
+
@waiter: Thread?
|
|
27
|
+
@group: bool
|
|
28
|
+
@windows: bool
|
|
29
|
+
|
|
30
|
+
def initialize: (Integer pid, ?Thread? waiter, ?group: bool, ?windows: bool) -> void
|
|
31
|
+
|
|
32
|
+
# The thread whose value is how it ended (a Process::Status).
|
|
33
|
+
def waiter: () -> Thread
|
|
34
|
+
|
|
35
|
+
# Waits for it on a thread from now on: returns itself.
|
|
36
|
+
def detach: () -> self
|
|
37
|
+
|
|
38
|
+
def alive?: () -> bool
|
|
39
|
+
|
|
40
|
+
# Waits for it, and returns how it ended, and its peak memory where
|
|
41
|
+
# that can be told.
|
|
42
|
+
def wait: () -> [Integer?, Integer?, Integer?]
|
|
43
|
+
|
|
44
|
+
# Sends it +signal+: on Windows, kills it with what it started.
|
|
45
|
+
def signal: (String signal) -> void
|
|
46
|
+
|
|
47
|
+
# Kills it, with what it started where that can be told, and waits
|
|
48
|
+
# for it to end.
|
|
49
|
+
def kill: () -> void
|
|
50
|
+
|
|
51
|
+
# Waits up to +seconds+ for it to end, then kills it.
|
|
52
|
+
def stop: (Numeric seconds) -> void
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Starts a process with Process.spawn, given its redirects and pipes.
|
|
56
|
+
module Directly
|
|
57
|
+
# Starts +command+, its output and errors in +log+.
|
|
58
|
+
def self.logged: (Hash[String, String?] env, Array[String] command, log: String, mode: String, **untyped options) -> Child
|
|
59
|
+
|
|
60
|
+
# Starts +command+ with a pipe each way: its Child, its input and its
|
|
61
|
+
# output.
|
|
62
|
+
def self.piped: (Hash[String, String?] env, Array[String] command, ?group: bool, **untyped options) -> [Child, IO, IO]
|
|
63
|
+
|
|
64
|
+
# Starts +command+, its output this process's.
|
|
65
|
+
def self.started: (Hash[String, String?] env, Array[String] command, **untyped options) -> Child
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Starts a process through Open3, as Directly does.
|
|
69
|
+
module ThroughOpen3
|
|
70
|
+
def self.logged: (Hash[String, String?] env, Array[String] command, log: String, mode: String, **untyped options) -> Child
|
|
71
|
+
|
|
72
|
+
def self.piped: (Hash[String, String?] env, Array[String] command, ?group: bool, **untyped options) -> [Child, IO, IO]
|
|
73
|
+
|
|
74
|
+
def self.started: (Hash[String, String?] env, Array[String] command, **untyped options) -> Child
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# How a process that failed ended: its exit status, or the signal that
|
|
78
|
+
# killed it.
|
|
79
|
+
def self.why: (Integer? code, Integer? signal) -> String
|
|
80
|
+
|
|
81
|
+
# The end of the log at +path+, for a failure to show.
|
|
82
|
+
def self.tail: (String path) -> String
|
|
83
|
+
|
|
84
|
+
# The log at +path+, opened for a process's output.
|
|
85
|
+
def self.log: (String path, String mode) -> File
|
|
86
|
+
|
|
87
|
+
# Open3's +name+d method, on JRuby on Windows with ENV and the
|
|
88
|
+
# program found.
|
|
89
|
+
def self.open3: (Symbol name, *untyped args, ?jruby_on_windows: bool, **untyped options) ?{ (*untyped) -> untyped } -> untyped
|
|
90
|
+
|
|
91
|
+
# +command+ as Windows runs it, its program found.
|
|
92
|
+
def self.program: (Array[String] command, ?path: String, ?exts: Array[String]) -> Array[String]
|
|
93
|
+
|
|
94
|
+
# +arg+ as a Windows program's command line has it.
|
|
95
|
+
def self.quote: (String arg) -> String
|
|
96
|
+
|
|
97
|
+
# Process.spawn's options that start a process in a group of its own:
|
|
98
|
+
# a process group, or on Windows, a console's.
|
|
99
|
+
def self.group: (?windows: bool) -> ({ pgroup: true } | { new_pgroup: true })
|
|
100
|
+
|
|
101
|
+
# Kills +pid+ and what it started: its process group, or on Windows,
|
|
102
|
+
# its tree of children.
|
|
103
|
+
def self.kill_tree: (Integer pid, ?windows: bool) -> untyped
|
|
104
|
+
|
|
105
|
+
# A thread copying +output+ to +out+, then closing both.
|
|
106
|
+
def self.copy: (IO output, IO out, ?close: bool) -> Thread
|
|
107
|
+
|
|
108
|
+
# A thread whose value is +waiter+'s, once +copy+ is done.
|
|
109
|
+
def self.ended: (Thread waiter, Thread copy, *IO outputs) -> Thread
|
|
110
|
+
end
|
|
111
|
+
end
|
data/sig/duel/summary.rbs
CHANGED
|
@@ -92,6 +92,9 @@ module Duel
|
|
|
92
92
|
# Every benchmark.
|
|
93
93
|
def table: () -> String?
|
|
94
94
|
|
|
95
|
+
# A table of +rows+ under +header+, without an empty last column.
|
|
96
|
+
def grid: (Array[String] header, Array[Array[String]] rows) -> String
|
|
97
|
+
|
|
95
98
|
# A table's row of +cells+.
|
|
96
99
|
def row: (Array[String] cells) -> String
|
|
97
100
|
|
|
@@ -104,8 +107,6 @@ module Duel
|
|
|
104
107
|
# Each counted measure that changed, an iteration.
|
|
105
108
|
def counts: (Result result) -> String
|
|
106
109
|
|
|
107
|
-
def mb: (Integer bytes) -> String
|
|
108
|
-
|
|
109
110
|
# The first +explained+ regressions, explained.
|
|
110
111
|
def regressions: (Integer explained) -> Array[String]
|
|
111
112
|
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
module Duel
|
|
2
|
+
# A worker process (Duel::Worker) running one version of the code.
|
|
3
|
+
# @api private
|
|
4
|
+
class WorkerProcess
|
|
5
|
+
# A worker's reply to a request (see Worker#handle), parsed from JSON.
|
|
6
|
+
type reply = untyped
|
|
7
|
+
|
|
8
|
+
LIB: String
|
|
9
|
+
POLL: Float
|
|
10
|
+
STOP_WAIT: Integer
|
|
11
|
+
# The gems that are Duel's: Duel, and its plugins.
|
|
12
|
+
DUEL_GEM: Regexp
|
|
13
|
+
|
|
14
|
+
@label: String
|
|
15
|
+
@timeout: Float
|
|
16
|
+
@root: String
|
|
17
|
+
@dir: String
|
|
18
|
+
@env: Hash[String, String?]
|
|
19
|
+
@ruby: String
|
|
20
|
+
@engine: String
|
|
21
|
+
@child: Spawn::Child
|
|
22
|
+
@launch: Array[String]?
|
|
23
|
+
@runs: ProcessSample?
|
|
24
|
+
@description: String
|
|
25
|
+
@extensions: Hash[String, String]
|
|
26
|
+
@twice: Hash[String, Array[String]]
|
|
27
|
+
@gem_dirs: Array[String]
|
|
28
|
+
@socket: UNIXSocket | TCPSocket
|
|
29
|
+
@benchmarks: Hash[String, Definition::summary]
|
|
30
|
+
@metrics: Array[Metric::spec]
|
|
31
|
+
@memory: bool
|
|
32
|
+
@profiler: bool
|
|
33
|
+
@yjit: bool
|
|
34
|
+
|
|
35
|
+
attr_reader label: String
|
|
36
|
+
|
|
37
|
+
attr_reader benchmarks: Hash[String, Definition::summary]
|
|
38
|
+
|
|
39
|
+
attr_reader metrics: Array[Metric::spec]
|
|
40
|
+
|
|
41
|
+
attr_reader memory: bool
|
|
42
|
+
|
|
43
|
+
attr_reader profiler: bool
|
|
44
|
+
|
|
45
|
+
attr_reader yjit: bool
|
|
46
|
+
|
|
47
|
+
# The Ruby it runs on (RUBY_DESCRIPTION).
|
|
48
|
+
attr_reader description: String
|
|
49
|
+
|
|
50
|
+
# The native extensions it loaded, by feature.
|
|
51
|
+
attr_reader extensions: Hash[String, String]
|
|
52
|
+
|
|
53
|
+
# Those it loaded from more than one file, by feature.
|
|
54
|
+
attr_reader twice: Hash[String, Array[String]]
|
|
55
|
+
|
|
56
|
+
# The directories gems are installed in, as real paths ending in "/".
|
|
57
|
+
attr_reader gem_dirs: Array[String]
|
|
58
|
+
|
|
59
|
+
# Where a worker on +ruby+ loads Duel from, ahead of anything else.
|
|
60
|
+
def self.load_path: (String ruby, ?specs: Hash[String, Gem::BasicSpecification]) -> Array[String]
|
|
61
|
+
|
|
62
|
+
def initialize: (String label, String root, String candidate_root, String dir, config_path: String?,
|
|
63
|
+
env: Hash[String, String?], timeout: Float, ruby: String, engine: String, ?unix: bool) -> void
|
|
64
|
+
|
|
65
|
+
# Sends a request, and returns the reply, or raises Failed with its
|
|
66
|
+
# error.
|
|
67
|
+
def request: (String op, String name, **untyped args) -> reply
|
|
68
|
+
|
|
69
|
+
def sample: (String name, Integer n, ?runs: Integer, ?count: bool) -> reply
|
|
70
|
+
|
|
71
|
+
# A run of a process benchmark: its process, fresh, started as this
|
|
72
|
+
# worker was.
|
|
73
|
+
def run: (Definition::process process) -> ProcessSample::sampled
|
|
74
|
+
|
|
75
|
+
# Closing the connection ends the worker. One still running +wait+
|
|
76
|
+
# seconds later is killed.
|
|
77
|
+
def stop: (?wait: Numeric) -> void
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
def accept: (UNIXServer | TCPServer server, String path) -> (UNIXSocket | TCPSocket)
|
|
82
|
+
|
|
83
|
+
def read: () -> reply
|
|
84
|
+
|
|
85
|
+
# Kills a worker that didn't +what+ within the timeout, and fails.
|
|
86
|
+
def hung: (String what) -> bot
|
|
87
|
+
|
|
88
|
+
def monotonic: () -> Float
|
|
89
|
+
end
|
|
90
|
+
end
|