duel 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +108 -1
- data/README.md +22 -5
- data/docs/benchmarks.md +2 -1
- data/docs/ci.md +150 -12
- data/docs/configuration.md +11 -4
- data/docs/how-it-works.md +32 -4
- data/docs/json.md +2 -0
- data/docs/plugins.md +2 -2
- data/docs/troubleshooting.md +28 -0
- data/lib/duel/baseline.rb +4 -4
- data/lib/duel/cli.rb +4 -1
- data/lib/duel/config.rb +13 -6
- data/lib/duel/context.rb +7 -6
- data/lib/duel/init.rb +4 -0
- data/lib/duel/json_report.rb +4 -3
- data/lib/duel/lock.rb +52 -8
- data/lib/duel/message.rb +25 -0
- data/lib/duel/native.rb +57 -10
- data/lib/duel/resource.rb +37 -2
- data/lib/duel/runner.rb +115 -17
- data/lib/duel/summary.rb +4 -1
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +23 -4
- data/lib/duel/templates/perf.yml.erb +86 -3
- data/lib/duel/testing.rb +2 -2
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +34 -14
- data/sig/duel/config.rbs +3 -0
- data/sig/duel/init.rbs +3 -0
- data/sig/duel/lock.rbs +11 -0
- data/sig/duel/message.rbs +11 -0
- data/sig/duel/native.rbs +13 -3
- data/sig/duel/resource.rbs +10 -1
- data/sig/duel/runner.rbs +41 -2
- data/sig/duel/summary.rbs +3 -0
- data/sig/duel/worker.rbs +11 -3
- metadata +4 -2
data/lib/duel/config.rb
CHANGED
|
@@ -31,6 +31,14 @@ module Duel
|
|
|
31
31
|
# Where the project is (and its config file).
|
|
32
32
|
attr_reader :root
|
|
33
33
|
|
|
34
|
+
# Where Duel keeps what it keeps for the project, tmp/duel: the lock
|
|
35
|
+
# that keeps two runs from sharing it (see Lock), what prepare blocks
|
|
36
|
+
# build once (Resource#cached), in cache/, and the baseline's checkout
|
|
37
|
+
# where it can't go next to the repository (see Baseline#checkout), in
|
|
38
|
+
# base/. A run removes the lock when it ends, and what that leaves
|
|
39
|
+
# empty. `duel --clean` removes the rest.
|
|
40
|
+
def state = File.join(root, "tmp", "duel")
|
|
41
|
+
|
|
34
42
|
# A glob, from the root, of the files defining benchmarks.
|
|
35
43
|
attr_accessor :benchmarks
|
|
36
44
|
|
|
@@ -312,18 +320,17 @@ module Duel
|
|
|
312
320
|
# stopped when it's done with. The block returns the environment
|
|
313
321
|
# variables it gets: a worker's, under its version's.
|
|
314
322
|
#
|
|
315
|
-
# A version's (named "
|
|
316
|
-
#
|
|
317
|
-
#
|
|
318
|
-
#
|
|
319
|
-
# its process benchmarks').
|
|
323
|
+
# A version's (named "base" or "head", its #root the version's tree
|
|
324
|
+
# and its #ruby the Ruby it runs on) is prepared once for a run, in
|
|
325
|
+
# the runner, before any of its workers start: installing its bundle,
|
|
326
|
+
# say. Its environment is its workers' (and its process benchmarks').
|
|
320
327
|
#
|
|
321
328
|
# c.prepare(:version) do |v|
|
|
322
329
|
# v.cached("bundle-#{Digest::SHA256.file(File.join(v.root, "Gemfile.lock"))}") { |_| v.run("bundle", "install", chdir: v.root) }
|
|
323
330
|
# {"BUNDLE_GEMFILE" => File.join(v.root, "Gemfile")}
|
|
324
331
|
# end
|
|
325
332
|
#
|
|
326
|
-
# A worker's (named "
|
|
333
|
+
# A worker's (named "base-1", say) is prepared by the runner as
|
|
327
334
|
# the worker starts: a server of its own, say.
|
|
328
335
|
#
|
|
329
336
|
# c.prepare(:worker) do |w|
|
data/lib/duel/context.rb
CHANGED
|
@@ -498,16 +498,17 @@ module Duel
|
|
|
498
498
|
end
|
|
499
499
|
|
|
500
500
|
# A stretch of a benchmark run for its peak memory: with the garbage
|
|
501
|
-
# collector working as usual, the most
|
|
502
|
-
#
|
|
503
|
-
# every
|
|
501
|
+
# collector working as usual, the most memory the process had in use
|
|
502
|
+
# (Native::Memory.used: resident memory, or on JRuby, the JVM's heap)
|
|
503
|
+
# above where it started (after a full collection), watched every
|
|
504
|
+
# millisecond.
|
|
504
505
|
# @api private
|
|
505
506
|
class MemorySample
|
|
506
507
|
attr_reader :started, :stopped, :peak
|
|
507
508
|
|
|
508
509
|
def start
|
|
509
510
|
@started = true
|
|
510
|
-
|
|
511
|
+
Native::Memory.collect
|
|
511
512
|
@base = resident
|
|
512
513
|
@peak = 0
|
|
513
514
|
@watcher = Thread.new do
|
|
@@ -570,8 +571,8 @@ module Duel
|
|
|
570
571
|
end
|
|
571
572
|
|
|
572
573
|
def resident
|
|
573
|
-
# (
|
|
574
|
-
Native::Memory.
|
|
574
|
+
# (The memory in use is known: see Lane#peak_memory.)
|
|
575
|
+
Native::Memory.used #: Integer
|
|
575
576
|
end
|
|
576
577
|
|
|
577
578
|
def watch
|
data/lib/duel/init.rb
CHANGED
|
@@ -249,5 +249,9 @@ module Duel
|
|
|
249
249
|
# What names the branches that mark the last commit Duel passed in a
|
|
250
250
|
# push, before the branch pushed's name (see Baseline#passed_branch).
|
|
251
251
|
def passed_branch = ["duel/passed", working_directory&.tr("/", "-")].compact.join("-")
|
|
252
|
+
|
|
253
|
+
# The branch that holds the workflow's badge: duel/badge, or for a
|
|
254
|
+
# project in a subdirectory, duel/badge-<its path>.
|
|
255
|
+
def badge_branch = ["duel/badge", working_directory&.tr("/", "-")].compact.join("-")
|
|
252
256
|
end
|
|
253
257
|
end
|
data/lib/duel/json_report.rb
CHANGED
|
@@ -22,7 +22,7 @@ module Duel
|
|
|
22
22
|
none = [] #: Array[value]
|
|
23
23
|
nothing = {} #: report
|
|
24
24
|
head(status, exit, message, settings, comment).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
|
|
25
|
-
"benchmarks" => none, "across" => nothing, "errors" => none)
|
|
25
|
+
"benchmarks" => none, "across" => nothing, "errors" => none, "cut_short" => nil)
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
# What every report starts with.
|
|
@@ -30,7 +30,7 @@ module Duel
|
|
|
30
30
|
{"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message, "comment" => comment,
|
|
31
31
|
"settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
|
|
32
32
|
"jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
|
|
33
|
-
"warm" => settings.warm, "force" => settings.force}}
|
|
33
|
+
"time_limit" => settings.time_limit, "warm" => settings.warm, "force" => settings.force}}
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
@@ -54,7 +54,8 @@ module Duel
|
|
|
54
54
|
"plugins" => @runner.plugins.map(&:to_s),
|
|
55
55
|
"benchmarks" => @results.map { |result| benchmark(result) },
|
|
56
56
|
"across" => across,
|
|
57
|
-
"errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} }
|
|
57
|
+
"errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} },
|
|
58
|
+
"cut_short" => @runner.cut_short
|
|
58
59
|
)
|
|
59
60
|
end
|
|
60
61
|
|
data/lib/duel/lock.rb
CHANGED
|
@@ -8,7 +8,9 @@ module Duel
|
|
|
8
8
|
# baseline's checkout, each checking out and building over the other's.
|
|
9
9
|
# A run holds an flock on tmp/duel/lock, which ends with its process
|
|
10
10
|
# however that ends, so a run that crashed leaves nothing to clear. The
|
|
11
|
-
# file says whose it is, for a run that finds it taken.
|
|
11
|
+
# file says whose it is, for a run that finds it taken. A run that ends
|
|
12
|
+
# removes it, and the directories that leaves empty: tmp/duel, and tmp
|
|
13
|
+
# if Duel made it.
|
|
12
14
|
# @api private
|
|
13
15
|
module Lock
|
|
14
16
|
# An exclusive lock, or none if another holds it.
|
|
@@ -17,21 +19,63 @@ module Duel
|
|
|
17
19
|
# Runs the block holding +dir+'s lock. Fails, naming the run that
|
|
18
20
|
# holds it, if another does.
|
|
19
21
|
def self.hold(dir)
|
|
22
|
+
made = missing(dir)
|
|
20
23
|
FileUtils.mkdir_p(dir)
|
|
21
|
-
|
|
22
|
-
file =
|
|
24
|
+
path = File.join(dir, "lock")
|
|
25
|
+
file = acquire(path, dir)
|
|
23
26
|
begin
|
|
24
|
-
unless file.flock(EXCLUSIVE)
|
|
25
|
-
# (Empty for a moment, while its run writes its pid.)
|
|
26
|
-
pid = file.read
|
|
27
|
-
raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
|
|
28
|
-
end
|
|
29
27
|
file.truncate(0)
|
|
30
28
|
file.write(Process.pid)
|
|
31
29
|
file.flush
|
|
32
30
|
yield
|
|
33
31
|
ensure
|
|
32
|
+
# (Removed while it's held, so a run that opens it after makes a new
|
|
33
|
+
# one. `duel --clean` may have removed it already.)
|
|
34
|
+
FileUtils.rm_f(path)
|
|
34
35
|
file.close
|
|
36
|
+
tidy(dir, made)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The lock file at +path+, locked: the one there now (a run that
|
|
41
|
+
# ended may have removed the one this opened, after this opened it).
|
|
42
|
+
def self.acquire(path, dir)
|
|
43
|
+
loop do
|
|
44
|
+
# (Read from the start, and written at the end: there's no more.)
|
|
45
|
+
file = File.new(path, "a+")
|
|
46
|
+
unless file.flock(EXCLUSIVE)
|
|
47
|
+
# (Empty for a moment, while its run writes its pid.)
|
|
48
|
+
pid = file.read
|
|
49
|
+
file.close
|
|
50
|
+
raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
|
|
51
|
+
end
|
|
52
|
+
return file if File.exist?(path) && File.identical?(path, file)
|
|
53
|
+
|
|
54
|
+
file.close
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The directories, +dir+ and those it's in, that aren't there yet,
|
|
59
|
+
# deepest first.
|
|
60
|
+
def self.missing(dir)
|
|
61
|
+
made = [] #: Array[String]
|
|
62
|
+
path = dir
|
|
63
|
+
until File.exist?(path)
|
|
64
|
+
made << path
|
|
65
|
+
path = File.dirname(path)
|
|
66
|
+
end
|
|
67
|
+
made
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Removes the empty directories in +dir+ (cache/, say: not what's in
|
|
71
|
+
# them, nor in a baseline's checkout there), then +dir+, then those of
|
|
72
|
+
# +made+ (Duel made them), where they're empty. (One that isn't empty
|
|
73
|
+
# holds those it's in.)
|
|
74
|
+
def self.tidy(dir, made)
|
|
75
|
+
[*Dir.glob(File.join(dir, "*/")), dir, *made].each do |path|
|
|
76
|
+
Dir.rmdir(path)
|
|
77
|
+
rescue SystemCallError # (not empty, or gone already)
|
|
78
|
+
nil
|
|
35
79
|
end
|
|
36
80
|
end
|
|
37
81
|
end
|
data/lib/duel/message.rb
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Duel
|
|
4
|
+
# What the runner and a worker say to each other, over the worker's
|
|
5
|
+
# socket: plain Hashes, Arrays, Strings, numbers, true, false and nil,
|
|
6
|
+
# marshaled. Not JSON: the worker would load json before it loads the
|
|
7
|
+
# version it measures, and in json's own repository, every baseline
|
|
8
|
+
# worker would load the candidate's json first, and the baseline's
|
|
9
|
+
# over it. Marshal is Ruby's own, with nothing to load (so a version's
|
|
10
|
+
# code is the first of it the worker loads), and it keeps a String's
|
|
11
|
+
# encoding. (Both ends are Duel's own processes, and the socket is in
|
|
12
|
+
# the run's own directory.)
|
|
13
|
+
# @api private
|
|
14
|
+
module Message
|
|
15
|
+
# Sends +message+ over +io+.
|
|
16
|
+
def self.write(io, message) = io.write(Marshal.dump(message))
|
|
17
|
+
|
|
18
|
+
# The next message over +io+, or nil once the other end has closed it.
|
|
19
|
+
def self.read(io)
|
|
20
|
+
Marshal.load(io)
|
|
21
|
+
rescue EOFError
|
|
22
|
+
nil
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
data/lib/duel/native.rb
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "etc"
|
|
4
|
-
require "fiddle"
|
|
5
3
|
require "rbconfig"
|
|
6
4
|
|
|
7
5
|
module Duel
|
|
@@ -16,9 +14,19 @@ module Duel
|
|
|
16
14
|
CPU = RbConfig::CONFIG["host_cpu"]
|
|
17
15
|
UINT64 = 8 # bytes, the size of every count read
|
|
18
16
|
|
|
17
|
+
# Loads Fiddle, as the first C function is made (see .function): not
|
|
18
|
+
# as Duel loads, which in a worker is before its version (in fiddle's
|
|
19
|
+
# own repository, fiddle).
|
|
20
|
+
# mutant:disable -- the tests' process has Fiddle loaded from the start,
|
|
21
|
+
# so a require changes nothing there: WorkerTest's
|
|
22
|
+
# test_loads_no_gem_a_version_could_be_before_the_version checks it, in
|
|
23
|
+
# a process of its own
|
|
24
|
+
def self.require_fiddle = require("fiddle")
|
|
25
|
+
|
|
19
26
|
# A C function of the process's own libraries, by name, taking and
|
|
20
27
|
# returning Fiddle types (:INT, :LONG, :VOIDP...).
|
|
21
28
|
def self.function(name, args, returns)
|
|
29
|
+
require_fiddle
|
|
22
30
|
Fiddle::Function.new(Fiddle.dlopen(nil)[name], args.map { |type| Fiddle.const_get(:"TYPE_#{type}") },
|
|
23
31
|
Fiddle.const_get(:"TYPE_#{returns}"))
|
|
24
32
|
end
|
|
@@ -145,18 +153,57 @@ module Duel
|
|
|
145
153
|
|
|
146
154
|
# This process's resident memory in bytes, or nil where it can't tell.
|
|
147
155
|
module Memory
|
|
148
|
-
|
|
149
|
-
#
|
|
150
|
-
|
|
156
|
+
# (Its VmRSS line, in kB: no page size to know, as /proc/self/statm's
|
|
157
|
+
# pages need, which would take Etc, a gem a worker would load before
|
|
158
|
+
# its version.)
|
|
159
|
+
STATUS = "/proc/self/status"
|
|
160
|
+
# macOS's counter, which reads resident memory there, or nil: opened
|
|
161
|
+
# as resident memory is first read, as opening it loads Fiddle.
|
|
162
|
+
# mutant:disable -- kept only so as not to open it again: it's the
|
|
163
|
+
# same counter however often it's opened
|
|
164
|
+
def self.macos_counter
|
|
165
|
+
return @macos if defined?(@macos)
|
|
166
|
+
|
|
167
|
+
# (Never false: Steep doesn't see that false || nil is nil.)
|
|
168
|
+
@macos = (FIDDLE && MacOS.open) || nil #: MacOS?
|
|
169
|
+
end
|
|
151
170
|
|
|
152
171
|
# From /proc (Linux), else from macOS's counter, else nil.
|
|
153
|
-
def self.resident(
|
|
154
|
-
return macos&.resident unless File.readable?(
|
|
172
|
+
def self.resident(status: STATUS, macos: macos_counter)
|
|
173
|
+
return macos&.resident unless File.readable?(status)
|
|
174
|
+
|
|
175
|
+
kb = File.read(status)[/^VmRSS:\s+(\d+) kB$/, 1] #: String
|
|
176
|
+
Integer(kb) * 1024
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# The memory a benchmark's peak is measured in (see MemorySample),
|
|
180
|
+
# in bytes, or nil where it can't tell: resident memory, but on
|
|
181
|
+
# JRuby, the JVM's heap in use.
|
|
182
|
+
def self.used = resident
|
|
183
|
+
|
|
184
|
+
# Collects every object that's garbage, so that what's in use (see
|
|
185
|
+
# .used) is what's live: a full collection, swept at once.
|
|
186
|
+
def self.collect = GC.start(full_mark: true, immediate_sweep: true)
|
|
187
|
+
end
|
|
155
188
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
189
|
+
# simplecov:disable JRuby's heap is the JVM's (JRuby's test run covers
|
|
190
|
+
# this).
|
|
191
|
+
if RUBY_ENGINE == "jruby"
|
|
192
|
+
module Memory
|
|
193
|
+
# The JVM takes memory for its heap ahead of time, and keeps what it
|
|
194
|
+
# used while it started, so resident memory barely moves as a
|
|
195
|
+
# benchmark's objects fill it: 24 MB more of them can show as none.
|
|
196
|
+
# The heap in use shows them.
|
|
197
|
+
HEAP = java.lang.management.ManagementFactory.getMemoryMXBean
|
|
198
|
+
|
|
199
|
+
def self.used = HEAP.getHeapMemoryUsage.getUsed
|
|
200
|
+
|
|
201
|
+
# JRuby's GC.start collects nothing. The JVM's own call does:
|
|
202
|
+
# without it, a peak was measured above garbage that the benchmark's
|
|
203
|
+
# objects then made room for, and came out tens of megabytes short.
|
|
204
|
+
def self.collect = java.lang.System.gc
|
|
159
205
|
end
|
|
160
206
|
end
|
|
207
|
+
# simplecov:enable
|
|
161
208
|
end
|
|
162
209
|
end
|
data/lib/duel/resource.rb
CHANGED
|
@@ -42,7 +42,36 @@ module Duel
|
|
|
42
42
|
# @api private
|
|
43
43
|
CACHE = Mutex.new
|
|
44
44
|
|
|
45
|
-
#
|
|
45
|
+
# What a cache key names, without the version and the hash it ends
|
|
46
|
+
# with: "mysql", of mysql-0.1.2-3f9c2a1b7e5d4c6a. Builds of one name
|
|
47
|
+
# are of one thing, and the one a run used replaces the rest (see
|
|
48
|
+
# .evict).
|
|
49
|
+
# @api private
|
|
50
|
+
NAME = /\A(.+?)(?:-\d[\w.]*)?-\h{8,}\z/
|
|
51
|
+
|
|
52
|
+
# The builds this process's runs used (#cached), by path.
|
|
53
|
+
# @api private
|
|
54
|
+
USED = Set.new
|
|
55
|
+
|
|
56
|
+
# Removes from +cache+ the builds that ones this process used replace:
|
|
57
|
+
# of the same name (see NAME), with another version or hash. A server
|
|
58
|
+
# or a plugin upgraded, or a lockfile changed, left its old build
|
|
59
|
+
# behind otherwise: tens of megabytes of a database's files, each
|
|
60
|
+
# time. A key that doesn't end with a hash is never removed, nor is
|
|
61
|
+
# any that this process used (each version's bundle, say). (Once a
|
|
62
|
+
# run's workers have started: none adds to USED then.)
|
|
63
|
+
# @api private
|
|
64
|
+
def self.evict(cache)
|
|
65
|
+
used = USED.select { |dir| File.dirname(dir) == cache }.map { |dir| File.basename(dir) }
|
|
66
|
+
names = used.filter_map { |key| key[NAME, 1] }
|
|
67
|
+
Dir.children(cache).each do |build|
|
|
68
|
+
FileUtils.rm_rf(File.join(cache, build)) if !used.include?(build) && names.include?(build[NAME, 1])
|
|
69
|
+
end
|
|
70
|
+
rescue Errno::ENOENT # (no cache)
|
|
71
|
+
nil
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The worker it's for ("base-1", say), or the version ("base",
|
|
46
75
|
# for Config#prepare(:version)), the directory it has of its own, the
|
|
47
76
|
# tree of the version that worker runs, and the Ruby it runs on (see
|
|
48
77
|
# Config#ruby).
|
|
@@ -88,11 +117,17 @@ module Duel
|
|
|
88
117
|
# block is given, and the rest get it built. A key names what's built
|
|
89
118
|
# (put a version in it), as a file name: tmp/duel/cache/+key+, which
|
|
90
119
|
# `duel --clean` removes. A build that fails leaves nothing behind.
|
|
120
|
+
# A key that ends with a hash (and a version before it, if you like:
|
|
121
|
+
# mysql-0.1.2-3f9c2a1b) replaces the builds of its name when the run
|
|
122
|
+
# ends (see .evict): the one a run used is the one that's kept.
|
|
91
123
|
def cached(key)
|
|
92
124
|
raise ArgumentError, "a cache key is a file name (letters, digits, - _ .), not #{key.inspect}" unless key.match?(KEY)
|
|
93
125
|
|
|
94
126
|
dir = File.join(@cache, key)
|
|
95
|
-
CACHE.synchronize
|
|
127
|
+
CACHE.synchronize do
|
|
128
|
+
USED << dir
|
|
129
|
+
build(dir) { |building| yield building } unless File.directory?(dir)
|
|
130
|
+
end
|
|
96
131
|
dir
|
|
97
132
|
end
|
|
98
133
|
|
data/lib/duel/runner.rb
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "fileutils"
|
|
4
|
-
require "json"
|
|
5
4
|
require "rbconfig"
|
|
6
5
|
require "socket"
|
|
7
6
|
require "tmpdir"
|
|
@@ -9,6 +8,7 @@ require_relative "../duel"
|
|
|
9
8
|
require_relative "stats"
|
|
10
9
|
require_relative "profile"
|
|
11
10
|
require_relative "across"
|
|
11
|
+
require_relative "message"
|
|
12
12
|
require_relative "resource"
|
|
13
13
|
require_relative "rubies"
|
|
14
14
|
require_relative "process_sample"
|
|
@@ -27,13 +27,17 @@ module Duel
|
|
|
27
27
|
# @api private
|
|
28
28
|
YJIT_WARM = 2.0
|
|
29
29
|
|
|
30
|
+
# The seconds a run measures for, at most, by default (see
|
|
31
|
+
# Settings#time_limit and Runner#out_of_time?).
|
|
32
|
+
TIME_LIMIT = 1800.0
|
|
33
|
+
|
|
30
34
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
31
35
|
# @api private
|
|
32
36
|
Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
|
|
33
|
-
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target) do
|
|
37
|
+
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit) do
|
|
34
38
|
# @type self: singleton(Settings)
|
|
35
39
|
def self.defaults = new(filter: //, jobs: 1, processes: 3, limit: 1.02, confidence: 0.99, warm: nil, force: false,
|
|
36
|
-
timeout: 60.0)
|
|
40
|
+
timeout: 60.0, time_limit: TIME_LIMIT)
|
|
37
41
|
end
|
|
38
42
|
|
|
39
43
|
# One version as a run has it (see Runner#prepare), +name+d "baseline"
|
|
@@ -51,6 +55,21 @@ module Duel
|
|
|
51
55
|
end
|
|
52
56
|
end
|
|
53
57
|
|
|
58
|
+
class Side
|
|
59
|
+
# What each version's workers, their directories and their sockets
|
|
60
|
+
# are named for (see #label): "base" and "head", as long as each
|
|
61
|
+
# other, so that the two versions' workers get arguments and
|
|
62
|
+
# environments of the same length. Where a process's stack starts
|
|
63
|
+
# depends on how long they are, and at some lengths, the same code
|
|
64
|
+
# runs the same instructions in more cycles: a version whose workers'
|
|
65
|
+
# were a byte longer could measure slower in every one.
|
|
66
|
+
LABELS = {"baseline" => "base", "candidate" => "head"}.freeze
|
|
67
|
+
|
|
68
|
+
# What its workers, and its version's resources, are named for: "base"
|
|
69
|
+
# or "head".
|
|
70
|
+
def label = LABELS.fetch(name)
|
|
71
|
+
end
|
|
72
|
+
|
|
54
73
|
# A worker process (Duel::Worker) running one version of the code.
|
|
55
74
|
# @api private
|
|
56
75
|
class WorkerProcess
|
|
@@ -66,9 +85,10 @@ module Duel
|
|
|
66
85
|
attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :yjit
|
|
67
86
|
|
|
68
87
|
# The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
|
|
69
|
-
# loaded (see Worker.extensions),
|
|
70
|
-
# installed in (see
|
|
71
|
-
|
|
88
|
+
# loaded (see Worker.extensions), those it loaded twice (see
|
|
89
|
+
# Worker.twice), and the directories gems are installed in (see
|
|
90
|
+
# Worker.gem_dirs).
|
|
91
|
+
attr_reader :description, :extensions, :twice, :gem_dirs
|
|
72
92
|
|
|
73
93
|
# Where a worker on +ruby+ loads Duel from, ahead of anything else
|
|
74
94
|
# (a version's own bundle, which may not have Duel, or an older one):
|
|
@@ -109,13 +129,15 @@ module Duel
|
|
|
109
129
|
@yjit = hello.fetch("yjit")
|
|
110
130
|
@description = hello.fetch("ruby")
|
|
111
131
|
@extensions = hello.fetch("extensions")
|
|
132
|
+
@twice = hello.fetch("twice")
|
|
112
133
|
@gem_dirs = hello.fetch("gem_dirs")
|
|
113
134
|
end
|
|
114
135
|
|
|
115
136
|
# Sends a request, and returns the reply, or raises Failed with its
|
|
116
137
|
# error.
|
|
117
138
|
def request(op, name, **args)
|
|
118
|
-
|
|
139
|
+
# (By name: +args+ come by keyword, as Symbols.)
|
|
140
|
+
Message.write(@socket, {"op" => op, "name" => name, **args.transform_keys(&:to_s)})
|
|
119
141
|
reply = read
|
|
120
142
|
raise Failed, "#{@label}: #{reply.fetch("error")}\n #{reply.fetch("backtrace").join("\n ")}" if reply.key?("error")
|
|
121
143
|
|
|
@@ -166,8 +188,7 @@ module Duel
|
|
|
166
188
|
|
|
167
189
|
def read
|
|
168
190
|
hung("answer") unless @socket.wait_readable(@timeout)
|
|
169
|
-
|
|
170
|
-
JSON.parse(line)
|
|
191
|
+
Message.read(@socket) or raise Failed, "#{@label} worker exited"
|
|
171
192
|
end
|
|
172
193
|
|
|
173
194
|
# Kills a worker that didn't +what+ within the timeout, and fails.
|
|
@@ -324,12 +345,12 @@ module Duel
|
|
|
324
345
|
# A worker of +side+, with its own resources, and its version's Ruby
|
|
325
346
|
# and environment.
|
|
326
347
|
def spawn(side, index)
|
|
327
|
-
label = "#{side}-#{index + 1}"
|
|
328
348
|
version = @sides.fetch(side)
|
|
349
|
+
label = "#{version.label}-#{index + 1}"
|
|
329
350
|
env = version.env
|
|
330
351
|
@config.prepares.fetch(:worker).each_with_index do |block, i|
|
|
331
352
|
resource = Resource.new(File.join(@dir, "#{label}-#{i + 1}"), label, version.root, timeout: @settings.timeout,
|
|
332
|
-
cache: File.join(@config.
|
|
353
|
+
cache: File.join(@config.state, "cache"), ruby: version.ruby, env: version.env)
|
|
333
354
|
keep(@resources, resource)
|
|
334
355
|
env = Resource.environment(label, env, block.call(resource))
|
|
335
356
|
end
|
|
@@ -347,10 +368,17 @@ module Duel
|
|
|
347
368
|
# be of its code. An installed gem's extension (in the directories
|
|
348
369
|
# gems are installed in, by the worker that loaded it) is in no
|
|
349
370
|
# version's tree, even in a bundle there (vendor/bundle, say, which
|
|
350
|
-
# both versions may load from).
|
|
371
|
+
# both versions may load from). And fails if a version's workers
|
|
372
|
+
# loaded one twice (see Worker.twice): the candidate's, that something
|
|
373
|
+
# required before Duel loaded the version, and then its own over it.
|
|
351
374
|
def check_builds
|
|
352
375
|
workers = {"baseline" => @bases.first, "candidate" => @candidates.first}
|
|
353
376
|
workers.each do |side, worker|
|
|
377
|
+
worker.twice.each do |feature, paths|
|
|
378
|
+
raise Failed, "the #{side} loaded #{feature} twice, from #{paths.join(" and ")}: something required it before " \
|
|
379
|
+
"Duel loaded the #{side}, and both stay loaded. Require what loads it in a block that runs once the version " \
|
|
380
|
+
"has loaded (c.in_worker, in .duel.rb), not as .duel.rb or a plugin loads"
|
|
381
|
+
end
|
|
354
382
|
other = (side == "baseline") ? "candidate" : "baseline"
|
|
355
383
|
theirs = workers.fetch(other).then { |w| w.extensions.select { |_, path| inside?(path, other, w) } }
|
|
356
384
|
worker.extensions.each do |feature, path|
|
|
@@ -851,23 +879,41 @@ module Duel
|
|
|
851
879
|
# seconds the run took. Set by #run.
|
|
852
880
|
attr_reader :rubies, :seconds
|
|
853
881
|
|
|
882
|
+
# What the run left undone when it reached its time limit
|
|
883
|
+
# (Settings#time_limit): {"unmeasured" => the benchmarks it didn't
|
|
884
|
+
# measure, "unconfirmed" => those it judged without every
|
|
885
|
+
# measurement again (see #confirm), "across" => whether it judged
|
|
886
|
+
# across the board without measuring every benchmark again (see
|
|
887
|
+
# #judge_across)}, or nil where it reached none. Set by #run.
|
|
888
|
+
def cut_short
|
|
889
|
+
return if @unmeasured.empty? && @unconfirmed.empty? && !@across_cut
|
|
890
|
+
|
|
891
|
+
{"unmeasured" => @unmeasured, "unconfirmed" => @unconfirmed, "across" => @across_cut}
|
|
892
|
+
end
|
|
893
|
+
|
|
854
894
|
def initialize(config, settings, base_root, out: $stdout)
|
|
855
895
|
@config = config
|
|
856
896
|
@settings = settings
|
|
857
897
|
@base_root = base_root
|
|
858
898
|
@out = out
|
|
859
899
|
@prepared = []
|
|
900
|
+
@started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
901
|
+
@lock = Mutex.new
|
|
902
|
+
@unmeasured = [] #: Array[String]
|
|
903
|
+
@unconfirmed = [] #: Array[String]
|
|
904
|
+
@across_cut = false
|
|
860
905
|
end
|
|
861
906
|
|
|
862
907
|
# Returns the Results.
|
|
863
908
|
def run
|
|
864
|
-
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
909
|
+
started = @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
865
910
|
@dir = Lane.tmpdir
|
|
866
911
|
begin
|
|
867
912
|
compare_all(started)
|
|
868
913
|
ensure
|
|
869
914
|
@prepared.reverse_each(&:stop)
|
|
870
915
|
FileUtils.rm_rf(@dir)
|
|
916
|
+
Resource.evict(File.join(@config.state, "cache"))
|
|
871
917
|
end
|
|
872
918
|
end
|
|
873
919
|
|
|
@@ -877,6 +923,27 @@ module Duel
|
|
|
877
923
|
|
|
878
924
|
def regressed_across = @across.select { |_, v| v["confirmed"] }.keys
|
|
879
925
|
|
|
926
|
+
# What a run cut short at its time limit left undone (see #cut_short),
|
|
927
|
+
# or nil where it wasn't.
|
|
928
|
+
def cut_short_note
|
|
929
|
+
return unless cut_short
|
|
930
|
+
|
|
931
|
+
undone = [("#{Runner.count(@unmeasured.size, "benchmark")} not measured" unless @unmeasured.empty?),
|
|
932
|
+
("#{Runner.count(@unconfirmed.size, "benchmark")} judged without measuring every one again" unless @unconfirmed.empty?),
|
|
933
|
+
("across the board judged without measuring again" if @across_cut)].compact
|
|
934
|
+
"the run measured for longer than #{limit_text}: #{undone.join(", ")}. Raise --time-limit, or make the slowest " \
|
|
935
|
+
"benchmarks quicker to settle"
|
|
936
|
+
end
|
|
937
|
+
|
|
938
|
+
# Whether the run has measured for longer than its time limit
|
|
939
|
+
# (Settings#time_limit). Then it starts nothing more: a benchmark it
|
|
940
|
+
# hasn't measured is an error, unmeasured, and what looked worse is
|
|
941
|
+
# judged on the measurements it has, without measuring it again. It
|
|
942
|
+
# fails what still looks worse in all of them, so a run cut short
|
|
943
|
+
# never passes a regression it would have failed. Each measurement it
|
|
944
|
+
# started runs to its end.
|
|
945
|
+
def out_of_time? = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started > @settings.time_limit
|
|
946
|
+
|
|
880
947
|
# The plugins the configuration used (Config#plugins), by name.
|
|
881
948
|
def plugins = @config.plugins
|
|
882
949
|
|
|
@@ -918,10 +985,10 @@ module Duel
|
|
|
918
985
|
side.ruby = Rubies.find(choice, side.root, side.name)
|
|
919
986
|
side.env = Rubies.environment(side.ruby)
|
|
920
987
|
@config.prepares.fetch(:version).each_with_index do |block, i|
|
|
921
|
-
resource = Resource.new(File.join(@dir, "#{side.
|
|
922
|
-
cache: File.join(@config.
|
|
988
|
+
resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
|
|
989
|
+
cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
|
|
923
990
|
@prepared << resource
|
|
924
|
-
side.env = Resource.environment(side.
|
|
991
|
+
side.env = Resource.environment(side.label, side.env, block.call(resource))
|
|
925
992
|
end
|
|
926
993
|
side
|
|
927
994
|
end
|
|
@@ -944,6 +1011,14 @@ module Duel
|
|
|
944
1011
|
slower = verdicts.select { |_, v| v.fetch("verdict") == "slower" }.keys
|
|
945
1012
|
return verdicts if slower.empty?
|
|
946
1013
|
|
|
1014
|
+
if out_of_time?
|
|
1015
|
+
# (Judged on the one measurement: slower in it, so slower.)
|
|
1016
|
+
@across_cut = true
|
|
1017
|
+
@out.puts "\nSlower across the board in #{slower.join(", ")}, not measured again: the run reached its time limit (#{limit_text})"
|
|
1018
|
+
slower.each { |metric| verdicts.fetch(metric)["confirmed"] = true }
|
|
1019
|
+
return verdicts
|
|
1020
|
+
end
|
|
1021
|
+
|
|
947
1022
|
@out.puts "\nMeasuring every benchmark again, in fresh processes: slower across the board in #{slower.join(", ")}?"
|
|
948
1023
|
again = Across.new(with_lanes { |lanes| measure_all(lanes, names) }, confidence: @settings.confidence).verdicts
|
|
949
1024
|
slower.each { |metric| verdicts.fetch(metric)["confirmed"] = again.dig(metric, "verdict") == "slower" }
|
|
@@ -960,6 +1035,11 @@ module Duel
|
|
|
960
1035
|
lanes.each_index.map do |i|
|
|
961
1036
|
Thread.new do
|
|
962
1037
|
while (name, index = queue.deq)
|
|
1038
|
+
if out_of_time?
|
|
1039
|
+
results[index] = unmeasured(name)
|
|
1040
|
+
next
|
|
1041
|
+
end
|
|
1042
|
+
|
|
963
1043
|
result = results[index] = attempt(lanes.fetch(i), name)
|
|
964
1044
|
@out.print "#{line(result)}\n" # (in one write: other threads write too)
|
|
965
1045
|
next unless result.error
|
|
@@ -979,6 +1059,16 @@ module Duel
|
|
|
979
1059
|
Result.new(name).tap { |r| r.error = "#{e.class}: #{e}" }
|
|
980
1060
|
end
|
|
981
1061
|
|
|
1062
|
+
# A benchmark the run reached its time limit before it measured: an
|
|
1063
|
+
# error, so the run fails. (Each lane's thread adds to the list.)
|
|
1064
|
+
# mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
|
|
1065
|
+
def unmeasured(name)
|
|
1066
|
+
@lock.synchronize { @unmeasured << name }
|
|
1067
|
+
Result.new(name).tap { |r| r.error = "not measured: the run reached its time limit (#{limit_text})" }
|
|
1068
|
+
end
|
|
1069
|
+
|
|
1070
|
+
def limit_text = "#{@settings.time_limit.round}s, --time-limit"
|
|
1071
|
+
|
|
982
1072
|
# Measures each suspect again, from scratch, in fresh processes, until
|
|
983
1073
|
# one measurement clears it or CONFIRM agree (and once more for each
|
|
984
1074
|
# too close to call, up to CONFIRM more: see Result#confirm).
|
|
@@ -989,6 +1079,12 @@ module Duel
|
|
|
989
1079
|
"#{CONFIRM} times each (#{CONFIRM + 1} if what's slower differs, and once more for each too close to call):"
|
|
990
1080
|
suspects.each do |result|
|
|
991
1081
|
while result.confirming?
|
|
1082
|
+
if out_of_time?
|
|
1083
|
+
@unconfirmed << result.name
|
|
1084
|
+
@out.puts format(" %-48s judged on %s: the run reached its time limit (%s)", result.name,
|
|
1085
|
+
Runner.count(result.confirmations.size + 1, "measurement"), limit_text)
|
|
1086
|
+
break
|
|
1087
|
+
end
|
|
992
1088
|
again = Lane.open(@config, @settings, @sides) { |lane| attempt(lane, result.name) }
|
|
993
1089
|
@out.puts line(again) + (result.confirm(again) ? " (too close to call: once more)" : "")
|
|
994
1090
|
end
|
|
@@ -1019,8 +1115,10 @@ module Duel
|
|
|
1019
1115
|
@out.puts format("across the board: %s %.3f [%.3f, %.3f] %s (limit %.4f, %d benchmarks)", metric, v.fetch("median"),
|
|
1020
1116
|
low, high, v.fetch("verdict"), v.fetch("limit"), v.fetch("benchmarks"))
|
|
1021
1117
|
end
|
|
1022
|
-
|
|
1118
|
+
both = @across_cut ? "slower, not measured again (the run reached its time limit)" : "slower in both measurements"
|
|
1119
|
+
regressed_across.each { |metric| @out.puts "REGRESSION across the board: #{metric} #{both}" }
|
|
1023
1120
|
errors.each { |r| @out.puts "ERROR #{r.name}: #{r.error}" }
|
|
1121
|
+
cut_short_note&.then { |note| @out.puts "TIME LIMIT: #{note}" }
|
|
1024
1122
|
regressed.each do |r|
|
|
1025
1123
|
@out.puts "REGRESSION #{r.name}: " +
|
|
1026
1124
|
(r.mixed? ? "slower in every measurement, by one of #{r.regressed.join(", ")}" : "worse #{r.regressed.join(", ")} in every measurement")
|