duel 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +135 -1
- data/README.md +5 -5
- data/docs/ci.md +145 -42
- data/docs/configuration.md +8 -4
- data/docs/how-it-works.md +123 -35
- data/docs/json.md +18 -12
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +25 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +19 -3
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/memory.rb +54 -0
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +383 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/perf-comment.yml.erb +53 -5
- data/lib/duel/templates/perf.yml.erb +85 -42
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +6 -0
- data/sig/duel/memory.rbs +30 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +129 -6
- data/sig/duel/stats.rbs +25 -0
- metadata +3 -1
data/lib/duel/init.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "erb"
|
|
4
4
|
require "fileutils"
|
|
5
|
+
require "tmpdir"
|
|
5
6
|
require "open3"
|
|
6
7
|
require "optparse"
|
|
7
8
|
require "pathname"
|
|
@@ -182,21 +183,36 @@ module Duel
|
|
|
182
183
|
def config = Config.new(@root)
|
|
183
184
|
|
|
184
185
|
# The project's settings, which decide the workflow's jobs: its
|
|
185
|
-
# .duel.rb's, or
|
|
186
|
-
#
|
|
186
|
+
# .duel.rb's, or where there's none yet or --force replaces it, those
|
|
187
|
+
# of the one it writes. Nil where it doesn't load (one
|
|
187
188
|
# of its plugins isn't installed, say), and then the workflow has
|
|
188
189
|
# both jobs for pushes, as if .duel.rb asked for them, and says so.
|
|
189
190
|
def settings = instance_variable_defined?(:@settings) ? @settings : (@settings = load_settings)
|
|
190
191
|
|
|
191
192
|
def load_settings
|
|
192
193
|
path = File.join(@root, ".duel.rb")
|
|
193
|
-
return
|
|
194
|
+
return written_settings if @force || !File.exist?(path)
|
|
194
195
|
|
|
195
196
|
Duel.load_config(path)
|
|
196
197
|
rescue ScriptError, StandardError => e
|
|
197
198
|
@err.puts "duel init: .duel.rb didn't load (#{e.class}: #{e}), so the workflow has both jobs for pushes, as if it asked for them"
|
|
198
199
|
end
|
|
199
200
|
|
|
201
|
+
# The settings of the .duel.rb it writes.
|
|
202
|
+
def written_settings
|
|
203
|
+
Dir.mktmpdir do |dir|
|
|
204
|
+
path = File.join(dir, ".duel.rb")
|
|
205
|
+
File.write(path, render("duel.rb"))
|
|
206
|
+
Duel.load_config(path)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# The label that accepts a regression (Config#accept_label), for the
|
|
211
|
+
# workflows: the Duel job's, to find what was accepted since the last
|
|
212
|
+
# release, and the comment's, to run the Duel job again when it's
|
|
213
|
+
# added or taken off. Nil without one, or where .duel.rb doesn't load.
|
|
214
|
+
def accept_label = settings&.accept_label
|
|
215
|
+
|
|
200
216
|
# Whether the workflow comments on a pushed commit: where a regression
|
|
201
217
|
# in a push passes (c.fail_on = :pull_requests), the only time Duel
|
|
202
218
|
# says to.
|
data/lib/duel/json_report.rb
CHANGED
|
@@ -77,7 +77,7 @@ module Duel
|
|
|
77
77
|
result.timed.to_h do |metric, t|
|
|
78
78
|
n = result.n #: Integer
|
|
79
79
|
[metric, {"ratio" => t.fetch("median"), "interval" => t.fetch("interval"), "limit" => t.fetch("limit"),
|
|
80
|
-
"pairs" => t.fetch("pairs"), "verdict" => t.fetch("verdict"), "baseline" => t.fetch("baseline").fdiv(n),
|
|
80
|
+
"pairs" => t.fetch("pairs"), "processes" => t["processes"], "verdict" => t.fetch("verdict"), "baseline" => t.fetch("baseline").fdiv(n),
|
|
81
81
|
"candidate" => t.fetch("candidate").fdiv(n), "unit" => t.fetch("unit"), "plural" => t.fetch("plural"),
|
|
82
82
|
"across" => t.fetch("across")}]
|
|
83
83
|
end
|
|
@@ -103,7 +103,7 @@ module Duel
|
|
|
103
103
|
def across
|
|
104
104
|
@runner.across.transform_values do |v|
|
|
105
105
|
{"ratio" => v.fetch("median"), "interval" => v.fetch("interval"), "limit" => v.fetch("limit"), "benchmarks" => v.fetch("benchmarks"),
|
|
106
|
-
"verdict" => v.fetch("verdict"), "confirmed" => v.fetch("confirmed", false)}
|
|
106
|
+
"processes" => v["processes"], "verdict" => v.fetch("verdict"), "confirmed" => v.fetch("confirmed", false)}
|
|
107
107
|
end
|
|
108
108
|
end
|
|
109
109
|
end
|
data/lib/duel/memory.rb
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Duel
|
|
4
|
+
# The memory a run's workers can take (see Runner#fit): what the
|
|
5
|
+
# machine has available, or what its cgroup's limit leaves, if that's
|
|
6
|
+
# less (a container's, say).
|
|
7
|
+
# @api private
|
|
8
|
+
module Memory
|
|
9
|
+
# The kinds of pages macOS's vm_stat counts that a process can have
|
|
10
|
+
# without the system swapping: free, and cached but unused.
|
|
11
|
+
MAC_PAGES = %w[free inactive speculative].freeze
|
|
12
|
+
|
|
13
|
+
# Where Linux's proc and sys are.
|
|
14
|
+
ROOT = "/"
|
|
15
|
+
|
|
16
|
+
class << self
|
|
17
|
+
# The bytes available, or nil where Duel can't tell: Linux's
|
|
18
|
+
# MemAvailable (in +root+'s proc/meminfo), at most what the cgroup
|
|
19
|
+
# mounted at sys/fs/cgroup leaves, or else macOS's free and
|
|
20
|
+
# inactive pages (in what +vm_stat+ prints).
|
|
21
|
+
def available(root: ROOT, vm_stat: -> { command("vm_stat") }) = linux(root) || mac(vm_stat.call)
|
|
22
|
+
|
|
23
|
+
# MemAvailable, at most what the cgroup leaves, or nil.
|
|
24
|
+
def linux(root)
|
|
25
|
+
kb = read(File.join(root, "proc", "meminfo"))&.[](/^MemAvailable:\s+(\d+) kB$/, 1) or return
|
|
26
|
+
[Integer(kb) * 1024, cgroup(File.join(root, "sys", "fs", "cgroup"))].compact.min
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# What a cgroup (v2) at +dir+ leaves of its limit, or nil where it
|
|
30
|
+
# has none ("max").
|
|
31
|
+
def cgroup(dir)
|
|
32
|
+
max, current = %w[memory.max memory.current].map { |file| Integer(read(File.join(dir, file)), exception: false) }
|
|
33
|
+
max - current if max && current
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The bytes in the pages of MAC_PAGES that vm_stat's +output+
|
|
37
|
+
# counts, or nil.
|
|
38
|
+
def mac(output)
|
|
39
|
+
page = output&.[](/page size of (\d+) bytes/, 1) or return
|
|
40
|
+
MAC_PAGES.sum { |kind| Integer(output[/^Pages #{kind}:\s+(\d+)\./, 1] || 0) } * Integer(page)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# A file's contents, or nil where there's none.
|
|
44
|
+
def read(path) = (File.read(path) if File.file?(path))
|
|
45
|
+
|
|
46
|
+
# What a command prints, or nil where there's no such command.
|
|
47
|
+
def command(*args)
|
|
48
|
+
IO.popen(args, err: File::NULL, &:read)
|
|
49
|
+
rescue SystemCallError
|
|
50
|
+
nil
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
data/lib/duel/metric.rb
CHANGED
|
@@ -424,11 +424,21 @@ module Duel
|
|
|
424
424
|
|
|
425
425
|
def self.process = java.lang.management.ManagementFactory.getOperatingSystemMXBean.getProcessCpuTime
|
|
426
426
|
end
|
|
427
|
+
|
|
428
|
+
# Bytes allocated (JRuby: the JVM's count, as it has none of objects):
|
|
429
|
+
# by the benchmark thread, or by every thread.
|
|
430
|
+
# @api private
|
|
431
|
+
module Allocated
|
|
432
|
+
def self.thread = java.lang.management.ManagementFactory.getThreadMXBean.getCurrentThreadAllocatedBytes
|
|
433
|
+
|
|
434
|
+
def self.process = java.lang.management.ManagementFactory.getThreadMXBean.getTotalThreadAllocatedBytes
|
|
435
|
+
end
|
|
427
436
|
end
|
|
428
437
|
# simplecov:enable
|
|
429
438
|
|
|
430
439
|
# The built-in metrics: their measures are named ("cpu",
|
|
431
|
-
# "instructions", "allocations"), as Config#measure
|
|
440
|
+
# "instructions", "allocations", "allocated_bytes"), as Config#measure
|
|
441
|
+
# takes them.
|
|
432
442
|
# @api private
|
|
433
443
|
module Metrics
|
|
434
444
|
# CPU time (see Definition#cpu), in nanoseconds: what other
|
|
@@ -477,6 +487,27 @@ module Duel
|
|
|
477
487
|
def stop_sample = (@counter.read - @started if @counting)
|
|
478
488
|
end
|
|
479
489
|
|
|
490
|
+
# Bytes allocated (JRuby, which counts no objects): the benchmark
|
|
491
|
+
# thread's, or for a benchmark with threads of its own, every
|
|
492
|
+
# thread's (see Definition#cpu), as the JVM counts them. It's the same
|
|
493
|
+
# in every worker, once the JIT has compiled the code (it can do away
|
|
494
|
+
# with an allocation, so the count falls as it warms up), but for a
|
|
495
|
+
# sample here and there some 31 bytes an iteration apart: so more by
|
|
496
|
+
# 32 bytes an iteration, about a small object's, isn't more yet.
|
|
497
|
+
class AllocatedBytes < Metric
|
|
498
|
+
def self.measures = {"allocated_bytes" => {tolerance: 32, unit: %w[byte bytes]}}
|
|
499
|
+
|
|
500
|
+
# mutant:disable -- mutant runs on CRuby, where this is always false
|
|
501
|
+
def self.available? = RUBY_ENGINE == "jruby"
|
|
502
|
+
|
|
503
|
+
def start_sample(sample)
|
|
504
|
+
@scope = sample.benchmark.cpu
|
|
505
|
+
@started = Allocated.public_send(@scope)
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
def stop_sample = Allocated.public_send(@scope) - @started
|
|
509
|
+
end
|
|
510
|
+
|
|
480
511
|
# Ruby objects allocated (CRuby). The same code's count can differ by
|
|
481
512
|
# a few from run to run (ids of random lengths, say), so it's one
|
|
482
513
|
# more per two iterations, not one more, that's a regression.
|
data/lib/duel/rubies.rb
CHANGED
|
@@ -35,6 +35,21 @@ module Duel
|
|
|
35
35
|
VARIABLES = %w[GEM_HOME GEM_PATH GEM_ROOT RUBY_ROOT RUBY_ENGINE RUBY_VERSION RUBY_PATCHLEVEL RBENV_VERSION
|
|
36
36
|
MISE_RUBY_VERSION ASDF_RUBY_VERSION].freeze
|
|
37
37
|
|
|
38
|
+
# The JVM's options for a JRuby worker. Each JVM's JIT compiles the
|
|
39
|
+
# same code a little differently, as its compiling in the background
|
|
40
|
+
# races its running, so one worker can be some 10% faster or slower
|
|
41
|
+
# than another for its whole life. -Xbatch compiles in the
|
|
42
|
+
# foreground, so what's compiled is less a matter of timing, and the
|
|
43
|
+
# serial collector runs no threads of its own beside the code:
|
|
44
|
+
# together they make JRuby's intervals over workers some half as
|
|
45
|
+
# wide.
|
|
46
|
+
JRUBY_WORKER = %w[-J-Xbatch -J-XX:+UseSerialGC].freeze
|
|
47
|
+
|
|
48
|
+
# A JVM option that chooses its garbage collector: the JVM won't start
|
|
49
|
+
# with two, so where JRUBY_OPTS chooses one, the serial one isn't
|
|
50
|
+
# added.
|
|
51
|
+
GARBAGE_COLLECTOR = /\A-J-XX:\+Use\w+GC\z/
|
|
52
|
+
|
|
38
53
|
class << self
|
|
39
54
|
# The Ruby (its executable) that +choice+ gives the version +name+d
|
|
40
55
|
# ("baseline") at +root+: one that isn't there fails.
|
|
@@ -50,6 +65,30 @@ module Duel
|
|
|
50
65
|
ruby
|
|
51
66
|
end
|
|
52
67
|
|
|
68
|
+
# The engine of +ruby+ (RUBY_ENGINE): the runner's, or what +ruby+
|
|
69
|
+
# says, without the runner's gems (or "ruby", where it can't say).
|
|
70
|
+
def engine(ruby)
|
|
71
|
+
return RUBY_ENGINE if runner?(ruby)
|
|
72
|
+
|
|
73
|
+
out = IO.popen(environment(ruby).merge("RUBYOPT" => nil), [ruby, "--disable-gems", "-e", "print RUBY_ENGINE"], err: File::NULL, &:read)
|
|
74
|
+
status = $? #: Process::Status
|
|
75
|
+
status.success? ? out : "ruby"
|
|
76
|
+
rescue SystemCallError
|
|
77
|
+
"ruby"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# What a worker on +engine+ starts with: +env+, and on JRuby, the
|
|
81
|
+
# JVM's options in JRUBY_WORKER (+env+'s JRUBY_OPTS, or else
|
|
82
|
+
# +inherited+, the runner's, before them).
|
|
83
|
+
def worker_environment(env, engine, inherited = ENV["JRUBY_OPTS"])
|
|
84
|
+
return env unless engine == "jruby"
|
|
85
|
+
|
|
86
|
+
options = env.fetch("JRUBY_OPTS") { inherited }.to_s.split
|
|
87
|
+
collector = options.any? { |option| option.match?(GARBAGE_COLLECTOR) }
|
|
88
|
+
added = JRUBY_WORKER.reject { |option| options.include?(option) || (collector && option.match?(GARBAGE_COLLECTOR)) }
|
|
89
|
+
env.merge("JRUBY_OPTS" => (options + added).join(" "))
|
|
90
|
+
end
|
|
91
|
+
|
|
53
92
|
# Whether +ruby+ is the runner's own.
|
|
54
93
|
def runner?(ruby) = File.realpath(ruby) == File.realpath(RbConfig.ruby)
|
|
55
94
|
|