duel 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +68 -31
- data/CHANGELOG.md +158 -1
- data/README.md +16 -13
- data/docs/README.md +1 -1
- data/docs/benchmarks.md +32 -3
- data/docs/ci.md +365 -167
- data/docs/configuration.md +11 -4
- data/docs/how-it-works.md +109 -35
- data/docs/json.md +16 -10
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +18 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +55 -1
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rake_task.rb +10 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +337 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/duel.rb.erb +7 -3
- data/lib/duel/templates/perf-comment.yml.erb +68 -4
- data/lib/duel/templates/perf.yml.erb +136 -38
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +23 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rake_task.rbs +3 -0
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +113 -7
- data/sig/duel/stats.rbs +25 -0
- metadata +1 -1
data/lib/duel/init.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "erb"
|
|
4
4
|
require "fileutils"
|
|
5
|
+
require "tmpdir"
|
|
5
6
|
require "open3"
|
|
6
7
|
require "optparse"
|
|
7
8
|
require "pathname"
|
|
@@ -11,7 +12,8 @@ module Duel
|
|
|
11
12
|
# `duel init`: sets Duel up in a gem or an app that isn't a Rails app
|
|
12
13
|
# (duel-rails's generator sets a Rails app up). It writes .duel.rb, a
|
|
13
14
|
# benchmark to start from, and GitHub Actions workflows (Duel's, and
|
|
14
|
-
# one that comments on a pull request with its report
|
|
15
|
+
# one that comments on a pull request with its report, and with --badge
|
|
16
|
+
# a job that keeps a badge), or with
|
|
15
17
|
# --gitlab, GitLab CI jobs to include, adds a Rake
|
|
16
18
|
# task to the Rakefile, and adds StackProf to the Gemfile (for profiles
|
|
17
19
|
# of regressions, on CRuby). It overwrites nothing without --force, and
|
|
@@ -79,6 +81,7 @@ module Duel
|
|
|
79
81
|
o.banner = USAGE
|
|
80
82
|
o.on("--force", "Overwrite files that exist") { @force = true }
|
|
81
83
|
o.on("--dry-run", "Say what it would do, and write nothing") { @dry_run = true }
|
|
84
|
+
o.on("--badge", "Write a job that keeps a badge for the README (it needs contents: write)") { @badge = true }
|
|
82
85
|
o.on("--gitlab", "Write GitLab CI jobs (to include from .gitlab-ci.yml), not GitHub Actions workflows") { @gitlab = true }
|
|
83
86
|
o.on("-h", "--help", "Print this help") do
|
|
84
87
|
@out.puts o
|
|
@@ -179,6 +182,57 @@ module Duel
|
|
|
179
182
|
# The project's configuration, as the conventions have it.
|
|
180
183
|
def config = Config.new(@root)
|
|
181
184
|
|
|
185
|
+
# The project's settings, which decide the workflow's jobs: its
|
|
186
|
+
# .duel.rb's, or where there's none yet or --force replaces it, those
|
|
187
|
+
# of the one it writes. Nil where it doesn't load (one
|
|
188
|
+
# of its plugins isn't installed, say), and then the workflow has
|
|
189
|
+
# both jobs for pushes, as if .duel.rb asked for them, and says so.
|
|
190
|
+
def settings = instance_variable_defined?(:@settings) ? @settings : (@settings = load_settings)
|
|
191
|
+
|
|
192
|
+
def load_settings
|
|
193
|
+
path = File.join(@root, ".duel.rb")
|
|
194
|
+
return written_settings if @force || !File.exist?(path)
|
|
195
|
+
|
|
196
|
+
Duel.load_config(path)
|
|
197
|
+
rescue ScriptError, StandardError => e
|
|
198
|
+
@err.puts "duel init: .duel.rb didn't load (#{e.class}: #{e}), so the workflow has both jobs for pushes, as if it asked for them"
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# The settings of the .duel.rb it writes.
|
|
202
|
+
def written_settings
|
|
203
|
+
Dir.mktmpdir do |dir|
|
|
204
|
+
path = File.join(dir, ".duel.rb")
|
|
205
|
+
File.write(path, render("duel.rb"))
|
|
206
|
+
Duel.load_config(path)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# The label that accepts a regression (Config#accept_label), for the
|
|
211
|
+
# workflows: the Duel job's, to find what was accepted since the last
|
|
212
|
+
# release, and the comment's, to run the Duel job again when it's
|
|
213
|
+
# added or taken off. Nil without one, or where .duel.rb doesn't load.
|
|
214
|
+
def accept_label = settings&.accept_label
|
|
215
|
+
|
|
216
|
+
# Whether the workflow comments on a pushed commit: where a regression
|
|
217
|
+
# in a push passes (c.fail_on = :pull_requests), the only time Duel
|
|
218
|
+
# says to.
|
|
219
|
+
def comment_on_pushes?
|
|
220
|
+
found = settings
|
|
221
|
+
found.nil? || found.fail_on == :pull_requests
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
# Whether the workflow marks a push passed: where a push is compared
|
|
225
|
+
# with the last commit Duel passed (c.push_base = :last_passed).
|
|
226
|
+
def mark_passed?
|
|
227
|
+
found = settings
|
|
228
|
+
found.nil? || found.push_base == :last_passed
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# Whether the workflow keeps a badge for the README (--badge): its job
|
|
232
|
+
# moves a branch, so it's the workflow's only job, without the jobs
|
|
233
|
+
# for pushes, that writes to the repository.
|
|
234
|
+
def badge? = @badge
|
|
235
|
+
|
|
182
236
|
# The library the gemspec names (see Config#library), or nil.
|
|
183
237
|
def library = config.library
|
|
184
238
|
|
data/lib/duel/json_report.rb
CHANGED
|
@@ -77,7 +77,7 @@ module Duel
|
|
|
77
77
|
result.timed.to_h do |metric, t|
|
|
78
78
|
n = result.n #: Integer
|
|
79
79
|
[metric, {"ratio" => t.fetch("median"), "interval" => t.fetch("interval"), "limit" => t.fetch("limit"),
|
|
80
|
-
"pairs" => t.fetch("pairs"), "verdict" => t.fetch("verdict"), "baseline" => t.fetch("baseline").fdiv(n),
|
|
80
|
+
"pairs" => t.fetch("pairs"), "processes" => t["processes"], "verdict" => t.fetch("verdict"), "baseline" => t.fetch("baseline").fdiv(n),
|
|
81
81
|
"candidate" => t.fetch("candidate").fdiv(n), "unit" => t.fetch("unit"), "plural" => t.fetch("plural"),
|
|
82
82
|
"across" => t.fetch("across")}]
|
|
83
83
|
end
|
|
@@ -103,7 +103,7 @@ module Duel
|
|
|
103
103
|
def across
|
|
104
104
|
@runner.across.transform_values do |v|
|
|
105
105
|
{"ratio" => v.fetch("median"), "interval" => v.fetch("interval"), "limit" => v.fetch("limit"), "benchmarks" => v.fetch("benchmarks"),
|
|
106
|
-
"verdict" => v.fetch("verdict"), "confirmed" => v.fetch("confirmed", false)}
|
|
106
|
+
"processes" => v["processes"], "verdict" => v.fetch("verdict"), "confirmed" => v.fetch("confirmed", false)}
|
|
107
107
|
end
|
|
108
108
|
end
|
|
109
109
|
end
|
data/lib/duel/metric.rb
CHANGED
|
@@ -424,11 +424,21 @@ module Duel
|
|
|
424
424
|
|
|
425
425
|
def self.process = java.lang.management.ManagementFactory.getOperatingSystemMXBean.getProcessCpuTime
|
|
426
426
|
end
|
|
427
|
+
|
|
428
|
+
# Bytes allocated (JRuby: the JVM's count, as it has none of objects):
|
|
429
|
+
# by the benchmark thread, or by every thread.
|
|
430
|
+
# @api private
|
|
431
|
+
module Allocated
|
|
432
|
+
def self.thread = java.lang.management.ManagementFactory.getThreadMXBean.getCurrentThreadAllocatedBytes
|
|
433
|
+
|
|
434
|
+
def self.process = java.lang.management.ManagementFactory.getThreadMXBean.getTotalThreadAllocatedBytes
|
|
435
|
+
end
|
|
427
436
|
end
|
|
428
437
|
# simplecov:enable
|
|
429
438
|
|
|
430
439
|
# The built-in metrics: their measures are named ("cpu",
|
|
431
|
-
# "instructions", "allocations"), as Config#measure
|
|
440
|
+
# "instructions", "allocations", "allocated_bytes"), as Config#measure
|
|
441
|
+
# takes them.
|
|
432
442
|
# @api private
|
|
433
443
|
module Metrics
|
|
434
444
|
# CPU time (see Definition#cpu), in nanoseconds: what other
|
|
@@ -477,6 +487,27 @@ module Duel
|
|
|
477
487
|
def stop_sample = (@counter.read - @started if @counting)
|
|
478
488
|
end
|
|
479
489
|
|
|
490
|
+
# Bytes allocated (JRuby, which counts no objects): the benchmark
|
|
491
|
+
# thread's, or for a benchmark with threads of its own, every
|
|
492
|
+
# thread's (see Definition#cpu), as the JVM counts them. It's the same
|
|
493
|
+
# in every worker, once the JIT has compiled the code (it can do away
|
|
494
|
+
# with an allocation, so the count falls as it warms up), but for a
|
|
495
|
+
# sample here and there some 31 bytes an iteration apart: so more by
|
|
496
|
+
# 32 bytes an iteration, about a small object's, isn't more yet.
|
|
497
|
+
class AllocatedBytes < Metric
|
|
498
|
+
def self.measures = {"allocated_bytes" => {tolerance: 32, unit: %w[byte bytes]}}
|
|
499
|
+
|
|
500
|
+
# mutant:disable -- mutant runs on CRuby, where this is always false
|
|
501
|
+
def self.available? = RUBY_ENGINE == "jruby"
|
|
502
|
+
|
|
503
|
+
def start_sample(sample)
|
|
504
|
+
@scope = sample.benchmark.cpu
|
|
505
|
+
@started = Allocated.public_send(@scope)
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
def stop_sample = Allocated.public_send(@scope) - @started
|
|
509
|
+
end
|
|
510
|
+
|
|
480
511
|
# Ruby objects allocated (CRuby). The same code's count can differ by
|
|
481
512
|
# a few from run to run (ids of random lengths, say), so it's one
|
|
482
513
|
# more per two iterations, not one more, that's a regression.
|
data/lib/duel/rake_task.rb
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "rake"
|
|
4
|
-
require_relative "cli"
|
|
5
4
|
|
|
6
5
|
module Duel
|
|
7
6
|
# A rake task running `duel` with +args+:
|
|
@@ -9,12 +8,22 @@ module Duel
|
|
|
9
8
|
# require "duel/rake_task"
|
|
10
9
|
# Duel::RakeTask.new(:perf)
|
|
11
10
|
# Duel::RakeTask.new(:perf_since_release, "--base", "last-release")
|
|
11
|
+
#
|
|
12
|
+
# Duel itself loads only as the task runs: a Rakefile that defines the
|
|
13
|
+
# task loads nothing more for its other tasks (rake test, say).
|
|
12
14
|
class RakeTask
|
|
13
15
|
include Rake::DSL
|
|
14
16
|
|
|
17
|
+
# Loads Duel's command, as the task runs.
|
|
18
|
+
# mutant:disable -- the tests' process has Duel loaded from the start, so
|
|
19
|
+
# a require changes nothing there: RakeTaskTest's
|
|
20
|
+
# test_the_task_loads_duel_as_it_runs checks it, in a process of its own
|
|
21
|
+
def self.load_duel = require_relative("cli")
|
|
22
|
+
|
|
15
23
|
def initialize(name = :duel, *args)
|
|
16
24
|
desc "Fail if this tree is slower than its baseline at any benchmark (duel)"
|
|
17
25
|
task(name) do
|
|
26
|
+
RakeTask.load_duel
|
|
18
27
|
case CLI.new(args).run
|
|
19
28
|
when 1 then abort "duel: slower than the baseline"
|
|
20
29
|
when 2 then abort "duel: couldn't compare"
|
data/lib/duel/rubies.rb
CHANGED
|
@@ -35,6 +35,21 @@ module Duel
|
|
|
35
35
|
VARIABLES = %w[GEM_HOME GEM_PATH GEM_ROOT RUBY_ROOT RUBY_ENGINE RUBY_VERSION RUBY_PATCHLEVEL RBENV_VERSION
|
|
36
36
|
MISE_RUBY_VERSION ASDF_RUBY_VERSION].freeze
|
|
37
37
|
|
|
38
|
+
# The JVM's options for a JRuby worker. Each JVM's JIT compiles the
|
|
39
|
+
# same code a little differently, as its compiling in the background
|
|
40
|
+
# races its running, so one worker can be some 10% faster or slower
|
|
41
|
+
# than another for its whole life. -Xbatch compiles in the
|
|
42
|
+
# foreground, so what's compiled is less a matter of timing, and the
|
|
43
|
+
# serial collector runs no threads of its own beside the code:
|
|
44
|
+
# together they make JRuby's intervals over workers some half as
|
|
45
|
+
# wide.
|
|
46
|
+
JRUBY_WORKER = %w[-J-Xbatch -J-XX:+UseSerialGC].freeze
|
|
47
|
+
|
|
48
|
+
# A JVM option that chooses its garbage collector: the JVM won't start
|
|
49
|
+
# with two, so where JRUBY_OPTS chooses one, the serial one isn't
|
|
50
|
+
# added.
|
|
51
|
+
GARBAGE_COLLECTOR = /\A-J-XX:\+Use\w+GC\z/
|
|
52
|
+
|
|
38
53
|
class << self
|
|
39
54
|
# The Ruby (its executable) that +choice+ gives the version +name+d
|
|
40
55
|
# ("baseline") at +root+: one that isn't there fails.
|
|
@@ -50,6 +65,30 @@ module Duel
|
|
|
50
65
|
ruby
|
|
51
66
|
end
|
|
52
67
|
|
|
68
|
+
# The engine of +ruby+ (RUBY_ENGINE): the runner's, or what +ruby+
|
|
69
|
+
# says, without the runner's gems (or "ruby", where it can't say).
|
|
70
|
+
def engine(ruby)
|
|
71
|
+
return RUBY_ENGINE if runner?(ruby)
|
|
72
|
+
|
|
73
|
+
out = IO.popen(environment(ruby).merge("RUBYOPT" => nil), [ruby, "--disable-gems", "-e", "print RUBY_ENGINE"], err: File::NULL, &:read)
|
|
74
|
+
status = $? #: Process::Status
|
|
75
|
+
status.success? ? out : "ruby"
|
|
76
|
+
rescue SystemCallError
|
|
77
|
+
"ruby"
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# What a worker on +engine+ starts with: +env+, and on JRuby, the
|
|
81
|
+
# JVM's options in JRUBY_WORKER (+env+'s JRUBY_OPTS, or else
|
|
82
|
+
# +inherited+, the runner's, before them).
|
|
83
|
+
def worker_environment(env, engine, inherited = ENV["JRUBY_OPTS"])
|
|
84
|
+
return env unless engine == "jruby"
|
|
85
|
+
|
|
86
|
+
options = env.fetch("JRUBY_OPTS") { inherited }.to_s.split
|
|
87
|
+
collector = options.any? { |option| option.match?(GARBAGE_COLLECTOR) }
|
|
88
|
+
added = JRUBY_WORKER.reject { |option| options.include?(option) || (collector && option.match?(GARBAGE_COLLECTOR)) }
|
|
89
|
+
env.merge("JRUBY_OPTS" => (options + added).join(" "))
|
|
90
|
+
end
|
|
91
|
+
|
|
53
92
|
# Whether +ruby+ is the runner's own.
|
|
54
93
|
def runner?(ruby) = File.realpath(ruby) == File.realpath(RbConfig.ruby)
|
|
55
94
|
|