duel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +51 -27
- data/CHANGELOG.md +247 -1
- data/README.md +30 -14
- data/docs/benchmarks.md +28 -24
- data/docs/ci.md +328 -362
- data/docs/configuration.md +40 -37
- data/docs/how-it-works.md +23 -10
- data/docs/json.md +21 -25
- data/docs/plugins.md +40 -22
- data/docs/troubleshooting.md +3 -2
- data/lib/duel/across.rb +18 -18
- data/lib/duel/badge.rb +174 -0
- data/lib/duel/baseline.rb +38 -15
- data/lib/duel/cli.rb +175 -107
- data/lib/duel/comment.rb +80 -71
- data/lib/duel/config.rb +145 -63
- data/lib/duel/context.rb +29 -24
- data/lib/duel/forge.rb +87 -0
- data/lib/duel/init.rb +44 -87
- data/lib/duel/json_report.rb +34 -25
- data/lib/duel/lane.rb +107 -61
- data/lib/duel/lock.rb +1 -1
- data/lib/duel/mark_passed.rb +132 -0
- data/lib/duel/metric.rb +28 -27
- data/lib/duel/native.rb +7 -5
- data/lib/duel/platform.rb +4 -0
- data/lib/duel/process_sample.rb +9 -6
- data/lib/duel/rake_task.rb +4 -5
- data/lib/duel/resource.rb +10 -11
- data/lib/duel/result.rb +22 -14
- data/lib/duel/rubies.rb +6 -2
- data/lib/duel/runner.rb +27 -15
- data/lib/duel/spawn.rb +29 -4
- data/lib/duel/stats.rb +14 -12
- data/lib/duel/summary.rb +31 -29
- data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
- data/lib/duel/templates/duel.rb.erb +7 -7
- data/lib/duel/templates/perf-caller.yml.erb +22 -5
- data/lib/duel/templates/perf-comment.yml.erb +11 -11
- data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
- data/lib/duel/templates/perf-push.yml.erb +50 -0
- data/lib/duel/templates/perf.yml.erb +113 -335
- data/lib/duel/testing.rb +20 -25
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +16 -9
- data/lib/duel/worker_process.rb +6 -6
- data/lib/duel.rb +88 -33
- data/sig/duel/across.rbs +4 -0
- data/sig/duel/badge.rbs +51 -0
- data/sig/duel/baseline.rbs +7 -2
- data/sig/duel/cli.rbs +31 -20
- data/sig/duel/comment.rbs +14 -15
- data/sig/duel/config.rbs +71 -29
- data/sig/duel/context.rbs +4 -4
- data/sig/duel/forge.rbs +42 -0
- data/sig/duel/init.rbs +8 -26
- data/sig/duel/json_report.rbs +7 -6
- data/sig/duel/lane.rbs +29 -6
- data/sig/duel/mark_passed.rbs +39 -0
- data/sig/duel/metric.rbs +10 -10
- data/sig/duel/native.rbs +3 -0
- data/sig/duel/platform.rbs +1 -0
- data/sig/duel/process_sample.rbs +3 -0
- data/sig/duel/resource.rbs +3 -3
- data/sig/duel/result.rbs +12 -5
- data/sig/duel/rubies.rbs +3 -0
- data/sig/duel/runner.rbs +23 -16
- data/sig/duel/spawn.rbs +7 -0
- data/sig/duel/stats.rbs +4 -1
- data/sig/duel/summary.rbs +8 -0
- data/sig/duel/testing.rbs +2 -2
- data/sig/duel/worker.rbs +6 -5
- data/sig/duel/worker_process.rbs +3 -3
- data/sig/duel.rbs +50 -31
- metadata +9 -1
data/lib/duel/context.rb
CHANGED
|
@@ -68,10 +68,10 @@ module Duel
|
|
|
68
68
|
body = definition.body
|
|
69
69
|
@body = bound(body)
|
|
70
70
|
@given_n = body.arity == 1
|
|
71
|
-
@setup =
|
|
72
|
-
@teardown =
|
|
73
|
-
@before_sample =
|
|
74
|
-
@after =
|
|
71
|
+
@setup = plain_hooks(:setup, hooks)
|
|
72
|
+
@teardown = plain_hooks(:teardown, hooks).reverse
|
|
73
|
+
@before_sample = plain_hooks(:before_sample, hooks)
|
|
74
|
+
@after = plain_hooks(:after_iterations, hooks)
|
|
75
75
|
# (Each around_iteration hook's Iteration, innermost first, made once:
|
|
76
76
|
# running one allocates nothing.)
|
|
77
77
|
iteration = nil #: Iteration?
|
|
@@ -188,27 +188,27 @@ module Duel
|
|
|
188
188
|
hooks.fetch(name) + [*file, *@definition.hooks[name]].map { |hook| bound(hook) }
|
|
189
189
|
end
|
|
190
190
|
|
|
191
|
-
# +
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
191
|
+
# The hooks +name+d (see #hooks), which Duel gives nothing: one that
|
|
192
|
+
# takes a parameter fails, saying so, not getting nil.
|
|
193
|
+
def plain_hooks(name, hooks)
|
|
194
|
+
hooks(name, hooks).each do |hook|
|
|
195
|
+
raise Error, "#{@definition.name}: a #{name} hook takes no parameter (give it a block without one)" unless hook.arity.zero?
|
|
196
|
+
end
|
|
197
197
|
end
|
|
198
198
|
|
|
199
199
|
# What pausing and resuming leaves in each of +metrics+' measures, an
|
|
200
200
|
# iteration: the least of what CALIBRATION samples of STRETCHES empty
|
|
201
201
|
# iterations left.
|
|
202
202
|
def paused(metrics)
|
|
203
|
-
|
|
203
|
+
samples = Array.new(Snapshot::CALIBRATION) do
|
|
204
204
|
sample = Sample.new(@definition, metrics, n: STRETCHES)
|
|
205
205
|
sample.start
|
|
206
206
|
sample.between { STRETCHES.times { sample.unpause {} } }
|
|
207
207
|
sample.stop
|
|
208
|
-
sample
|
|
208
|
+
sample
|
|
209
209
|
end
|
|
210
210
|
# (Two stretches more: from the start, and to the stop.)
|
|
211
|
-
least(
|
|
211
|
+
least(samples, STRETCHES + 2)
|
|
212
212
|
end
|
|
213
213
|
|
|
214
214
|
# What the around_iteration hooks add to each of +metrics+' measures
|
|
@@ -220,16 +220,19 @@ module Duel
|
|
|
220
220
|
def hooked(metrics)
|
|
221
221
|
return NO_LEAK if metrics.empty?
|
|
222
222
|
|
|
223
|
-
|
|
223
|
+
samples = Array.new(Snapshot::CALIBRATION) do
|
|
224
224
|
sample = Sample.new(@definition, metrics, n: HOOKED)
|
|
225
225
|
run(HOOKED, Hooked.new(sample))
|
|
226
|
-
sample
|
|
226
|
+
sample
|
|
227
227
|
end
|
|
228
|
-
least(
|
|
228
|
+
least(samples, HOOKED)
|
|
229
229
|
end
|
|
230
230
|
|
|
231
|
-
# Each measure's least in +
|
|
232
|
-
def least(
|
|
231
|
+
# Each measure's least in +samples+, over +stretches+.
|
|
232
|
+
def least(samples, stretches)
|
|
233
|
+
runs = samples.map { |sample| sample.timed.merge(sample.counted) }
|
|
234
|
+
runs.first.to_h { |name, _| [name, runs.map { |run| run.fetch(name) }.min.fdiv(stretches)] }
|
|
235
|
+
end
|
|
233
236
|
|
|
234
237
|
# Runs +n+ iterations, then its after_iterations hooks, each inside
|
|
235
238
|
# the around_iteration hooks if it has them, with +sample+ paused
|
|
@@ -248,17 +251,19 @@ module Duel
|
|
|
248
251
|
end
|
|
249
252
|
end
|
|
250
253
|
end
|
|
251
|
-
|
|
254
|
+
return
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
if @given_n
|
|
252
258
|
@body.call(n)
|
|
253
|
-
@after.each(&:call)
|
|
254
259
|
else
|
|
255
260
|
i = 0
|
|
256
261
|
while i < n
|
|
257
262
|
@body.call
|
|
258
263
|
i += 1
|
|
259
264
|
end
|
|
260
|
-
@after.each(&:call)
|
|
261
265
|
end
|
|
266
|
+
@after.each(&:call)
|
|
262
267
|
end
|
|
263
268
|
|
|
264
269
|
# +block+ as a method of the context's (of a module of its own, not
|
|
@@ -350,7 +355,7 @@ module Duel
|
|
|
350
355
|
@benchmark = benchmark
|
|
351
356
|
@n = n
|
|
352
357
|
@counting = counting
|
|
353
|
-
@metrics = metrics.select { |m| m.kinds.value?(:
|
|
358
|
+
@metrics = metrics.select { |m| m.kinds.value?(:timed) || counting }
|
|
354
359
|
# (Those paused around each iteration, inside around_iteration hooks:
|
|
355
360
|
# not those costly to read. Without such hooks, or such a metric, the
|
|
356
361
|
# same Array: nothing more is allocated, or worked out for every
|
|
@@ -447,9 +452,9 @@ module Duel
|
|
|
447
452
|
return if measured.nil?
|
|
448
453
|
|
|
449
454
|
kind = kinds.fetch(name)
|
|
450
|
-
return unless kind == :
|
|
455
|
+
return unless kind == :timed || @counting
|
|
451
456
|
|
|
452
|
-
totals = (kind == :
|
|
457
|
+
totals = (kind == :timed) ? @timed : @counted
|
|
453
458
|
totals[name] = totals.fetch(name, 0) + measured
|
|
454
459
|
end
|
|
455
460
|
end
|
data/lib/duel/forge.rb
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require_relative "spawn"
|
|
6
|
+
|
|
7
|
+
module Duel
|
|
8
|
+
# What the commands that run in CI's jobs of their own (Comment,
|
|
9
|
+
# MarkPassed, Badge) share: GitHub's API, through gh, and GitLab's,
|
|
10
|
+
# over HTTP, each saying why what it asked failed, as the command
|
|
11
|
+
# (#command) it's for. An includer has @env and @err, and names its
|
|
12
|
+
# GitLab token (#gitlab_token).
|
|
13
|
+
# @api private
|
|
14
|
+
module Forge
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
# What gh api prints, given +args+ (and +input+ on its standard
|
|
18
|
+
# input), or nil where it fails (or isn't there), which it says why,
|
|
19
|
+
# unless +quiet+ (where failing says what's there, a branch that
|
|
20
|
+
# isn't, say).
|
|
21
|
+
def gh(*args, quiet: false)
|
|
22
|
+
out, err, status = Spawn.open3(:capture3, "gh", "api", *args)
|
|
23
|
+
return out if status.success?
|
|
24
|
+
|
|
25
|
+
@err.puts "#{command}: gh api #{args.first}: #{err.strip}" unless quiet
|
|
26
|
+
rescue SystemCallError => e
|
|
27
|
+
@err.puts "#{command}: gh: #{e}"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# What GitLab's API says to +verb+ (:Get, :Post or :Delete) +path+
|
|
31
|
+
# (under CI_API_V4_URL), given +body+ (as JSON), signed in with
|
|
32
|
+
# #gitlab_token, or nil where it fails, which it says why, unless
|
|
33
|
+
# +quiet+. With +pages+, that and the next page's number
|
|
34
|
+
# (X-Next-Page), if there's one.
|
|
35
|
+
def gitlab(verb, path, body = nil, pages: false, quiet: false)
|
|
36
|
+
uri = URI("#{@env.fetch("CI_API_V4_URL")}#{path}")
|
|
37
|
+
request = Net::HTTP.const_get(verb).new(uri)
|
|
38
|
+
request["PRIVATE-TOKEN"] = @env.fetch(gitlab_token)
|
|
39
|
+
request["Content-Type"] = "application/json"
|
|
40
|
+
request.body = JSON.generate(body) if body
|
|
41
|
+
http = connection(uri)
|
|
42
|
+
http.use_ssl = uri.scheme == "https"
|
|
43
|
+
response = http.request(request)
|
|
44
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
45
|
+
return if quiet
|
|
46
|
+
|
|
47
|
+
return @err.puts("#{command}: GitLab: #{verb.upcase} #{path}: #{[response.code, response.body.to_s.strip].reject(&:empty?).join(" ")}")
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
text = response.body.to_s
|
|
51
|
+
none = {} #: Hash[String, untyped]
|
|
52
|
+
# (Nothing, as a deletion says.)
|
|
53
|
+
parsed = text.empty? ? none : JSON.parse(text)
|
|
54
|
+
pages ? [parsed, response["X-Next-Page"].to_s[/\A[0-9]+\z/]] : parsed
|
|
55
|
+
rescue SystemCallError, IOError, Timeout::Error, JSON::ParserError, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
|
|
56
|
+
@err.puts "#{command}: GitLab: #{verb.upcase} #{path}: #{e}"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Names the branch (of those +branch+'s name is in, which the block
|
|
60
|
+
# says is there) that keeps Git from making +branch+: a branch named
|
|
61
|
+
# duel blocks duel/passed/main. Returns 1.
|
|
62
|
+
def blocked(branch)
|
|
63
|
+
prefixes = branch.split("/").then { |parts| (1...parts.size).map { |n| parts.first(n).join("/") } }
|
|
64
|
+
prefixes.reverse_each.select { |prefix| yield(prefix) }.each do |prefix|
|
|
65
|
+
@out.puts "#{"::error::" unless gitlab?}The branch #{prefix} keeps Duel from making #{branch}, as Git can't have both. " \
|
|
66
|
+
"Delete it (git push origin --delete #{prefix}), or rename it."
|
|
67
|
+
end
|
|
68
|
+
1
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# A connection to +uri+'s host, not yet started.
|
|
72
|
+
# mutant:disable -- a nil host is Net::HTTP's localhost, where the
|
|
73
|
+
# tests' server is, so only another host could tell them apart
|
|
74
|
+
def connection(uri) = Net::HTTP.new(uri.host.to_s, uri.port)
|
|
75
|
+
|
|
76
|
+
# Whether it runs on GitLab CI (GITLAB_CI, set and not empty).
|
|
77
|
+
def gitlab? = !@env["GITLAB_CI"].to_s.empty?
|
|
78
|
+
|
|
79
|
+
def repository = @env.fetch("GITHUB_REPOSITORY")
|
|
80
|
+
|
|
81
|
+
# The project's path in GitLab's API.
|
|
82
|
+
def project = "/projects/#{@env.fetch("CI_PROJECT_ID")}"
|
|
83
|
+
|
|
84
|
+
# +ref+, as a path of GitLab's API takes a branch's name.
|
|
85
|
+
def encoded(ref) = ref.gsub("/", "%2F")
|
|
86
|
+
end
|
|
87
|
+
end
|
data/lib/duel/init.rb
CHANGED
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
require "erb"
|
|
4
4
|
require "json"
|
|
5
5
|
require "fileutils"
|
|
6
|
-
require "tmpdir"
|
|
7
6
|
require "open3"
|
|
8
7
|
require "optparse"
|
|
9
8
|
require "pathname"
|
|
@@ -20,9 +19,9 @@ module Duel
|
|
|
20
19
|
# a job that keeps a badge), or with
|
|
21
20
|
# --gitlab, GitLab CI jobs to include, adds a Rake
|
|
22
21
|
# task to the Rakefile, and adds StackProf to the Gemfile (for profiles
|
|
23
|
-
# of regressions, on CRuby).
|
|
24
|
-
#
|
|
25
|
-
#
|
|
22
|
+
# of regressions, on CRuby). Its workflows are short ones that call
|
|
23
|
+
# Duel's reusable workflows (see Reusable), passing on what .duel.rb
|
|
24
|
+
# sets, or with --standalone, whole ones, to change as you like. It overwrites nothing without --force, and
|
|
26
25
|
# writes nothing with --dry-run, or with --check, which says how the
|
|
27
26
|
# workflows differ from what it writes.
|
|
28
27
|
# @api private
|
|
@@ -50,7 +49,8 @@ module Duel
|
|
|
50
49
|
LABEL_COLOR = "FBCA04"
|
|
51
50
|
|
|
52
51
|
# Where Duel's reusable workflows are (see Reusable), which the
|
|
53
|
-
# workflows it writes
|
|
52
|
+
# workflows it writes call (without --standalone), at this release's
|
|
53
|
+
# tag.
|
|
54
54
|
REUSABLE = "sferik/duel-ruby/.github/workflows"
|
|
55
55
|
|
|
56
56
|
def initialize(argv, out: $stdout, err: $stderr, input: $stdin, root: Dir.pwd)
|
|
@@ -67,7 +67,6 @@ module Duel
|
|
|
67
67
|
def run
|
|
68
68
|
rest = catch(:help) { parser.parse(@argv) } or return 0
|
|
69
69
|
raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
|
|
70
|
-
raise OptionParser::InvalidOption, "--reusable with --gitlab (GitLab CI has no reusable workflows)" if @reusable && @gitlab
|
|
71
70
|
return rails if File.exist?(File.join(@root, "config", "application.rb"))
|
|
72
71
|
|
|
73
72
|
return check if @check
|
|
@@ -91,8 +90,8 @@ module Duel
|
|
|
91
90
|
return written.merge(top.join(gitlab_file).to_s => render("duel.gitlab-ci.yml")) if @gitlab
|
|
92
91
|
|
|
93
92
|
workflows = top.join(".github", "workflows")
|
|
94
|
-
# (
|
|
95
|
-
perf, comment = @
|
|
93
|
+
# (Ones that call Duel's, or with --standalone, whole ones.)
|
|
94
|
+
perf, comment = @standalone ? %w[perf.yml perf-comment.yml] : %w[perf-caller.yml perf-comment-caller.yml]
|
|
96
95
|
written.merge(workflows.join(workflow_name).to_s => render(perf),
|
|
97
96
|
workflows.join(workflow_name.sub(/\.yml\z/, "-comment.yml")).to_s => render(comment))
|
|
98
97
|
end
|
|
@@ -107,7 +106,7 @@ module Duel
|
|
|
107
106
|
o.on("--check", "Say how the workflows differ from what it writes (with the same options), and write nothing") { @check = true }
|
|
108
107
|
o.on("--badge", "Write a job that keeps a badge for the README (it needs contents: write)") { @badge = true }
|
|
109
108
|
o.on("--gitlab", "Write GitLab CI jobs (to include from .gitlab-ci.yml), not GitHub Actions workflows") { @gitlab = true }
|
|
110
|
-
o.on("--
|
|
109
|
+
o.on("--standalone", "Write whole workflows, not ones that call Duel's reusable ones (at this release)") { @standalone = true }
|
|
111
110
|
o.on("-h", "--help", "Print this help") do
|
|
112
111
|
@out.puts o
|
|
113
112
|
throw :help
|
|
@@ -316,7 +315,7 @@ module Duel
|
|
|
316
315
|
"Commit and push. On a pull request, the workflow compares the code with the branch it targets, and comments with the " \
|
|
317
316
|
"report when a benchmark fails (c.comment in .duel.rb says when)."
|
|
318
317
|
end
|
|
319
|
-
"\nNext:\n#{steps.
|
|
318
|
+
"\nNext:\n#{steps.each.with_index(1).map { |step, i| " #{i}. #{step}\n" }.join}"
|
|
320
319
|
end
|
|
321
320
|
|
|
322
321
|
# The template +name+, rendered. (Read with LF line endings, as Git
|
|
@@ -329,33 +328,20 @@ module Duel
|
|
|
329
328
|
# mutant:disable -- nil (a mutation's) renders as false does
|
|
330
329
|
def reusable? = false
|
|
331
330
|
|
|
332
|
-
#
|
|
333
|
-
#
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
# What the workflow it writes with --reusable passes on to Duel's
|
|
337
|
-
# Perf workflow, as its inputs (see Reusable): what .duel.rb sets that
|
|
338
|
-
# isn't the input's default, each as YAML has it.
|
|
331
|
+
# What the workflow it writes (without --standalone) passes on to
|
|
332
|
+
# Duel's Perf workflow, as its inputs (see Reusable): what isn't the
|
|
333
|
+
# input's default, each as YAML has it. (What .duel.rb sets, Duel
|
|
334
|
+
# reads.)
|
|
339
335
|
def perf_inputs
|
|
340
336
|
{
|
|
341
337
|
"ruby-version" => ruby_version,
|
|
342
338
|
"version-file" => (true if version_file),
|
|
343
|
-
"working-directory" => working_directory
|
|
344
|
-
"accept-label" => accept_label,
|
|
345
|
-
"release-tags" => (set_release_tags unless set_release_tags == "v*"),
|
|
346
|
-
"comment-on-pushes" => (true if comment_on_pushes?),
|
|
347
|
-
"mark-passed" => (true if mark_passed?),
|
|
348
|
-
"badge" => (true if badge?),
|
|
349
|
-
"passed-branch" => (passed_branch if working_directory),
|
|
350
|
-
"badge-branch" => (badge_branch if working_directory && badge?),
|
|
351
|
-
"marker" => (marker if working_directory)
|
|
339
|
+
"working-directory" => working_directory
|
|
352
340
|
}.compact.transform_values(&:to_json)
|
|
353
341
|
end
|
|
354
342
|
|
|
355
|
-
# The
|
|
356
|
-
|
|
357
|
-
# conventions', and say to set RELEASE_TAGS to match.)
|
|
358
|
-
def set_release_tags = settings&.release_tags
|
|
343
|
+
# The same, for Duel's push workflow, besides what the Duel job said.
|
|
344
|
+
def push_inputs = {"working-directory" => working_directory, "badge" => (true if badge?)}.compact.transform_values(&:to_json)
|
|
359
345
|
|
|
360
346
|
# The same, for Duel's Perf comment workflow.
|
|
361
347
|
def comment_inputs
|
|
@@ -369,63 +355,31 @@ module Duel
|
|
|
369
355
|
# The project's configuration, as the conventions have it.
|
|
370
356
|
def config = Config.new(@root)
|
|
371
357
|
|
|
372
|
-
# The project's settings, which decide the workflow
|
|
373
|
-
#
|
|
374
|
-
#
|
|
375
|
-
#
|
|
376
|
-
#
|
|
358
|
+
# The project's settings, which decide what the comment workflow runs
|
|
359
|
+
# the Duel job again for (the label that accepts a regression): its
|
|
360
|
+
# .duel.rb's, or nil where there's none yet or --force replaces it
|
|
361
|
+
# (the one it writes sets nothing), or it doesn't load (one of its
|
|
362
|
+
# plugins isn't installed, say, which it says): Duel's own.
|
|
377
363
|
def settings = instance_variable_defined?(:@settings) ? @settings : (@settings = load_settings)
|
|
378
364
|
|
|
379
365
|
def load_settings
|
|
380
366
|
path = File.join(@root, ".duel.rb")
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
Duel.load_config(path)
|
|
367
|
+
Duel.load_config(path) if !@force && File.exist?(path)
|
|
384
368
|
rescue ScriptError, StandardError => e
|
|
385
|
-
@err.puts "duel init: .duel.rb didn't load (#{e.class}: #{e}), so the
|
|
386
|
-
end
|
|
387
|
-
|
|
388
|
-
# The settings of the .duel.rb it writes.
|
|
389
|
-
def written_settings
|
|
390
|
-
Dir.mktmpdir do |dir|
|
|
391
|
-
path = File.join(dir, ".duel.rb")
|
|
392
|
-
File.write(path, render("duel.rb"))
|
|
393
|
-
Duel.load_config(path)
|
|
394
|
-
end
|
|
369
|
+
@err.puts "duel init: .duel.rb didn't load (#{e.class}: #{e}), so the workflows take Duel's defaults"
|
|
395
370
|
end
|
|
396
371
|
|
|
397
372
|
# The label that accepts a regression (Config#accept_label), for the
|
|
398
|
-
#
|
|
399
|
-
#
|
|
400
|
-
#
|
|
401
|
-
def accept_label = settings
|
|
402
|
-
|
|
403
|
-
# Whether the workflow comments on a pushed commit: where a regression
|
|
404
|
-
# in a push passes (c.fail_on = :pull_requests), the only time Duel
|
|
405
|
-
# says to.
|
|
406
|
-
def comment_on_pushes?
|
|
407
|
-
found = settings
|
|
408
|
-
found.nil? || found.fail_on == :pull_requests
|
|
409
|
-
end
|
|
373
|
+
# comment workflow, to run the Duel job again when it's added or taken
|
|
374
|
+
# off: .duel.rb's, or where it doesn't load, Duel's own. Nil without
|
|
375
|
+
# one.
|
|
376
|
+
def accept_label = (settings || config).accept_label
|
|
410
377
|
|
|
411
|
-
# Whether the workflow
|
|
412
|
-
# with the last commit Duel passed (c.push_base = :last_passed).
|
|
413
|
-
def mark_passed?
|
|
414
|
-
found = settings
|
|
415
|
-
found.nil? || found.push_base == :last_passed
|
|
416
|
-
end
|
|
417
|
-
|
|
418
|
-
# Whether the workflow keeps a badge for the README (--badge): its job
|
|
419
|
-
# moves a branch, so it's the workflow's only job, without the jobs
|
|
420
|
-
# for pushes, that writes to the repository.
|
|
378
|
+
# Whether the workflow keeps a badge for the README (--badge).
|
|
421
379
|
def badge? = @badge
|
|
422
380
|
|
|
423
|
-
# The library the gemspec names (see Config#
|
|
424
|
-
def library = config.
|
|
425
|
-
|
|
426
|
-
# The glob release tags match (Config#release_tags), for the workflow
|
|
427
|
-
# to find the last release as Duel does.
|
|
428
|
-
def release_tags = config.release_tags
|
|
381
|
+
# The library the gemspec names (see Config#gem_library), or nil.
|
|
382
|
+
def library = config.gem_library
|
|
429
383
|
|
|
430
384
|
# The library, without a gemspec: lib/'s only Ruby file, or nil.
|
|
431
385
|
def guess
|
|
@@ -500,8 +454,8 @@ module Duel
|
|
|
500
454
|
# project in a subdirectory, duel/badge-<its path>.
|
|
501
455
|
def badge_branch = suffixed("duel/badge")
|
|
502
456
|
|
|
503
|
-
# Duel's reusable workflows, which the workflows `duel init
|
|
504
|
-
#
|
|
457
|
+
# Duel's reusable workflows, which the workflows `duel init` writes
|
|
458
|
+
# call: .github/workflows/duel.yml (the Perf workflow's jobs)
|
|
505
459
|
# and duel-comment.yml (the Perf comment workflow's), in Duel's
|
|
506
460
|
# repository. They're the templates `duel init` writes from, rendered
|
|
507
461
|
# with each setting of .duel.rb they need as an input (and a job or a
|
|
@@ -513,33 +467,36 @@ module Duel
|
|
|
513
467
|
# The files, from the repository's root, to what's in them.
|
|
514
468
|
def files
|
|
515
469
|
workflows = File.join(".github", "workflows")
|
|
516
|
-
{File.join(workflows, "duel.yml") => render("perf.yml"), File.join(workflows, "duel-comment.yml") => render("perf-comment.yml")
|
|
470
|
+
{File.join(workflows, "duel.yml") => render("perf.yml"), File.join(workflows, "duel-comment.yml") => render("perf-comment.yml"),
|
|
471
|
+
File.join(workflows, "duel-push.yml") => render("perf-push.yml")}
|
|
517
472
|
end
|
|
518
473
|
|
|
519
474
|
private
|
|
520
475
|
|
|
476
|
+
# The whole Perf workflow's jobs for a push, as the push workflow's:
|
|
477
|
+
# what they take from the Duel job (needs.duel), its inputs.
|
|
478
|
+
def push_jobs = render("perf-push-jobs.yml").gsub(/^ needs: duel\n/, "").gsub("needs.duel.outputs.", "inputs.").gsub("needs.duel.result", "inputs.result")
|
|
479
|
+
|
|
521
480
|
def reusable? = true
|
|
522
481
|
|
|
523
482
|
# Each an input's (see the templates' workflow_call).
|
|
524
483
|
def accept_label = "${{ inputs.accept-label }}"
|
|
525
484
|
|
|
526
|
-
def release_tags = "${{ inputs.release-tags }}"
|
|
527
|
-
|
|
528
485
|
def ruby_version = "${{ inputs.ruby-version }}"
|
|
529
486
|
|
|
530
487
|
def working_directory = "${{ inputs.working-directory }}"
|
|
531
488
|
|
|
532
|
-
|
|
489
|
+
# (As the comments name them, for a project at the root: the jobs
|
|
490
|
+
# work them out from working-directory.)
|
|
491
|
+
def passed_branch = "duel/passed"
|
|
492
|
+
|
|
493
|
+
def badge_branch = "duel/badge"
|
|
533
494
|
|
|
534
|
-
def
|
|
495
|
+
def workflow_name = "perf.yml"
|
|
535
496
|
|
|
536
497
|
# Each in, run only where its input says to.
|
|
537
498
|
def version_file = true
|
|
538
499
|
|
|
539
|
-
def comment_on_pushes? = true
|
|
540
|
-
|
|
541
|
-
def mark_passed? = true
|
|
542
|
-
|
|
543
500
|
def badge? = true
|
|
544
501
|
end
|
|
545
502
|
end
|
data/lib/duel/json_report.rb
CHANGED
|
@@ -13,48 +13,50 @@ module Duel
|
|
|
13
13
|
# isn't a new key.
|
|
14
14
|
# @api private
|
|
15
15
|
class JSONReport
|
|
16
|
-
SCHEMA =
|
|
16
|
+
SCHEMA = 2
|
|
17
17
|
|
|
18
18
|
# A run that didn't compare: +status+ ("nothing_to_compare", or
|
|
19
|
-
# "error"), its +exit+ status, +message+, what it said
|
|
20
|
-
|
|
21
|
-
def self.note(status, exit, message, settings, comment)
|
|
19
|
+
# "error"), its +exit+ status, and +message+, what it said.
|
|
20
|
+
def self.note(status, exit, message, settings)
|
|
22
21
|
none = [] #: Array[value]
|
|
23
22
|
nothing = {} #: report
|
|
24
|
-
head(status, exit, message, settings
|
|
25
|
-
"benchmarks" => none, "across" => nothing, "
|
|
23
|
+
head(status, exit, nil, message, settings).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
|
|
24
|
+
"benchmarks" => none, "across" => nothing, "cut_short" => nil)
|
|
26
25
|
end
|
|
27
26
|
|
|
28
27
|
# What every report starts with.
|
|
29
|
-
def self.head(status, exit, message, settings
|
|
30
|
-
{"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "
|
|
31
|
-
"settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
|
|
32
|
-
|
|
33
|
-
|
|
28
|
+
def self.head(status, exit, passed_anyway, message, settings)
|
|
29
|
+
{"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "passed_anyway" => passed_anyway,
|
|
30
|
+
"message" => message, "settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
|
|
31
|
+
"jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
|
|
32
|
+
"time_limit" => settings.time_limit, "warm" => settings.warm, "force" => settings.force}}
|
|
34
33
|
end
|
|
35
34
|
|
|
36
35
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
37
36
|
# +candidate+ (each {"ref", "commit", "root"} or {"commit",
|
|
38
|
-
# "uncommitted_changes", "root"}) as +settings+ say,
|
|
39
|
-
# +exit+, and
|
|
40
|
-
|
|
37
|
+
# "uncommitted_changes", "root"}) as +settings+ say, and whether they
|
|
38
|
+
# +passed+, exits with +exit+, and if they failed, passed anyway as
|
|
39
|
+
# +passed_anyway+ says (see CLI#slower): "accept_label" (a pull
|
|
40
|
+
# request's label accepted it), "fail_on" (only a pull request
|
|
41
|
+
# fails), or nil.
|
|
42
|
+
def initialize(runner, results, settings:, passed:, exit:, passed_anyway:, baseline:, candidate:)
|
|
41
43
|
@runner = runner
|
|
42
44
|
@results = results
|
|
43
45
|
@settings = settings
|
|
46
|
+
@passed = passed
|
|
44
47
|
@exit = exit
|
|
45
|
-
@
|
|
48
|
+
@passed_anyway = passed_anyway
|
|
46
49
|
@versions = {"baseline" => baseline, "candidate" => candidate}
|
|
47
50
|
end
|
|
48
51
|
|
|
49
52
|
def to_h
|
|
50
53
|
rubies = @runner.rubies
|
|
51
|
-
JSONReport.head(@
|
|
54
|
+
JSONReport.head(@passed ? "passed" : "failed", @exit, @passed_anyway, nil, @settings).merge(
|
|
52
55
|
"seconds" => @runner.seconds,
|
|
53
56
|
**@versions.to_h { |side, version| [side, version.merge("ruby" => rubies.fetch(side))] },
|
|
54
57
|
"plugins" => @runner.plugins.map(&:to_s),
|
|
55
58
|
"benchmarks" => @results.map { |result| benchmark(result) },
|
|
56
59
|
"across" => across,
|
|
57
|
-
"errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} },
|
|
58
60
|
"cut_short" => @runner.cut_short
|
|
59
61
|
)
|
|
60
62
|
end
|
|
@@ -64,7 +66,7 @@ module Duel
|
|
|
64
66
|
# (Runner#explained says nothing of a benchmark that didn't regress.)
|
|
65
67
|
def benchmark(result)
|
|
66
68
|
{"name" => result.name, "verdict" => Summary.verdict(result), "passed" => result.passed?, "regressed" => result.regressed,
|
|
67
|
-
"
|
|
69
|
+
"iterations" => result.n, "seconds" => result.seconds,
|
|
68
70
|
"confirmations" => result.confirmations.size, "timed" => timed(result), "counted" => counted(result),
|
|
69
71
|
"peak_memory" => peak_memory(result), "details" => result.details,
|
|
70
72
|
"explanation" => @runner.explained(result), "error" => result.error, "skipped" => result.skipped}
|
|
@@ -74,8 +76,8 @@ module Duel
|
|
|
74
76
|
# each version's median an iteration.
|
|
75
77
|
def timed(result)
|
|
76
78
|
# (Timed, so measured, with an n.)
|
|
79
|
+
n = result.n #: Integer
|
|
77
80
|
result.timed.to_h do |metric, t|
|
|
78
|
-
n = result.n #: Integer
|
|
79
81
|
[metric, {"ratio" => t.fetch("median"), "interval" => t.fetch("interval"), "limit" => t.fetch("limit"),
|
|
80
82
|
"pairs" => t.fetch("pairs"), "processes" => t["processes"], "verdict" => t.fetch("verdict"), "baseline" => t.fetch("baseline").fdiv(n),
|
|
81
83
|
"candidate" => t.fetch("candidate").fdiv(n), "unit" => t.fetch("unit"), "plural" => t.fetch("plural"),
|
|
@@ -83,20 +85,27 @@ module Duel
|
|
|
83
85
|
end
|
|
84
86
|
end
|
|
85
87
|
|
|
86
|
-
# Each counted measure: each version's fewest an iteration, and
|
|
87
|
-
#
|
|
88
|
+
# Each counted measure: each version's fewest an iteration, and its
|
|
89
|
+
# verdict: "more" (worse: more than the metric tolerates), "fewer",
|
|
90
|
+
# or "same" (or more, within what it tolerates).
|
|
88
91
|
def counted(result)
|
|
89
92
|
worse = result.worse
|
|
90
93
|
result.counted.to_h do |metric, c|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
+
base, candidate = Result.per_iteration(c)
|
|
95
|
+
verdict = if worse.include?(metric) then "more"
|
|
96
|
+
elsif candidate < base then "fewer"
|
|
97
|
+
else "same"
|
|
98
|
+
end
|
|
99
|
+
[metric, {"baseline" => base, "candidate" => candidate, "unit" => c.fetch("unit"), "plural" => c.fetch("plural"), "verdict" => verdict}]
|
|
94
100
|
end
|
|
95
101
|
end
|
|
96
102
|
|
|
103
|
+
# Each version's peak memory, and its verdict: "more" (than Duel
|
|
104
|
+
# tolerates), or "same".
|
|
97
105
|
def peak_memory(result)
|
|
98
106
|
result.memory&.then do |memory|
|
|
99
|
-
{"baseline" => memory.fetch("baseline"), "candidate" => memory.fetch("candidate"), "unit" => "B",
|
|
107
|
+
{"baseline" => memory.fetch("baseline"), "candidate" => memory.fetch("candidate"), "unit" => "B",
|
|
108
|
+
"verdict" => result.more_memory? ? "more" : "same"}
|
|
100
109
|
end
|
|
101
110
|
end
|
|
102
111
|
|