duel 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +146 -1
- data/README.md +35 -0
- data/docs/ci.md +96 -3
- data/docs/configuration.md +35 -1
- data/docs/how-it-works.md +14 -8
- data/docs/plugins.md +16 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +45 -19
- data/lib/duel/comment.rb +71 -10
- data/lib/duel/config.rb +113 -2
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +142 -18
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +13 -1039
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +10 -10
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +45 -3
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +27 -3
- data/sig/duel/cli.rbs +8 -0
- data/sig/duel/comment.rbs +16 -1
- data/sig/duel/config.rbs +52 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +56 -0
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +3 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +3 -2
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -107
data/lib/duel/cli.rb
CHANGED
|
@@ -11,6 +11,7 @@ require_relative "summary"
|
|
|
11
11
|
require_relative "json_report"
|
|
12
12
|
require_relative "init"
|
|
13
13
|
require_relative "comment"
|
|
14
|
+
require_relative "spawn"
|
|
14
15
|
|
|
15
16
|
module Duel
|
|
16
17
|
# The `duel` command (and `duel init` and `duel comment`: see Init and
|
|
@@ -80,6 +81,8 @@ module Duel
|
|
|
80
81
|
|
|
81
82
|
settings = @settings = catch(:version) { parse } or return 0
|
|
82
83
|
config = @config = settings.config_path ? Duel.load_config(settings.config_path) : Config.new
|
|
84
|
+
# (.duel.rb's, but where an option says: c.limit, unless --limit.)
|
|
85
|
+
config.run_settings.each { |name, value| set(settings, name, value) unless @given.include?(name) }
|
|
83
86
|
Lock.hold(config.state) { compare(config, settings) }
|
|
84
87
|
rescue Error => e
|
|
85
88
|
@err.puts "duel: #{e}"
|
|
@@ -126,6 +129,7 @@ module Duel
|
|
|
126
129
|
|
|
127
130
|
# The settings, from the environment and then the options.
|
|
128
131
|
def parse
|
|
132
|
+
@given = [] # (the RUN_SETTINGS options given: see Config)
|
|
129
133
|
settings = Settings.defaults
|
|
130
134
|
settings.jobs = [Etc.nprocessors / CPUS_PER_LANE, 1].max
|
|
131
135
|
settings.fit = %i[jobs processes]
|
|
@@ -227,11 +231,10 @@ module Duel
|
|
|
227
231
|
# (CI_MERGE_REQUEST_LABELS, by commas), or none, where neither is
|
|
228
232
|
# set, or DUEL_LABELS isn't a JSON array.
|
|
229
233
|
def labels
|
|
230
|
-
|
|
231
|
-
return
|
|
232
|
-
return [] unless set.key?("DUEL_LABELS")
|
|
234
|
+
json = variable("DUEL_LABELS")
|
|
235
|
+
return variable("CI_MERGE_REQUEST_LABELS").to_s.split(",") unless json
|
|
233
236
|
|
|
234
|
-
names = JSON.parse(
|
|
237
|
+
names = JSON.parse(json)
|
|
235
238
|
names.instance_of?(Array) ? names : []
|
|
236
239
|
rescue JSON::ParserError
|
|
237
240
|
[]
|
|
@@ -334,14 +337,15 @@ module Duel
|
|
|
334
337
|
# writes them there too, as variables.
|
|
335
338
|
def publish(status, markdown, json)
|
|
336
339
|
comment = commenting(status)
|
|
337
|
-
set = variables
|
|
338
340
|
outputs = "comment=#{comment}\nstatus=#{MARKED.fetch(status)}\n#{"passed=true\n" if @passed}"
|
|
339
|
-
|
|
340
|
-
File.write(
|
|
341
|
+
step_output = variable(STEP_OUTPUT)
|
|
342
|
+
File.write(step_output, outputs, mode: "a") if step_output
|
|
343
|
+
output = variable(OUTPUT)
|
|
344
|
+
File.write(output, outputs.gsub(/^\w+=/) { |name| "DUEL_#{name.upcase}" }, mode: "a") if output
|
|
341
345
|
path = @json
|
|
342
346
|
write(path, "#{JSON.pretty_generate(json.call(comment))}\n") if path
|
|
343
347
|
path = @markdown
|
|
344
|
-
summary = (
|
|
348
|
+
summary = variable(STEP_SUMMARY)
|
|
345
349
|
return unless path || summary
|
|
346
350
|
|
|
347
351
|
text = markdown.call
|
|
@@ -464,7 +468,7 @@ module Duel
|
|
|
464
468
|
|
|
465
469
|
# Git's output, and its status, in +dir+.
|
|
466
470
|
def git(dir, *args)
|
|
467
|
-
out, _, status =
|
|
471
|
+
out, _, status = Spawn.open3(:capture3, "git", "-C", dir, *args)
|
|
468
472
|
[out, status]
|
|
469
473
|
end
|
|
470
474
|
|
|
@@ -490,6 +494,25 @@ module Duel
|
|
|
490
494
|
# The environment's variables that are set (not empty).
|
|
491
495
|
def variables = @env.reject { |_, value| value.empty? }
|
|
492
496
|
|
|
497
|
+
# Sets +settings+' +name+d setting (one of Config::RUN_SETTINGS) to
|
|
498
|
+
# +value+, an option's, over .duel.rb's: one it doesn't take is an
|
|
499
|
+
# invalid option.
|
|
500
|
+
def given(settings, name, value)
|
|
501
|
+
problem = Config.problem(name, value)
|
|
502
|
+
# (OptionParser says which option.)
|
|
503
|
+
raise OptionParser::InvalidArgument, "#{value}: give #{problem}" if problem
|
|
504
|
+
|
|
505
|
+
set(settings, name, value)
|
|
506
|
+
@given << name
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
# Sets +settings+' +name+d setting to +value+: a number of processes,
|
|
510
|
+
# that many, not as many as fit in memory.
|
|
511
|
+
def set(settings, name, value)
|
|
512
|
+
settings[name] = value
|
|
513
|
+
settings.fit -= [name]
|
|
514
|
+
end
|
|
515
|
+
|
|
493
516
|
def parser(settings)
|
|
494
517
|
OptionParser.new do |o|
|
|
495
518
|
o.banner = USAGE
|
|
@@ -503,19 +526,22 @@ module Duel
|
|
|
503
526
|
o.on("--candidate-dir DIR", "Candidate tree (default: this one)") { |v| settings.candidate_dir = v }
|
|
504
527
|
o.on("--filter REGEXP", "Only benchmarks whose names match") { |v| settings.filter = CLI.regexp(v) }
|
|
505
528
|
o.on("--jobs N", Integer, "Benchmarks measured at once (default: one for every #{CPUS_PER_LANE} CPUs, as many as fit in memory)") do |v|
|
|
506
|
-
settings
|
|
507
|
-
|
|
529
|
+
set(settings, :jobs, v)
|
|
530
|
+
end
|
|
531
|
+
o.on("--processes N", Integer, "Worker processes per version (default: c.processes, or #{PROCESSES}, as many as fit in memory)") do |v|
|
|
532
|
+
given(settings, :processes, v)
|
|
533
|
+
end
|
|
534
|
+
o.on("--timeout SECONDS", Float, "Most a worker may take to load or answer (default: c.timeout, or #{TIMEOUT.round})") do |v|
|
|
535
|
+
given(settings, :timeout, v)
|
|
508
536
|
end
|
|
509
|
-
o.on("--
|
|
510
|
-
settings
|
|
511
|
-
settings.fit -= [:processes]
|
|
537
|
+
o.on("--time-limit SECONDS", Float, "Most a run measures for, then fails what's unmeasured or looks worse (default: c.time_limit, or 7200)") do |v|
|
|
538
|
+
given(settings, :time_limit, v)
|
|
512
539
|
end
|
|
513
|
-
o.on("--
|
|
514
|
-
o.on("--
|
|
515
|
-
|
|
540
|
+
o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: c.limit, or 1.02)") { |v| given(settings, :limit, v) }
|
|
541
|
+
o.on("--confidence LEVEL", Float, "Confidence of every interval (default: c.confidence, or 0.99)") { |v| given(settings, :confidence, v) }
|
|
542
|
+
o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: c.warm, or until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT on, else 0)") do |v|
|
|
543
|
+
given(settings, :warm, v)
|
|
516
544
|
end
|
|
517
|
-
o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: 1.02)") { |v| settings.limit = v }
|
|
518
|
-
o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT on, else 0)") { |v| settings.warm = v }
|
|
519
545
|
o.on("--ruby version-file|PATH", "Each version's Ruby: the one its version file names (.ruby-version, say), or a",
|
|
520
546
|
"path (default: c.ruby, or this one)") { |v| settings.ruby = (v == "version-file") ? :version_file : v }
|
|
521
547
|
o.on("--force", "Compare even if nothing watched changed") { settings.force = true }
|
data/lib/duel/comment.rb
CHANGED
|
@@ -5,6 +5,7 @@ require "net/http"
|
|
|
5
5
|
require "open3"
|
|
6
6
|
require "optparse"
|
|
7
7
|
require "uri"
|
|
8
|
+
require_relative "spawn"
|
|
8
9
|
|
|
9
10
|
module Duel
|
|
10
11
|
# `duel comment`: what the Perf comment workflow runs, once the Perf
|
|
@@ -34,9 +35,11 @@ module Duel
|
|
|
34
35
|
"request's head is still SHA: what the Perf comment workflow runs (or with --gitlab, the comment job).\n\n"
|
|
35
36
|
|
|
36
37
|
# The first release of Duel whose `duel comment` the Perf comment
|
|
37
|
-
# workflow that this one writes can run
|
|
38
|
-
#
|
|
39
|
-
|
|
38
|
+
# workflow that this one writes can run (0.2.0's merges the reports
|
|
39
|
+
# of a workflow that called Duel's reusable one more than once): it
|
|
40
|
+
# installs the newest release of the same minor version (~>), so a
|
|
41
|
+
# fix comes with one.
|
|
42
|
+
SINCE = "0.2.0"
|
|
40
43
|
|
|
41
44
|
# Where GitLab CI has what Duel said (comment.env's, on GitHub): the
|
|
42
45
|
# merge request's, and the Duel job's dotenv report's.
|
|
@@ -45,6 +48,15 @@ module Duel
|
|
|
45
48
|
# What Duel says in comment.env, as each line must have it.
|
|
46
49
|
SAID = {"pr" => /\A[0-9]+\z/, "comment" => /\A(?:post|change)\z/, "status" => /\A(?:passed|failed|neutral)\z/}.freeze
|
|
47
50
|
|
|
51
|
+
# The characters a comment of reports has room for (GitHub takes
|
|
52
|
+
# 65,536, the marker and the names' headings among them), which each
|
|
53
|
+
# report's cut to its share of.
|
|
54
|
+
ROOM = 65_000
|
|
55
|
+
|
|
56
|
+
# A report's, in a comment of them, where its Duel job failed before
|
|
57
|
+
# it compared.
|
|
58
|
+
NONE = "No report: its Duel job failed before it compared.\n"
|
|
59
|
+
|
|
48
60
|
# The comments on a pull request, a page at a time (gh api graphql
|
|
49
61
|
# --paginate follows endCursor): GraphQL's, which say whether each is
|
|
50
62
|
# hidden, as REST's don't.
|
|
@@ -94,7 +106,7 @@ module Duel
|
|
|
94
106
|
def parser
|
|
95
107
|
OptionParser.new do |o|
|
|
96
108
|
o.banner = USAGE
|
|
97
|
-
o.on("--report DIR", "The directory of the Perf run's report (duel.md and comment.env)") { |v| @report = v }
|
|
109
|
+
o.on("--report DIR", "The directory of the Perf run's report (duel.md and comment.env), or of its reports (duel-report*/)") { |v| @report = v }
|
|
98
110
|
o.on("--head SHA", "The commit the run compared, the pull request's head") { |v| @head = v }
|
|
99
111
|
o.on("--marker TEXT", "What marks Duel's comments (default: duel)") { |v| @marker = v }
|
|
100
112
|
o.on("--gitlab", "Comment on a GitLab merge request, from GitLab CI (with DUEL_COMMENT_TOKEN)") { @gitlab = true }
|
|
@@ -105,10 +117,11 @@ module Duel
|
|
|
105
117
|
end
|
|
106
118
|
end
|
|
107
119
|
|
|
108
|
-
# Comments with the report in +
|
|
109
|
-
# request's head is still the run's.
|
|
110
|
-
def comment(
|
|
111
|
-
|
|
120
|
+
# Comments with the report in +dir+ (or the reports: see reports), if
|
|
121
|
+
# Duel said to, and the pull request's head is still the run's.
|
|
122
|
+
def comment(dir)
|
|
123
|
+
reports = reports(dir)
|
|
124
|
+
pr, mode, status = SAID.keys.map { |name| merged(name, reports.map { |report| said(name, report) }) }
|
|
112
125
|
return quiet("Duel said not to") unless pr && mode && status
|
|
113
126
|
|
|
114
127
|
head = (@gitlab ? merge_request(pr)&.fetch("sha") : gh("repos/#{repository}/pulls/#{pr}", "--jq", ".head.sha")&.chomp) or return 1
|
|
@@ -120,7 +133,7 @@ module Duel
|
|
|
120
133
|
stale = last && !last.eql?("#{marker} status=#{status} -->")
|
|
121
134
|
return quiet("the last comment says #{status} already") unless mode == "post" || stale
|
|
122
135
|
|
|
123
|
-
post(pr, "#{marker} status=#{status} -->\n#{
|
|
136
|
+
post(pr, "#{marker} status=#{status} -->\n#{body(reports)}") or return 1
|
|
124
137
|
@out.puts "duel comment: commented on #{@gitlab ? "merge request !" : "pull request "}#{pr}"
|
|
125
138
|
ours.reject { |earlier| earlier.fetch("hidden") }.each { |earlier| hide(earlier.fetch("id")) } unless @gitlab
|
|
126
139
|
0
|
|
@@ -134,6 +147,54 @@ module Duel
|
|
|
134
147
|
gitlab(:Post, "#{project}/merge_requests/#{pr}/notes", {"body" => body})
|
|
135
148
|
end
|
|
136
149
|
|
|
150
|
+
# The reports' directories in +dir+: +dir+, or where there's no
|
|
151
|
+
# report there (no comment.env), each of the Perf run's in a directory
|
|
152
|
+
# of its own, as the reusable Perf comment workflow gets them,
|
|
153
|
+
# duel-report and duel-report-NAME (where a call of the Perf workflow
|
|
154
|
+
# named its report), in order (Dir.glob's).
|
|
155
|
+
def reports(dir)
|
|
156
|
+
return [dir] if @gitlab || File.file?(File.join(dir, "comment.env"))
|
|
157
|
+
|
|
158
|
+
Dir.glob("duel-report{,-*}", base: dir).map { |name| File.join(dir, name) }
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# What +values+, each report's, say together for +name+: the pull
|
|
162
|
+
# request, where every one that names one names the same; a post, if
|
|
163
|
+
# any says to (or else a change); and a status, failed if any failed,
|
|
164
|
+
# passed if every one passed, and otherwise neutral. Nil where none
|
|
165
|
+
# says it.
|
|
166
|
+
def merged(name, values)
|
|
167
|
+
said = values.compact
|
|
168
|
+
return if said.empty?
|
|
169
|
+
|
|
170
|
+
case name
|
|
171
|
+
when "pr" then said.first if said.uniq.one?
|
|
172
|
+
when "comment" then said.max # (post, where any says to)
|
|
173
|
+
else
|
|
174
|
+
return "failed" if said.include?("failed")
|
|
175
|
+
|
|
176
|
+
values.all?("passed") ? "passed" : "neutral"
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# The comment's Markdown: the report's, or for more than one, each
|
|
181
|
+
# under its name (its directory's, after duel-report-, or Duel, for
|
|
182
|
+
# the one not named), cut to its share of ROOM, or where it has none,
|
|
183
|
+
# NONE.
|
|
184
|
+
def body(reports)
|
|
185
|
+
case reports
|
|
186
|
+
in [report] then File.read(File.join(report, "duel.md"))
|
|
187
|
+
else
|
|
188
|
+
share = ROOM / reports.size
|
|
189
|
+
reports.map do |each|
|
|
190
|
+
name = File.basename(each).delete_prefix("duel-report").delete_prefix("-")
|
|
191
|
+
path = File.join(each, "duel.md")
|
|
192
|
+
text = File.file?(path) ? File.read(path) : NONE
|
|
193
|
+
"## #{name.empty? ? "Duel" : name}\n\n#{text[0, share].chomp}\n"
|
|
194
|
+
end.join("\n")
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
137
198
|
# What Duel said, as SAID has it, or nil: in +report+'s comment.env
|
|
138
199
|
# (the last line that says it), or on GitLab CI, as GITLAB has it.
|
|
139
200
|
def said(name, report)
|
|
@@ -182,7 +243,7 @@ module Duel
|
|
|
182
243
|
# What gh api prints, given +args+, or nil where it fails (or isn't
|
|
183
244
|
# there), which it says why.
|
|
184
245
|
def gh(*args)
|
|
185
|
-
out, err, status =
|
|
246
|
+
out, err, status = Spawn.open3(:capture3, "gh", "api", *args)
|
|
186
247
|
return out if status.success?
|
|
187
248
|
|
|
188
249
|
@err.puts "duel comment: gh api #{args.first}: #{err.strip}"
|
data/lib/duel/config.rb
CHANGED
|
@@ -39,6 +39,11 @@ module Duel
|
|
|
39
39
|
# empty. `duel --clean` removes the rest.
|
|
40
40
|
def state = File.join(root, "tmp", "duel")
|
|
41
41
|
|
|
42
|
+
# Where what a resource builds once is kept (see Resource#cached):
|
|
43
|
+
# tmp/duel/cache.
|
|
44
|
+
# @api private
|
|
45
|
+
def cache = File.join(state, "cache")
|
|
46
|
+
|
|
42
47
|
# A glob, from the root, of the files defining benchmarks.
|
|
43
48
|
attr_accessor :benchmarks
|
|
44
49
|
|
|
@@ -128,6 +133,114 @@ module Duel
|
|
|
128
133
|
@fail_on = runs
|
|
129
134
|
end
|
|
130
135
|
|
|
136
|
+
# The settings of a run that .duel.rb can set too, as the options of
|
|
137
|
+
# their names do (--limit, --confidence, --processes, --timeout,
|
|
138
|
+
# --time-limit, --warm): for a run not given that option, as in CI,
|
|
139
|
+
# where the workflow runs Duel with none. Each is nil, by default, for
|
|
140
|
+
# Duel's own default. For each, what it takes, and whether a value is
|
|
141
|
+
# that: a number, as a Float, but a number of processes, a whole one.
|
|
142
|
+
# @api private
|
|
143
|
+
RUN_SETTINGS = {
|
|
144
|
+
limit: ["a ratio of 1 or more", ->(value) { value >= 1 }],
|
|
145
|
+
confidence: ["a level above 0 and below 1", ->(value) { value.positive? && value < 1 }],
|
|
146
|
+
processes: ["a whole number, 1 or more", ->(value) { value.is_a?(Integer) && value.positive? }],
|
|
147
|
+
timeout: ["seconds, more than 0", ->(value) { value.positive? }],
|
|
148
|
+
time_limit: ["seconds, more than 0", ->(value) { value.positive? }],
|
|
149
|
+
warm: ["seconds, 0 or more", ->(value) { value >= 0 }]
|
|
150
|
+
}.freeze #: Hash[Symbol, [String, ^(Numeric) -> bool]]
|
|
151
|
+
|
|
152
|
+
# The ratio above which a timing is slower (--limit), for every
|
|
153
|
+
# measure without a limit of its own (see #measure): 1.02 by default.
|
|
154
|
+
#
|
|
155
|
+
# c.limit = 1.05
|
|
156
|
+
attr_reader :limit
|
|
157
|
+
|
|
158
|
+
def limit=(ratio)
|
|
159
|
+
@limit = Config.setting(:limit, ratio)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# The confidence of every interval Duel judges with (--confidence):
|
|
163
|
+
# 0.99 by default. A higher one fails less code that isn't slower, and
|
|
164
|
+
# takes more samples, and workers, to fail what is: judging by process
|
|
165
|
+
# (see Lane.by_process?) takes 5 workers of each version at 0.99, and
|
|
166
|
+
# 7 at 0.999.
|
|
167
|
+
#
|
|
168
|
+
# c.confidence = 0.995
|
|
169
|
+
attr_reader :confidence
|
|
170
|
+
|
|
171
|
+
def confidence=(level)
|
|
172
|
+
@confidence = Config.setting(:confidence, level)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# The worker processes of each version (--processes): 12 by default (8
|
|
176
|
+
# on JRuby and TruffleRuby), as many as fit in memory. One set is that
|
|
177
|
+
# many, however much memory there is.
|
|
178
|
+
#
|
|
179
|
+
# c.processes = 6
|
|
180
|
+
attr_reader :processes
|
|
181
|
+
|
|
182
|
+
def processes=(count)
|
|
183
|
+
@processes = Config.setting(:processes, count)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# The most seconds a worker may take to load, or to answer a request,
|
|
187
|
+
# before it's taken for hung (--timeout): 60 by default, 300 on JRuby
|
|
188
|
+
# and TruffleRuby.
|
|
189
|
+
#
|
|
190
|
+
# c.timeout = 120
|
|
191
|
+
attr_reader :timeout
|
|
192
|
+
|
|
193
|
+
def timeout=(seconds)
|
|
194
|
+
@timeout = Config.setting(:timeout, seconds)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# The most seconds a run measures for, before it fails what it hasn't
|
|
198
|
+
# measured, or what looks worse (--time-limit): 7200 by default.
|
|
199
|
+
#
|
|
200
|
+
# c.time_limit = 3600
|
|
201
|
+
attr_reader :time_limit
|
|
202
|
+
|
|
203
|
+
def time_limit=(seconds)
|
|
204
|
+
@time_limit = Config.setting(:time_limit, seconds)
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# The seconds each benchmark warms up for, before it's measured
|
|
208
|
+
# (--warm), or nil (the default) for until it's steady, for up to a
|
|
209
|
+
# minute on JRuby and TruffleRuby, ten seconds with YJIT on, and none
|
|
210
|
+
# otherwise.
|
|
211
|
+
#
|
|
212
|
+
# c.warm = 10
|
|
213
|
+
attr_reader :warm
|
|
214
|
+
|
|
215
|
+
def warm=(seconds)
|
|
216
|
+
@warm = Config.setting(:warm, seconds)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# +value+, set as the RUN_SETTINGS +name+d, as that takes it (a Float,
|
|
220
|
+
# or an Integer), or nil. One that's neither fails, saying what to
|
|
221
|
+
# give.
|
|
222
|
+
# @api private
|
|
223
|
+
def self.setting(name, value)
|
|
224
|
+
return if value.nil?
|
|
225
|
+
|
|
226
|
+
problem = problem(name, value)
|
|
227
|
+
raise Error, "c.#{name} = #{value.inspect}: give #{problem}, or nil" if problem
|
|
228
|
+
|
|
229
|
+
(name == :processes) ? value : Float(value)
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# What to give instead of +value+ for the RUN_SETTINGS +name+d, or
|
|
233
|
+
# nil if it's one that takes.
|
|
234
|
+
# @api private
|
|
235
|
+
def self.problem(name, value)
|
|
236
|
+
what, valid = RUN_SETTINGS.fetch(name)
|
|
237
|
+
what unless value.is_a?(Numeric) && valid.call(value)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# The RUN_SETTINGS .duel.rb set, by name.
|
|
241
|
+
# @api private
|
|
242
|
+
def run_settings = RUN_SETTINGS.to_h { |name| [name, public_send(name)] }.compact
|
|
243
|
+
|
|
131
244
|
# The name of a label that accepts a regression in a pull request
|
|
132
245
|
# (or nil, the default, for none): on GitHub Actions, the workflow
|
|
133
246
|
# that `duel init` writes passes Duel the pull request's labels
|
|
@@ -307,8 +420,6 @@ module Duel
|
|
|
307
420
|
# A plugin as #use configured it: with +options+, returning +value+.
|
|
308
421
|
# @api private
|
|
309
422
|
class Used
|
|
310
|
-
attr_reader :options
|
|
311
|
-
|
|
312
423
|
attr_accessor :value
|
|
313
424
|
|
|
314
425
|
def initialize(options)
|
data/lib/duel/context.rb
CHANGED
|
@@ -339,7 +339,7 @@ module Duel
|
|
|
339
339
|
attr_reader :benchmark, :n
|
|
340
340
|
|
|
341
341
|
# @api private
|
|
342
|
-
attr_reader :
|
|
342
|
+
attr_reader :timed, :counted, :heap, :details
|
|
343
343
|
|
|
344
344
|
# +metrics+: the worker's metric instances, outermost first (CPU, the
|
|
345
345
|
# innermost, last). +counting+: whether counted measures are taken
|
|
@@ -365,7 +365,6 @@ module Duel
|
|
|
365
365
|
|
|
366
366
|
# @api private
|
|
367
367
|
def start
|
|
368
|
-
@started = true
|
|
369
368
|
QUIET_GC.call
|
|
370
369
|
@heap = -HEAP.call
|
|
371
370
|
@metrics.each { |m| m.start_sample(self) }
|
|
@@ -377,7 +376,6 @@ module Duel
|
|
|
377
376
|
@heap += HEAP.call
|
|
378
377
|
GC.enable
|
|
379
378
|
@metrics.each { |m| explain(m) }
|
|
380
|
-
@stopped = true
|
|
381
379
|
end
|
|
382
380
|
|
|
383
381
|
# Runs the block with the metrics paused (what they measured so far
|
|
@@ -484,11 +482,11 @@ module Duel
|
|
|
484
482
|
# A run of a benchmark that measures nothing: to see that it runs.
|
|
485
483
|
# @api private
|
|
486
484
|
class Trial
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
def start = @started = true
|
|
485
|
+
def start
|
|
486
|
+
end
|
|
490
487
|
|
|
491
|
-
def stop
|
|
488
|
+
def stop
|
|
489
|
+
end
|
|
492
490
|
|
|
493
491
|
def pause = yield
|
|
494
492
|
|
|
@@ -504,10 +502,13 @@ module Duel
|
|
|
504
502
|
# millisecond.
|
|
505
503
|
# @api private
|
|
506
504
|
class MemorySample
|
|
507
|
-
attr_reader :
|
|
505
|
+
attr_reader :peak
|
|
508
506
|
|
|
509
507
|
def start
|
|
510
|
-
|
|
508
|
+
# (Held as the watcher reads, and as it's paused and resumed: on
|
|
509
|
+
# JRuby, whose threads run at once, a reading could land between
|
|
510
|
+
# the two otherwise.)
|
|
511
|
+
@lock = Mutex.new
|
|
511
512
|
Native::Memory.collect
|
|
512
513
|
@base = resident
|
|
513
514
|
@peak = 0
|
|
@@ -523,7 +524,6 @@ module Duel
|
|
|
523
524
|
@watcher.kill
|
|
524
525
|
@watcher.join
|
|
525
526
|
watch
|
|
526
|
-
@stopped = true
|
|
527
527
|
end
|
|
528
528
|
|
|
529
529
|
# Runs the block (an untimed one, or a benchmark's once: see
|
|
@@ -559,15 +559,19 @@ module Duel
|
|
|
559
559
|
# From now on, nothing is watched, and what's left resident isn't
|
|
560
560
|
# counted.
|
|
561
561
|
def unwatched
|
|
562
|
-
@
|
|
563
|
-
|
|
562
|
+
@lock.synchronize do
|
|
563
|
+
@paused = true
|
|
564
|
+
@unwatched = resident
|
|
565
|
+
end
|
|
564
566
|
end
|
|
565
567
|
|
|
566
568
|
# From now on, what's resident is watched again, above what was left
|
|
567
569
|
# while it wasn't.
|
|
568
570
|
def watched
|
|
569
|
-
@
|
|
570
|
-
|
|
571
|
+
@lock.synchronize do
|
|
572
|
+
@base += resident - @unwatched
|
|
573
|
+
@paused = false
|
|
574
|
+
end
|
|
571
575
|
end
|
|
572
576
|
|
|
573
577
|
def resident
|
|
@@ -576,10 +580,12 @@ module Duel
|
|
|
576
580
|
end
|
|
577
581
|
|
|
578
582
|
def watch
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
+
@lock.synchronize do
|
|
584
|
+
unless @paused
|
|
585
|
+
above = resident - @base
|
|
586
|
+
@peak = above if above > @peak
|
|
587
|
+
end
|
|
588
|
+
end
|
|
583
589
|
end
|
|
584
590
|
end
|
|
585
591
|
|
|
@@ -593,26 +599,19 @@ module Duel
|
|
|
593
599
|
class ProfileSample
|
|
594
600
|
INTERVAL = 100 # microseconds between StackProf's samples
|
|
595
601
|
|
|
596
|
-
# What StackProf samples by on +platform
|
|
597
|
-
#
|
|
598
|
-
#
|
|
599
|
-
#
|
|
600
|
-
#
|
|
601
|
-
|
|
602
|
+
# What StackProf samples by on +platform+ (the host's system, as
|
|
603
|
+
# Platform tells it): CPU time, but on macOS, elapsed time. macOS
|
|
604
|
+
# signals that CPU time is up only as the thread enters the kernel, so
|
|
605
|
+
# its samples pile up on whatever makes system calls, the most
|
|
606
|
+
# CPU-hungry code or not. A benchmark busy on the CPU spends elapsed
|
|
607
|
+
# time the same way.
|
|
608
|
+
def self.mode(platform: Platform::OS) = platform.include?("darwin") ? :wall : :cpu
|
|
602
609
|
|
|
603
610
|
MODE = mode
|
|
604
611
|
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
def start
|
|
608
|
-
@started = true
|
|
609
|
-
Sample::QUIET_GC.call
|
|
610
|
-
end
|
|
612
|
+
def start = Sample::QUIET_GC.call
|
|
611
613
|
|
|
612
|
-
def stop
|
|
613
|
-
GC.enable
|
|
614
|
-
@stopped = true
|
|
615
|
-
end
|
|
614
|
+
def stop = GC.enable
|
|
616
615
|
|
|
617
616
|
# Runs the block (an untimed one, or the around_iteration hooks of a
|
|
618
617
|
# benchmark with them: see #between): what it does is sampled in this
|
data/lib/duel/control.rb
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
|
|
5
|
+
module Duel
|
|
6
|
+
# The runner's connection to a worker: a Unix socket at a path, or
|
|
7
|
+
# where there are no Unix sockets (JRuby on Windows has none), a port of
|
|
8
|
+
# 127.0.0.1's, its address "127.0.0.1:PORT". (As open to the machine's
|
|
9
|
+
# other processes as the plugins' servers are there.)
|
|
10
|
+
# @api private
|
|
11
|
+
module Control
|
|
12
|
+
HOST = "127.0.0.1"
|
|
13
|
+
UNIX = !defined?(UNIXServer).nil?
|
|
14
|
+
TCP = /\A127\.0\.0\.1:(\d+)\z/
|
|
15
|
+
|
|
16
|
+
# Whether a worker can reach a server on a Unix socket (see
|
|
17
|
+
# Duel.unix_sockets?): not on Windows (+windows+), whose builds of
|
|
18
|
+
# servers (Cygwin's, MSYS2's) have Unix sockets of their own, which
|
|
19
|
+
# Windows's Ruby can't reach, nor where Ruby has none (+unix+: JRuby's,
|
|
20
|
+
# on Windows).
|
|
21
|
+
def self.unix_sockets?(windows:, unix: UNIX) = !windows && unix
|
|
22
|
+
|
|
23
|
+
# A server for a worker to connect to, at +path+ (or a port, unless
|
|
24
|
+
# +unix+), and its address.
|
|
25
|
+
def self.listen(path, unix:)
|
|
26
|
+
return [UNIXServer.new(path), path] if unix
|
|
27
|
+
|
|
28
|
+
server = tcp_server
|
|
29
|
+
[server, "#{HOST}:#{server.addr.fetch(1)}"]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# A server on a port of 127.0.0.1's that the system chose.
|
|
33
|
+
# mutant:disable -- a port of nil is 0's alike: any that's free
|
|
34
|
+
def self.tcp_server
|
|
35
|
+
# (RBS's TCPServer.new takes a port alone, but Ruby's takes the host
|
|
36
|
+
# before it.)
|
|
37
|
+
TCPServer.new(HOST, 0) # steep:ignore UnexpectedPositionalArgument
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# A connection to the server at +address+.
|
|
41
|
+
# mutant:disable -- a host of nil is localhost, which is 127.0.0.1
|
|
42
|
+
# where the tests run
|
|
43
|
+
def self.connect(address)
|
|
44
|
+
port = address[TCP, 1]
|
|
45
|
+
port ? TCPSocket.new(HOST, Integer(port)) : UNIXSocket.new(address)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Removes the server's socket file, once it's done with: a Unix
|
|
49
|
+
# socket's.
|
|
50
|
+
def self.remove(address) = (File.delete(address) unless address.match?(TCP))
|
|
51
|
+
end
|
|
52
|
+
end
|