duel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +51 -27
- data/CHANGELOG.md +247 -1
- data/README.md +30 -14
- data/docs/benchmarks.md +28 -24
- data/docs/ci.md +328 -362
- data/docs/configuration.md +40 -37
- data/docs/how-it-works.md +23 -10
- data/docs/json.md +21 -25
- data/docs/plugins.md +40 -22
- data/docs/troubleshooting.md +3 -2
- data/lib/duel/across.rb +18 -18
- data/lib/duel/badge.rb +174 -0
- data/lib/duel/baseline.rb +38 -15
- data/lib/duel/cli.rb +175 -107
- data/lib/duel/comment.rb +80 -71
- data/lib/duel/config.rb +145 -63
- data/lib/duel/context.rb +29 -24
- data/lib/duel/forge.rb +87 -0
- data/lib/duel/init.rb +44 -87
- data/lib/duel/json_report.rb +34 -25
- data/lib/duel/lane.rb +107 -61
- data/lib/duel/lock.rb +1 -1
- data/lib/duel/mark_passed.rb +132 -0
- data/lib/duel/metric.rb +28 -27
- data/lib/duel/native.rb +7 -5
- data/lib/duel/platform.rb +4 -0
- data/lib/duel/process_sample.rb +9 -6
- data/lib/duel/rake_task.rb +4 -5
- data/lib/duel/resource.rb +10 -11
- data/lib/duel/result.rb +22 -14
- data/lib/duel/rubies.rb +6 -2
- data/lib/duel/runner.rb +27 -15
- data/lib/duel/spawn.rb +29 -4
- data/lib/duel/stats.rb +14 -12
- data/lib/duel/summary.rb +31 -29
- data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
- data/lib/duel/templates/duel.rb.erb +7 -7
- data/lib/duel/templates/perf-caller.yml.erb +22 -5
- data/lib/duel/templates/perf-comment.yml.erb +11 -11
- data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
- data/lib/duel/templates/perf-push.yml.erb +50 -0
- data/lib/duel/templates/perf.yml.erb +113 -335
- data/lib/duel/testing.rb +20 -25
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +16 -9
- data/lib/duel/worker_process.rb +6 -6
- data/lib/duel.rb +88 -33
- data/sig/duel/across.rbs +4 -0
- data/sig/duel/badge.rbs +51 -0
- data/sig/duel/baseline.rbs +7 -2
- data/sig/duel/cli.rbs +31 -20
- data/sig/duel/comment.rbs +14 -15
- data/sig/duel/config.rbs +71 -29
- data/sig/duel/context.rbs +4 -4
- data/sig/duel/forge.rbs +42 -0
- data/sig/duel/init.rbs +8 -26
- data/sig/duel/json_report.rbs +7 -6
- data/sig/duel/lane.rbs +29 -6
- data/sig/duel/mark_passed.rbs +39 -0
- data/sig/duel/metric.rbs +10 -10
- data/sig/duel/native.rbs +3 -0
- data/sig/duel/platform.rbs +1 -0
- data/sig/duel/process_sample.rbs +3 -0
- data/sig/duel/resource.rbs +3 -3
- data/sig/duel/result.rbs +12 -5
- data/sig/duel/rubies.rbs +3 -0
- data/sig/duel/runner.rbs +23 -16
- data/sig/duel/spawn.rbs +7 -0
- data/sig/duel/stats.rbs +4 -1
- data/sig/duel/summary.rbs +8 -0
- data/sig/duel/testing.rbs +2 -2
- data/sig/duel/worker.rbs +6 -5
- data/sig/duel/worker_process.rbs +3 -3
- data/sig/duel.rbs +50 -31
- metadata +9 -1
data/lib/duel/comment.rb
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "json"
|
|
4
|
-
require "net/http"
|
|
5
|
-
require "open3"
|
|
6
3
|
require "optparse"
|
|
7
4
|
require "uri"
|
|
8
|
-
require_relative "
|
|
5
|
+
require_relative "forge"
|
|
9
6
|
|
|
10
7
|
module Duel
|
|
11
8
|
# `duel comment`: what the Perf comment workflow runs, once the Perf
|
|
@@ -22,24 +19,22 @@ module Duel
|
|
|
22
19
|
# collapses, so the newest is the one in view. It talks to GitHub with
|
|
23
20
|
# gh, which GH_TOKEN signs in, in GITHUB_REPOSITORY.
|
|
24
21
|
#
|
|
25
|
-
#
|
|
26
|
-
# merge request: what Duel said is the Duel job's dotenv
|
|
27
|
-
# (DUEL_COMMENT and DUEL_STATUS), it talks to GitLab's API
|
|
22
|
+
# On GitLab CI (GITLAB_CI), it's what the GitLab CI jobs' comment job
|
|
23
|
+
# runs, on a merge request: what Duel said is the Duel job's dotenv
|
|
24
|
+
# report's (DUEL_COMMENT and DUEL_STATUS), it talks to GitLab's API
|
|
28
25
|
# (CI_API_V4_URL) with DUEL_COMMENT_TOKEN, and it hides nothing, as only
|
|
29
26
|
# a Developer can resolve a thread, and a merge request's token is a
|
|
30
|
-
# Reporter's.
|
|
27
|
+
# Reporter's. With --commit, it signs in with DUEL_TOKEN.
|
|
31
28
|
# @api private
|
|
32
29
|
class Comment
|
|
33
|
-
|
|
34
|
-
"Comments on a pull request with the report a Perf run kept in DIR, when Duel said to, and only if the pull\n" \
|
|
35
|
-
"request's head is still SHA: what the Perf comment workflow runs (or with --gitlab, the comment job).\n\n"
|
|
30
|
+
include Forge
|
|
36
31
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
32
|
+
USAGE = "Usage: duel comment --report DIR --head SHA [--marker TEXT]\n" \
|
|
33
|
+
" duel comment --report DIR --commit SHA [--marker TEXT]\n\n" \
|
|
34
|
+
"Comments on a pull request with the report a Perf run kept in DIR, when Duel said to, and only if the pull\n" \
|
|
35
|
+
"request's head is still SHA: what the Perf comment workflow runs (or on GitLab CI, where GITLAB_CI is set,\n" \
|
|
36
|
+
"on a merge request, with DUEL_COMMENT_TOKEN: the comment job). With --commit, on the pushed commit SHA, as\n" \
|
|
37
|
+
"DUEL_COMMENT and DUEL_STATUS say: what the Comment on the commit job runs.\n\n"
|
|
43
38
|
|
|
44
39
|
# Where GitLab CI has what Duel said (comment.env's, on GitHub): the
|
|
45
40
|
# merge request's, and the Duel job's dotenv report's.
|
|
@@ -90,10 +85,15 @@ module Duel
|
|
|
90
85
|
# Returns the exit status: 0 commented (or had nothing to say), 1 GitHub
|
|
91
86
|
# wouldn't take it, 2 a bad option.
|
|
92
87
|
def run
|
|
93
|
-
|
|
88
|
+
# (--gitlab is taken, and unlisted: the comment jobs that duel init
|
|
89
|
+
# wrote before, which install the newest Duel of their release,
|
|
90
|
+
# pass it. GITLAB_CI says it's GitLab's now.)
|
|
91
|
+
rest = catch(:help) { parser.parse(@argv - ["--gitlab"]) } or return 0
|
|
94
92
|
raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
|
|
95
93
|
report = @report
|
|
96
|
-
|
|
94
|
+
commit = @commit
|
|
95
|
+
return commit_comment(report, commit) if report && commit
|
|
96
|
+
raise OptionParser::MissingArgument, "--report, and --head or --commit, are required" unless report && @head
|
|
97
97
|
|
|
98
98
|
comment(report)
|
|
99
99
|
rescue OptionParser::ParseError => e
|
|
@@ -108,8 +108,8 @@ module Duel
|
|
|
108
108
|
o.banner = USAGE
|
|
109
109
|
o.on("--report DIR", "The directory of the Perf run's report (duel.md and comment.env), or of its reports (duel-report*/)") { |v| @report = v }
|
|
110
110
|
o.on("--head SHA", "The commit the run compared, the pull request's head") { |v| @head = v }
|
|
111
|
+
o.on("--commit SHA", "Comment on this pushed commit instead") { |v| @commit = v }
|
|
111
112
|
o.on("--marker TEXT", "What marks Duel's comments (default: duel)") { |v| @marker = v }
|
|
112
|
-
o.on("--gitlab", "Comment on a GitLab merge request, from GitLab CI (with DUEL_COMMENT_TOKEN)") { @gitlab = true }
|
|
113
113
|
o.on("-h", "--help", "Print this help") do
|
|
114
114
|
@out.puts o
|
|
115
115
|
throw :help
|
|
@@ -124,25 +124,71 @@ module Duel
|
|
|
124
124
|
pr, mode, status = SAID.keys.map { |name| merged(name, reports.map { |report| said(name, report) }) }
|
|
125
125
|
return quiet("Duel said not to") unless pr && mode && status
|
|
126
126
|
|
|
127
|
-
head = (
|
|
128
|
-
return quiet("the #{
|
|
127
|
+
head = (gitlab? ? merge_request(pr)&.fetch("sha") : gh("repos/#{repository}/pulls/#{pr}", "--jq", ".head.sha")&.chomp) or return 1
|
|
128
|
+
return quiet("the #{gitlab? ? "merge" : "pull"} request has moved on since the run") unless head == @head
|
|
129
129
|
|
|
130
|
-
ours = (
|
|
130
|
+
ours = (gitlab? ? notes(pr) : earlier(pr)) or return 1
|
|
131
131
|
ours = ours.select { |c| c.fetch("marked").start_with?("#{marker} status=") || c.fetch("marked") == "#{marker} -->" }
|
|
132
132
|
last = ours.last&.fetch("marked")
|
|
133
133
|
stale = last && !last.eql?("#{marker} status=#{status} -->")
|
|
134
134
|
return quiet("the last comment says #{status} already") unless mode == "post" || stale
|
|
135
135
|
|
|
136
136
|
post(pr, "#{marker} status=#{status} -->\n#{body(reports)}") or return 1
|
|
137
|
-
@out.puts "duel comment: commented on #{
|
|
138
|
-
ours.reject { |earlier| earlier.fetch("hidden") }.each { |earlier| hide(earlier.fetch("id")) } unless
|
|
137
|
+
@out.puts "duel comment: commented on #{gitlab? ? "merge request !" : "pull request "}#{pr}"
|
|
138
|
+
ours.reject { |earlier| earlier.fetch("hidden") }.each { |earlier| hide(earlier.fetch("id")) } unless gitlab?
|
|
139
139
|
0
|
|
140
140
|
end
|
|
141
141
|
|
|
142
|
+
# Comments on the pushed commit +sha+ with the report in +dir+, where
|
|
143
|
+
# Duel said to (DUEL_COMMENT and DUEL_STATUS, the Duel job's outputs):
|
|
144
|
+
# a new comment, which notifies its author, where it says to post, and
|
|
145
|
+
# otherwise only where the last comment's status isn't this run's.
|
|
146
|
+
# Its last comment is the last one on any commit (on GitLab, on the
|
|
147
|
+
# commit before, CI_COMMIT_BEFORE_SHA).
|
|
148
|
+
def commit_comment(dir, sha)
|
|
149
|
+
mode, status = %w[comment status].map { |name| @env[GITLAB.fetch(name)].to_s[SAID.fetch(name)] }
|
|
150
|
+
return quiet("Duel said not to") unless mode && status
|
|
151
|
+
|
|
152
|
+
report = File.join(dir, "duel.md")
|
|
153
|
+
return quiet("no report: the Duel job failed before it compared") unless File.file?(report)
|
|
154
|
+
|
|
155
|
+
marked = (gitlab? ? commit_notes : commit_comments) or return 1
|
|
156
|
+
last = marked.reverse.find { |first| first.start_with?("#{marker} status=") }
|
|
157
|
+
stale = last && !last.eql?("#{marker} status=#{status} -->")
|
|
158
|
+
return quiet("the last comment says #{status} already") unless mode == "post" || stale
|
|
159
|
+
|
|
160
|
+
body = "#{marker} status=#{status} -->\n#{File.read(report)}"
|
|
161
|
+
posted = gitlab? ? gitlab(:Post, "#{project}/repository/commits/#{sha}/comments", {"note" => body}) :
|
|
162
|
+
gh("repos/#{repository}/commits/#{sha}/comments", "--method", "POST", "-f", "body=#{body}")
|
|
163
|
+
return 1 unless posted
|
|
164
|
+
|
|
165
|
+
@out.puts "duel comment: commented on commit #{sha}"
|
|
166
|
+
0
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# The markers (first lines) of GitHub Actions' comments on the
|
|
170
|
+
# repository's commits, oldest first, or nil where GitHub wouldn't
|
|
171
|
+
# list them.
|
|
172
|
+
def commit_comments
|
|
173
|
+
listed = gh("--paginate", "repos/#{repository}/comments", "--jq",
|
|
174
|
+
'.[] | select(.user.login == "github-actions[bot]") | .body | split("\n")[0] | rtrimstr("\r")') or return
|
|
175
|
+
listed.lines(chomp: true)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# The markers of the token's comments on the commit before the push
|
|
179
|
+
# (CI_COMMIT_BEFORE_SHA), oldest first, or nil where GitLab wouldn't
|
|
180
|
+
# list them.
|
|
181
|
+
def commit_notes
|
|
182
|
+
me = gitlab(:Get, "/user") or return
|
|
183
|
+
before = @env.fetch("CI_COMMIT_BEFORE_SHA")
|
|
184
|
+
comments = gitlab(:Get, "#{project}/repository/commits/#{before}/comments?per_page=100") or return
|
|
185
|
+
comments.select { |comment| comment.dig("author", "id") == me.fetch("id") }.map { |comment| comment.fetch("note").lines.first.to_s.chomp }
|
|
186
|
+
end
|
|
187
|
+
|
|
142
188
|
# Posts +body+ as a new comment on the pull request (or merge request)
|
|
143
189
|
# +pr+: what GitHub (or GitLab) said, or nil.
|
|
144
190
|
def post(pr, body)
|
|
145
|
-
return gh("repos/#{repository}/issues/#{pr}/comments", "--method", "POST", "-f", "body=#{body}") unless
|
|
191
|
+
return gh("repos/#{repository}/issues/#{pr}/comments", "--method", "POST", "-f", "body=#{body}") unless gitlab?
|
|
146
192
|
|
|
147
193
|
gitlab(:Post, "#{project}/merge_requests/#{pr}/notes", {"body" => body})
|
|
148
194
|
end
|
|
@@ -153,7 +199,7 @@ module Duel
|
|
|
153
199
|
# duel-report and duel-report-NAME (where a call of the Perf workflow
|
|
154
200
|
# named its report), in order (Dir.glob's).
|
|
155
201
|
def reports(dir)
|
|
156
|
-
return [dir] if
|
|
202
|
+
return [dir] if gitlab? || File.file?(File.join(dir, "comment.env"))
|
|
157
203
|
|
|
158
204
|
Dir.glob("duel-report{,-*}", base: dir).map { |name| File.join(dir, name) }
|
|
159
205
|
end
|
|
@@ -198,7 +244,7 @@ module Duel
|
|
|
198
244
|
# What Duel said, as SAID has it, or nil: in +report+'s comment.env
|
|
199
245
|
# (the last line that says it), or on GitLab CI, as GITLAB has it.
|
|
200
246
|
def said(name, report)
|
|
201
|
-
return @env[GITLAB.fetch(name)].to_s[SAID.fetch(name)] if
|
|
247
|
+
return @env[GITLAB.fetch(name)].to_s[SAID.fetch(name)] if gitlab?
|
|
202
248
|
|
|
203
249
|
path = File.join(report, "comment.env")
|
|
204
250
|
lines = File.file?(path) ? File.readlines(path, chomp: true) : [] #: Array[String]
|
|
@@ -240,46 +286,6 @@ module Duel
|
|
|
240
286
|
gh("graphql", "-f", "query=#{query}", "-f", "id=#{id}") || @err.puts("duel comment: couldn't hide an earlier comment")
|
|
241
287
|
end
|
|
242
288
|
|
|
243
|
-
# What gh api prints, given +args+, or nil where it fails (or isn't
|
|
244
|
-
# there), which it says why.
|
|
245
|
-
def gh(*args)
|
|
246
|
-
out, err, status = Spawn.open3(:capture3, "gh", "api", *args)
|
|
247
|
-
return out if status.success?
|
|
248
|
-
|
|
249
|
-
@err.puts "duel comment: gh api #{args.first}: #{err.strip}"
|
|
250
|
-
rescue SystemCallError => e
|
|
251
|
-
@err.puts "duel comment: gh: #{e}"
|
|
252
|
-
end
|
|
253
|
-
|
|
254
|
-
# What GitLab's API says to +verb+ (:Get or :Post) +path+ (under
|
|
255
|
-
# CI_API_V4_URL), given +body+ (as JSON), signed in with
|
|
256
|
-
# DUEL_COMMENT_TOKEN, or nil where it fails, which it says why. With
|
|
257
|
-
# +pages+, that and the next page's number (X-Next-Page), if there's
|
|
258
|
-
# one.
|
|
259
|
-
def gitlab(verb, path, body = nil, pages: false)
|
|
260
|
-
uri = URI("#{@env.fetch("CI_API_V4_URL")}#{path}")
|
|
261
|
-
request = Net::HTTP.const_get(verb).new(uri)
|
|
262
|
-
request["PRIVATE-TOKEN"] = @env.fetch("DUEL_COMMENT_TOKEN")
|
|
263
|
-
request["Content-Type"] = "application/json"
|
|
264
|
-
request.body = JSON.generate(body) if body
|
|
265
|
-
http = connection(uri)
|
|
266
|
-
http.use_ssl = uri.scheme == "https"
|
|
267
|
-
response = http.request(request)
|
|
268
|
-
unless response.is_a?(Net::HTTPSuccess)
|
|
269
|
-
return @err.puts("duel comment: GitLab: #{verb.upcase} #{path}: #{[response.code, response.body.to_s.strip].reject(&:empty?).join(" ")}")
|
|
270
|
-
end
|
|
271
|
-
|
|
272
|
-
parsed = JSON.parse(response.body)
|
|
273
|
-
pages ? [parsed, response["X-Next-Page"].to_s[/\A[0-9]+\z/]] : parsed
|
|
274
|
-
rescue SystemCallError, IOError, Timeout::Error, JSON::ParserError, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
|
|
275
|
-
@err.puts "duel comment: GitLab: #{verb.upcase} #{path}: #{e}"
|
|
276
|
-
end
|
|
277
|
-
|
|
278
|
-
# A connection to +uri+'s host, not yet started.
|
|
279
|
-
# mutant:disable -- a nil host is Net::HTTP's localhost, where the
|
|
280
|
-
# tests' server is, so only another host could tell them apart
|
|
281
|
-
def connection(uri) = Net::HTTP.new(uri.host.to_s, uri.port)
|
|
282
|
-
|
|
283
289
|
# Says why it doesn't comment, and returns 0.
|
|
284
290
|
def quiet(why)
|
|
285
291
|
@out.puts "duel comment: no comment (#{why})"
|
|
@@ -288,9 +294,12 @@ module Duel
|
|
|
288
294
|
|
|
289
295
|
def marker = "<!-- #{@marker}"
|
|
290
296
|
|
|
291
|
-
|
|
297
|
+
# (For Forge.)
|
|
298
|
+
def command = "duel comment"
|
|
292
299
|
|
|
293
|
-
#
|
|
294
|
-
|
|
300
|
+
# (A merge request's comment signs in with a token of its own, which
|
|
301
|
+
# an unprotected pipeline has: a Reporter's. A push's, with the
|
|
302
|
+
# Developer's that moves the branches.)
|
|
303
|
+
def gitlab_token = @commit ? "DUEL_TOKEN" : "DUEL_COMMENT_TOKEN"
|
|
295
304
|
end
|
|
296
305
|
end
|
data/lib/duel/config.rb
CHANGED
|
@@ -36,7 +36,7 @@ module Duel
|
|
|
36
36
|
# build once (Resource#cached), in cache/, and the baseline's checkout
|
|
37
37
|
# where it can't go next to the repository (see Baseline#checkout), in
|
|
38
38
|
# base/. A run removes the lock when it ends, and what that leaves
|
|
39
|
-
# empty. `duel
|
|
39
|
+
# empty. `duel clean` removes the rest.
|
|
40
40
|
def state = File.join(root, "tmp", "duel")
|
|
41
41
|
|
|
42
42
|
# Where what a resource builds once is kept (see Resource#cached):
|
|
@@ -44,29 +44,65 @@ module Duel
|
|
|
44
44
|
# @api private
|
|
45
45
|
def cache = File.join(state, "cache")
|
|
46
46
|
|
|
47
|
-
# A glob, from the root, of the files defining benchmarks
|
|
48
|
-
|
|
47
|
+
# A glob, from the root, of the files defining benchmarks, or a list
|
|
48
|
+
# of them:
|
|
49
|
+
#
|
|
50
|
+
# c.benchmarks = %w[perf/**/*_duel.rb test/perf/**/*_duel.rb]
|
|
51
|
+
attr_reader :benchmarks
|
|
52
|
+
|
|
53
|
+
def benchmarks=(globs)
|
|
54
|
+
@benchmarks = Config.given(:benchmarks, globs, "a glob, or a list of them") { Config.strings?(globs) }
|
|
55
|
+
end
|
|
49
56
|
|
|
50
|
-
#
|
|
51
|
-
# about: when none of them changed since the
|
|
52
|
-
# decides how it's measured (the benchmarks'
|
|
53
|
-
# configuration's file, and with each version on its
|
|
54
|
-
# version files at the root), there's nothing to
|
|
55
|
-
|
|
57
|
+
# A path, from the root, of the code whose changes a comparison is
|
|
58
|
+
# about, or a list of them: when none of them changed since the
|
|
59
|
+
# baseline, nor what decides how it's measured (the benchmarks'
|
|
60
|
+
# directory, the configuration's file, and with each version on its
|
|
61
|
+
# own Ruby, the version files at the root), there's nothing to
|
|
62
|
+
# compare.
|
|
63
|
+
attr_reader :watch
|
|
56
64
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
65
|
+
def watch=(paths)
|
|
66
|
+
@watch = Config.given(:watch, paths, "a path, or a list of them") { Config.strings?(paths, empty: true) }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# How each version loads: the library to require from it, as
|
|
70
|
+
# `require` takes it (with its lib/ first on the load path), nil for
|
|
71
|
+
# none (benchmarks require what they need), or a lambda that loads the
|
|
72
|
+
# version from the root of its tree, which it's given, instead (see
|
|
73
|
+
# #load_into):
|
|
74
|
+
#
|
|
75
|
+
# c.library = "my_gem"
|
|
76
|
+
# c.library = ->(root) { require File.join(root, "config", "boot") }
|
|
77
|
+
def library=(library)
|
|
78
|
+
@library = Config.given(:library, library, "a library's name, a lambda (->(root) { ... }), or nil") do
|
|
79
|
+
library.nil? || library.instance_of?(String) || library.is_a?(Proc)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
60
82
|
|
|
61
83
|
# A shell command that builds a checked-out tree, run in it, or nil
|
|
62
84
|
# for none.
|
|
63
|
-
|
|
85
|
+
def build=(command)
|
|
86
|
+
@build = Config.given(:build, command, "a shell command, or nil") { command.nil? || command.instance_of?(String) }
|
|
87
|
+
end
|
|
64
88
|
|
|
65
89
|
# The branch a comparison defaults to (nil: the one origin/HEAD names).
|
|
66
|
-
|
|
90
|
+
attr_reader :main_branch
|
|
67
91
|
|
|
68
|
-
|
|
69
|
-
|
|
92
|
+
def main_branch=(branch)
|
|
93
|
+
@main_branch = Config.given(:main_branch, branch, "a branch (origin/main, say), or nil") { branch.nil? || branch.instance_of?(String) }
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# The glob release tags match (for `--base last-release`), or a list
|
|
97
|
+
# of them, any of which a tag may match (a gem that renamed its tags,
|
|
98
|
+
# say):
|
|
99
|
+
#
|
|
100
|
+
# c.release_tags = %w[v* my_gem-v*]
|
|
101
|
+
attr_reader :release_tags
|
|
102
|
+
|
|
103
|
+
def release_tags=(globs)
|
|
104
|
+
@release_tags = Config.given(:release_tags, globs, "a glob, or a list of them") { Config.strings?(globs) }
|
|
105
|
+
end
|
|
70
106
|
|
|
71
107
|
# The Ruby each version runs on (`duel --ruby` overrides it). It's nil
|
|
72
108
|
# for the runner's own, or :version_file for the one the version's own
|
|
@@ -77,10 +113,16 @@ module Duel
|
|
|
77
113
|
#
|
|
78
114
|
# c.ruby = :version_file
|
|
79
115
|
# c.ruby = ->(root) { File.join(Dir.home, ".rubies", File.read(File.join(root, ".ruby-version")).strip, "bin", "ruby") }
|
|
80
|
-
|
|
116
|
+
attr_reader :ruby
|
|
117
|
+
|
|
118
|
+
def ruby=(ruby)
|
|
119
|
+
@ruby = Config.given(:ruby, ruby, "a Ruby's path, :version_file, a lambda (->(root) { ... }), or nil") do
|
|
120
|
+
ruby.nil? || ruby == :version_file || ruby.instance_of?(String) || ruby.is_a?(Proc)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
81
123
|
|
|
82
124
|
# When the workflow that `duel init` writes comments on a pull request
|
|
83
|
-
# with the report
|
|
125
|
+
# with the report:
|
|
84
126
|
#
|
|
85
127
|
# - :always, on every run
|
|
86
128
|
# - :results, when a run passed or failed, but not with nothing
|
|
@@ -104,12 +146,10 @@ module Duel
|
|
|
104
146
|
COMMENTS = %i[always results failures never].freeze #: Array[comment]
|
|
105
147
|
|
|
106
148
|
def comment=(mode)
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
@comment = mode
|
|
149
|
+
@comment = Config.one_of(:comment, mode, COMMENTS)
|
|
110
150
|
end
|
|
111
151
|
|
|
112
|
-
# Which runs a regression fails
|
|
152
|
+
# Which runs a regression fails:
|
|
113
153
|
#
|
|
114
154
|
# - :every_run (the default)
|
|
115
155
|
# - :pull_requests, only a pull request's (or a merge queue's), and a
|
|
@@ -128,21 +168,21 @@ module Duel
|
|
|
128
168
|
FAIL_ON = %i[every_run pull_requests].freeze #: Array[fail_on]
|
|
129
169
|
|
|
130
170
|
def fail_on=(runs)
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
@fail_on = runs
|
|
171
|
+
@fail_on = Config.one_of(:fail_on, runs, FAIL_ON)
|
|
134
172
|
end
|
|
135
173
|
|
|
136
174
|
# The settings of a run that .duel.rb can set too, as the options of
|
|
137
|
-
# their names do (--limit, --confidence, --
|
|
138
|
-
# --time-limit, --warm)
|
|
139
|
-
#
|
|
140
|
-
#
|
|
141
|
-
#
|
|
175
|
+
# their names do (--limit, --confidence, --jobs, --processes,
|
|
176
|
+
# --timeout, --time-limit, --warm), and their variables (DUEL_LIMIT,
|
|
177
|
+
# say): for a run given neither, as in CI, where the workflow runs Duel
|
|
178
|
+
# with none. Each is nil, by default, for Duel's own default. For
|
|
179
|
+
# each, what it takes, and whether a value is that: a number, as a
|
|
180
|
+
# Float, but a number of jobs or processes, a whole one.
|
|
142
181
|
# @api private
|
|
143
182
|
RUN_SETTINGS = {
|
|
144
183
|
limit: ["a ratio of 1 or more", ->(value) { value >= 1 }],
|
|
145
184
|
confidence: ["a level above 0 and below 1", ->(value) { value.positive? && value < 1 }],
|
|
185
|
+
jobs: ["a whole number, 1 or more", ->(value) { value.is_a?(Integer) && value.positive? }],
|
|
146
186
|
processes: ["a whole number, 1 or more", ->(value) { value.is_a?(Integer) && value.positive? }],
|
|
147
187
|
timeout: ["seconds, more than 0", ->(value) { value.positive? }],
|
|
148
188
|
time_limit: ["seconds, more than 0", ->(value) { value.positive? }],
|
|
@@ -172,6 +212,17 @@ module Duel
|
|
|
172
212
|
@confidence = Config.setting(:confidence, level)
|
|
173
213
|
end
|
|
174
214
|
|
|
215
|
+
# The benchmarks measured at once (--jobs): by default, one for every
|
|
216
|
+
# CPUS_PER_LANE CPUs, as many as fit in memory. One set is
|
|
217
|
+
# that many, however much memory there is.
|
|
218
|
+
#
|
|
219
|
+
# c.jobs = 2
|
|
220
|
+
attr_reader :jobs
|
|
221
|
+
|
|
222
|
+
def jobs=(count)
|
|
223
|
+
@jobs = Config.setting(:jobs, count)
|
|
224
|
+
end
|
|
225
|
+
|
|
175
226
|
# The worker processes of each version (--processes): 12 by default (8
|
|
176
227
|
# on JRuby and TruffleRuby), as many as fit in memory. One set is that
|
|
177
228
|
# many, however much memory there is.
|
|
@@ -206,8 +257,8 @@ module Duel
|
|
|
206
257
|
|
|
207
258
|
# The seconds each benchmark warms up for, before it's measured
|
|
208
259
|
# (--warm), or nil (the default) for until it's steady, for up to a
|
|
209
|
-
# minute on JRuby and TruffleRuby, ten seconds with YJIT on,
|
|
210
|
-
# otherwise.
|
|
260
|
+
# minute on JRuby and TruffleRuby, ten seconds with YJIT or ZJIT on,
|
|
261
|
+
# and none otherwise.
|
|
211
262
|
#
|
|
212
263
|
# c.warm = 10
|
|
213
264
|
attr_reader :warm
|
|
@@ -216,6 +267,31 @@ module Duel
|
|
|
216
267
|
@warm = Config.setting(:warm, seconds)
|
|
217
268
|
end
|
|
218
269
|
|
|
270
|
+
# +value+, set as the setting +name+d, if the block says it's one it
|
|
271
|
+
# takes. One that isn't fails, saying to +give+ one that is, before
|
|
272
|
+
# anything runs.
|
|
273
|
+
# @api private
|
|
274
|
+
def self.given(name, value, give)
|
|
275
|
+
raise Error, "c.#{name} = #{value.inspect}: give #{give}" unless yield
|
|
276
|
+
|
|
277
|
+
value
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# Whether +value+ is a String, or a list of them (+empty+, or not).
|
|
281
|
+
# @api private
|
|
282
|
+
def self.strings?(value, empty: false)
|
|
283
|
+
value.instance_of?(String) || (value.instance_of?(Array) && (empty || !value.empty?) && value.all? { |each| each.instance_of?(String) })
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# +value+, set as the setting +name+d, if it's one of +values+, the
|
|
287
|
+
# ones that takes. One that isn't fails, saying which to give.
|
|
288
|
+
# @api private
|
|
289
|
+
def self.one_of(name, value, values)
|
|
290
|
+
raise Error, "c.#{name} = #{value.inspect}: give #{values.map(&:inspect).join(", ")}" unless values.include?(value)
|
|
291
|
+
|
|
292
|
+
value
|
|
293
|
+
end
|
|
294
|
+
|
|
219
295
|
# +value+, set as the RUN_SETTINGS +name+d, as that takes it (a Float,
|
|
220
296
|
# or an Integer), or nil. One that's neither fails, saying what to
|
|
221
297
|
# give.
|
|
@@ -226,9 +302,13 @@ module Duel
|
|
|
226
302
|
problem = problem(name, value)
|
|
227
303
|
raise Error, "c.#{name} = #{value.inspect}: give #{problem}, or nil" if problem
|
|
228
304
|
|
|
229
|
-
(name
|
|
305
|
+
WHOLE.include?(name) ? value : Float(value)
|
|
230
306
|
end
|
|
231
307
|
|
|
308
|
+
# The RUN_SETTINGS that are whole numbers: the rest are Floats.
|
|
309
|
+
# @api private
|
|
310
|
+
WHOLE = %i[jobs processes].freeze
|
|
311
|
+
|
|
232
312
|
# What to give instead of +value+ for the RUN_SETTINGS +name+d, or
|
|
233
313
|
# nil if it's one that takes.
|
|
234
314
|
# @api private
|
|
@@ -241,10 +321,10 @@ module Duel
|
|
|
241
321
|
# @api private
|
|
242
322
|
def run_settings = RUN_SETTINGS.to_h { |name| [name, public_send(name)] }.compact
|
|
243
323
|
|
|
244
|
-
# The name of a label that accepts a regression in a pull request
|
|
245
|
-
#
|
|
246
|
-
# that `duel init` writes passes Duel the pull request's
|
|
247
|
-
# (DUEL_LABELS), and with this one, a regression passes, with a
|
|
324
|
+
# The name of a label that accepts a regression in a pull request:
|
|
325
|
+
# "duel: accept" by default, or nil for none. On GitHub Actions, the
|
|
326
|
+
# workflow that `duel init` writes passes Duel the pull request's
|
|
327
|
+
# labels (DUEL_LABELS), and with this one, a regression passes, with a
|
|
248
328
|
# warning, and the report says it was accepted. For a push, it's the
|
|
249
329
|
# labels of the pull request that merged it, so where #push_base is
|
|
250
330
|
# :last_passed, the push is marked passed too. The labels are read
|
|
@@ -284,9 +364,7 @@ module Duel
|
|
|
284
364
|
PUSH_BASES = %i[before last_passed].freeze #: Array[push_base]
|
|
285
365
|
|
|
286
366
|
def push_base=(base)
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
@push_base = base
|
|
367
|
+
@push_base = Config.one_of(:push_base, base, PUSH_BASES)
|
|
290
368
|
end
|
|
291
369
|
|
|
292
370
|
# The metrics #metric added, in order (classes: a Duel::Metric
|
|
@@ -316,6 +394,7 @@ module Duel
|
|
|
316
394
|
@comment = :failures
|
|
317
395
|
@fail_on = :every_run
|
|
318
396
|
@push_base = :before
|
|
397
|
+
@accept_label = "duel: accept"
|
|
319
398
|
@prepares = SCOPES.to_h { |scope| [scope, []] }
|
|
320
399
|
@metrics = []
|
|
321
400
|
@in_workers = []
|
|
@@ -331,12 +410,19 @@ module Duel
|
|
|
331
410
|
# @api private
|
|
332
411
|
SCOPES = %i[version worker].freeze
|
|
333
412
|
|
|
334
|
-
# The library: the gem the project's only .gemspec names (
|
|
335
|
-
#
|
|
413
|
+
# The library: the gem the project's only .gemspec names (see
|
|
414
|
+
# #gem_library), unless set.
|
|
336
415
|
def library
|
|
337
|
-
given = @library
|
|
338
|
-
|
|
416
|
+
case (given = @library)
|
|
417
|
+
when :conventional then gem_library
|
|
418
|
+
else given
|
|
419
|
+
end
|
|
420
|
+
end
|
|
339
421
|
|
|
422
|
+
# The gem the project's only .gemspec names (its name's dashes are
|
|
423
|
+
# directories: my-gem is my/gem), or nil.
|
|
424
|
+
# @api private
|
|
425
|
+
def gem_library
|
|
340
426
|
specs = Dir[File.join(root, "*.gemspec")]
|
|
341
427
|
return unless specs.size == 1
|
|
342
428
|
|
|
@@ -353,32 +439,24 @@ module Duel
|
|
|
353
439
|
"bundle exec rake compile" if Dir.exist?(File.join(root, "ext")) && File.exist?(File.join(root, "Rakefile"))
|
|
354
440
|
end
|
|
355
441
|
|
|
356
|
-
#
|
|
357
|
-
#
|
|
358
|
-
#
|
|
359
|
-
# c.load { |root| require File.join(root, "config", "boot") }
|
|
360
|
-
#
|
|
361
|
-
# (It hides Kernel#load from the configuration's own methods, which
|
|
362
|
-
# never call it: a file is loaded by Duel.load_config.)
|
|
363
|
-
def load(&block) = @loader = block
|
|
364
|
-
|
|
365
|
-
# Loads the version at +root+: its lib/ goes first on the load path,
|
|
442
|
+
# Loads the version at +version_root+: with a lambda for the library,
|
|
443
|
+
# as that does, and otherwise, its lib/ goes first on the load path,
|
|
366
444
|
# and the library is required from it. Under `bundle exec`, Bundler
|
|
367
445
|
# has already loaded the candidate's gemspec, which may have required
|
|
368
446
|
# files of the candidate's (its version, say): for another version
|
|
369
447
|
# they're forgotten, and constants they defined are redefined quietly.
|
|
370
448
|
# @api private
|
|
371
449
|
def load_into(version_root)
|
|
372
|
-
|
|
373
|
-
return
|
|
450
|
+
loaded = library
|
|
451
|
+
return loaded.call(version_root) if loaded.is_a?(Proc)
|
|
374
452
|
|
|
375
453
|
$LOAD_PATH.unshift(File.join(version_root, "lib"))
|
|
376
|
-
return unless
|
|
377
|
-
return require(
|
|
454
|
+
return unless loaded
|
|
455
|
+
return require(loaded) if File.expand_path(version_root) == File.expand_path(root)
|
|
378
456
|
|
|
379
457
|
candidate_lib = File.join(File.expand_path(root), "lib", "")
|
|
380
458
|
$LOADED_FEATURES.reject! { |feature| feature.start_with?(candidate_lib) }
|
|
381
|
-
QuietRedefinition.during { require
|
|
459
|
+
QuietRedefinition.during { require loaded }
|
|
382
460
|
end
|
|
383
461
|
|
|
384
462
|
# Uses a plugin, by its name: requires it (duel/+name+, the plugin's
|
|
@@ -474,7 +552,7 @@ module Duel
|
|
|
474
552
|
# name or several by their settings:
|
|
475
553
|
#
|
|
476
554
|
# c.metric("sql_queries") { queries }
|
|
477
|
-
# c.metric("redis_cpu" => {kind: :
|
|
555
|
+
# c.metric("redis_cpu" => {kind: :timed, unit: "µs"}, "redis_commands" => {kind: :counted}) do
|
|
478
556
|
# {"redis_cpu" => usec, "redis_commands" => calls}
|
|
479
557
|
# end
|
|
480
558
|
#
|
|
@@ -514,8 +592,7 @@ module Duel
|
|
|
514
592
|
# The hooks (see Duel::HOOKS), for every benchmark, run in each worker
|
|
515
593
|
# before its file's and its own: setup once before a benchmark's
|
|
516
594
|
# samples, and teardown after them (last first), and before_sample
|
|
517
|
-
# before every sample (the warm-up's too), not measured
|
|
518
|
-
# the benchmark (see Sample#benchmark) if it takes it.
|
|
595
|
+
# before every sample (the warm-up's too), not measured.
|
|
519
596
|
# around_iteration runs around each iteration, given it to run, and
|
|
520
597
|
# after_iterations after each sample's iterations (inside each
|
|
521
598
|
# iteration, with around_iteration), measured with them: to do the
|
|
@@ -546,7 +623,12 @@ module Duel
|
|
|
546
623
|
|
|
547
624
|
# The files that define benchmarks, in order.
|
|
548
625
|
# @api private
|
|
549
|
-
def benchmark_files = Dir[File.join(root,
|
|
626
|
+
def benchmark_files = Config.list(benchmarks).flat_map { |glob| Dir[File.join(root, glob)] }.uniq
|
|
627
|
+
|
|
628
|
+
# A setting that takes one or a list (c.benchmarks, c.watch,
|
|
629
|
+
# c.release_tags), as a list.
|
|
630
|
+
# @api private
|
|
631
|
+
def self.list(given) = given.instance_of?(String) ? [given] : given
|
|
550
632
|
|
|
551
633
|
private
|
|
552
634
|
|