duel 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +203 -1
- data/README.md +41 -1
- data/docs/ci.md +170 -76
- data/docs/configuration.md +38 -2
- data/docs/how-it-works.md +18 -9
- data/docs/plugins.md +23 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +96 -28
- data/lib/duel/comment.rb +296 -0
- data/lib/duel/config.rb +124 -4
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +255 -20
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +33 -1044
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +21 -15
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +61 -44
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +35 -3
- data/sig/duel/cli.rbs +19 -0
- data/sig/duel/comment.rbs +97 -0
- data/sig/duel/config.rbs +57 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +95 -1
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +6 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +6 -3
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -105
data/lib/duel/comment.rb
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "open3"
|
|
6
|
+
require "optparse"
|
|
7
|
+
require "uri"
|
|
8
|
+
require_relative "spawn"
|
|
9
|
+
|
|
10
|
+
module Duel
|
|
11
|
+
# `duel comment`: what the Perf comment workflow runs, once the Perf
|
|
12
|
+
# workflow has, to comment on a pull request with Duel's report. It runs
|
|
13
|
+
# with the repository's own workflow and a token that can write, and the
|
|
14
|
+
# pull request's code ran in the Duel job, which could have written
|
|
15
|
+
# anything in what it kept (the report's directory), so it trusts only
|
|
16
|
+
# what Duel could have said there: the pull request's number, if it
|
|
17
|
+
# names the pull request whose head the run compared, and a comment
|
|
18
|
+
# (post or change: see CLI.comment) and a status (see CLI::MARKED) that
|
|
19
|
+
# Duel says. The report itself is the comment, as Markdown. It never
|
|
20
|
+
# edits or deletes a comment: once it has posted one, it hides Duel's
|
|
21
|
+
# earlier ones that aren't hidden yet as outdated, which GitHub
|
|
22
|
+
# collapses, so the newest is the one in view. It talks to GitHub with
|
|
23
|
+
# gh, which GH_TOKEN signs in, in GITHUB_REPOSITORY.
|
|
24
|
+
#
|
|
25
|
+
# With --gitlab, it's what the GitLab CI jobs' comment job runs, on a
|
|
26
|
+
# merge request: what Duel said is the Duel job's dotenv report's
|
|
27
|
+
# (DUEL_COMMENT and DUEL_STATUS), it talks to GitLab's API
|
|
28
|
+
# (CI_API_V4_URL) with DUEL_COMMENT_TOKEN, and it hides nothing, as only
|
|
29
|
+
# a Developer can resolve a thread, and a merge request's token is a
|
|
30
|
+
# Reporter's.
|
|
31
|
+
# @api private
|
|
32
|
+
class Comment
|
|
33
|
+
USAGE = "Usage: duel comment --report DIR --head SHA [--marker TEXT] [--gitlab]\n\n" \
|
|
34
|
+
"Comments on a pull request with the report a Perf run kept in DIR, when Duel said to, and only if the pull\n" \
|
|
35
|
+
"request's head is still SHA: what the Perf comment workflow runs (or with --gitlab, the comment job).\n\n"
|
|
36
|
+
|
|
37
|
+
# The first release of Duel whose `duel comment` the Perf comment
|
|
38
|
+
# workflow that this one writes can run (0.2.0's merges the reports
|
|
39
|
+
# of a workflow that called Duel's reusable one more than once): it
|
|
40
|
+
# installs the newest release of the same minor version (~>), so a
|
|
41
|
+
# fix comes with one.
|
|
42
|
+
SINCE = "0.2.0"
|
|
43
|
+
|
|
44
|
+
# Where GitLab CI has what Duel said (comment.env's, on GitHub): the
|
|
45
|
+
# merge request's, and the Duel job's dotenv report's.
|
|
46
|
+
GITLAB = {"pr" => "CI_MERGE_REQUEST_IID", "comment" => "DUEL_COMMENT", "status" => "DUEL_STATUS"}.freeze
|
|
47
|
+
|
|
48
|
+
# What Duel says in comment.env, as each line must have it.
|
|
49
|
+
SAID = {"pr" => /\A[0-9]+\z/, "comment" => /\A(?:post|change)\z/, "status" => /\A(?:passed|failed|neutral)\z/}.freeze
|
|
50
|
+
|
|
51
|
+
# The characters a comment of reports has room for (GitHub takes
|
|
52
|
+
# 65,536, the marker and the names' headings among them), which each
|
|
53
|
+
# report's cut to its share of.
|
|
54
|
+
ROOM = 65_000
|
|
55
|
+
|
|
56
|
+
# A report's, in a comment of them, where its Duel job failed before
|
|
57
|
+
# it compared.
|
|
58
|
+
NONE = "No report: its Duel job failed before it compared.\n"
|
|
59
|
+
|
|
60
|
+
# The comments on a pull request, a page at a time (gh api graphql
|
|
61
|
+
# --paginate follows endCursor): GraphQL's, which say whether each is
|
|
62
|
+
# hidden, as REST's don't.
|
|
63
|
+
COMMENTS = <<~GRAPHQL
|
|
64
|
+
query($owner: String!, $name: String!, $pr: Int!, $endCursor: String) {
|
|
65
|
+
repository(owner: $owner, name: $name) {
|
|
66
|
+
pullRequest(number: $pr) {
|
|
67
|
+
comments(first: 100, after: $endCursor) {
|
|
68
|
+
nodes { id isMinimized author { login } body }
|
|
69
|
+
pageInfo { hasNextPage endCursor }
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
GRAPHQL
|
|
75
|
+
|
|
76
|
+
# The jq that picks, of COMMENTS, the ones GitHub Actions' token
|
|
77
|
+
# posted, each with its id, whether it's hidden, and its marker (its
|
|
78
|
+
# body's first line).
|
|
79
|
+
LISTED = '.data.repository.pullRequest.comments.nodes[] | select(.author.login == "github-actions") | ' \
|
|
80
|
+
'{id, hidden: .isMinimized, marked: (.body | split("\n")[0] | rtrimstr("\r"))}'
|
|
81
|
+
|
|
82
|
+
def initialize(argv, env:, out:, err:)
|
|
83
|
+
@argv = argv
|
|
84
|
+
@env = env
|
|
85
|
+
@out = out
|
|
86
|
+
@err = err
|
|
87
|
+
@marker = "duel"
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Returns the exit status: 0 commented (or had nothing to say), 1 GitHub
|
|
91
|
+
# wouldn't take it, 2 a bad option.
|
|
92
|
+
def run
|
|
93
|
+
rest = catch(:help) { parser.parse(@argv) } or return 0
|
|
94
|
+
raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
|
|
95
|
+
report = @report
|
|
96
|
+
raise OptionParser::MissingArgument, "--report and --head are required" unless report && @head
|
|
97
|
+
|
|
98
|
+
comment(report)
|
|
99
|
+
rescue OptionParser::ParseError => e
|
|
100
|
+
@err.puts "duel comment: #{e}"
|
|
101
|
+
2
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
private
|
|
105
|
+
|
|
106
|
+
def parser
|
|
107
|
+
OptionParser.new do |o|
|
|
108
|
+
o.banner = USAGE
|
|
109
|
+
o.on("--report DIR", "The directory of the Perf run's report (duel.md and comment.env), or of its reports (duel-report*/)") { |v| @report = v }
|
|
110
|
+
o.on("--head SHA", "The commit the run compared, the pull request's head") { |v| @head = v }
|
|
111
|
+
o.on("--marker TEXT", "What marks Duel's comments (default: duel)") { |v| @marker = v }
|
|
112
|
+
o.on("--gitlab", "Comment on a GitLab merge request, from GitLab CI (with DUEL_COMMENT_TOKEN)") { @gitlab = true }
|
|
113
|
+
o.on("-h", "--help", "Print this help") do
|
|
114
|
+
@out.puts o
|
|
115
|
+
throw :help
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Comments with the report in +dir+ (or the reports: see reports), if
|
|
121
|
+
# Duel said to, and the pull request's head is still the run's.
|
|
122
|
+
def comment(dir)
|
|
123
|
+
reports = reports(dir)
|
|
124
|
+
pr, mode, status = SAID.keys.map { |name| merged(name, reports.map { |report| said(name, report) }) }
|
|
125
|
+
return quiet("Duel said not to") unless pr && mode && status
|
|
126
|
+
|
|
127
|
+
head = (@gitlab ? merge_request(pr)&.fetch("sha") : gh("repos/#{repository}/pulls/#{pr}", "--jq", ".head.sha")&.chomp) or return 1
|
|
128
|
+
return quiet("the #{@gitlab ? "merge" : "pull"} request has moved on since the run") unless head == @head
|
|
129
|
+
|
|
130
|
+
ours = (@gitlab ? notes(pr) : earlier(pr)) or return 1
|
|
131
|
+
ours = ours.select { |c| c.fetch("marked").start_with?("#{marker} status=") || c.fetch("marked") == "#{marker} -->" }
|
|
132
|
+
last = ours.last&.fetch("marked")
|
|
133
|
+
stale = last && !last.eql?("#{marker} status=#{status} -->")
|
|
134
|
+
return quiet("the last comment says #{status} already") unless mode == "post" || stale
|
|
135
|
+
|
|
136
|
+
post(pr, "#{marker} status=#{status} -->\n#{body(reports)}") or return 1
|
|
137
|
+
@out.puts "duel comment: commented on #{@gitlab ? "merge request !" : "pull request "}#{pr}"
|
|
138
|
+
ours.reject { |earlier| earlier.fetch("hidden") }.each { |earlier| hide(earlier.fetch("id")) } unless @gitlab
|
|
139
|
+
0
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Posts +body+ as a new comment on the pull request (or merge request)
|
|
143
|
+
# +pr+: what GitHub (or GitLab) said, or nil.
|
|
144
|
+
def post(pr, body)
|
|
145
|
+
return gh("repos/#{repository}/issues/#{pr}/comments", "--method", "POST", "-f", "body=#{body}") unless @gitlab
|
|
146
|
+
|
|
147
|
+
gitlab(:Post, "#{project}/merge_requests/#{pr}/notes", {"body" => body})
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# The reports' directories in +dir+: +dir+, or where there's no
|
|
151
|
+
# report there (no comment.env), each of the Perf run's in a directory
|
|
152
|
+
# of its own, as the reusable Perf comment workflow gets them,
|
|
153
|
+
# duel-report and duel-report-NAME (where a call of the Perf workflow
|
|
154
|
+
# named its report), in order (Dir.glob's).
|
|
155
|
+
def reports(dir)
|
|
156
|
+
return [dir] if @gitlab || File.file?(File.join(dir, "comment.env"))
|
|
157
|
+
|
|
158
|
+
Dir.glob("duel-report{,-*}", base: dir).map { |name| File.join(dir, name) }
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# What +values+, each report's, say together for +name+: the pull
|
|
162
|
+
# request, where every one that names one names the same; a post, if
|
|
163
|
+
# any says to (or else a change); and a status, failed if any failed,
|
|
164
|
+
# passed if every one passed, and otherwise neutral. Nil where none
|
|
165
|
+
# says it.
|
|
166
|
+
def merged(name, values)
|
|
167
|
+
said = values.compact
|
|
168
|
+
return if said.empty?
|
|
169
|
+
|
|
170
|
+
case name
|
|
171
|
+
when "pr" then said.first if said.uniq.one?
|
|
172
|
+
when "comment" then said.max # (post, where any says to)
|
|
173
|
+
else
|
|
174
|
+
return "failed" if said.include?("failed")
|
|
175
|
+
|
|
176
|
+
values.all?("passed") ? "passed" : "neutral"
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# The comment's Markdown: the report's, or for more than one, each
|
|
181
|
+
# under its name (its directory's, after duel-report-, or Duel, for
|
|
182
|
+
# the one not named), cut to its share of ROOM, or where it has none,
|
|
183
|
+
# NONE.
|
|
184
|
+
def body(reports)
|
|
185
|
+
case reports
|
|
186
|
+
in [report] then File.read(File.join(report, "duel.md"))
|
|
187
|
+
else
|
|
188
|
+
share = ROOM / reports.size
|
|
189
|
+
reports.map do |each|
|
|
190
|
+
name = File.basename(each).delete_prefix("duel-report").delete_prefix("-")
|
|
191
|
+
path = File.join(each, "duel.md")
|
|
192
|
+
text = File.file?(path) ? File.read(path) : NONE
|
|
193
|
+
"## #{name.empty? ? "Duel" : name}\n\n#{text[0, share].chomp}\n"
|
|
194
|
+
end.join("\n")
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# What Duel said, as SAID has it, or nil: in +report+'s comment.env
|
|
199
|
+
# (the last line that says it), or on GitLab CI, as GITLAB has it.
|
|
200
|
+
def said(name, report)
|
|
201
|
+
return @env[GITLAB.fetch(name)].to_s[SAID.fetch(name)] if @gitlab
|
|
202
|
+
|
|
203
|
+
path = File.join(report, "comment.env")
|
|
204
|
+
lines = File.file?(path) ? File.readlines(path, chomp: true) : [] #: Array[String]
|
|
205
|
+
lines.map { |line| line[/\A#{name}=(.*)\z/, 1] }.grep(SAID.fetch(name)).last
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# Duel's comments on the pull request, oldest first, each with its id,
|
|
209
|
+
# whether it's hidden, and its marker (its first line), or nil where
|
|
210
|
+
# GitHub wouldn't list them.
|
|
211
|
+
def earlier(pr)
|
|
212
|
+
owner, name = repository.split("/")
|
|
213
|
+
listed = gh("graphql", "--paginate", "-f", "query=#{COMMENTS}", "-F", "pr=#{pr}", "-f", "owner=#{owner}", "-f", "name=#{name}",
|
|
214
|
+
"--jq", LISTED) or return
|
|
215
|
+
listed.lines.map { |line| JSON.parse(line) }
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# The merge request +pr+, as GitLab's API has it, or nil.
|
|
219
|
+
def merge_request(pr) = gitlab(:Get, "#{project}/merge_requests/#{pr}")
|
|
220
|
+
|
|
221
|
+
# The token's notes on the merge request, oldest first, each with its
|
|
222
|
+
# marker (its first line), or nil where GitLab wouldn't list them.
|
|
223
|
+
def notes(pr)
|
|
224
|
+
me = gitlab(:Get, "/user") or return
|
|
225
|
+
notes = [] #: Array[Hash[String, untyped]]
|
|
226
|
+
page = "1"
|
|
227
|
+
while page
|
|
228
|
+
listed, page = gitlab(:Get, "#{project}/merge_requests/#{pr}/notes?sort=asc&order_by=created_at&per_page=100&page=#{page}", pages: true)
|
|
229
|
+
return unless listed
|
|
230
|
+
|
|
231
|
+
notes.concat(listed)
|
|
232
|
+
end
|
|
233
|
+
notes.select { |note| note.dig("author", "id") == me.fetch("id") }.map { |note| {"marked" => note.fetch("body").lines.first.to_s.chomp} }
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Hides an earlier comment as outdated, which GitHub collapses, or
|
|
237
|
+
# says it couldn't (it's still there, and that's all).
|
|
238
|
+
def hide(id)
|
|
239
|
+
query = "mutation($id: ID!) { minimizeComment(input: {subjectId: $id, classifier: OUTDATED}) { clientMutationId } }"
|
|
240
|
+
gh("graphql", "-f", "query=#{query}", "-f", "id=#{id}") || @err.puts("duel comment: couldn't hide an earlier comment")
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# What gh api prints, given +args+, or nil where it fails (or isn't
|
|
244
|
+
# there), which it says why.
|
|
245
|
+
def gh(*args)
|
|
246
|
+
out, err, status = Spawn.open3(:capture3, "gh", "api", *args)
|
|
247
|
+
return out if status.success?
|
|
248
|
+
|
|
249
|
+
@err.puts "duel comment: gh api #{args.first}: #{err.strip}"
|
|
250
|
+
rescue SystemCallError => e
|
|
251
|
+
@err.puts "duel comment: gh: #{e}"
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
# What GitLab's API says to +verb+ (:Get or :Post) +path+ (under
|
|
255
|
+
# CI_API_V4_URL), given +body+ (as JSON), signed in with
|
|
256
|
+
# DUEL_COMMENT_TOKEN, or nil where it fails, which it says why. With
|
|
257
|
+
# +pages+, that and the next page's number (X-Next-Page), if there's
|
|
258
|
+
# one.
|
|
259
|
+
def gitlab(verb, path, body = nil, pages: false)
|
|
260
|
+
uri = URI("#{@env.fetch("CI_API_V4_URL")}#{path}")
|
|
261
|
+
request = Net::HTTP.const_get(verb).new(uri)
|
|
262
|
+
request["PRIVATE-TOKEN"] = @env.fetch("DUEL_COMMENT_TOKEN")
|
|
263
|
+
request["Content-Type"] = "application/json"
|
|
264
|
+
request.body = JSON.generate(body) if body
|
|
265
|
+
http = connection(uri)
|
|
266
|
+
http.use_ssl = uri.scheme == "https"
|
|
267
|
+
response = http.request(request)
|
|
268
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
269
|
+
return @err.puts("duel comment: GitLab: #{verb.upcase} #{path}: #{[response.code, response.body.to_s.strip].reject(&:empty?).join(" ")}")
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
parsed = JSON.parse(response.body)
|
|
273
|
+
pages ? [parsed, response["X-Next-Page"].to_s[/\A[0-9]+\z/]] : parsed
|
|
274
|
+
rescue SystemCallError, IOError, Timeout::Error, JSON::ParserError, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
|
|
275
|
+
@err.puts "duel comment: GitLab: #{verb.upcase} #{path}: #{e}"
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# A connection to +uri+'s host, not yet started.
|
|
279
|
+
# mutant:disable -- a nil host is Net::HTTP's localhost, where the
|
|
280
|
+
# tests' server is, so only another host could tell them apart
|
|
281
|
+
def connection(uri) = Net::HTTP.new(uri.host.to_s, uri.port)
|
|
282
|
+
|
|
283
|
+
# Says why it doesn't comment, and returns 0.
|
|
284
|
+
def quiet(why)
|
|
285
|
+
@out.puts "duel comment: no comment (#{why})"
|
|
286
|
+
0
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
def marker = "<!-- #{@marker}"
|
|
290
|
+
|
|
291
|
+
def repository = @env.fetch("GITHUB_REPOSITORY")
|
|
292
|
+
|
|
293
|
+
# The project's path in GitLab's API.
|
|
294
|
+
def project = "/projects/#{@env.fetch("CI_PROJECT_ID")}"
|
|
295
|
+
end
|
|
296
|
+
end
|
data/lib/duel/config.rb
CHANGED
|
@@ -39,6 +39,11 @@ module Duel
|
|
|
39
39
|
# empty. `duel --clean` removes the rest.
|
|
40
40
|
def state = File.join(root, "tmp", "duel")
|
|
41
41
|
|
|
42
|
+
# Where what a resource builds once is kept (see Resource#cached):
|
|
43
|
+
# tmp/duel/cache.
|
|
44
|
+
# @api private
|
|
45
|
+
def cache = File.join(state, "cache")
|
|
46
|
+
|
|
42
47
|
# A glob, from the root, of the files defining benchmarks.
|
|
43
48
|
attr_accessor :benchmarks
|
|
44
49
|
|
|
@@ -88,8 +93,9 @@ module Duel
|
|
|
88
93
|
# keeps them all. Once it has commented, a run that it wouldn't
|
|
89
94
|
# comment on still does, but :never's, when its status differs from
|
|
90
95
|
# the last comment's, so the last comment is never stale: after a
|
|
91
|
-
# failure, the first run that passes says so.
|
|
92
|
-
#
|
|
96
|
+
# failure, the first run that passes says so. A regression that
|
|
97
|
+
# #accept_label accepts counts as a run that passed. Duel tells the
|
|
98
|
+
# workflow which to do (see CLI.comment).
|
|
93
99
|
#
|
|
94
100
|
# c.comment = :results
|
|
95
101
|
attr_reader :comment
|
|
@@ -127,6 +133,114 @@ module Duel
|
|
|
127
133
|
@fail_on = runs
|
|
128
134
|
end
|
|
129
135
|
|
|
136
|
+
# The settings of a run that .duel.rb can set too, as the options of
|
|
137
|
+
# their names do (--limit, --confidence, --processes, --timeout,
|
|
138
|
+
# --time-limit, --warm): for a run not given that option, as in CI,
|
|
139
|
+
# where the workflow runs Duel with none. Each is nil, by default, for
|
|
140
|
+
# Duel's own default. For each, what it takes, and whether a value is
|
|
141
|
+
# that: a number, as a Float, but a number of processes, a whole one.
|
|
142
|
+
# @api private
|
|
143
|
+
RUN_SETTINGS = {
|
|
144
|
+
limit: ["a ratio of 1 or more", ->(value) { value >= 1 }],
|
|
145
|
+
confidence: ["a level above 0 and below 1", ->(value) { value.positive? && value < 1 }],
|
|
146
|
+
processes: ["a whole number, 1 or more", ->(value) { value.is_a?(Integer) && value.positive? }],
|
|
147
|
+
timeout: ["seconds, more than 0", ->(value) { value.positive? }],
|
|
148
|
+
time_limit: ["seconds, more than 0", ->(value) { value.positive? }],
|
|
149
|
+
warm: ["seconds, 0 or more", ->(value) { value >= 0 }]
|
|
150
|
+
}.freeze #: Hash[Symbol, [String, ^(Numeric) -> bool]]
|
|
151
|
+
|
|
152
|
+
# The ratio above which a timing is slower (--limit), for every
|
|
153
|
+
# measure without a limit of its own (see #measure): 1.02 by default.
|
|
154
|
+
#
|
|
155
|
+
# c.limit = 1.05
|
|
156
|
+
attr_reader :limit
|
|
157
|
+
|
|
158
|
+
def limit=(ratio)
|
|
159
|
+
@limit = Config.setting(:limit, ratio)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# The confidence of every interval Duel judges with (--confidence):
|
|
163
|
+
# 0.99 by default. A higher one fails less code that isn't slower, and
|
|
164
|
+
# takes more samples, and workers, to fail what is: judging by process
|
|
165
|
+
# (see Lane.by_process?) takes 5 workers of each version at 0.99, and
|
|
166
|
+
# 7 at 0.999.
|
|
167
|
+
#
|
|
168
|
+
# c.confidence = 0.995
|
|
169
|
+
attr_reader :confidence
|
|
170
|
+
|
|
171
|
+
def confidence=(level)
|
|
172
|
+
@confidence = Config.setting(:confidence, level)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# The worker processes of each version (--processes): 12 by default (8
|
|
176
|
+
# on JRuby and TruffleRuby), as many as fit in memory. One set is that
|
|
177
|
+
# many, however much memory there is.
|
|
178
|
+
#
|
|
179
|
+
# c.processes = 6
|
|
180
|
+
attr_reader :processes
|
|
181
|
+
|
|
182
|
+
def processes=(count)
|
|
183
|
+
@processes = Config.setting(:processes, count)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# The most seconds a worker may take to load, or to answer a request,
|
|
187
|
+
# before it's taken for hung (--timeout): 60 by default, 300 on JRuby
|
|
188
|
+
# and TruffleRuby.
|
|
189
|
+
#
|
|
190
|
+
# c.timeout = 120
|
|
191
|
+
attr_reader :timeout
|
|
192
|
+
|
|
193
|
+
def timeout=(seconds)
|
|
194
|
+
@timeout = Config.setting(:timeout, seconds)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# The most seconds a run measures for, before it fails what it hasn't
|
|
198
|
+
# measured, or what looks worse (--time-limit): 7200 by default.
|
|
199
|
+
#
|
|
200
|
+
# c.time_limit = 3600
|
|
201
|
+
attr_reader :time_limit
|
|
202
|
+
|
|
203
|
+
def time_limit=(seconds)
|
|
204
|
+
@time_limit = Config.setting(:time_limit, seconds)
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# The seconds each benchmark warms up for, before it's measured
|
|
208
|
+
# (--warm), or nil (the default) for until it's steady, for up to a
|
|
209
|
+
# minute on JRuby and TruffleRuby, ten seconds with YJIT on, and none
|
|
210
|
+
# otherwise.
|
|
211
|
+
#
|
|
212
|
+
# c.warm = 10
|
|
213
|
+
attr_reader :warm
|
|
214
|
+
|
|
215
|
+
def warm=(seconds)
|
|
216
|
+
@warm = Config.setting(:warm, seconds)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# +value+, set as the RUN_SETTINGS +name+d, as that takes it (a Float,
|
|
220
|
+
# or an Integer), or nil. One that's neither fails, saying what to
|
|
221
|
+
# give.
|
|
222
|
+
# @api private
|
|
223
|
+
def self.setting(name, value)
|
|
224
|
+
return if value.nil?
|
|
225
|
+
|
|
226
|
+
problem = problem(name, value)
|
|
227
|
+
raise Error, "c.#{name} = #{value.inspect}: give #{problem}, or nil" if problem
|
|
228
|
+
|
|
229
|
+
(name == :processes) ? value : Float(value)
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# What to give instead of +value+ for the RUN_SETTINGS +name+d, or
|
|
233
|
+
# nil if it's one that takes.
|
|
234
|
+
# @api private
|
|
235
|
+
def self.problem(name, value)
|
|
236
|
+
what, valid = RUN_SETTINGS.fetch(name)
|
|
237
|
+
what unless value.is_a?(Numeric) && valid.call(value)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# The RUN_SETTINGS .duel.rb set, by name.
|
|
241
|
+
# @api private
|
|
242
|
+
def run_settings = RUN_SETTINGS.to_h { |name| [name, public_send(name)] }.compact
|
|
243
|
+
|
|
130
244
|
# The name of a label that accepts a regression in a pull request
|
|
131
245
|
# (or nil, the default, for none): on GitHub Actions, the workflow
|
|
132
246
|
# that `duel init` writes passes Duel the pull request's labels
|
|
@@ -185,6 +299,11 @@ module Duel
|
|
|
185
299
|
# @api private
|
|
186
300
|
attr_reader :prepares, :in_workers, :hooks, :measures
|
|
187
301
|
|
|
302
|
+
# The memory that what the plugins used start for each worker takes at
|
|
303
|
+
# most, as they say (see Duel.plugin), in bytes: 0 where none says.
|
|
304
|
+
# @api private
|
|
305
|
+
attr_reader :worker_memory
|
|
306
|
+
|
|
188
307
|
# A configuration of the conventions, for the project at +root+ (a
|
|
189
308
|
# plugin's tests make one to use it in).
|
|
190
309
|
def initialize(root = Dir.pwd)
|
|
@@ -202,6 +321,7 @@ module Duel
|
|
|
202
321
|
@in_workers = []
|
|
203
322
|
@hooks = HOOKS.to_h { |hook| [hook, []] }
|
|
204
323
|
@used = {}
|
|
324
|
+
@worker_memory = 0
|
|
205
325
|
@measures = Hash.new do |all, name|
|
|
206
326
|
all[name] = {} #: Hash[Symbol, untyped]
|
|
207
327
|
end
|
|
@@ -289,6 +409,8 @@ module Duel
|
|
|
289
409
|
DSL.add(plugin::DSL) if plugin.const_defined?(:DSL, false)
|
|
290
410
|
used = @used[name] = Used.new(options)
|
|
291
411
|
used.value = plugin.configure(self, **options)
|
|
412
|
+
@worker_memory += plugin.worker_memory if plugin.respond_to?(:worker_memory)
|
|
413
|
+
used.value
|
|
292
414
|
end
|
|
293
415
|
|
|
294
416
|
# The plugins used (see #use), by name, in the order they were first
|
|
@@ -298,8 +420,6 @@ module Duel
|
|
|
298
420
|
# A plugin as #use configured it: with +options+, returning +value+.
|
|
299
421
|
# @api private
|
|
300
422
|
class Used
|
|
301
|
-
attr_reader :options
|
|
302
|
-
|
|
303
423
|
attr_accessor :value
|
|
304
424
|
|
|
305
425
|
def initialize(options)
|
data/lib/duel/context.rb
CHANGED
|
@@ -339,7 +339,7 @@ module Duel
|
|
|
339
339
|
attr_reader :benchmark, :n
|
|
340
340
|
|
|
341
341
|
# @api private
|
|
342
|
-
attr_reader :
|
|
342
|
+
attr_reader :timed, :counted, :heap, :details
|
|
343
343
|
|
|
344
344
|
# +metrics+: the worker's metric instances, outermost first (CPU, the
|
|
345
345
|
# innermost, last). +counting+: whether counted measures are taken
|
|
@@ -365,7 +365,6 @@ module Duel
|
|
|
365
365
|
|
|
366
366
|
# @api private
|
|
367
367
|
def start
|
|
368
|
-
@started = true
|
|
369
368
|
QUIET_GC.call
|
|
370
369
|
@heap = -HEAP.call
|
|
371
370
|
@metrics.each { |m| m.start_sample(self) }
|
|
@@ -377,7 +376,6 @@ module Duel
|
|
|
377
376
|
@heap += HEAP.call
|
|
378
377
|
GC.enable
|
|
379
378
|
@metrics.each { |m| explain(m) }
|
|
380
|
-
@stopped = true
|
|
381
379
|
end
|
|
382
380
|
|
|
383
381
|
# Runs the block with the metrics paused (what they measured so far
|
|
@@ -484,11 +482,11 @@ module Duel
|
|
|
484
482
|
# A run of a benchmark that measures nothing: to see that it runs.
|
|
485
483
|
# @api private
|
|
486
484
|
class Trial
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
def start = @started = true
|
|
485
|
+
def start
|
|
486
|
+
end
|
|
490
487
|
|
|
491
|
-
def stop
|
|
488
|
+
def stop
|
|
489
|
+
end
|
|
492
490
|
|
|
493
491
|
def pause = yield
|
|
494
492
|
|
|
@@ -504,10 +502,13 @@ module Duel
|
|
|
504
502
|
# millisecond.
|
|
505
503
|
# @api private
|
|
506
504
|
class MemorySample
|
|
507
|
-
attr_reader :
|
|
505
|
+
attr_reader :peak
|
|
508
506
|
|
|
509
507
|
def start
|
|
510
|
-
|
|
508
|
+
# (Held as the watcher reads, and as it's paused and resumed: on
|
|
509
|
+
# JRuby, whose threads run at once, a reading could land between
|
|
510
|
+
# the two otherwise.)
|
|
511
|
+
@lock = Mutex.new
|
|
511
512
|
Native::Memory.collect
|
|
512
513
|
@base = resident
|
|
513
514
|
@peak = 0
|
|
@@ -523,7 +524,6 @@ module Duel
|
|
|
523
524
|
@watcher.kill
|
|
524
525
|
@watcher.join
|
|
525
526
|
watch
|
|
526
|
-
@stopped = true
|
|
527
527
|
end
|
|
528
528
|
|
|
529
529
|
# Runs the block (an untimed one, or a benchmark's once: see
|
|
@@ -559,15 +559,19 @@ module Duel
|
|
|
559
559
|
# From now on, nothing is watched, and what's left resident isn't
|
|
560
560
|
# counted.
|
|
561
561
|
def unwatched
|
|
562
|
-
@
|
|
563
|
-
|
|
562
|
+
@lock.synchronize do
|
|
563
|
+
@paused = true
|
|
564
|
+
@unwatched = resident
|
|
565
|
+
end
|
|
564
566
|
end
|
|
565
567
|
|
|
566
568
|
# From now on, what's resident is watched again, above what was left
|
|
567
569
|
# while it wasn't.
|
|
568
570
|
def watched
|
|
569
|
-
@
|
|
570
|
-
|
|
571
|
+
@lock.synchronize do
|
|
572
|
+
@base += resident - @unwatched
|
|
573
|
+
@paused = false
|
|
574
|
+
end
|
|
571
575
|
end
|
|
572
576
|
|
|
573
577
|
def resident
|
|
@@ -576,10 +580,12 @@ module Duel
|
|
|
576
580
|
end
|
|
577
581
|
|
|
578
582
|
def watch
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
+
@lock.synchronize do
|
|
584
|
+
unless @paused
|
|
585
|
+
above = resident - @base
|
|
586
|
+
@peak = above if above > @peak
|
|
587
|
+
end
|
|
588
|
+
end
|
|
583
589
|
end
|
|
584
590
|
end
|
|
585
591
|
|
|
@@ -593,26 +599,19 @@ module Duel
|
|
|
593
599
|
class ProfileSample
|
|
594
600
|
INTERVAL = 100 # microseconds between StackProf's samples
|
|
595
601
|
|
|
596
|
-
# What StackProf samples by on +platform
|
|
597
|
-
#
|
|
598
|
-
#
|
|
599
|
-
#
|
|
600
|
-
#
|
|
601
|
-
|
|
602
|
+
# What StackProf samples by on +platform+ (the host's system, as
|
|
603
|
+
# Platform tells it): CPU time, but on macOS, elapsed time. macOS
|
|
604
|
+
# signals that CPU time is up only as the thread enters the kernel, so
|
|
605
|
+
# its samples pile up on whatever makes system calls, the most
|
|
606
|
+
# CPU-hungry code or not. A benchmark busy on the CPU spends elapsed
|
|
607
|
+
# time the same way.
|
|
608
|
+
def self.mode(platform: Platform::OS) = platform.include?("darwin") ? :wall : :cpu
|
|
602
609
|
|
|
603
610
|
MODE = mode
|
|
604
611
|
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
def start
|
|
608
|
-
@started = true
|
|
609
|
-
Sample::QUIET_GC.call
|
|
610
|
-
end
|
|
612
|
+
def start = Sample::QUIET_GC.call
|
|
611
613
|
|
|
612
|
-
def stop
|
|
613
|
-
GC.enable
|
|
614
|
-
@stopped = true
|
|
615
|
-
end
|
|
614
|
+
def stop = GC.enable
|
|
616
615
|
|
|
617
616
|
# Runs the block (an untimed one, or the around_iteration hooks of a
|
|
618
617
|
# benchmark with them: see #between): what it does is sampled in this
|
data/lib/duel/control.rb
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
|
|
5
|
+
module Duel
|
|
6
|
+
# The runner's connection to a worker: a Unix socket at a path, or
|
|
7
|
+
# where there are no Unix sockets (JRuby on Windows has none), a port of
|
|
8
|
+
# 127.0.0.1's, its address "127.0.0.1:PORT". (As open to the machine's
|
|
9
|
+
# other processes as the plugins' servers are there.)
|
|
10
|
+
# @api private
|
|
11
|
+
module Control
|
|
12
|
+
HOST = "127.0.0.1"
|
|
13
|
+
UNIX = !defined?(UNIXServer).nil?
|
|
14
|
+
TCP = /\A127\.0\.0\.1:(\d+)\z/
|
|
15
|
+
|
|
16
|
+
# Whether a worker can reach a server on a Unix socket (see
|
|
17
|
+
# Duel.unix_sockets?): not on Windows (+windows+), whose builds of
|
|
18
|
+
# servers (Cygwin's, MSYS2's) have Unix sockets of their own, which
|
|
19
|
+
# Windows's Ruby can't reach, nor where Ruby has none (+unix+: JRuby's,
|
|
20
|
+
# on Windows).
|
|
21
|
+
def self.unix_sockets?(windows:, unix: UNIX) = !windows && unix
|
|
22
|
+
|
|
23
|
+
# A server for a worker to connect to, at +path+ (or a port, unless
|
|
24
|
+
# +unix+), and its address.
|
|
25
|
+
def self.listen(path, unix:)
|
|
26
|
+
return [UNIXServer.new(path), path] if unix
|
|
27
|
+
|
|
28
|
+
server = tcp_server
|
|
29
|
+
[server, "#{HOST}:#{server.addr.fetch(1)}"]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# A server on a port of 127.0.0.1's that the system chose.
|
|
33
|
+
# mutant:disable -- a port of nil is 0's alike: any that's free
|
|
34
|
+
def self.tcp_server
|
|
35
|
+
# (RBS's TCPServer.new takes a port alone, but Ruby's takes the host
|
|
36
|
+
# before it.)
|
|
37
|
+
TCPServer.new(HOST, 0) # steep:ignore UnexpectedPositionalArgument
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# A connection to the server at +address+.
|
|
41
|
+
# mutant:disable -- a host of nil is localhost, which is 127.0.0.1
|
|
42
|
+
# where the tests run
|
|
43
|
+
def self.connect(address)
|
|
44
|
+
port = address[TCP, 1]
|
|
45
|
+
port ? TCPSocket.new(HOST, Integer(port)) : UNIXSocket.new(address)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Removes the server's socket file, once it's done with: a Unix
|
|
49
|
+
# socket's.
|
|
50
|
+
def self.remove(address) = (File.delete(address) unless address.match?(TCP))
|
|
51
|
+
end
|
|
52
|
+
end
|