duel 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +83 -1
- data/README.md +6 -1
- data/docs/ci.md +102 -93
- data/docs/configuration.md +3 -1
- data/docs/how-it-works.md +17 -0
- data/docs/json.md +2 -2
- data/docs/plugins.md +7 -0
- data/lib/duel/cli.rb +60 -11
- data/lib/duel/comment.rb +235 -0
- data/lib/duel/config.rb +11 -2
- data/lib/duel/init.rb +116 -5
- data/lib/duel/memory.rb +54 -0
- data/lib/duel/runner.rb +63 -2
- data/lib/duel/summary.rb +11 -5
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-comment.yml.erb +17 -42
- data/lib/duel/templates/perf.yml.erb +14 -9
- data/lib/duel/version.rb +1 -1
- data/lib/duel.rb +8 -0
- data/sig/duel/cli.rbs +11 -0
- data/sig/duel/comment.rbs +82 -0
- data/sig/duel/config.rbs +5 -0
- data/sig/duel/init.rbs +39 -1
- data/sig/duel/memory.rbs +30 -0
- data/sig/duel/runner.rbs +22 -2
- data/sig/duel/summary.rbs +3 -1
- metadata +5 -1
data/lib/duel/cli.rb
CHANGED
|
@@ -10,11 +10,13 @@ require_relative "lock"
|
|
|
10
10
|
require_relative "summary"
|
|
11
11
|
require_relative "json_report"
|
|
12
12
|
require_relative "init"
|
|
13
|
+
require_relative "comment"
|
|
13
14
|
|
|
14
15
|
module Duel
|
|
15
|
-
# The `duel` command (and `duel init`: see Init
|
|
16
|
-
# rake task or a CI expression can't pass
|
|
17
|
-
# the environment (DUEL_BASE, DUEL_FILTER,
|
|
16
|
+
# The `duel` command (and `duel init` and `duel comment`: see Init and
|
|
17
|
+
# Comment). What a run behind a rake task or a CI expression can't pass
|
|
18
|
+
# as an option can be set in the environment (DUEL_BASE, DUEL_FILTER,
|
|
19
|
+
# DUEL_FORCE), and options win.
|
|
18
20
|
# On GitHub Actions (GITHUB_STEP_SUMMARY), it adds its report to the
|
|
19
21
|
# job's summary, in Markdown (see Summary), and tells the step's
|
|
20
22
|
# outputs (GITHUB_OUTPUT) what to do about the comment on a pull
|
|
@@ -24,8 +26,9 @@ module Duel
|
|
|
24
26
|
# outcome, but not for a command line that isn't valid.
|
|
25
27
|
# @api private
|
|
26
28
|
class CLI
|
|
27
|
-
USAGE = "Usage: duel [options]\n duel init [options]\n\n" \
|
|
28
|
-
"Fails if this tree is slower than a baseline at any benchmark. `duel init` sets Duel up in a project
|
|
29
|
+
USAGE = "Usage: duel [options]\n duel init [options]\n duel comment [options]\n\n" \
|
|
30
|
+
"Fails if this tree is slower than a baseline at any benchmark. `duel init` sets Duel up in a project, and\n" \
|
|
31
|
+
"`duel comment` is what its Perf comment workflow runs.\n\n"
|
|
29
32
|
|
|
30
33
|
# Where GitHub Actions has a step add to its job's summary.
|
|
31
34
|
STEP_SUMMARY = "GITHUB_STEP_SUMMARY"
|
|
@@ -73,6 +76,7 @@ module Duel
|
|
|
73
76
|
# warned (see WARNED).
|
|
74
77
|
def run
|
|
75
78
|
return Init.new(@argv.drop(1), out: @out, err: @err).run if @argv.first == "init"
|
|
79
|
+
return Comment.new(@argv.drop(1), env: @env, out: @out, err: @err).run if @argv.first == "comment"
|
|
76
80
|
|
|
77
81
|
settings = @settings = catch(:version) { parse } or return 0
|
|
78
82
|
config = @config = settings.config_path ? Duel.load_config(settings.config_path) : Config.new
|
|
@@ -124,6 +128,7 @@ module Duel
|
|
|
124
128
|
def parse
|
|
125
129
|
settings = Settings.defaults
|
|
126
130
|
settings.jobs = [Etc.nprocessors / CPUS_PER_LANE, 1].max
|
|
131
|
+
settings.fit = %i[jobs processes]
|
|
127
132
|
parser(settings).parse!(env_options + @argv)
|
|
128
133
|
branch = variable("GITHUB_REF_NAME", "CI_COMMIT_BRANCH")
|
|
129
134
|
base = variable("DUEL_BASE")
|
|
@@ -170,7 +175,7 @@ module Duel
|
|
|
170
175
|
passed ? "passed" : "failed",
|
|
171
176
|
-> {
|
|
172
177
|
Summary.new(runner, results, settings: settings, baseline: described_baseline(config, settings, baseline),
|
|
173
|
-
candidate: described_candidate(config, settings), note: @note).markdown
|
|
178
|
+
candidate: described_candidate(config, settings), note: @note, accepted: @accepted).markdown
|
|
174
179
|
},
|
|
175
180
|
->(comment) {
|
|
176
181
|
JSONReport.new(runner, results, settings: settings, exit: status, comment: comment,
|
|
@@ -189,6 +194,7 @@ module Duel
|
|
|
189
194
|
if accepted?(config)
|
|
190
195
|
# (Set: see #accepted?.)
|
|
191
196
|
label = config.accept_label #: String
|
|
197
|
+
@accepted = true
|
|
192
198
|
return passing("accepted with the #{label} label", "Accepted with the #{Summary.code(label)} label.")
|
|
193
199
|
end
|
|
194
200
|
return 1 unless pull_requests_only?(config, settings)
|
|
@@ -302,14 +308,17 @@ module Duel
|
|
|
302
308
|
# JSON report's: as --comment says, or else the configuration, or the
|
|
303
309
|
# default where the configuration didn't load. It comments on a push
|
|
304
310
|
# only where a regression in one passes (Config#fail_on), so the
|
|
305
|
-
# comment says what the check doesn't.
|
|
311
|
+
# comment says what the check doesn't. A regression the label
|
|
312
|
+
# accepts (Config#accept_label) is commented on as a run that passed
|
|
313
|
+
# would be: whoever added the label knows, so by default it posts
|
|
314
|
+
# only where the last comment's status isn't its own.
|
|
306
315
|
def commenting(status)
|
|
307
316
|
# (Parsed, so set: see #run.)
|
|
308
317
|
settings = @settings #: Settings
|
|
309
318
|
config = @config || Config.new
|
|
310
319
|
return "none" if pushed? && !pull_requests_only?(config, settings)
|
|
311
320
|
|
|
312
|
-
CLI.comment(settings.comment || config.comment, status)
|
|
321
|
+
CLI.comment(settings.comment || config.comment, @accepted ? "passed" : status)
|
|
313
322
|
end
|
|
314
323
|
|
|
315
324
|
# Writes the reports of a run whose +status+ is the JSON report's
|
|
@@ -391,7 +400,41 @@ module Duel
|
|
|
391
400
|
return Summary.code(dir) unless found.success?
|
|
392
401
|
|
|
393
402
|
_, clean = git(dir, "diff", "--quiet", "HEAD", "--", *Baseline.new(config, settings).watched)
|
|
394
|
-
"#{head.chomp}
|
|
403
|
+
return "#{head.chomp} (with changes not committed)" unless clean.success?
|
|
404
|
+
|
|
405
|
+
merged(dir) || head.chomp
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
# On GitHub Actions, a pull request's run checks out the commit that
|
|
409
|
+
# merges it with the branch it targets, which isn't in the pull
|
|
410
|
+
# request, so its head is named instead, as merged (or nil, where HEAD
|
|
411
|
+
# is no such commit). So does a merge request's merged results
|
|
412
|
+
# pipeline, on GitLab CI.
|
|
413
|
+
def merged(dir)
|
|
414
|
+
pull = (gitlab? ? merge_request : pull_request) or return
|
|
415
|
+
sha, branch = pull.dig("head", "sha"), pull.dig("base", "ref")
|
|
416
|
+
return unless branch.instance_of?(String) && git(dir, "rev-parse", "HEAD^2").first.chomp == sha
|
|
417
|
+
|
|
418
|
+
"#{git(dir, "rev-parse", "--short", sha).first.chomp} (merged with #{Summary.code(branch)})"
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
# A GitLab merge request's merged results pipeline's head and the
|
|
422
|
+
# branch it targets, as #pull_request has a pull request's (each nil
|
|
423
|
+
# in a merge request's other pipelines, whose HEAD is its head).
|
|
424
|
+
def merge_request = {"head" => {"sha" => variable("CI_MERGE_REQUEST_SOURCE_BRANCH_SHA")},
|
|
425
|
+
"base" => {"ref" => variable("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")}}
|
|
426
|
+
|
|
427
|
+
# The pull request a pull_request event's run is for, as GitHub
|
|
428
|
+
# Actions describes it (GITHUB_EVENT_PATH), or nil.
|
|
429
|
+
def pull_request
|
|
430
|
+
path = variable("GITHUB_EVENT_PATH") if event == "pull_request"
|
|
431
|
+
return unless path && File.file?(path)
|
|
432
|
+
|
|
433
|
+
payload = JSON.parse(File.read(path))
|
|
434
|
+
pull = payload["pull_request"] if payload.instance_of?(Hash)
|
|
435
|
+
pull if pull.instance_of?(Hash)
|
|
436
|
+
rescue JSON::ParserError
|
|
437
|
+
nil
|
|
395
438
|
end
|
|
396
439
|
|
|
397
440
|
# The baseline, for JSON: the ref that named it (the main branch,
|
|
@@ -459,8 +502,14 @@ module Duel
|
|
|
459
502
|
end
|
|
460
503
|
o.on("--candidate-dir DIR", "Candidate tree (default: this one)") { |v| settings.candidate_dir = v }
|
|
461
504
|
o.on("--filter REGEXP", "Only benchmarks whose names match") { |v| settings.filter = CLI.regexp(v) }
|
|
462
|
-
o.on("--jobs N", Integer, "Benchmarks measured at once (default: one for every #{CPUS_PER_LANE} CPUs)")
|
|
463
|
-
|
|
505
|
+
o.on("--jobs N", Integer, "Benchmarks measured at once (default: one for every #{CPUS_PER_LANE} CPUs, as many as fit in memory)") do |v|
|
|
506
|
+
settings.jobs = v
|
|
507
|
+
settings.fit -= [:jobs]
|
|
508
|
+
end
|
|
509
|
+
o.on("--processes N", Integer, "Worker processes per version (default: #{PROCESSES}, as many as fit in memory)") do |v|
|
|
510
|
+
settings.processes = v
|
|
511
|
+
settings.fit -= [:processes]
|
|
512
|
+
end
|
|
464
513
|
o.on("--timeout SECONDS", Float, "Most a worker may take to load or answer (default: #{TIMEOUT.round})") { |v| settings.timeout = v }
|
|
465
514
|
o.on("--time-limit SECONDS", Float, "Most a run measures for, then fails what's unmeasured or looks worse (default: 7200)") do |v|
|
|
466
515
|
settings.time_limit = v
|
data/lib/duel/comment.rb
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "open3"
|
|
6
|
+
require "optparse"
|
|
7
|
+
require "uri"
|
|
8
|
+
|
|
9
|
+
module Duel
|
|
10
|
+
# `duel comment`: what the Perf comment workflow runs, once the Perf
|
|
11
|
+
# workflow has, to comment on a pull request with Duel's report. It runs
|
|
12
|
+
# with the repository's own workflow and a token that can write, and the
|
|
13
|
+
# pull request's code ran in the Duel job, which could have written
|
|
14
|
+
# anything in what it kept (the report's directory), so it trusts only
|
|
15
|
+
# what Duel could have said there: the pull request's number, if it
|
|
16
|
+
# names the pull request whose head the run compared, and a comment
|
|
17
|
+
# (post or change: see CLI.comment) and a status (see CLI::MARKED) that
|
|
18
|
+
# Duel says. The report itself is the comment, as Markdown. It never
|
|
19
|
+
# edits or deletes a comment: once it has posted one, it hides Duel's
|
|
20
|
+
# earlier ones that aren't hidden yet as outdated, which GitHub
|
|
21
|
+
# collapses, so the newest is the one in view. It talks to GitHub with
|
|
22
|
+
# gh, which GH_TOKEN signs in, in GITHUB_REPOSITORY.
|
|
23
|
+
#
|
|
24
|
+
# With --gitlab, it's what the GitLab CI jobs' comment job runs, on a
|
|
25
|
+
# merge request: what Duel said is the Duel job's dotenv report's
|
|
26
|
+
# (DUEL_COMMENT and DUEL_STATUS), it talks to GitLab's API
|
|
27
|
+
# (CI_API_V4_URL) with DUEL_COMMENT_TOKEN, and it hides nothing, as only
|
|
28
|
+
# a Developer can resolve a thread, and a merge request's token is a
|
|
29
|
+
# Reporter's.
|
|
30
|
+
# @api private
|
|
31
|
+
class Comment
|
|
32
|
+
USAGE = "Usage: duel comment --report DIR --head SHA [--marker TEXT] [--gitlab]\n\n" \
|
|
33
|
+
"Comments on a pull request with the report a Perf run kept in DIR, when Duel said to, and only if the pull\n" \
|
|
34
|
+
"request's head is still SHA: what the Perf comment workflow runs (or with --gitlab, the comment job).\n\n"
|
|
35
|
+
|
|
36
|
+
# The first release of Duel whose `duel comment` the Perf comment
|
|
37
|
+
# workflow that this one writes can run: it installs the newest
|
|
38
|
+
# release of the same minor version (~>), so a fix comes with one.
|
|
39
|
+
SINCE = "0.1.8"
|
|
40
|
+
|
|
41
|
+
# Where GitLab CI has what Duel said (comment.env's, on GitHub): the
|
|
42
|
+
# merge request's, and the Duel job's dotenv report's.
|
|
43
|
+
GITLAB = {"pr" => "CI_MERGE_REQUEST_IID", "comment" => "DUEL_COMMENT", "status" => "DUEL_STATUS"}.freeze
|
|
44
|
+
|
|
45
|
+
# What Duel says in comment.env, as each line must have it.
|
|
46
|
+
SAID = {"pr" => /\A[0-9]+\z/, "comment" => /\A(?:post|change)\z/, "status" => /\A(?:passed|failed|neutral)\z/}.freeze
|
|
47
|
+
|
|
48
|
+
# The comments on a pull request, a page at a time (gh api graphql
|
|
49
|
+
# --paginate follows endCursor): GraphQL's, which say whether each is
|
|
50
|
+
# hidden, as REST's don't.
|
|
51
|
+
COMMENTS = <<~GRAPHQL
|
|
52
|
+
query($owner: String!, $name: String!, $pr: Int!, $endCursor: String) {
|
|
53
|
+
repository(owner: $owner, name: $name) {
|
|
54
|
+
pullRequest(number: $pr) {
|
|
55
|
+
comments(first: 100, after: $endCursor) {
|
|
56
|
+
nodes { id isMinimized author { login } body }
|
|
57
|
+
pageInfo { hasNextPage endCursor }
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
GRAPHQL
|
|
63
|
+
|
|
64
|
+
# The jq that picks, of COMMENTS, the ones GitHub Actions' token
|
|
65
|
+
# posted, each with its id, whether it's hidden, and its marker (its
|
|
66
|
+
# body's first line).
|
|
67
|
+
LISTED = '.data.repository.pullRequest.comments.nodes[] | select(.author.login == "github-actions") | ' \
|
|
68
|
+
'{id, hidden: .isMinimized, marked: (.body | split("\n")[0] | rtrimstr("\r"))}'
|
|
69
|
+
|
|
70
|
+
def initialize(argv, env:, out:, err:)
|
|
71
|
+
@argv = argv
|
|
72
|
+
@env = env
|
|
73
|
+
@out = out
|
|
74
|
+
@err = err
|
|
75
|
+
@marker = "duel"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Returns the exit status: 0 commented (or had nothing to say), 1 GitHub
|
|
79
|
+
# wouldn't take it, 2 a bad option.
|
|
80
|
+
def run
|
|
81
|
+
rest = catch(:help) { parser.parse(@argv) } or return 0
|
|
82
|
+
raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
|
|
83
|
+
report = @report
|
|
84
|
+
raise OptionParser::MissingArgument, "--report and --head are required" unless report && @head
|
|
85
|
+
|
|
86
|
+
comment(report)
|
|
87
|
+
rescue OptionParser::ParseError => e
|
|
88
|
+
@err.puts "duel comment: #{e}"
|
|
89
|
+
2
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private
|
|
93
|
+
|
|
94
|
+
def parser
|
|
95
|
+
OptionParser.new do |o|
|
|
96
|
+
o.banner = USAGE
|
|
97
|
+
o.on("--report DIR", "The directory of the Perf run's report (duel.md and comment.env)") { |v| @report = v }
|
|
98
|
+
o.on("--head SHA", "The commit the run compared, the pull request's head") { |v| @head = v }
|
|
99
|
+
o.on("--marker TEXT", "What marks Duel's comments (default: duel)") { |v| @marker = v }
|
|
100
|
+
o.on("--gitlab", "Comment on a GitLab merge request, from GitLab CI (with DUEL_COMMENT_TOKEN)") { @gitlab = true }
|
|
101
|
+
o.on("-h", "--help", "Print this help") do
|
|
102
|
+
@out.puts o
|
|
103
|
+
throw :help
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Comments with the report in +report+, if Duel said to, and the pull
|
|
109
|
+
# request's head is still the run's.
|
|
110
|
+
def comment(report)
|
|
111
|
+
pr, mode, status = SAID.keys.map { |name| said(name, report) }
|
|
112
|
+
return quiet("Duel said not to") unless pr && mode && status
|
|
113
|
+
|
|
114
|
+
head = (@gitlab ? merge_request(pr)&.fetch("sha") : gh("repos/#{repository}/pulls/#{pr}", "--jq", ".head.sha")&.chomp) or return 1
|
|
115
|
+
return quiet("the #{@gitlab ? "merge" : "pull"} request has moved on since the run") unless head == @head
|
|
116
|
+
|
|
117
|
+
ours = (@gitlab ? notes(pr) : earlier(pr)) or return 1
|
|
118
|
+
ours = ours.select { |c| c.fetch("marked").start_with?("#{marker} status=") || c.fetch("marked") == "#{marker} -->" }
|
|
119
|
+
last = ours.last&.fetch("marked")
|
|
120
|
+
stale = last && !last.eql?("#{marker} status=#{status} -->")
|
|
121
|
+
return quiet("the last comment says #{status} already") unless mode == "post" || stale
|
|
122
|
+
|
|
123
|
+
post(pr, "#{marker} status=#{status} -->\n#{File.read(File.join(report, "duel.md"))}") or return 1
|
|
124
|
+
@out.puts "duel comment: commented on #{@gitlab ? "merge request !" : "pull request "}#{pr}"
|
|
125
|
+
ours.reject { |earlier| earlier.fetch("hidden") }.each { |earlier| hide(earlier.fetch("id")) } unless @gitlab
|
|
126
|
+
0
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Posts +body+ as a new comment on the pull request (or merge request)
|
|
130
|
+
# +pr+: what GitHub (or GitLab) said, or nil.
|
|
131
|
+
def post(pr, body)
|
|
132
|
+
return gh("repos/#{repository}/issues/#{pr}/comments", "--method", "POST", "-f", "body=#{body}") unless @gitlab
|
|
133
|
+
|
|
134
|
+
gitlab(:Post, "#{project}/merge_requests/#{pr}/notes", {"body" => body})
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# What Duel said, as SAID has it, or nil: in +report+'s comment.env
|
|
138
|
+
# (the last line that says it), or on GitLab CI, as GITLAB has it.
|
|
139
|
+
def said(name, report)
|
|
140
|
+
return @env[GITLAB.fetch(name)].to_s[SAID.fetch(name)] if @gitlab
|
|
141
|
+
|
|
142
|
+
path = File.join(report, "comment.env")
|
|
143
|
+
lines = File.file?(path) ? File.readlines(path, chomp: true) : [] #: Array[String]
|
|
144
|
+
lines.map { |line| line[/\A#{name}=(.*)\z/, 1] }.grep(SAID.fetch(name)).last
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Duel's comments on the pull request, oldest first, each with its id,
|
|
148
|
+
# whether it's hidden, and its marker (its first line), or nil where
|
|
149
|
+
# GitHub wouldn't list them.
|
|
150
|
+
def earlier(pr)
|
|
151
|
+
owner, name = repository.split("/")
|
|
152
|
+
listed = gh("graphql", "--paginate", "-f", "query=#{COMMENTS}", "-F", "pr=#{pr}", "-f", "owner=#{owner}", "-f", "name=#{name}",
|
|
153
|
+
"--jq", LISTED) or return
|
|
154
|
+
listed.lines.map { |line| JSON.parse(line) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# The merge request +pr+, as GitLab's API has it, or nil.
|
|
158
|
+
def merge_request(pr) = gitlab(:Get, "#{project}/merge_requests/#{pr}")
|
|
159
|
+
|
|
160
|
+
# The token's notes on the merge request, oldest first, each with its
|
|
161
|
+
# marker (its first line), or nil where GitLab wouldn't list them.
|
|
162
|
+
def notes(pr)
|
|
163
|
+
me = gitlab(:Get, "/user") or return
|
|
164
|
+
notes = [] #: Array[Hash[String, untyped]]
|
|
165
|
+
page = "1"
|
|
166
|
+
while page
|
|
167
|
+
listed, page = gitlab(:Get, "#{project}/merge_requests/#{pr}/notes?sort=asc&order_by=created_at&per_page=100&page=#{page}", pages: true)
|
|
168
|
+
return unless listed
|
|
169
|
+
|
|
170
|
+
notes.concat(listed)
|
|
171
|
+
end
|
|
172
|
+
notes.select { |note| note.dig("author", "id") == me.fetch("id") }.map { |note| {"marked" => note.fetch("body").lines.first.to_s.chomp} }
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Hides an earlier comment as outdated, which GitHub collapses, or
|
|
176
|
+
# says it couldn't (it's still there, and that's all).
|
|
177
|
+
def hide(id)
|
|
178
|
+
query = "mutation($id: ID!) { minimizeComment(input: {subjectId: $id, classifier: OUTDATED}) { clientMutationId } }"
|
|
179
|
+
gh("graphql", "-f", "query=#{query}", "-f", "id=#{id}") || @err.puts("duel comment: couldn't hide an earlier comment")
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# What gh api prints, given +args+, or nil where it fails (or isn't
|
|
183
|
+
# there), which it says why.
|
|
184
|
+
def gh(*args)
|
|
185
|
+
out, err, status = Open3.capture3("gh", "api", *args)
|
|
186
|
+
return out if status.success?
|
|
187
|
+
|
|
188
|
+
@err.puts "duel comment: gh api #{args.first}: #{err.strip}"
|
|
189
|
+
rescue SystemCallError => e
|
|
190
|
+
@err.puts "duel comment: gh: #{e}"
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# What GitLab's API says to +verb+ (:Get or :Post) +path+ (under
|
|
194
|
+
# CI_API_V4_URL), given +body+ (as JSON), signed in with
|
|
195
|
+
# DUEL_COMMENT_TOKEN, or nil where it fails, which it says why. With
|
|
196
|
+
# +pages+, that and the next page's number (X-Next-Page), if there's
|
|
197
|
+
# one.
|
|
198
|
+
def gitlab(verb, path, body = nil, pages: false)
|
|
199
|
+
uri = URI("#{@env.fetch("CI_API_V4_URL")}#{path}")
|
|
200
|
+
request = Net::HTTP.const_get(verb).new(uri)
|
|
201
|
+
request["PRIVATE-TOKEN"] = @env.fetch("DUEL_COMMENT_TOKEN")
|
|
202
|
+
request["Content-Type"] = "application/json"
|
|
203
|
+
request.body = JSON.generate(body) if body
|
|
204
|
+
http = connection(uri)
|
|
205
|
+
http.use_ssl = uri.scheme == "https"
|
|
206
|
+
response = http.request(request)
|
|
207
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
208
|
+
return @err.puts("duel comment: GitLab: #{verb.upcase} #{path}: #{[response.code, response.body.to_s.strip].reject(&:empty?).join(" ")}")
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
parsed = JSON.parse(response.body)
|
|
212
|
+
pages ? [parsed, response["X-Next-Page"].to_s[/\A[0-9]+\z/]] : parsed
|
|
213
|
+
rescue SystemCallError, IOError, Timeout::Error, JSON::ParserError, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
|
|
214
|
+
@err.puts "duel comment: GitLab: #{verb.upcase} #{path}: #{e}"
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# A connection to +uri+'s host, not yet started.
|
|
218
|
+
# mutant:disable -- a nil host is Net::HTTP's localhost, where the
|
|
219
|
+
# tests' server is, so only another host could tell them apart
|
|
220
|
+
def connection(uri) = Net::HTTP.new(uri.host.to_s, uri.port)
|
|
221
|
+
|
|
222
|
+
# Says why it doesn't comment, and returns 0.
|
|
223
|
+
def quiet(why)
|
|
224
|
+
@out.puts "duel comment: no comment (#{why})"
|
|
225
|
+
0
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def marker = "<!-- #{@marker}"
|
|
229
|
+
|
|
230
|
+
def repository = @env.fetch("GITHUB_REPOSITORY")
|
|
231
|
+
|
|
232
|
+
# The project's path in GitLab's API.
|
|
233
|
+
def project = "/projects/#{@env.fetch("CI_PROJECT_ID")}"
|
|
234
|
+
end
|
|
235
|
+
end
|
data/lib/duel/config.rb
CHANGED
|
@@ -88,8 +88,9 @@ module Duel
|
|
|
88
88
|
# keeps them all. Once it has commented, a run that it wouldn't
|
|
89
89
|
# comment on still does, but :never's, when its status differs from
|
|
90
90
|
# the last comment's, so the last comment is never stale: after a
|
|
91
|
-
# failure, the first run that passes says so.
|
|
92
|
-
#
|
|
91
|
+
# failure, the first run that passes says so. A regression that
|
|
92
|
+
# #accept_label accepts counts as a run that passed. Duel tells the
|
|
93
|
+
# workflow which to do (see CLI.comment).
|
|
93
94
|
#
|
|
94
95
|
# c.comment = :results
|
|
95
96
|
attr_reader :comment
|
|
@@ -185,6 +186,11 @@ module Duel
|
|
|
185
186
|
# @api private
|
|
186
187
|
attr_reader :prepares, :in_workers, :hooks, :measures
|
|
187
188
|
|
|
189
|
+
# The memory that what the plugins used start for each worker takes at
|
|
190
|
+
# most, as they say (see Duel.plugin), in bytes: 0 where none says.
|
|
191
|
+
# @api private
|
|
192
|
+
attr_reader :worker_memory
|
|
193
|
+
|
|
188
194
|
# A configuration of the conventions, for the project at +root+ (a
|
|
189
195
|
# plugin's tests make one to use it in).
|
|
190
196
|
def initialize(root = Dir.pwd)
|
|
@@ -202,6 +208,7 @@ module Duel
|
|
|
202
208
|
@in_workers = []
|
|
203
209
|
@hooks = HOOKS.to_h { |hook| [hook, []] }
|
|
204
210
|
@used = {}
|
|
211
|
+
@worker_memory = 0
|
|
205
212
|
@measures = Hash.new do |all, name|
|
|
206
213
|
all[name] = {} #: Hash[Symbol, untyped]
|
|
207
214
|
end
|
|
@@ -289,6 +296,8 @@ module Duel
|
|
|
289
296
|
DSL.add(plugin::DSL) if plugin.const_defined?(:DSL, false)
|
|
290
297
|
used = @used[name] = Used.new(options)
|
|
291
298
|
used.value = plugin.configure(self, **options)
|
|
299
|
+
@worker_memory += plugin.worker_memory if plugin.respond_to?(:worker_memory)
|
|
300
|
+
used.value
|
|
292
301
|
end
|
|
293
302
|
|
|
294
303
|
# The plugins used (see #use), by name, in the order they were first
|
data/lib/duel/init.rb
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "erb"
|
|
4
|
+
require "json"
|
|
4
5
|
require "fileutils"
|
|
5
6
|
require "tmpdir"
|
|
6
7
|
require "open3"
|
|
7
8
|
require "optparse"
|
|
8
9
|
require "pathname"
|
|
9
10
|
require_relative "../duel"
|
|
11
|
+
require_relative "comment"
|
|
10
12
|
|
|
11
13
|
module Duel
|
|
12
14
|
# `duel init`: sets Duel up in a gem or an app that isn't a Rails app
|
|
@@ -17,7 +19,8 @@ module Duel
|
|
|
17
19
|
# --gitlab, GitLab CI jobs to include, adds a Rake
|
|
18
20
|
# task to the Rakefile, and adds StackProf to the Gemfile (for profiles
|
|
19
21
|
# of regressions, on CRuby). It overwrites nothing without --force, and
|
|
20
|
-
# writes nothing with --dry-run
|
|
22
|
+
# writes nothing with --dry-run, or with --check, which says how the
|
|
23
|
+
# workflows differ from what it writes.
|
|
21
24
|
# @api private
|
|
22
25
|
class Init
|
|
23
26
|
USAGE = "Usage: duel init [options]\n\n" \
|
|
@@ -38,24 +41,33 @@ module Duel
|
|
|
38
41
|
# indentation of its first line.
|
|
39
42
|
DEVELOPMENT = /^([ \t]*)group[ \t(]+(?::development(?:,[ \t]*:test)?|:test,[ \t]*:development)[ \t)]+do[ \t]*\n/
|
|
40
43
|
|
|
41
|
-
|
|
44
|
+
# The color of the label it creates (see #create_label): GitHub's
|
|
45
|
+
# yellow, a warning's.
|
|
46
|
+
LABEL_COLOR = "FBCA04"
|
|
47
|
+
|
|
48
|
+
def initialize(argv, out: $stdout, err: $stderr, input: $stdin, root: Dir.pwd)
|
|
42
49
|
@argv = argv
|
|
43
50
|
@out = out
|
|
44
51
|
@err = err
|
|
52
|
+
@input = input
|
|
45
53
|
@root = root
|
|
46
54
|
end
|
|
47
55
|
|
|
48
|
-
# Returns the exit status: 0 set up
|
|
49
|
-
#
|
|
56
|
+
# Returns the exit status: 0 set up (or with --check, its workflows
|
|
57
|
+
# are what it writes), 1 a Rails app (not set up, or checked), or with
|
|
58
|
+
# --check, a workflow that differs, 2 a bad option.
|
|
50
59
|
def run
|
|
51
60
|
rest = catch(:help) { parser.parse(@argv) } or return 0
|
|
52
61
|
raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
|
|
53
62
|
return rails if File.exist?(File.join(@root, "config", "application.rb"))
|
|
54
63
|
|
|
64
|
+
return check if @check
|
|
65
|
+
|
|
55
66
|
@out.puts "duel init: a dry run, so nothing is written" if @dry_run
|
|
56
67
|
files.each { |path, content| write(path, content) }
|
|
57
68
|
add_rake_task
|
|
58
69
|
add_stackprof
|
|
70
|
+
create_label
|
|
59
71
|
@out.print next_steps
|
|
60
72
|
0
|
|
61
73
|
rescue OptionParser::ParseError => e
|
|
@@ -81,6 +93,7 @@ module Duel
|
|
|
81
93
|
o.banner = USAGE
|
|
82
94
|
o.on("--force", "Overwrite files that exist") { @force = true }
|
|
83
95
|
o.on("--dry-run", "Say what it would do, and write nothing") { @dry_run = true }
|
|
96
|
+
o.on("--check", "Say how the workflows differ from what it writes (with the same options), and write nothing") { @check = true }
|
|
84
97
|
o.on("--badge", "Write a job that keeps a badge for the README (it needs contents: write)") { @badge = true }
|
|
85
98
|
o.on("--gitlab", "Write GitLab CI jobs (to include from .gitlab-ci.yml), not GitHub Actions workflows") { @gitlab = true }
|
|
86
99
|
o.on("-h", "--help", "Print this help") do
|
|
@@ -93,10 +106,106 @@ module Duel
|
|
|
93
106
|
# Says to use duel-rails's generator instead.
|
|
94
107
|
def rails
|
|
95
108
|
@err.puts "duel init: this is a Rails app (config/application.rb). Add duel-rails to the Gemfile's development group, " \
|
|
96
|
-
"and run `bin/rails generate duel:install` instead"
|
|
109
|
+
"and run `bin/rails generate duel:install#{" --check" if @check}` instead"
|
|
97
110
|
1
|
|
98
111
|
end
|
|
99
112
|
|
|
113
|
+
# Says whether each workflow (or with --gitlab, the GitLab CI jobs) is
|
|
114
|
+
# what it writes, or how it differs, or that it's missing: they're the
|
|
115
|
+
# files a project rarely changes, but Duel's releases do. Returns 0
|
|
116
|
+
# where each is, or else 1.
|
|
117
|
+
def check
|
|
118
|
+
written = files.except(".duel.rb", benchmark_file)
|
|
119
|
+
written.count { |path, content| !same?(path, content) }.zero? ? 0 : 1
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Whether the file at +path+ is +content+, saying so, or how it
|
|
123
|
+
# differs (as diff -u has it), or that it's missing.
|
|
124
|
+
def same?(path, content)
|
|
125
|
+
full = File.join(@root, path)
|
|
126
|
+
unless File.file?(full)
|
|
127
|
+
said("missing", path)
|
|
128
|
+
return false
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
differences, = Open3.capture2("diff", "-u", "-L", path, "-L", "#{path} (duel init)", full, "-", stdin_data: content)
|
|
132
|
+
said(differences.empty? ? "same" : "differs", path)
|
|
133
|
+
@out.print differences
|
|
134
|
+
differences.empty?
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Creates the label that accepts a regression (Config#accept_label) in
|
|
138
|
+
# the project's GitHub repository, with gh (or with --gitlab, its
|
|
139
|
+
# GitLab project, with glab), if it's missing and the answer to
|
|
140
|
+
# whether to is yes: it asks only at a terminal, and not on a dry run.
|
|
141
|
+
# Where it doesn't, the next steps say to.
|
|
142
|
+
def create_label
|
|
143
|
+
label = accept_label
|
|
144
|
+
return unless label && !@dry_run && @input.tty?
|
|
145
|
+
|
|
146
|
+
repo = (@gitlab ? gitlab_project : cli("gh", "repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner")&.chomp) or return
|
|
147
|
+
names = (@gitlab ? gitlab_labels(repo) : github_labels(repo, label)) or return
|
|
148
|
+
if names.include?(label)
|
|
149
|
+
said("skip", "label #{label} (#{repo} has it)")
|
|
150
|
+
return @label = true
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
@out.print "Create the label #{label.inspect} in #{repo} with #{@gitlab ? "glab" : "gh"}, so a #{request} can accept a regression? [y/N] "
|
|
154
|
+
return unless @input.gets.to_s.strip.match?(/\Ay(?:es)?\z/i)
|
|
155
|
+
|
|
156
|
+
described = ["--repo", repo, "--description", "Accepts a regression that Duel found"]
|
|
157
|
+
created = @gitlab ? cli("glab", "label", "create", "--name", label, "--color", "##{LABEL_COLOR}", *described) :
|
|
158
|
+
cli("gh", "label", "create", label, "--color", LABEL_COLOR, *described)
|
|
159
|
+
return unless created
|
|
160
|
+
|
|
161
|
+
said("create", "label #{label} (#{repo})")
|
|
162
|
+
@label = true
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# The names of the GitHub repository +repo+'s labels that gh finds by
|
|
166
|
+
# +label+, or nil.
|
|
167
|
+
def github_labels(repo, label) = cli("gh", "label", "list", "--repo", repo, "--search", label, "--limit", "100", "--json", "name",
|
|
168
|
+
"--jq", ".[].name")&.lines(chomp: true)
|
|
169
|
+
|
|
170
|
+
# The project's GitLab project, as glab has it (group/project), or nil.
|
|
171
|
+
def gitlab_project
|
|
172
|
+
project = cli("glab", "repo", "view", "--output", "json") or return
|
|
173
|
+
parsed = JSON.parse(project)
|
|
174
|
+
parsed["path_with_namespace"] if parsed.instance_of?(Hash)
|
|
175
|
+
rescue JSON::ParserError
|
|
176
|
+
nil
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# The names of the GitLab project +repo+'s labels, every page of them,
|
|
180
|
+
# or nil where glab can't list a page.
|
|
181
|
+
def gitlab_labels(repo)
|
|
182
|
+
names = [] #: Array[String]
|
|
183
|
+
page = 1
|
|
184
|
+
while (listed = cli("glab", "label", "list", "--repo", repo, "--output", "json", "--per-page", "100", "--page", page.to_s))
|
|
185
|
+
labels = JSON.parse(listed)
|
|
186
|
+
names.concat(labels.map { |label| label.fetch("name") })
|
|
187
|
+
return names if labels.size < 100
|
|
188
|
+
|
|
189
|
+
page += 1
|
|
190
|
+
end
|
|
191
|
+
rescue JSON::ParserError
|
|
192
|
+
nil
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# What +program+ (gh, or glab) prints, run in the project, or nil where
|
|
196
|
+
# it fails (which it says) or isn't there.
|
|
197
|
+
def cli(program, *args)
|
|
198
|
+
out, err, status = Open3.capture3(program, *args, chdir: @root)
|
|
199
|
+
return out if status.success?
|
|
200
|
+
|
|
201
|
+
@err.puts "duel init: #{program} #{args.first(2).join(" ")}: #{err.strip}"
|
|
202
|
+
rescue SystemCallError
|
|
203
|
+
nil
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# What Duel compares: a pull request, or on GitLab, a merge request.
|
|
207
|
+
def request = @gitlab ? "merge request" : "pull request"
|
|
208
|
+
|
|
100
209
|
# Writes +content+ to +path+, unless it exists (without --force), and
|
|
101
210
|
# says what it did.
|
|
102
211
|
def write(path, content)
|
|
@@ -166,6 +275,8 @@ module Duel
|
|
|
166
275
|
end
|
|
167
276
|
steps << "Set c.library in .duel.rb to what each version's workers require." unless required
|
|
168
277
|
steps << "Replace the example in #{benchmark_file} with benchmarks of your own code."
|
|
278
|
+
steps << "Create the label #{accept_label.inspect} in your repository, so a #{request} can accept a regression (c.accept_label)." if
|
|
279
|
+
accept_label && !@label
|
|
169
280
|
steps << "Run `bundle exec duel --force`. It compares the code with itself, and passes when the benchmarks are sound."
|
|
170
281
|
steps << if @gitlab
|
|
171
282
|
"Include #{gitlab_file} from .gitlab-ci.yml, and add the tokens it says to, for comments. On a merge request, the job " \
|
data/lib/duel/memory.rb
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Duel
|
|
4
|
+
# The memory a run's workers can take (see Runner#fit): what the
|
|
5
|
+
# machine has available, or what its cgroup's limit leaves, if that's
|
|
6
|
+
# less (a container's, say).
|
|
7
|
+
# @api private
|
|
8
|
+
module Memory
|
|
9
|
+
# The kinds of pages macOS's vm_stat counts that a process can have
|
|
10
|
+
# without the system swapping: free, and cached but unused.
|
|
11
|
+
MAC_PAGES = %w[free inactive speculative].freeze
|
|
12
|
+
|
|
13
|
+
# Where Linux's proc and sys are.
|
|
14
|
+
ROOT = "/"
|
|
15
|
+
|
|
16
|
+
class << self
|
|
17
|
+
# The bytes available, or nil where Duel can't tell: Linux's
|
|
18
|
+
# MemAvailable (in +root+'s proc/meminfo), at most what the cgroup
|
|
19
|
+
# mounted at sys/fs/cgroup leaves, or else macOS's free and
|
|
20
|
+
# inactive pages (in what +vm_stat+ prints).
|
|
21
|
+
def available(root: ROOT, vm_stat: -> { command("vm_stat") }) = linux(root) || mac(vm_stat.call)
|
|
22
|
+
|
|
23
|
+
# MemAvailable, at most what the cgroup leaves, or nil.
|
|
24
|
+
def linux(root)
|
|
25
|
+
kb = read(File.join(root, "proc", "meminfo"))&.[](/^MemAvailable:\s+(\d+) kB$/, 1) or return
|
|
26
|
+
[Integer(kb) * 1024, cgroup(File.join(root, "sys", "fs", "cgroup"))].compact.min
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# What a cgroup (v2) at +dir+ leaves of its limit, or nil where it
|
|
30
|
+
# has none ("max").
|
|
31
|
+
def cgroup(dir)
|
|
32
|
+
max, current = %w[memory.max memory.current].map { |file| Integer(read(File.join(dir, file)), exception: false) }
|
|
33
|
+
max - current if max && current
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The bytes in the pages of MAC_PAGES that vm_stat's +output+
|
|
37
|
+
# counts, or nil.
|
|
38
|
+
def mac(output)
|
|
39
|
+
page = output&.[](/page size of (\d+) bytes/, 1) or return
|
|
40
|
+
MAC_PAGES.sum { |kind| Integer(output[/^Pages #{kind}:\s+(\d+)\./, 1] || 0) } * Integer(page)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# A file's contents, or nil where there's none.
|
|
44
|
+
def read(path) = (File.read(path) if File.file?(path))
|
|
45
|
+
|
|
46
|
+
# What a command prints, or nil where there's no such command.
|
|
47
|
+
def command(*args)
|
|
48
|
+
IO.popen(args, err: File::NULL, &:read)
|
|
49
|
+
rescue SystemCallError
|
|
50
|
+
nil
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|