duel 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +58 -1
- data/README.md +6 -1
- data/docs/ci.md +77 -76
- data/docs/configuration.md +3 -1
- data/docs/how-it-works.md +4 -1
- data/docs/plugins.md +7 -0
- data/lib/duel/cli.rb +51 -9
- data/lib/duel/comment.rb +235 -0
- data/lib/duel/config.rb +11 -2
- data/lib/duel/init.rb +116 -5
- data/lib/duel/runner.rb +20 -5
- data/lib/duel/summary.rb +11 -5
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-comment.yml.erb +17 -42
- data/lib/duel/version.rb +1 -1
- data/lib/duel.rb +8 -0
- data/sig/duel/cli.rbs +11 -0
- data/sig/duel/comment.rbs +82 -0
- data/sig/duel/config.rbs +5 -0
- data/sig/duel/init.rbs +39 -1
- data/sig/duel/runner.rbs +3 -0
- data/sig/duel/summary.rbs +3 -1
- metadata +3 -1
data/lib/duel/comment.rb
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "open3"
|
|
6
|
+
require "optparse"
|
|
7
|
+
require "uri"
|
|
8
|
+
|
|
9
|
+
module Duel
|
|
10
|
+
# `duel comment`: what the Perf comment workflow runs, once the Perf
|
|
11
|
+
# workflow has, to comment on a pull request with Duel's report. It runs
|
|
12
|
+
# with the repository's own workflow and a token that can write, and the
|
|
13
|
+
# pull request's code ran in the Duel job, which could have written
|
|
14
|
+
# anything in what it kept (the report's directory), so it trusts only
|
|
15
|
+
# what Duel could have said there: the pull request's number, if it
|
|
16
|
+
# names the pull request whose head the run compared, and a comment
|
|
17
|
+
# (post or change: see CLI.comment) and a status (see CLI::MARKED) that
|
|
18
|
+
# Duel says. The report itself is the comment, as Markdown. It never
|
|
19
|
+
# edits or deletes a comment: once it has posted one, it hides Duel's
|
|
20
|
+
# earlier ones that aren't hidden yet as outdated, which GitHub
|
|
21
|
+
# collapses, so the newest is the one in view. It talks to GitHub with
|
|
22
|
+
# gh, which GH_TOKEN signs in, in GITHUB_REPOSITORY.
|
|
23
|
+
#
|
|
24
|
+
# With --gitlab, it's what the GitLab CI jobs' comment job runs, on a
|
|
25
|
+
# merge request: what Duel said is the Duel job's dotenv report's
|
|
26
|
+
# (DUEL_COMMENT and DUEL_STATUS), it talks to GitLab's API
|
|
27
|
+
# (CI_API_V4_URL) with DUEL_COMMENT_TOKEN, and it hides nothing, as only
|
|
28
|
+
# a Developer can resolve a thread, and a merge request's token is a
|
|
29
|
+
# Reporter's.
|
|
30
|
+
# @api private
|
|
31
|
+
class Comment
|
|
32
|
+
USAGE = "Usage: duel comment --report DIR --head SHA [--marker TEXT] [--gitlab]\n\n" \
|
|
33
|
+
"Comments on a pull request with the report a Perf run kept in DIR, when Duel said to, and only if the pull\n" \
|
|
34
|
+
"request's head is still SHA: what the Perf comment workflow runs (or with --gitlab, the comment job).\n\n"
|
|
35
|
+
|
|
36
|
+
# The first release of Duel whose `duel comment` the Perf comment
|
|
37
|
+
# workflow that this one writes can run: it installs the newest
|
|
38
|
+
# release of the same minor version (~>), so a fix comes with one.
|
|
39
|
+
SINCE = "0.1.8"
|
|
40
|
+
|
|
41
|
+
# Where GitLab CI has what Duel said (comment.env's, on GitHub): the
|
|
42
|
+
# merge request's, and the Duel job's dotenv report's.
|
|
43
|
+
GITLAB = {"pr" => "CI_MERGE_REQUEST_IID", "comment" => "DUEL_COMMENT", "status" => "DUEL_STATUS"}.freeze
|
|
44
|
+
|
|
45
|
+
# What Duel says in comment.env, as each line must have it.
|
|
46
|
+
SAID = {"pr" => /\A[0-9]+\z/, "comment" => /\A(?:post|change)\z/, "status" => /\A(?:passed|failed|neutral)\z/}.freeze
|
|
47
|
+
|
|
48
|
+
# The comments on a pull request, a page at a time (gh api graphql
|
|
49
|
+
# --paginate follows endCursor): GraphQL's, which say whether each is
|
|
50
|
+
# hidden, as REST's don't.
|
|
51
|
+
COMMENTS = <<~GRAPHQL
|
|
52
|
+
query($owner: String!, $name: String!, $pr: Int!, $endCursor: String) {
|
|
53
|
+
repository(owner: $owner, name: $name) {
|
|
54
|
+
pullRequest(number: $pr) {
|
|
55
|
+
comments(first: 100, after: $endCursor) {
|
|
56
|
+
nodes { id isMinimized author { login } body }
|
|
57
|
+
pageInfo { hasNextPage endCursor }
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
GRAPHQL
|
|
63
|
+
|
|
64
|
+
# The jq that picks, of COMMENTS, the ones GitHub Actions' token
|
|
65
|
+
# posted, each with its id, whether it's hidden, and its marker (its
|
|
66
|
+
# body's first line).
|
|
67
|
+
LISTED = '.data.repository.pullRequest.comments.nodes[] | select(.author.login == "github-actions") | ' \
|
|
68
|
+
'{id, hidden: .isMinimized, marked: (.body | split("\n")[0] | rtrimstr("\r"))}'
|
|
69
|
+
|
|
70
|
+
def initialize(argv, env:, out:, err:)
|
|
71
|
+
@argv = argv
|
|
72
|
+
@env = env
|
|
73
|
+
@out = out
|
|
74
|
+
@err = err
|
|
75
|
+
@marker = "duel"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Returns the exit status: 0 commented (or had nothing to say), 1 GitHub
|
|
79
|
+
# wouldn't take it, 2 a bad option.
|
|
80
|
+
def run
|
|
81
|
+
rest = catch(:help) { parser.parse(@argv) } or return 0
|
|
82
|
+
raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
|
|
83
|
+
report = @report
|
|
84
|
+
raise OptionParser::MissingArgument, "--report and --head are required" unless report && @head
|
|
85
|
+
|
|
86
|
+
comment(report)
|
|
87
|
+
rescue OptionParser::ParseError => e
|
|
88
|
+
@err.puts "duel comment: #{e}"
|
|
89
|
+
2
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private
|
|
93
|
+
|
|
94
|
+
def parser
|
|
95
|
+
OptionParser.new do |o|
|
|
96
|
+
o.banner = USAGE
|
|
97
|
+
o.on("--report DIR", "The directory of the Perf run's report (duel.md and comment.env)") { |v| @report = v }
|
|
98
|
+
o.on("--head SHA", "The commit the run compared, the pull request's head") { |v| @head = v }
|
|
99
|
+
o.on("--marker TEXT", "What marks Duel's comments (default: duel)") { |v| @marker = v }
|
|
100
|
+
o.on("--gitlab", "Comment on a GitLab merge request, from GitLab CI (with DUEL_COMMENT_TOKEN)") { @gitlab = true }
|
|
101
|
+
o.on("-h", "--help", "Print this help") do
|
|
102
|
+
@out.puts o
|
|
103
|
+
throw :help
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Comments with the report in +report+, if Duel said to, and the pull
|
|
109
|
+
# request's head is still the run's.
|
|
110
|
+
def comment(report)
|
|
111
|
+
pr, mode, status = SAID.keys.map { |name| said(name, report) }
|
|
112
|
+
return quiet("Duel said not to") unless pr && mode && status
|
|
113
|
+
|
|
114
|
+
head = (@gitlab ? merge_request(pr)&.fetch("sha") : gh("repos/#{repository}/pulls/#{pr}", "--jq", ".head.sha")&.chomp) or return 1
|
|
115
|
+
return quiet("the #{@gitlab ? "merge" : "pull"} request has moved on since the run") unless head == @head
|
|
116
|
+
|
|
117
|
+
ours = (@gitlab ? notes(pr) : earlier(pr)) or return 1
|
|
118
|
+
ours = ours.select { |c| c.fetch("marked").start_with?("#{marker} status=") || c.fetch("marked") == "#{marker} -->" }
|
|
119
|
+
last = ours.last&.fetch("marked")
|
|
120
|
+
stale = last && !last.eql?("#{marker} status=#{status} -->")
|
|
121
|
+
return quiet("the last comment says #{status} already") unless mode == "post" || stale
|
|
122
|
+
|
|
123
|
+
post(pr, "#{marker} status=#{status} -->\n#{File.read(File.join(report, "duel.md"))}") or return 1
|
|
124
|
+
@out.puts "duel comment: commented on #{@gitlab ? "merge request !" : "pull request "}#{pr}"
|
|
125
|
+
ours.reject { |earlier| earlier.fetch("hidden") }.each { |earlier| hide(earlier.fetch("id")) } unless @gitlab
|
|
126
|
+
0
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Posts +body+ as a new comment on the pull request (or merge request)
|
|
130
|
+
# +pr+: what GitHub (or GitLab) said, or nil.
|
|
131
|
+
def post(pr, body)
|
|
132
|
+
return gh("repos/#{repository}/issues/#{pr}/comments", "--method", "POST", "-f", "body=#{body}") unless @gitlab
|
|
133
|
+
|
|
134
|
+
gitlab(:Post, "#{project}/merge_requests/#{pr}/notes", {"body" => body})
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# What Duel said, as SAID has it, or nil: in +report+'s comment.env
|
|
138
|
+
# (the last line that says it), or on GitLab CI, as GITLAB has it.
|
|
139
|
+
def said(name, report)
|
|
140
|
+
return @env[GITLAB.fetch(name)].to_s[SAID.fetch(name)] if @gitlab
|
|
141
|
+
|
|
142
|
+
path = File.join(report, "comment.env")
|
|
143
|
+
lines = File.file?(path) ? File.readlines(path, chomp: true) : [] #: Array[String]
|
|
144
|
+
lines.map { |line| line[/\A#{name}=(.*)\z/, 1] }.grep(SAID.fetch(name)).last
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Duel's comments on the pull request, oldest first, each with its id,
|
|
148
|
+
# whether it's hidden, and its marker (its first line), or nil where
|
|
149
|
+
# GitHub wouldn't list them.
|
|
150
|
+
def earlier(pr)
|
|
151
|
+
owner, name = repository.split("/")
|
|
152
|
+
listed = gh("graphql", "--paginate", "-f", "query=#{COMMENTS}", "-F", "pr=#{pr}", "-f", "owner=#{owner}", "-f", "name=#{name}",
|
|
153
|
+
"--jq", LISTED) or return
|
|
154
|
+
listed.lines.map { |line| JSON.parse(line) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# The merge request +pr+, as GitLab's API has it, or nil.
|
|
158
|
+
def merge_request(pr) = gitlab(:Get, "#{project}/merge_requests/#{pr}")
|
|
159
|
+
|
|
160
|
+
# The token's notes on the merge request, oldest first, each with its
|
|
161
|
+
# marker (its first line), or nil where GitLab wouldn't list them.
|
|
162
|
+
def notes(pr)
|
|
163
|
+
me = gitlab(:Get, "/user") or return
|
|
164
|
+
notes = [] #: Array[Hash[String, untyped]]
|
|
165
|
+
page = "1"
|
|
166
|
+
while page
|
|
167
|
+
listed, page = gitlab(:Get, "#{project}/merge_requests/#{pr}/notes?sort=asc&order_by=created_at&per_page=100&page=#{page}", pages: true)
|
|
168
|
+
return unless listed
|
|
169
|
+
|
|
170
|
+
notes.concat(listed)
|
|
171
|
+
end
|
|
172
|
+
notes.select { |note| note.dig("author", "id") == me.fetch("id") }.map { |note| {"marked" => note.fetch("body").lines.first.to_s.chomp} }
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Hides an earlier comment as outdated, which GitHub collapses, or
|
|
176
|
+
# says it couldn't (it's still there, and that's all).
|
|
177
|
+
def hide(id)
|
|
178
|
+
query = "mutation($id: ID!) { minimizeComment(input: {subjectId: $id, classifier: OUTDATED}) { clientMutationId } }"
|
|
179
|
+
gh("graphql", "-f", "query=#{query}", "-f", "id=#{id}") || @err.puts("duel comment: couldn't hide an earlier comment")
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# What gh api prints, given +args+, or nil where it fails (or isn't
|
|
183
|
+
# there), which it says why.
|
|
184
|
+
def gh(*args)
|
|
185
|
+
out, err, status = Open3.capture3("gh", "api", *args)
|
|
186
|
+
return out if status.success?
|
|
187
|
+
|
|
188
|
+
@err.puts "duel comment: gh api #{args.first}: #{err.strip}"
|
|
189
|
+
rescue SystemCallError => e
|
|
190
|
+
@err.puts "duel comment: gh: #{e}"
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# What GitLab's API says to +verb+ (:Get or :Post) +path+ (under
|
|
194
|
+
# CI_API_V4_URL), given +body+ (as JSON), signed in with
|
|
195
|
+
# DUEL_COMMENT_TOKEN, or nil where it fails, which it says why. With
|
|
196
|
+
# +pages+, that and the next page's number (X-Next-Page), if there's
|
|
197
|
+
# one.
|
|
198
|
+
def gitlab(verb, path, body = nil, pages: false)
|
|
199
|
+
uri = URI("#{@env.fetch("CI_API_V4_URL")}#{path}")
|
|
200
|
+
request = Net::HTTP.const_get(verb).new(uri)
|
|
201
|
+
request["PRIVATE-TOKEN"] = @env.fetch("DUEL_COMMENT_TOKEN")
|
|
202
|
+
request["Content-Type"] = "application/json"
|
|
203
|
+
request.body = JSON.generate(body) if body
|
|
204
|
+
http = connection(uri)
|
|
205
|
+
http.use_ssl = uri.scheme == "https"
|
|
206
|
+
response = http.request(request)
|
|
207
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
208
|
+
return @err.puts("duel comment: GitLab: #{verb.upcase} #{path}: #{[response.code, response.body.to_s.strip].reject(&:empty?).join(" ")}")
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
parsed = JSON.parse(response.body)
|
|
212
|
+
pages ? [parsed, response["X-Next-Page"].to_s[/\A[0-9]+\z/]] : parsed
|
|
213
|
+
rescue SystemCallError, IOError, Timeout::Error, JSON::ParserError, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
|
|
214
|
+
@err.puts "duel comment: GitLab: #{verb.upcase} #{path}: #{e}"
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# A connection to +uri+'s host, not yet started.
|
|
218
|
+
# mutant:disable -- a nil host is Net::HTTP's localhost, where the
|
|
219
|
+
# tests' server is, so only another host could tell them apart
|
|
220
|
+
def connection(uri) = Net::HTTP.new(uri.host.to_s, uri.port)
|
|
221
|
+
|
|
222
|
+
# Says why it doesn't comment, and returns 0.
|
|
223
|
+
def quiet(why)
|
|
224
|
+
@out.puts "duel comment: no comment (#{why})"
|
|
225
|
+
0
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def marker = "<!-- #{@marker}"
|
|
229
|
+
|
|
230
|
+
def repository = @env.fetch("GITHUB_REPOSITORY")
|
|
231
|
+
|
|
232
|
+
# The project's path in GitLab's API.
|
|
233
|
+
def project = "/projects/#{@env.fetch("CI_PROJECT_ID")}"
|
|
234
|
+
end
|
|
235
|
+
end
|
data/lib/duel/config.rb
CHANGED
|
@@ -88,8 +88,9 @@ module Duel
|
|
|
88
88
|
# keeps them all. Once it has commented, a run that it wouldn't
|
|
89
89
|
# comment on still does, but :never's, when its status differs from
|
|
90
90
|
# the last comment's, so the last comment is never stale: after a
|
|
91
|
-
# failure, the first run that passes says so.
|
|
92
|
-
#
|
|
91
|
+
# failure, the first run that passes says so. A regression that
|
|
92
|
+
# #accept_label accepts counts as a run that passed. Duel tells the
|
|
93
|
+
# workflow which to do (see CLI.comment).
|
|
93
94
|
#
|
|
94
95
|
# c.comment = :results
|
|
95
96
|
attr_reader :comment
|
|
@@ -185,6 +186,11 @@ module Duel
|
|
|
185
186
|
# @api private
|
|
186
187
|
attr_reader :prepares, :in_workers, :hooks, :measures
|
|
187
188
|
|
|
189
|
+
# The memory that what the plugins used start for each worker takes at
|
|
190
|
+
# most, as they say (see Duel.plugin), in bytes: 0 where none says.
|
|
191
|
+
# @api private
|
|
192
|
+
attr_reader :worker_memory
|
|
193
|
+
|
|
188
194
|
# A configuration of the conventions, for the project at +root+ (a
|
|
189
195
|
# plugin's tests make one to use it in).
|
|
190
196
|
def initialize(root = Dir.pwd)
|
|
@@ -202,6 +208,7 @@ module Duel
|
|
|
202
208
|
@in_workers = []
|
|
203
209
|
@hooks = HOOKS.to_h { |hook| [hook, []] }
|
|
204
210
|
@used = {}
|
|
211
|
+
@worker_memory = 0
|
|
205
212
|
@measures = Hash.new do |all, name|
|
|
206
213
|
all[name] = {} #: Hash[Symbol, untyped]
|
|
207
214
|
end
|
|
@@ -289,6 +296,8 @@ module Duel
|
|
|
289
296
|
DSL.add(plugin::DSL) if plugin.const_defined?(:DSL, false)
|
|
290
297
|
used = @used[name] = Used.new(options)
|
|
291
298
|
used.value = plugin.configure(self, **options)
|
|
299
|
+
@worker_memory += plugin.worker_memory if plugin.respond_to?(:worker_memory)
|
|
300
|
+
used.value
|
|
292
301
|
end
|
|
293
302
|
|
|
294
303
|
# The plugins used (see #use), by name, in the order they were first
|
data/lib/duel/init.rb
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "erb"
|
|
4
|
+
require "json"
|
|
4
5
|
require "fileutils"
|
|
5
6
|
require "tmpdir"
|
|
6
7
|
require "open3"
|
|
7
8
|
require "optparse"
|
|
8
9
|
require "pathname"
|
|
9
10
|
require_relative "../duel"
|
|
11
|
+
require_relative "comment"
|
|
10
12
|
|
|
11
13
|
module Duel
|
|
12
14
|
# `duel init`: sets Duel up in a gem or an app that isn't a Rails app
|
|
@@ -17,7 +19,8 @@ module Duel
|
|
|
17
19
|
# --gitlab, GitLab CI jobs to include, adds a Rake
|
|
18
20
|
# task to the Rakefile, and adds StackProf to the Gemfile (for profiles
|
|
19
21
|
# of regressions, on CRuby). It overwrites nothing without --force, and
|
|
20
|
-
# writes nothing with --dry-run
|
|
22
|
+
# writes nothing with --dry-run, or with --check, which says how the
|
|
23
|
+
# workflows differ from what it writes.
|
|
21
24
|
# @api private
|
|
22
25
|
class Init
|
|
23
26
|
USAGE = "Usage: duel init [options]\n\n" \
|
|
@@ -38,24 +41,33 @@ module Duel
|
|
|
38
41
|
# indentation of its first line.
|
|
39
42
|
DEVELOPMENT = /^([ \t]*)group[ \t(]+(?::development(?:,[ \t]*:test)?|:test,[ \t]*:development)[ \t)]+do[ \t]*\n/
|
|
40
43
|
|
|
41
|
-
|
|
44
|
+
# The color of the label it creates (see #create_label): GitHub's
|
|
45
|
+
# yellow, a warning's.
|
|
46
|
+
LABEL_COLOR = "FBCA04"
|
|
47
|
+
|
|
48
|
+
def initialize(argv, out: $stdout, err: $stderr, input: $stdin, root: Dir.pwd)
|
|
42
49
|
@argv = argv
|
|
43
50
|
@out = out
|
|
44
51
|
@err = err
|
|
52
|
+
@input = input
|
|
45
53
|
@root = root
|
|
46
54
|
end
|
|
47
55
|
|
|
48
|
-
# Returns the exit status: 0 set up
|
|
49
|
-
#
|
|
56
|
+
# Returns the exit status: 0 set up (or with --check, its workflows
|
|
57
|
+
# are what it writes), 1 a Rails app (not set up, or checked), or with
|
|
58
|
+
# --check, a workflow that differs, 2 a bad option.
|
|
50
59
|
def run
|
|
51
60
|
rest = catch(:help) { parser.parse(@argv) } or return 0
|
|
52
61
|
raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
|
|
53
62
|
return rails if File.exist?(File.join(@root, "config", "application.rb"))
|
|
54
63
|
|
|
64
|
+
return check if @check
|
|
65
|
+
|
|
55
66
|
@out.puts "duel init: a dry run, so nothing is written" if @dry_run
|
|
56
67
|
files.each { |path, content| write(path, content) }
|
|
57
68
|
add_rake_task
|
|
58
69
|
add_stackprof
|
|
70
|
+
create_label
|
|
59
71
|
@out.print next_steps
|
|
60
72
|
0
|
|
61
73
|
rescue OptionParser::ParseError => e
|
|
@@ -81,6 +93,7 @@ module Duel
|
|
|
81
93
|
o.banner = USAGE
|
|
82
94
|
o.on("--force", "Overwrite files that exist") { @force = true }
|
|
83
95
|
o.on("--dry-run", "Say what it would do, and write nothing") { @dry_run = true }
|
|
96
|
+
o.on("--check", "Say how the workflows differ from what it writes (with the same options), and write nothing") { @check = true }
|
|
84
97
|
o.on("--badge", "Write a job that keeps a badge for the README (it needs contents: write)") { @badge = true }
|
|
85
98
|
o.on("--gitlab", "Write GitLab CI jobs (to include from .gitlab-ci.yml), not GitHub Actions workflows") { @gitlab = true }
|
|
86
99
|
o.on("-h", "--help", "Print this help") do
|
|
@@ -93,10 +106,106 @@ module Duel
|
|
|
93
106
|
# Says to use duel-rails's generator instead.
|
|
94
107
|
def rails
|
|
95
108
|
@err.puts "duel init: this is a Rails app (config/application.rb). Add duel-rails to the Gemfile's development group, " \
|
|
96
|
-
"and run `bin/rails generate duel:install` instead"
|
|
109
|
+
"and run `bin/rails generate duel:install#{" --check" if @check}` instead"
|
|
97
110
|
1
|
|
98
111
|
end
|
|
99
112
|
|
|
113
|
+
# Says whether each workflow (or with --gitlab, the GitLab CI jobs) is
|
|
114
|
+
# what it writes, or how it differs, or that it's missing: they're the
|
|
115
|
+
# files a project rarely changes, but Duel's releases do. Returns 0
|
|
116
|
+
# where each is, or else 1.
|
|
117
|
+
def check
|
|
118
|
+
written = files.except(".duel.rb", benchmark_file)
|
|
119
|
+
written.count { |path, content| !same?(path, content) }.zero? ? 0 : 1
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Whether the file at +path+ is +content+, saying so, or how it
|
|
123
|
+
# differs (as diff -u has it), or that it's missing.
|
|
124
|
+
def same?(path, content)
|
|
125
|
+
full = File.join(@root, path)
|
|
126
|
+
unless File.file?(full)
|
|
127
|
+
said("missing", path)
|
|
128
|
+
return false
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
differences, = Open3.capture2("diff", "-u", "-L", path, "-L", "#{path} (duel init)", full, "-", stdin_data: content)
|
|
132
|
+
said(differences.empty? ? "same" : "differs", path)
|
|
133
|
+
@out.print differences
|
|
134
|
+
differences.empty?
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Creates the label that accepts a regression (Config#accept_label) in
|
|
138
|
+
# the project's GitHub repository, with gh (or with --gitlab, its
|
|
139
|
+
# GitLab project, with glab), if it's missing and the answer to
|
|
140
|
+
# whether to is yes: it asks only at a terminal, and not on a dry run.
|
|
141
|
+
# Where it doesn't, the next steps say to.
|
|
142
|
+
def create_label
|
|
143
|
+
label = accept_label
|
|
144
|
+
return unless label && !@dry_run && @input.tty?
|
|
145
|
+
|
|
146
|
+
repo = (@gitlab ? gitlab_project : cli("gh", "repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner")&.chomp) or return
|
|
147
|
+
names = (@gitlab ? gitlab_labels(repo) : github_labels(repo, label)) or return
|
|
148
|
+
if names.include?(label)
|
|
149
|
+
said("skip", "label #{label} (#{repo} has it)")
|
|
150
|
+
return @label = true
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
@out.print "Create the label #{label.inspect} in #{repo} with #{@gitlab ? "glab" : "gh"}, so a #{request} can accept a regression? [y/N] "
|
|
154
|
+
return unless @input.gets.to_s.strip.match?(/\Ay(?:es)?\z/i)
|
|
155
|
+
|
|
156
|
+
described = ["--repo", repo, "--description", "Accepts a regression that Duel found"]
|
|
157
|
+
created = @gitlab ? cli("glab", "label", "create", "--name", label, "--color", "##{LABEL_COLOR}", *described) :
|
|
158
|
+
cli("gh", "label", "create", label, "--color", LABEL_COLOR, *described)
|
|
159
|
+
return unless created
|
|
160
|
+
|
|
161
|
+
said("create", "label #{label} (#{repo})")
|
|
162
|
+
@label = true
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# The names of the GitHub repository +repo+'s labels that gh finds by
|
|
166
|
+
# +label+, or nil.
|
|
167
|
+
def github_labels(repo, label) = cli("gh", "label", "list", "--repo", repo, "--search", label, "--limit", "100", "--json", "name",
|
|
168
|
+
"--jq", ".[].name")&.lines(chomp: true)
|
|
169
|
+
|
|
170
|
+
# The project's GitLab project, as glab has it (group/project), or nil.
|
|
171
|
+
def gitlab_project
|
|
172
|
+
project = cli("glab", "repo", "view", "--output", "json") or return
|
|
173
|
+
parsed = JSON.parse(project)
|
|
174
|
+
parsed["path_with_namespace"] if parsed.instance_of?(Hash)
|
|
175
|
+
rescue JSON::ParserError
|
|
176
|
+
nil
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# The names of the GitLab project +repo+'s labels, every page of them,
|
|
180
|
+
# or nil where glab can't list a page.
|
|
181
|
+
def gitlab_labels(repo)
|
|
182
|
+
names = [] #: Array[String]
|
|
183
|
+
page = 1
|
|
184
|
+
while (listed = cli("glab", "label", "list", "--repo", repo, "--output", "json", "--per-page", "100", "--page", page.to_s))
|
|
185
|
+
labels = JSON.parse(listed)
|
|
186
|
+
names.concat(labels.map { |label| label.fetch("name") })
|
|
187
|
+
return names if labels.size < 100
|
|
188
|
+
|
|
189
|
+
page += 1
|
|
190
|
+
end
|
|
191
|
+
rescue JSON::ParserError
|
|
192
|
+
nil
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# What +program+ (gh, or glab) prints, run in the project, or nil where
|
|
196
|
+
# it fails (which it says) or isn't there.
|
|
197
|
+
def cli(program, *args)
|
|
198
|
+
out, err, status = Open3.capture3(program, *args, chdir: @root)
|
|
199
|
+
return out if status.success?
|
|
200
|
+
|
|
201
|
+
@err.puts "duel init: #{program} #{args.first(2).join(" ")}: #{err.strip}"
|
|
202
|
+
rescue SystemCallError
|
|
203
|
+
nil
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# What Duel compares: a pull request, or on GitLab, a merge request.
|
|
207
|
+
def request = @gitlab ? "merge request" : "pull request"
|
|
208
|
+
|
|
100
209
|
# Writes +content+ to +path+, unless it exists (without --force), and
|
|
101
210
|
# says what it did.
|
|
102
211
|
def write(path, content)
|
|
@@ -166,6 +275,8 @@ module Duel
|
|
|
166
275
|
end
|
|
167
276
|
steps << "Set c.library in .duel.rb to what each version's workers require." unless required
|
|
168
277
|
steps << "Replace the example in #{benchmark_file} with benchmarks of your own code."
|
|
278
|
+
steps << "Create the label #{accept_label.inspect} in your repository, so a #{request} can accept a regression (c.accept_label)." if
|
|
279
|
+
accept_label && !@label
|
|
169
280
|
steps << "Run `bundle exec duel --force`. It compares the code with itself, and passes when the benchmarks are sound."
|
|
170
281
|
steps << if @gitlab
|
|
171
282
|
"Include #{gitlab_file} from .gitlab-ci.yml, and add the tokens it says to, for comments. On a merge request, the job " \
|
data/lib/duel/runner.rb
CHANGED
|
@@ -1291,21 +1291,33 @@ module Duel
|
|
|
1291
1291
|
# Runner#fit's reckoning.
|
|
1292
1292
|
HEADROOM = 1.25
|
|
1293
1293
|
|
|
1294
|
+
# How many times what a worker took as it started one on a JIT's
|
|
1295
|
+
# engine takes once it's warm, by Runner#fit's reckoning, as the JIT
|
|
1296
|
+
# compiles: a TruffleRuby worker that started at some 510 MB grew to
|
|
1297
|
+
# some 1.5 GB (and HEADROOM covers the rest).
|
|
1298
|
+
JIT_GROWTH = {"truffleruby" => 2.5}.freeze
|
|
1299
|
+
|
|
1294
1300
|
# Lowers Settings#jobs, and then Settings#processes, as far as
|
|
1295
1301
|
# Settings#fit lets it, where their workers (and each one's
|
|
1296
1302
|
# resources, a database server, say) wouldn't fit in the memory
|
|
1297
1303
|
# available: more than that and the machine swaps, or a CI runner
|
|
1298
1304
|
# stops answering and its job is lost. What one worker of each
|
|
1299
1305
|
# version takes is found by starting them (a lane of them), and every
|
|
1300
|
-
# pair is taken to need that, HEADROOM over
|
|
1306
|
+
# pair is taken to need that, HEADROOM over (and on a JIT's engine,
|
|
1307
|
+
# JIT_GROWTH times that, as its workers grow as they warm up), or what
|
|
1308
|
+
# the plugins say each worker's resources take at most
|
|
1309
|
+
# (Config#worker_memory), if that's more: a database server takes
|
|
1310
|
+
# more as it's used.
|
|
1301
1311
|
def fit
|
|
1302
1312
|
return if @settings.fit.empty?
|
|
1303
1313
|
|
|
1304
1314
|
available = Memory.available or return
|
|
1305
1315
|
taken = Lane.open(@config, @settings, @sides, processes: 1) { available - (Memory.available || available) }
|
|
1306
|
-
|
|
1316
|
+
said = 2 * @config.worker_memory
|
|
1317
|
+
growth = @sides.each_value.map { |side| JIT_GROWTH.fetch(side.engine, 1.0) }.max
|
|
1318
|
+
pair = [(taken * HEADROOM * growth).ceil, said].max
|
|
1319
|
+
return unless pair.positive?
|
|
1307
1320
|
|
|
1308
|
-
pair = (taken * HEADROOM).ceil
|
|
1309
1321
|
pairs = [(available - RESERVE) / pair, 1].max
|
|
1310
1322
|
processes = @settings.fit.include?(:processes) ? [@settings.processes, pairs].min : @settings.processes
|
|
1311
1323
|
# (More jobs than the settings' are never taken: only fewer are.)
|
|
@@ -1314,8 +1326,11 @@ module Duel
|
|
|
1314
1326
|
("#{Runner.count(processes, "worker")} of each version (not #{@settings.processes})" if processes < @settings.processes)].compact
|
|
1315
1327
|
return if fewer.empty?
|
|
1316
1328
|
|
|
1317
|
-
|
|
1318
|
-
|
|
1329
|
+
takes = if said == pair then "#{gigabytes(said)} GB, with its resources, as its plugins say"
|
|
1330
|
+
elsif growth > 1 then "#{gigabytes(taken)} GB, with its resources, as it starts, and some #{growth}x that once warm"
|
|
1331
|
+
else "#{gigabytes(taken)} GB, with its resources"
|
|
1332
|
+
end
|
|
1333
|
+
@out.puts "A worker of each version takes #{takes}, of #{gigabytes(available)} GB available, so to fit: #{fewer.join(", and ")}"
|
|
1319
1334
|
@settings.jobs = jobs
|
|
1320
1335
|
@settings.processes = processes
|
|
1321
1336
|
end
|
data/lib/duel/summary.rb
CHANGED
|
@@ -27,9 +27,11 @@ module Duel
|
|
|
27
27
|
FAILED = %w[regressed error].freeze
|
|
28
28
|
|
|
29
29
|
# What the heading starts with: a run that passed, one that failed (or
|
|
30
|
-
# couldn't compare),
|
|
30
|
+
# couldn't compare), one that failed but was accepted (see
|
|
31
|
+
# Config#accept_label), and one with nothing compared.
|
|
31
32
|
PASSED_MARK = "🟢"
|
|
32
33
|
FAILED_MARK = "🔻"
|
|
34
|
+
ACCEPTED_MARK = "🟡"
|
|
33
35
|
NEUTRAL_MARK = "⬜"
|
|
34
36
|
|
|
35
37
|
# What an unsettled timing means.
|
|
@@ -91,9 +93,11 @@ module Duel
|
|
|
91
93
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
92
94
|
# +candidate+ (each a description, in Markdown, whose commit stands
|
|
93
95
|
# bare, so GitHub links it) as +settings+ say, with a +note+ (in
|
|
94
|
-
# Markdown) under the heading, if there's one
|
|
95
|
-
|
|
96
|
+
# Markdown) under the heading, if there's one, and whether what failed
|
|
97
|
+
# was +accepted+ (see Config#accept_label).
|
|
98
|
+
def initialize(runner, results, settings:, baseline:, candidate:, note: nil, accepted: nil)
|
|
96
99
|
@note = note
|
|
100
|
+
@accepted = accepted
|
|
97
101
|
@runner = runner
|
|
98
102
|
@results = results
|
|
99
103
|
@settings = settings
|
|
@@ -120,14 +124,16 @@ module Duel
|
|
|
120
124
|
end
|
|
121
125
|
|
|
122
126
|
# The verdict: what failed (benchmarks, or a metric across the board),
|
|
123
|
-
# or that every benchmark compared
|
|
127
|
+
# and whether that was accepted, or that every benchmark compared
|
|
128
|
+
# passed, and those skipped.
|
|
124
129
|
def heading
|
|
125
130
|
skipped = @results.count(&:skipped)
|
|
126
131
|
compared = @results.size > skipped
|
|
127
132
|
slower = @runner.regressed_across
|
|
128
133
|
said = [("#{Runner.count(@failed.size, "benchmark")} failed" unless @failed.empty?),
|
|
129
134
|
("slower across the board in #{slower.join(", ")}" unless slower.empty?)].compact
|
|
130
|
-
|
|
135
|
+
said << "accepted" if @accepted && said.any?
|
|
136
|
+
mark = if said.any? then @accepted ? ACCEPTED_MARK : FAILED_MARK
|
|
131
137
|
elsif compared then PASSED_MARK
|
|
132
138
|
else NEUTRAL_MARK
|
|
133
139
|
end
|
|
@@ -64,9 +64,11 @@
|
|
|
64
64
|
# The report, as a new comment on the merge request, when Duel says to:
|
|
65
65
|
# always (post), or only when its last comment's status differs from this
|
|
66
66
|
# run's (change), so the latest comment is never stale. It never edits or
|
|
67
|
-
# deletes a comment.
|
|
67
|
+
# deletes a comment. It runs `duel comment`, from RubyGems (none of the
|
|
68
|
+
# merge request's code), so a fix to how it comments comes with a release
|
|
69
|
+
# of Duel.
|
|
68
70
|
<%= gitlab_job %>:comment:
|
|
69
|
-
image:
|
|
71
|
+
image: ruby:slim
|
|
70
72
|
stage: .post
|
|
71
73
|
needs: [<%= gitlab_job %>]
|
|
72
74
|
rules:
|
|
@@ -75,23 +77,10 @@
|
|
|
75
77
|
variables:
|
|
76
78
|
GIT_STRATEGY: none # it runs none of the code
|
|
77
79
|
script:
|
|
78
|
-
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
[ -f "$report" ] || exit 0
|
|
83
|
-
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
84
|
-
token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
|
|
85
|
-
marker="<!-- <%= marker %>"
|
|
86
|
-
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
87
|
-
# The first line of the last comment Duel posted here: its marker,
|
|
88
|
-
# with its status.
|
|
89
|
-
last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
|
|
90
|
-
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
|
|
91
|
-
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
92
|
-
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
|
|
93
|
-
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
|
|
94
|
-
fi
|
|
80
|
+
- gem install duel --version '~> <%= Duel::Comment::SINCE %>' --no-document
|
|
81
|
+
# (The merge request's head: in a merged results pipeline, not the
|
|
82
|
+
# commit that merges it.)
|
|
83
|
+
- duel comment --gitlab --report <%= working_directory || "." %> --head "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --marker '<%= marker.gsub("'") { "'\\''" } %>'
|
|
95
84
|
|
|
96
85
|
# The report, as a new comment on a pushed commit, when Duel says to, as
|
|
97
86
|
# for a merge request: only where a regression in a push passes (see
|