duel 0.1.7 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,235 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "net/http"
5
+ require "open3"
6
+ require "optparse"
7
+ require "uri"
8
+
9
+ module Duel
10
+ # `duel comment`: what the Perf comment workflow runs, once the Perf
11
+ # workflow has, to comment on a pull request with Duel's report. It runs
12
+ # with the repository's own workflow and a token that can write, and the
13
+ # pull request's code ran in the Duel job, which could have written
14
+ # anything in what it kept (the report's directory), so it trusts only
15
+ # what Duel could have said there: the pull request's number, if it
16
+ # names the pull request whose head the run compared, and a comment
17
+ # (post or change: see CLI.comment) and a status (see CLI::MARKED) that
18
+ # Duel says. The report itself is the comment, as Markdown. It never
19
+ # edits or deletes a comment: once it has posted one, it hides Duel's
20
+ # earlier ones that aren't hidden yet as outdated, which GitHub
21
+ # collapses, so the newest is the one in view. It talks to GitHub with
22
+ # gh, which GH_TOKEN signs in, in GITHUB_REPOSITORY.
23
+ #
24
+ # With --gitlab, it's what the GitLab CI jobs' comment job runs, on a
25
+ # merge request: what Duel said is the Duel job's dotenv report's
26
+ # (DUEL_COMMENT and DUEL_STATUS), it talks to GitLab's API
27
+ # (CI_API_V4_URL) with DUEL_COMMENT_TOKEN, and it hides nothing, as only
28
+ # a Developer can resolve a thread, and a merge request's token is a
29
+ # Reporter's.
30
+ # @api private
31
+ class Comment
32
+ USAGE = "Usage: duel comment --report DIR --head SHA [--marker TEXT] [--gitlab]\n\n" \
33
+ "Comments on a pull request with the report a Perf run kept in DIR, when Duel said to, and only if the pull\n" \
34
+ "request's head is still SHA: what the Perf comment workflow runs (or with --gitlab, the comment job).\n\n"
35
+
36
+ # The first release of Duel whose `duel comment` the Perf comment
37
+ # workflow that this one writes can run: it installs the newest
38
+ # release of the same minor version (~>), so a fix comes with one.
39
+ SINCE = "0.1.8"
40
+
41
+ # Where GitLab CI has what Duel said (comment.env's, on GitHub): the
42
+ # merge request's, and the Duel job's dotenv report's.
43
+ GITLAB = {"pr" => "CI_MERGE_REQUEST_IID", "comment" => "DUEL_COMMENT", "status" => "DUEL_STATUS"}.freeze
44
+
45
+ # What Duel says in comment.env, as each line must have it.
46
+ SAID = {"pr" => /\A[0-9]+\z/, "comment" => /\A(?:post|change)\z/, "status" => /\A(?:passed|failed|neutral)\z/}.freeze
47
+
48
+ # The comments on a pull request, a page at a time (gh api graphql
49
+ # --paginate follows endCursor): GraphQL's, which say whether each is
50
+ # hidden, as REST's don't.
51
+ COMMENTS = <<~GRAPHQL
52
+ query($owner: String!, $name: String!, $pr: Int!, $endCursor: String) {
53
+ repository(owner: $owner, name: $name) {
54
+ pullRequest(number: $pr) {
55
+ comments(first: 100, after: $endCursor) {
56
+ nodes { id isMinimized author { login } body }
57
+ pageInfo { hasNextPage endCursor }
58
+ }
59
+ }
60
+ }
61
+ }
62
+ GRAPHQL
63
+
64
+ # The jq that picks, of COMMENTS, the ones GitHub Actions' token
65
+ # posted, each with its id, whether it's hidden, and its marker (its
66
+ # body's first line).
67
+ LISTED = '.data.repository.pullRequest.comments.nodes[] | select(.author.login == "github-actions") | ' \
68
+ '{id, hidden: .isMinimized, marked: (.body | split("\n")[0] | rtrimstr("\r"))}'
69
+
70
+ def initialize(argv, env:, out:, err:)
71
+ @argv = argv
72
+ @env = env
73
+ @out = out
74
+ @err = err
75
+ @marker = "duel"
76
+ end
77
+
78
+ # Returns the exit status: 0 commented (or had nothing to say), 1 GitHub
79
+ # wouldn't take it, 2 a bad option.
80
+ def run
81
+ rest = catch(:help) { parser.parse(@argv) } or return 0
82
+ raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
83
+ report = @report
84
+ raise OptionParser::MissingArgument, "--report and --head are required" unless report && @head
85
+
86
+ comment(report)
87
+ rescue OptionParser::ParseError => e
88
+ @err.puts "duel comment: #{e}"
89
+ 2
90
+ end
91
+
92
+ private
93
+
94
+ def parser
95
+ OptionParser.new do |o|
96
+ o.banner = USAGE
97
+ o.on("--report DIR", "The directory of the Perf run's report (duel.md and comment.env)") { |v| @report = v }
98
+ o.on("--head SHA", "The commit the run compared, the pull request's head") { |v| @head = v }
99
+ o.on("--marker TEXT", "What marks Duel's comments (default: duel)") { |v| @marker = v }
100
+ o.on("--gitlab", "Comment on a GitLab merge request, from GitLab CI (with DUEL_COMMENT_TOKEN)") { @gitlab = true }
101
+ o.on("-h", "--help", "Print this help") do
102
+ @out.puts o
103
+ throw :help
104
+ end
105
+ end
106
+ end
107
+
108
+ # Comments with the report in +report+, if Duel said to, and the pull
109
+ # request's head is still the run's.
110
+ def comment(report)
111
+ pr, mode, status = SAID.keys.map { |name| said(name, report) }
112
+ return quiet("Duel said not to") unless pr && mode && status
113
+
114
+ head = (@gitlab ? merge_request(pr)&.fetch("sha") : gh("repos/#{repository}/pulls/#{pr}", "--jq", ".head.sha")&.chomp) or return 1
115
+ return quiet("the #{@gitlab ? "merge" : "pull"} request has moved on since the run") unless head == @head
116
+
117
+ ours = (@gitlab ? notes(pr) : earlier(pr)) or return 1
118
+ ours = ours.select { |c| c.fetch("marked").start_with?("#{marker} status=") || c.fetch("marked") == "#{marker} -->" }
119
+ last = ours.last&.fetch("marked")
120
+ stale = last && !last.eql?("#{marker} status=#{status} -->")
121
+ return quiet("the last comment says #{status} already") unless mode == "post" || stale
122
+
123
+ post(pr, "#{marker} status=#{status} -->\n#{File.read(File.join(report, "duel.md"))}") or return 1
124
+ @out.puts "duel comment: commented on #{@gitlab ? "merge request !" : "pull request "}#{pr}"
125
+ ours.reject { |earlier| earlier.fetch("hidden") }.each { |earlier| hide(earlier.fetch("id")) } unless @gitlab
126
+ 0
127
+ end
128
+
129
+ # Posts +body+ as a new comment on the pull request (or merge request)
130
+ # +pr+: what GitHub (or GitLab) said, or nil.
131
+ def post(pr, body)
132
+ return gh("repos/#{repository}/issues/#{pr}/comments", "--method", "POST", "-f", "body=#{body}") unless @gitlab
133
+
134
+ gitlab(:Post, "#{project}/merge_requests/#{pr}/notes", {"body" => body})
135
+ end
136
+
137
+ # What Duel said, as SAID has it, or nil: in +report+'s comment.env
138
+ # (the last line that says it), or on GitLab CI, as GITLAB has it.
139
+ def said(name, report)
140
+ return @env[GITLAB.fetch(name)].to_s[SAID.fetch(name)] if @gitlab
141
+
142
+ path = File.join(report, "comment.env")
143
+ lines = File.file?(path) ? File.readlines(path, chomp: true) : [] #: Array[String]
144
+ lines.map { |line| line[/\A#{name}=(.*)\z/, 1] }.grep(SAID.fetch(name)).last
145
+ end
146
+
147
+ # Duel's comments on the pull request, oldest first, each with its id,
148
+ # whether it's hidden, and its marker (its first line), or nil where
149
+ # GitHub wouldn't list them.
150
+ def earlier(pr)
151
+ owner, name = repository.split("/")
152
+ listed = gh("graphql", "--paginate", "-f", "query=#{COMMENTS}", "-F", "pr=#{pr}", "-f", "owner=#{owner}", "-f", "name=#{name}",
153
+ "--jq", LISTED) or return
154
+ listed.lines.map { |line| JSON.parse(line) }
155
+ end
156
+
157
+ # The merge request +pr+, as GitLab's API has it, or nil.
158
+ def merge_request(pr) = gitlab(:Get, "#{project}/merge_requests/#{pr}")
159
+
160
+ # The token's notes on the merge request, oldest first, each with its
161
+ # marker (its first line), or nil where GitLab wouldn't list them.
162
+ def notes(pr)
163
+ me = gitlab(:Get, "/user") or return
164
+ notes = [] #: Array[Hash[String, untyped]]
165
+ page = "1"
166
+ while page
167
+ listed, page = gitlab(:Get, "#{project}/merge_requests/#{pr}/notes?sort=asc&order_by=created_at&per_page=100&page=#{page}", pages: true)
168
+ return unless listed
169
+
170
+ notes.concat(listed)
171
+ end
172
+ notes.select { |note| note.dig("author", "id") == me.fetch("id") }.map { |note| {"marked" => note.fetch("body").lines.first.to_s.chomp} }
173
+ end
174
+
175
+ # Hides an earlier comment as outdated, which GitHub collapses, or
176
+ # says it couldn't (it's still there, and that's all).
177
+ def hide(id)
178
+ query = "mutation($id: ID!) { minimizeComment(input: {subjectId: $id, classifier: OUTDATED}) { clientMutationId } }"
179
+ gh("graphql", "-f", "query=#{query}", "-f", "id=#{id}") || @err.puts("duel comment: couldn't hide an earlier comment")
180
+ end
181
+
182
+ # What gh api prints, given +args+, or nil where it fails (or isn't
183
+ # there), which it says why.
184
+ def gh(*args)
185
+ out, err, status = Open3.capture3("gh", "api", *args)
186
+ return out if status.success?
187
+
188
+ @err.puts "duel comment: gh api #{args.first}: #{err.strip}"
189
+ rescue SystemCallError => e
190
+ @err.puts "duel comment: gh: #{e}"
191
+ end
192
+
193
+ # What GitLab's API says to +verb+ (:Get or :Post) +path+ (under
194
+ # CI_API_V4_URL), given +body+ (as JSON), signed in with
195
+ # DUEL_COMMENT_TOKEN, or nil where it fails, which it says why. With
196
+ # +pages+, that and the next page's number (X-Next-Page), if there's
197
+ # one.
198
+ def gitlab(verb, path, body = nil, pages: false)
199
+ uri = URI("#{@env.fetch("CI_API_V4_URL")}#{path}")
200
+ request = Net::HTTP.const_get(verb).new(uri)
201
+ request["PRIVATE-TOKEN"] = @env.fetch("DUEL_COMMENT_TOKEN")
202
+ request["Content-Type"] = "application/json"
203
+ request.body = JSON.generate(body) if body
204
+ http = connection(uri)
205
+ http.use_ssl = uri.scheme == "https"
206
+ response = http.request(request)
207
+ unless response.is_a?(Net::HTTPSuccess)
208
+ return @err.puts("duel comment: GitLab: #{verb.upcase} #{path}: #{[response.code, response.body.to_s.strip].reject(&:empty?).join(" ")}")
209
+ end
210
+
211
+ parsed = JSON.parse(response.body)
212
+ pages ? [parsed, response["X-Next-Page"].to_s[/\A[0-9]+\z/]] : parsed
213
+ rescue SystemCallError, IOError, Timeout::Error, JSON::ParserError, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
214
+ @err.puts "duel comment: GitLab: #{verb.upcase} #{path}: #{e}"
215
+ end
216
+
217
+ # A connection to +uri+'s host, not yet started.
218
+ # mutant:disable -- a nil host is Net::HTTP's localhost, where the
219
+ # tests' server is, so only another host could tell them apart
220
+ def connection(uri) = Net::HTTP.new(uri.host.to_s, uri.port)
221
+
222
+ # Says why it doesn't comment, and returns 0.
223
+ def quiet(why)
224
+ @out.puts "duel comment: no comment (#{why})"
225
+ 0
226
+ end
227
+
228
+ def marker = "<!-- #{@marker}"
229
+
230
+ def repository = @env.fetch("GITHUB_REPOSITORY")
231
+
232
+ # The project's path in GitLab's API.
233
+ def project = "/projects/#{@env.fetch("CI_PROJECT_ID")}"
234
+ end
235
+ end
data/lib/duel/config.rb CHANGED
@@ -88,8 +88,9 @@ module Duel
88
88
  # keeps them all. Once it has commented, a run that it wouldn't
89
89
  # comment on still does, but :never's, when its status differs from
90
90
  # the last comment's, so the last comment is never stale: after a
91
- # failure, the first run that passes says so. Duel tells the workflow
92
- # which to do (see CLI.comment).
91
+ # failure, the first run that passes says so. A regression that
92
+ # #accept_label accepts counts as a run that passed. Duel tells the
93
+ # workflow which to do (see CLI.comment).
93
94
  #
94
95
  # c.comment = :results
95
96
  attr_reader :comment
@@ -185,6 +186,11 @@ module Duel
185
186
  # @api private
186
187
  attr_reader :prepares, :in_workers, :hooks, :measures
187
188
 
189
+ # The memory that what the plugins used start for each worker takes at
190
+ # most, as they say (see Duel.plugin), in bytes: 0 where none says.
191
+ # @api private
192
+ attr_reader :worker_memory
193
+
188
194
  # A configuration of the conventions, for the project at +root+ (a
189
195
  # plugin's tests make one to use it in).
190
196
  def initialize(root = Dir.pwd)
@@ -202,6 +208,7 @@ module Duel
202
208
  @in_workers = []
203
209
  @hooks = HOOKS.to_h { |hook| [hook, []] }
204
210
  @used = {}
211
+ @worker_memory = 0
205
212
  @measures = Hash.new do |all, name|
206
213
  all[name] = {} #: Hash[Symbol, untyped]
207
214
  end
@@ -289,6 +296,8 @@ module Duel
289
296
  DSL.add(plugin::DSL) if plugin.const_defined?(:DSL, false)
290
297
  used = @used[name] = Used.new(options)
291
298
  used.value = plugin.configure(self, **options)
299
+ @worker_memory += plugin.worker_memory if plugin.respond_to?(:worker_memory)
300
+ used.value
292
301
  end
293
302
 
294
303
  # The plugins used (see #use), by name, in the order they were first
data/lib/duel/init.rb CHANGED
@@ -1,12 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "erb"
4
+ require "json"
4
5
  require "fileutils"
5
6
  require "tmpdir"
6
7
  require "open3"
7
8
  require "optparse"
8
9
  require "pathname"
9
10
  require_relative "../duel"
11
+ require_relative "comment"
10
12
 
11
13
  module Duel
12
14
  # `duel init`: sets Duel up in a gem or an app that isn't a Rails app
@@ -17,7 +19,8 @@ module Duel
17
19
  # --gitlab, GitLab CI jobs to include, adds a Rake
18
20
  # task to the Rakefile, and adds StackProf to the Gemfile (for profiles
19
21
  # of regressions, on CRuby). It overwrites nothing without --force, and
20
- # writes nothing with --dry-run.
22
+ # writes nothing with --dry-run, or with --check, which says how the
23
+ # workflows differ from what it writes.
21
24
  # @api private
22
25
  class Init
23
26
  USAGE = "Usage: duel init [options]\n\n" \
@@ -38,24 +41,33 @@ module Duel
38
41
  # indentation of its first line.
39
42
  DEVELOPMENT = /^([ \t]*)group[ \t(]+(?::development(?:,[ \t]*:test)?|:test,[ \t]*:development)[ \t)]+do[ \t]*\n/
40
43
 
41
- def initialize(argv, out: $stdout, err: $stderr, root: Dir.pwd)
44
+ # The color of the label it creates (see #create_label): GitHub's
45
+ # yellow, a warning's.
46
+ LABEL_COLOR = "FBCA04"
47
+
48
+ def initialize(argv, out: $stdout, err: $stderr, input: $stdin, root: Dir.pwd)
42
49
  @argv = argv
43
50
  @out = out
44
51
  @err = err
52
+ @input = input
45
53
  @root = root
46
54
  end
47
55
 
48
- # Returns the exit status: 0 set up, 1 a Rails app (not set up), 2 a
49
- # bad option.
56
+ # Returns the exit status: 0 set up (or with --check, its workflows
57
+ # are what it writes), 1 a Rails app (not set up, or checked), or with
58
+ # --check, a workflow that differs, 2 a bad option.
50
59
  def run
51
60
  rest = catch(:help) { parser.parse(@argv) } or return 0
52
61
  raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
53
62
  return rails if File.exist?(File.join(@root, "config", "application.rb"))
54
63
 
64
+ return check if @check
65
+
55
66
  @out.puts "duel init: a dry run, so nothing is written" if @dry_run
56
67
  files.each { |path, content| write(path, content) }
57
68
  add_rake_task
58
69
  add_stackprof
70
+ create_label
59
71
  @out.print next_steps
60
72
  0
61
73
  rescue OptionParser::ParseError => e
@@ -81,6 +93,7 @@ module Duel
81
93
  o.banner = USAGE
82
94
  o.on("--force", "Overwrite files that exist") { @force = true }
83
95
  o.on("--dry-run", "Say what it would do, and write nothing") { @dry_run = true }
96
+ o.on("--check", "Say how the workflows differ from what it writes (with the same options), and write nothing") { @check = true }
84
97
  o.on("--badge", "Write a job that keeps a badge for the README (it needs contents: write)") { @badge = true }
85
98
  o.on("--gitlab", "Write GitLab CI jobs (to include from .gitlab-ci.yml), not GitHub Actions workflows") { @gitlab = true }
86
99
  o.on("-h", "--help", "Print this help") do
@@ -93,10 +106,106 @@ module Duel
93
106
  # Says to use duel-rails's generator instead.
94
107
  def rails
95
108
  @err.puts "duel init: this is a Rails app (config/application.rb). Add duel-rails to the Gemfile's development group, " \
96
- "and run `bin/rails generate duel:install` instead"
109
+ "and run `bin/rails generate duel:install#{" --check" if @check}` instead"
97
110
  1
98
111
  end
99
112
 
113
+ # Says whether each workflow (or with --gitlab, the GitLab CI jobs) is
114
+ # what it writes, or how it differs, or that it's missing: they're the
115
+ # files a project rarely changes, but Duel's releases do. Returns 0
116
+ # where each is, or else 1.
117
+ def check
118
+ written = files.except(".duel.rb", benchmark_file)
119
+ written.count { |path, content| !same?(path, content) }.zero? ? 0 : 1
120
+ end
121
+
122
+ # Whether the file at +path+ is +content+, saying so, or how it
123
+ # differs (as diff -u has it), or that it's missing.
124
+ def same?(path, content)
125
+ full = File.join(@root, path)
126
+ unless File.file?(full)
127
+ said("missing", path)
128
+ return false
129
+ end
130
+
131
+ differences, = Open3.capture2("diff", "-u", "-L", path, "-L", "#{path} (duel init)", full, "-", stdin_data: content)
132
+ said(differences.empty? ? "same" : "differs", path)
133
+ @out.print differences
134
+ differences.empty?
135
+ end
136
+
137
+ # Creates the label that accepts a regression (Config#accept_label) in
138
+ # the project's GitHub repository, with gh (or with --gitlab, its
139
+ # GitLab project, with glab), if it's missing and the answer to
140
+ # whether to is yes: it asks only at a terminal, and not on a dry run.
141
+ # Where it doesn't, the next steps say to.
142
+ def create_label
143
+ label = accept_label
144
+ return unless label && !@dry_run && @input.tty?
145
+
146
+ repo = (@gitlab ? gitlab_project : cli("gh", "repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner")&.chomp) or return
147
+ names = (@gitlab ? gitlab_labels(repo) : github_labels(repo, label)) or return
148
+ if names.include?(label)
149
+ said("skip", "label #{label} (#{repo} has it)")
150
+ return @label = true
151
+ end
152
+
153
+ @out.print "Create the label #{label.inspect} in #{repo} with #{@gitlab ? "glab" : "gh"}, so a #{request} can accept a regression? [y/N] "
154
+ return unless @input.gets.to_s.strip.match?(/\Ay(?:es)?\z/i)
155
+
156
+ described = ["--repo", repo, "--description", "Accepts a regression that Duel found"]
157
+ created = @gitlab ? cli("glab", "label", "create", "--name", label, "--color", "##{LABEL_COLOR}", *described) :
158
+ cli("gh", "label", "create", label, "--color", LABEL_COLOR, *described)
159
+ return unless created
160
+
161
+ said("create", "label #{label} (#{repo})")
162
+ @label = true
163
+ end
164
+
165
+ # The names of the GitHub repository +repo+'s labels that gh finds by
166
+ # +label+, or nil.
167
+ def github_labels(repo, label) = cli("gh", "label", "list", "--repo", repo, "--search", label, "--limit", "100", "--json", "name",
168
+ "--jq", ".[].name")&.lines(chomp: true)
169
+
170
+ # The project's GitLab project, as glab has it (group/project), or nil.
171
+ def gitlab_project
172
+ project = cli("glab", "repo", "view", "--output", "json") or return
173
+ parsed = JSON.parse(project)
174
+ parsed["path_with_namespace"] if parsed.instance_of?(Hash)
175
+ rescue JSON::ParserError
176
+ nil
177
+ end
178
+
179
+ # The names of the GitLab project +repo+'s labels, every page of them,
180
+ # or nil where glab can't list a page.
181
+ def gitlab_labels(repo)
182
+ names = [] #: Array[String]
183
+ page = 1
184
+ while (listed = cli("glab", "label", "list", "--repo", repo, "--output", "json", "--per-page", "100", "--page", page.to_s))
185
+ labels = JSON.parse(listed)
186
+ names.concat(labels.map { |label| label.fetch("name") })
187
+ return names if labels.size < 100
188
+
189
+ page += 1
190
+ end
191
+ rescue JSON::ParserError
192
+ nil
193
+ end
194
+
195
+ # What +program+ (gh, or glab) prints, run in the project, or nil where
196
+ # it fails (which it says) or isn't there.
197
+ def cli(program, *args)
198
+ out, err, status = Open3.capture3(program, *args, chdir: @root)
199
+ return out if status.success?
200
+
201
+ @err.puts "duel init: #{program} #{args.first(2).join(" ")}: #{err.strip}"
202
+ rescue SystemCallError
203
+ nil
204
+ end
205
+
206
+ # What Duel compares: a pull request, or on GitLab, a merge request.
207
+ def request = @gitlab ? "merge request" : "pull request"
208
+
100
209
  # Writes +content+ to +path+, unless it exists (without --force), and
101
210
  # says what it did.
102
211
  def write(path, content)
@@ -166,6 +275,8 @@ module Duel
166
275
  end
167
276
  steps << "Set c.library in .duel.rb to what each version's workers require." unless required
168
277
  steps << "Replace the example in #{benchmark_file} with benchmarks of your own code."
278
+ steps << "Create the label #{accept_label.inspect} in your repository, so a #{request} can accept a regression (c.accept_label)." if
279
+ accept_label && !@label
169
280
  steps << "Run `bundle exec duel --force`. It compares the code with itself, and passes when the benchmarks are sound."
170
281
  steps << if @gitlab
171
282
  "Include #{gitlab_file} from .gitlab-ci.yml, and add the tokens it says to, for comments. On a merge request, the job " \
data/lib/duel/runner.rb CHANGED
@@ -1291,21 +1291,33 @@ module Duel
1291
1291
  # Runner#fit's reckoning.
1292
1292
  HEADROOM = 1.25
1293
1293
 
1294
+ # How many times what a worker took as it started one on a JIT's
1295
+ # engine takes once it's warm, by Runner#fit's reckoning, as the JIT
1296
+ # compiles: a TruffleRuby worker that started at some 510 MB grew to
1297
+ # some 1.5 GB (and HEADROOM covers the rest).
1298
+ JIT_GROWTH = {"truffleruby" => 2.5}.freeze
1299
+
1294
1300
  # Lowers Settings#jobs, and then Settings#processes, as far as
1295
1301
  # Settings#fit lets it, where their workers (and each one's
1296
1302
  # resources, a database server, say) wouldn't fit in the memory
1297
1303
  # available: more than that and the machine swaps, or a CI runner
1298
1304
  # stops answering and its job is lost. What one worker of each
1299
1305
  # version takes is found by starting them (a lane of them), and every
1300
- # pair is taken to need that, HEADROOM over.
1306
+ # pair is taken to need that, HEADROOM over (and on a JIT's engine,
1307
+ # JIT_GROWTH times that, as its workers grow as they warm up), or what
1308
+ # the plugins say each worker's resources take at most
1309
+ # (Config#worker_memory), if that's more: a database server takes
1310
+ # more as it's used.
1301
1311
  def fit
1302
1312
  return if @settings.fit.empty?
1303
1313
 
1304
1314
  available = Memory.available or return
1305
1315
  taken = Lane.open(@config, @settings, @sides, processes: 1) { available - (Memory.available || available) }
1306
- return unless taken.positive?
1316
+ said = 2 * @config.worker_memory
1317
+ growth = @sides.each_value.map { |side| JIT_GROWTH.fetch(side.engine, 1.0) }.max
1318
+ pair = [(taken * HEADROOM * growth).ceil, said].max
1319
+ return unless pair.positive?
1307
1320
 
1308
- pair = (taken * HEADROOM).ceil
1309
1321
  pairs = [(available - RESERVE) / pair, 1].max
1310
1322
  processes = @settings.fit.include?(:processes) ? [@settings.processes, pairs].min : @settings.processes
1311
1323
  # (More jobs than the settings' are never taken: only fewer are.)
@@ -1314,8 +1326,11 @@ module Duel
1314
1326
  ("#{Runner.count(processes, "worker")} of each version (not #{@settings.processes})" if processes < @settings.processes)].compact
1315
1327
  return if fewer.empty?
1316
1328
 
1317
- @out.puts "A worker of each version takes #{gigabytes(taken)} GB, with its resources, of #{gigabytes(available)} GB available, " \
1318
- "so to fit: #{fewer.join(", and ")}"
1329
+ takes = if said == pair then "#{gigabytes(said)} GB, with its resources, as its plugins say"
1330
+ elsif growth > 1 then "#{gigabytes(taken)} GB, with its resources, as it starts, and some #{growth}x that once warm"
1331
+ else "#{gigabytes(taken)} GB, with its resources"
1332
+ end
1333
+ @out.puts "A worker of each version takes #{takes}, of #{gigabytes(available)} GB available, so to fit: #{fewer.join(", and ")}"
1319
1334
  @settings.jobs = jobs
1320
1335
  @settings.processes = processes
1321
1336
  end
data/lib/duel/summary.rb CHANGED
@@ -27,9 +27,11 @@ module Duel
27
27
  FAILED = %w[regressed error].freeze
28
28
 
29
29
  # What the heading starts with: a run that passed, one that failed (or
30
- # couldn't compare), and one with nothing compared.
30
+ # couldn't compare), one that failed but was accepted (see
31
+ # Config#accept_label), and one with nothing compared.
31
32
  PASSED_MARK = "🟢"
32
33
  FAILED_MARK = "🔻"
34
+ ACCEPTED_MARK = "🟡"
33
35
  NEUTRAL_MARK = "⬜"
34
36
 
35
37
  # What an unsettled timing means.
@@ -91,9 +93,11 @@ module Duel
91
93
  # +runner+ ran, and made +results+, comparing +baseline+ with
92
94
  # +candidate+ (each a description, in Markdown, whose commit stands
93
95
  # bare, so GitHub links it) as +settings+ say, with a +note+ (in
94
- # Markdown) under the heading, if there's one.
95
- def initialize(runner, results, settings:, baseline:, candidate:, note: nil)
96
+ # Markdown) under the heading, if there's one, and whether what failed
97
+ # was +accepted+ (see Config#accept_label).
98
+ def initialize(runner, results, settings:, baseline:, candidate:, note: nil, accepted: nil)
96
99
  @note = note
100
+ @accepted = accepted
97
101
  @runner = runner
98
102
  @results = results
99
103
  @settings = settings
@@ -120,14 +124,16 @@ module Duel
120
124
  end
121
125
 
122
126
  # The verdict: what failed (benchmarks, or a metric across the board),
123
- # or that every benchmark compared passed, and those skipped.
127
+ # and whether that was accepted, or that every benchmark compared
128
+ # passed, and those skipped.
124
129
  def heading
125
130
  skipped = @results.count(&:skipped)
126
131
  compared = @results.size > skipped
127
132
  slower = @runner.regressed_across
128
133
  said = [("#{Runner.count(@failed.size, "benchmark")} failed" unless @failed.empty?),
129
134
  ("slower across the board in #{slower.join(", ")}" unless slower.empty?)].compact
130
- mark = if said.any? then FAILED_MARK
135
+ said << "accepted" if @accepted && said.any?
136
+ mark = if said.any? then @accepted ? ACCEPTED_MARK : FAILED_MARK
131
137
  elsif compared then PASSED_MARK
132
138
  else NEUTRAL_MARK
133
139
  end
@@ -64,9 +64,11 @@
64
64
  # The report, as a new comment on the merge request, when Duel says to:
65
65
  # always (post), or only when its last comment's status differs from this
66
66
  # run's (change), so the latest comment is never stale. It never edits or
67
- # deletes a comment.
67
+ # deletes a comment. It runs `duel comment`, from RubyGems (none of the
68
+ # merge request's code), so a fix to how it comments comes with a release
69
+ # of Duel.
68
70
  <%= gitlab_job %>:comment:
69
- image: alpine:3
71
+ image: ruby:slim
70
72
  stage: .post
71
73
  needs: [<%= gitlab_job %>]
72
74
  rules:
@@ -75,23 +77,10 @@
75
77
  variables:
76
78
  GIT_STRATEGY: none # it runs none of the code
77
79
  script:
78
- - apk add --no-cache --quiet curl jq
79
- - |
80
- case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
81
- report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
82
- [ -f "$report" ] || exit 0
83
- api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
84
- token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
85
- marker="<!-- <%= marker %>"
86
- me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
87
- # The first line of the last comment Duel posted here: its marker,
88
- # with its status.
89
- last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
90
- jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
91
- if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
92
- { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
93
- curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
94
- fi
80
+ - gem install duel --version '~> <%= Duel::Comment::SINCE %>' --no-document
81
+ # (The merge request's head: in a merged results pipeline, not the
82
+ # commit that merges it.)
83
+ - duel comment --gitlab --report <%= working_directory || "." %> --head "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --marker '<%= marker.gsub("'") { "'\\''" } %>'
95
84
 
96
85
  # The report, as a new comment on a pushed commit, when Duel says to, as
97
86
  # for a merge request: only where a regression in a push passes (see