duel 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +2 -2
  3. data/CHANGELOG.md +203 -1
  4. data/README.md +41 -1
  5. data/docs/ci.md +170 -76
  6. data/docs/configuration.md +38 -2
  7. data/docs/how-it-works.md +18 -9
  8. data/docs/plugins.md +23 -0
  9. data/lib/duel/across.rb +1 -6
  10. data/lib/duel/baseline.rb +8 -7
  11. data/lib/duel/cli.rb +96 -28
  12. data/lib/duel/comment.rb +296 -0
  13. data/lib/duel/config.rb +124 -4
  14. data/lib/duel/context.rb +33 -34
  15. data/lib/duel/control.rb +52 -0
  16. data/lib/duel/init.rb +255 -20
  17. data/lib/duel/lane.rb +734 -0
  18. data/lib/duel/lock.rb +34 -4
  19. data/lib/duel/memory.rb +14 -5
  20. data/lib/duel/metric.rb +97 -26
  21. data/lib/duel/native.rb +199 -32
  22. data/lib/duel/platform.rb +31 -0
  23. data/lib/duel/process_sample.rb +74 -49
  24. data/lib/duel/resource.rb +71 -42
  25. data/lib/duel/result.rb +165 -0
  26. data/lib/duel/rubies.rb +21 -3
  27. data/lib/duel/runner.rb +33 -1044
  28. data/lib/duel/spawn.rb +327 -0
  29. data/lib/duel/summary.rb +21 -15
  30. data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
  31. data/lib/duel/templates/perf-caller.yml.erb +43 -0
  32. data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
  33. data/lib/duel/templates/perf-comment.yml.erb +61 -44
  34. data/lib/duel/templates/perf.yml.erb +143 -7
  35. data/lib/duel/testing.rb +3 -0
  36. data/lib/duel/version.rb +1 -1
  37. data/lib/duel/worker.rb +13 -4
  38. data/lib/duel/worker_process.rb +142 -0
  39. data/lib/duel.rb +35 -3
  40. data/sig/duel/cli.rbs +19 -0
  41. data/sig/duel/comment.rbs +97 -0
  42. data/sig/duel/config.rbs +57 -2
  43. data/sig/duel/context.rbs +4 -31
  44. data/sig/duel/control.rbs +27 -0
  45. data/sig/duel/init.rbs +95 -1
  46. data/sig/duel/lane.rbs +202 -0
  47. data/sig/duel/lock.rbs +8 -1
  48. data/sig/duel/memory.rbs +1 -1
  49. data/sig/duel/metric.rbs +53 -10
  50. data/sig/duel/native.rbs +109 -9
  51. data/sig/duel/platform.rbs +13 -0
  52. data/sig/duel/process_sample.rbs +7 -9
  53. data/sig/duel/resource.rbs +28 -9
  54. data/sig/duel/result.rbs +164 -0
  55. data/sig/duel/rubies.rbs +4 -0
  56. data/sig/duel/runner.rbs +6 -460
  57. data/sig/duel/spawn.rbs +111 -0
  58. data/sig/duel/summary.rbs +6 -3
  59. data/sig/duel/worker_process.rbs +90 -0
  60. data/sig/duel.rbs +17 -3
  61. metadata +121 -105
@@ -0,0 +1,296 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "net/http"
5
+ require "open3"
6
+ require "optparse"
7
+ require "uri"
8
+ require_relative "spawn"
9
+
10
+ module Duel
11
+ # `duel comment`: what the Perf comment workflow runs, once the Perf
12
+ # workflow has, to comment on a pull request with Duel's report. It runs
13
+ # with the repository's own workflow and a token that can write, and the
14
+ # pull request's code ran in the Duel job, which could have written
15
+ # anything in what it kept (the report's directory), so it trusts only
16
+ # what Duel could have said there: the pull request's number, if it
17
+ # names the pull request whose head the run compared, and a comment
18
+ # (post or change: see CLI.comment) and a status (see CLI::MARKED) that
19
+ # Duel says. The report itself is the comment, as Markdown. It never
20
+ # edits or deletes a comment: once it has posted one, it hides Duel's
21
+ # earlier ones that aren't hidden yet as outdated, which GitHub
22
+ # collapses, so the newest is the one in view. It talks to GitHub with
23
+ # gh, which GH_TOKEN signs in, in GITHUB_REPOSITORY.
24
+ #
25
+ # With --gitlab, it's what the GitLab CI jobs' comment job runs, on a
26
+ # merge request: what Duel said is the Duel job's dotenv report's
27
+ # (DUEL_COMMENT and DUEL_STATUS), it talks to GitLab's API
28
+ # (CI_API_V4_URL) with DUEL_COMMENT_TOKEN, and it hides nothing, as only
29
+ # a Developer can resolve a thread, and a merge request's token is a
30
+ # Reporter's.
31
+ # @api private
32
+ class Comment
33
+ USAGE = "Usage: duel comment --report DIR --head SHA [--marker TEXT] [--gitlab]\n\n" \
34
+ "Comments on a pull request with the report a Perf run kept in DIR, when Duel said to, and only if the pull\n" \
35
+ "request's head is still SHA: what the Perf comment workflow runs (or with --gitlab, the comment job).\n\n"
36
+
37
+ # The first release of Duel whose `duel comment` the Perf comment
38
+ # workflow that this one writes can run (0.2.0's merges the reports
39
+ # of a workflow that called Duel's reusable one more than once): it
40
+ # installs the newest release of the same minor version (~>), so a
41
+ # fix comes with one.
42
+ SINCE = "0.2.0"
43
+
44
+ # Where GitLab CI has what Duel said (comment.env's, on GitHub): the
45
+ # merge request's, and the Duel job's dotenv report's.
46
+ GITLAB = {"pr" => "CI_MERGE_REQUEST_IID", "comment" => "DUEL_COMMENT", "status" => "DUEL_STATUS"}.freeze
47
+
48
+ # What Duel says in comment.env, as each line must have it.
49
+ SAID = {"pr" => /\A[0-9]+\z/, "comment" => /\A(?:post|change)\z/, "status" => /\A(?:passed|failed|neutral)\z/}.freeze
50
+
51
+ # The characters a comment of reports has room for (GitHub takes
52
+ # 65,536, the marker and the names' headings among them), which each
53
+ # report's cut to its share of.
54
+ ROOM = 65_000
55
+
56
+ # A report's, in a comment of them, where its Duel job failed before
57
+ # it compared.
58
+ NONE = "No report: its Duel job failed before it compared.\n"
59
+
60
+ # The comments on a pull request, a page at a time (gh api graphql
61
+ # --paginate follows endCursor): GraphQL's, which say whether each is
62
+ # hidden, as REST's don't.
63
+ COMMENTS = <<~GRAPHQL
64
+ query($owner: String!, $name: String!, $pr: Int!, $endCursor: String) {
65
+ repository(owner: $owner, name: $name) {
66
+ pullRequest(number: $pr) {
67
+ comments(first: 100, after: $endCursor) {
68
+ nodes { id isMinimized author { login } body }
69
+ pageInfo { hasNextPage endCursor }
70
+ }
71
+ }
72
+ }
73
+ }
74
+ GRAPHQL
75
+
76
+ # The jq that picks, of COMMENTS, the ones GitHub Actions' token
77
+ # posted, each with its id, whether it's hidden, and its marker (its
78
+ # body's first line).
79
+ LISTED = '.data.repository.pullRequest.comments.nodes[] | select(.author.login == "github-actions") | ' \
80
+ '{id, hidden: .isMinimized, marked: (.body | split("\n")[0] | rtrimstr("\r"))}'
81
+
82
+ def initialize(argv, env:, out:, err:)
83
+ @argv = argv
84
+ @env = env
85
+ @out = out
86
+ @err = err
87
+ @marker = "duel"
88
+ end
89
+
90
+ # Returns the exit status: 0 commented (or had nothing to say), 1 GitHub
91
+ # wouldn't take it, 2 a bad option.
92
+ def run
93
+ rest = catch(:help) { parser.parse(@argv) } or return 0
94
+ raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
95
+ report = @report
96
+ raise OptionParser::MissingArgument, "--report and --head are required" unless report && @head
97
+
98
+ comment(report)
99
+ rescue OptionParser::ParseError => e
100
+ @err.puts "duel comment: #{e}"
101
+ 2
102
+ end
103
+
104
+ private
105
+
106
+ def parser
107
+ OptionParser.new do |o|
108
+ o.banner = USAGE
109
+ o.on("--report DIR", "The directory of the Perf run's report (duel.md and comment.env), or of its reports (duel-report*/)") { |v| @report = v }
110
+ o.on("--head SHA", "The commit the run compared, the pull request's head") { |v| @head = v }
111
+ o.on("--marker TEXT", "What marks Duel's comments (default: duel)") { |v| @marker = v }
112
+ o.on("--gitlab", "Comment on a GitLab merge request, from GitLab CI (with DUEL_COMMENT_TOKEN)") { @gitlab = true }
113
+ o.on("-h", "--help", "Print this help") do
114
+ @out.puts o
115
+ throw :help
116
+ end
117
+ end
118
+ end
119
+
120
+ # Comments with the report in +dir+ (or the reports: see reports), if
121
+ # Duel said to, and the pull request's head is still the run's.
122
+ def comment(dir)
123
+ reports = reports(dir)
124
+ pr, mode, status = SAID.keys.map { |name| merged(name, reports.map { |report| said(name, report) }) }
125
+ return quiet("Duel said not to") unless pr && mode && status
126
+
127
+ head = (@gitlab ? merge_request(pr)&.fetch("sha") : gh("repos/#{repository}/pulls/#{pr}", "--jq", ".head.sha")&.chomp) or return 1
128
+ return quiet("the #{@gitlab ? "merge" : "pull"} request has moved on since the run") unless head == @head
129
+
130
+ ours = (@gitlab ? notes(pr) : earlier(pr)) or return 1
131
+ ours = ours.select { |c| c.fetch("marked").start_with?("#{marker} status=") || c.fetch("marked") == "#{marker} -->" }
132
+ last = ours.last&.fetch("marked")
133
+ stale = last && !last.eql?("#{marker} status=#{status} -->")
134
+ return quiet("the last comment says #{status} already") unless mode == "post" || stale
135
+
136
+ post(pr, "#{marker} status=#{status} -->\n#{body(reports)}") or return 1
137
+ @out.puts "duel comment: commented on #{@gitlab ? "merge request !" : "pull request "}#{pr}"
138
+ ours.reject { |earlier| earlier.fetch("hidden") }.each { |earlier| hide(earlier.fetch("id")) } unless @gitlab
139
+ 0
140
+ end
141
+
142
+ # Posts +body+ as a new comment on the pull request (or merge request)
143
+ # +pr+: what GitHub (or GitLab) said, or nil.
144
+ def post(pr, body)
145
+ return gh("repos/#{repository}/issues/#{pr}/comments", "--method", "POST", "-f", "body=#{body}") unless @gitlab
146
+
147
+ gitlab(:Post, "#{project}/merge_requests/#{pr}/notes", {"body" => body})
148
+ end
149
+
150
+ # The reports' directories in +dir+: +dir+, or where there's no
151
+ # report there (no comment.env), each of the Perf run's in a directory
152
+ # of its own, as the reusable Perf comment workflow gets them,
153
+ # duel-report and duel-report-NAME (where a call of the Perf workflow
154
+ # named its report), in order (Dir.glob's).
155
+ def reports(dir)
156
+ return [dir] if @gitlab || File.file?(File.join(dir, "comment.env"))
157
+
158
+ Dir.glob("duel-report{,-*}", base: dir).map { |name| File.join(dir, name) }
159
+ end
160
+
161
+ # What +values+, each report's, say together for +name+: the pull
162
+ # request, where every one that names one names the same; a post, if
163
+ # any says to (or else a change); and a status, failed if any failed,
164
+ # passed if every one passed, and otherwise neutral. Nil where none
165
+ # says it.
166
+ def merged(name, values)
167
+ said = values.compact
168
+ return if said.empty?
169
+
170
+ case name
171
+ when "pr" then said.first if said.uniq.one?
172
+ when "comment" then said.max # (post, where any says to)
173
+ else
174
+ return "failed" if said.include?("failed")
175
+
176
+ values.all?("passed") ? "passed" : "neutral"
177
+ end
178
+ end
179
+
180
+ # The comment's Markdown: the report's, or for more than one, each
181
+ # under its name (its directory's, after duel-report-, or Duel, for
182
+ # the one not named), cut to its share of ROOM, or where it has none,
183
+ # NONE.
184
+ def body(reports)
185
+ case reports
186
+ in [report] then File.read(File.join(report, "duel.md"))
187
+ else
188
+ share = ROOM / reports.size
189
+ reports.map do |each|
190
+ name = File.basename(each).delete_prefix("duel-report").delete_prefix("-")
191
+ path = File.join(each, "duel.md")
192
+ text = File.file?(path) ? File.read(path) : NONE
193
+ "## #{name.empty? ? "Duel" : name}\n\n#{text[0, share].chomp}\n"
194
+ end.join("\n")
195
+ end
196
+ end
197
+
198
+ # What Duel said, as SAID has it, or nil: in +report+'s comment.env
199
+ # (the last line that says it), or on GitLab CI, as GITLAB has it.
200
+ def said(name, report)
201
+ return @env[GITLAB.fetch(name)].to_s[SAID.fetch(name)] if @gitlab
202
+
203
+ path = File.join(report, "comment.env")
204
+ lines = File.file?(path) ? File.readlines(path, chomp: true) : [] #: Array[String]
205
+ lines.map { |line| line[/\A#{name}=(.*)\z/, 1] }.grep(SAID.fetch(name)).last
206
+ end
207
+
208
+ # Duel's comments on the pull request, oldest first, each with its id,
209
+ # whether it's hidden, and its marker (its first line), or nil where
210
+ # GitHub wouldn't list them.
211
+ def earlier(pr)
212
+ owner, name = repository.split("/")
213
+ listed = gh("graphql", "--paginate", "-f", "query=#{COMMENTS}", "-F", "pr=#{pr}", "-f", "owner=#{owner}", "-f", "name=#{name}",
214
+ "--jq", LISTED) or return
215
+ listed.lines.map { |line| JSON.parse(line) }
216
+ end
217
+
218
+ # The merge request +pr+, as GitLab's API has it, or nil.
219
+ def merge_request(pr) = gitlab(:Get, "#{project}/merge_requests/#{pr}")
220
+
221
+ # The token's notes on the merge request, oldest first, each with its
222
+ # marker (its first line), or nil where GitLab wouldn't list them.
223
+ def notes(pr)
224
+ me = gitlab(:Get, "/user") or return
225
+ notes = [] #: Array[Hash[String, untyped]]
226
+ page = "1"
227
+ while page
228
+ listed, page = gitlab(:Get, "#{project}/merge_requests/#{pr}/notes?sort=asc&order_by=created_at&per_page=100&page=#{page}", pages: true)
229
+ return unless listed
230
+
231
+ notes.concat(listed)
232
+ end
233
+ notes.select { |note| note.dig("author", "id") == me.fetch("id") }.map { |note| {"marked" => note.fetch("body").lines.first.to_s.chomp} }
234
+ end
235
+
236
+ # Hides an earlier comment as outdated, which GitHub collapses, or
237
+ # says it couldn't (it's still there, and that's all).
238
+ def hide(id)
239
+ query = "mutation($id: ID!) { minimizeComment(input: {subjectId: $id, classifier: OUTDATED}) { clientMutationId } }"
240
+ gh("graphql", "-f", "query=#{query}", "-f", "id=#{id}") || @err.puts("duel comment: couldn't hide an earlier comment")
241
+ end
242
+
243
+ # What gh api prints, given +args+, or nil where it fails (or isn't
244
+ # there), which it says why.
245
+ def gh(*args)
246
+ out, err, status = Spawn.open3(:capture3, "gh", "api", *args)
247
+ return out if status.success?
248
+
249
+ @err.puts "duel comment: gh api #{args.first}: #{err.strip}"
250
+ rescue SystemCallError => e
251
+ @err.puts "duel comment: gh: #{e}"
252
+ end
253
+
254
+ # What GitLab's API says to +verb+ (:Get or :Post) +path+ (under
255
+ # CI_API_V4_URL), given +body+ (as JSON), signed in with
256
+ # DUEL_COMMENT_TOKEN, or nil where it fails, which it says why. With
257
+ # +pages+, that and the next page's number (X-Next-Page), if there's
258
+ # one.
259
+ def gitlab(verb, path, body = nil, pages: false)
260
+ uri = URI("#{@env.fetch("CI_API_V4_URL")}#{path}")
261
+ request = Net::HTTP.const_get(verb).new(uri)
262
+ request["PRIVATE-TOKEN"] = @env.fetch("DUEL_COMMENT_TOKEN")
263
+ request["Content-Type"] = "application/json"
264
+ request.body = JSON.generate(body) if body
265
+ http = connection(uri)
266
+ http.use_ssl = uri.scheme == "https"
267
+ response = http.request(request)
268
+ unless response.is_a?(Net::HTTPSuccess)
269
+ return @err.puts("duel comment: GitLab: #{verb.upcase} #{path}: #{[response.code, response.body.to_s.strip].reject(&:empty?).join(" ")}")
270
+ end
271
+
272
+ parsed = JSON.parse(response.body)
273
+ pages ? [parsed, response["X-Next-Page"].to_s[/\A[0-9]+\z/]] : parsed
274
+ rescue SystemCallError, IOError, Timeout::Error, JSON::ParserError, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
275
+ @err.puts "duel comment: GitLab: #{verb.upcase} #{path}: #{e}"
276
+ end
277
+
278
+ # A connection to +uri+'s host, not yet started.
279
+ # mutant:disable -- a nil host is Net::HTTP's localhost, where the
280
+ # tests' server is, so only another host could tell them apart
281
+ def connection(uri) = Net::HTTP.new(uri.host.to_s, uri.port)
282
+
283
+ # Says why it doesn't comment, and returns 0.
284
+ def quiet(why)
285
+ @out.puts "duel comment: no comment (#{why})"
286
+ 0
287
+ end
288
+
289
+ def marker = "<!-- #{@marker}"
290
+
291
+ def repository = @env.fetch("GITHUB_REPOSITORY")
292
+
293
+ # The project's path in GitLab's API.
294
+ def project = "/projects/#{@env.fetch("CI_PROJECT_ID")}"
295
+ end
296
+ end
data/lib/duel/config.rb CHANGED
@@ -39,6 +39,11 @@ module Duel
39
39
  # empty. `duel --clean` removes the rest.
40
40
  def state = File.join(root, "tmp", "duel")
41
41
 
42
+ # Where what a resource builds once is kept (see Resource#cached):
43
+ # tmp/duel/cache.
44
+ # @api private
45
+ def cache = File.join(state, "cache")
46
+
42
47
  # A glob, from the root, of the files defining benchmarks.
43
48
  attr_accessor :benchmarks
44
49
 
@@ -88,8 +93,9 @@ module Duel
88
93
  # keeps them all. Once it has commented, a run that it wouldn't
89
94
  # comment on still does, but :never's, when its status differs from
90
95
  # the last comment's, so the last comment is never stale: after a
91
- # failure, the first run that passes says so. Duel tells the workflow
92
- # which to do (see CLI.comment).
96
+ # failure, the first run that passes says so. A regression that
97
+ # #accept_label accepts counts as a run that passed. Duel tells the
98
+ # workflow which to do (see CLI.comment).
93
99
  #
94
100
  # c.comment = :results
95
101
  attr_reader :comment
@@ -127,6 +133,114 @@ module Duel
127
133
  @fail_on = runs
128
134
  end
129
135
 
136
+ # The settings of a run that .duel.rb can set too, as the options of
137
+ # their names do (--limit, --confidence, --processes, --timeout,
138
+ # --time-limit, --warm): for a run not given that option, as in CI,
139
+ # where the workflow runs Duel with none. Each is nil, by default, for
140
+ # Duel's own default. For each, what it takes, and whether a value is
141
+ # that: a number, as a Float, but a number of processes, a whole one.
142
+ # @api private
143
+ RUN_SETTINGS = {
144
+ limit: ["a ratio of 1 or more", ->(value) { value >= 1 }],
145
+ confidence: ["a level above 0 and below 1", ->(value) { value.positive? && value < 1 }],
146
+ processes: ["a whole number, 1 or more", ->(value) { value.is_a?(Integer) && value.positive? }],
147
+ timeout: ["seconds, more than 0", ->(value) { value.positive? }],
148
+ time_limit: ["seconds, more than 0", ->(value) { value.positive? }],
149
+ warm: ["seconds, 0 or more", ->(value) { value >= 0 }]
150
+ }.freeze #: Hash[Symbol, [String, ^(Numeric) -> bool]]
151
+
152
+ # The ratio above which a timing is slower (--limit), for every
153
+ # measure without a limit of its own (see #measure): 1.02 by default.
154
+ #
155
+ # c.limit = 1.05
156
+ attr_reader :limit
157
+
158
+ def limit=(ratio)
159
+ @limit = Config.setting(:limit, ratio)
160
+ end
161
+
162
+ # The confidence of every interval Duel judges with (--confidence):
163
+ # 0.99 by default. A higher one fails less code that isn't slower, and
164
+ # takes more samples, and workers, to fail what is: judging by process
165
+ # (see Lane.by_process?) takes 5 workers of each version at 0.99, and
166
+ # 7 at 0.999.
167
+ #
168
+ # c.confidence = 0.995
169
+ attr_reader :confidence
170
+
171
+ def confidence=(level)
172
+ @confidence = Config.setting(:confidence, level)
173
+ end
174
+
175
+ # The worker processes of each version (--processes): 12 by default (8
176
+ # on JRuby and TruffleRuby), as many as fit in memory. One set is that
177
+ # many, however much memory there is.
178
+ #
179
+ # c.processes = 6
180
+ attr_reader :processes
181
+
182
+ def processes=(count)
183
+ @processes = Config.setting(:processes, count)
184
+ end
185
+
186
+ # The most seconds a worker may take to load, or to answer a request,
187
+ # before it's taken for hung (--timeout): 60 by default, 300 on JRuby
188
+ # and TruffleRuby.
189
+ #
190
+ # c.timeout = 120
191
+ attr_reader :timeout
192
+
193
+ def timeout=(seconds)
194
+ @timeout = Config.setting(:timeout, seconds)
195
+ end
196
+
197
+ # The most seconds a run measures for, before it fails what it hasn't
198
+ # measured, or what looks worse (--time-limit): 7200 by default.
199
+ #
200
+ # c.time_limit = 3600
201
+ attr_reader :time_limit
202
+
203
+ def time_limit=(seconds)
204
+ @time_limit = Config.setting(:time_limit, seconds)
205
+ end
206
+
207
+ # The seconds each benchmark warms up for, before it's measured
208
+ # (--warm), or nil (the default) for until it's steady, for up to a
209
+ # minute on JRuby and TruffleRuby, ten seconds with YJIT on, and none
210
+ # otherwise.
211
+ #
212
+ # c.warm = 10
213
+ attr_reader :warm
214
+
215
+ def warm=(seconds)
216
+ @warm = Config.setting(:warm, seconds)
217
+ end
218
+
219
+ # +value+, set as the RUN_SETTINGS +name+d, as that takes it (a Float,
220
+ # or an Integer), or nil. One that's neither fails, saying what to
221
+ # give.
222
+ # @api private
223
+ def self.setting(name, value)
224
+ return if value.nil?
225
+
226
+ problem = problem(name, value)
227
+ raise Error, "c.#{name} = #{value.inspect}: give #{problem}, or nil" if problem
228
+
229
+ (name == :processes) ? value : Float(value)
230
+ end
231
+
232
+ # What to give instead of +value+ for the RUN_SETTINGS +name+d, or
233
+ # nil if it's one that takes.
234
+ # @api private
235
+ def self.problem(name, value)
236
+ what, valid = RUN_SETTINGS.fetch(name)
237
+ what unless value.is_a?(Numeric) && valid.call(value)
238
+ end
239
+
240
+ # The RUN_SETTINGS .duel.rb set, by name.
241
+ # @api private
242
+ def run_settings = RUN_SETTINGS.to_h { |name| [name, public_send(name)] }.compact
243
+
130
244
  # The name of a label that accepts a regression in a pull request
131
245
  # (or nil, the default, for none): on GitHub Actions, the workflow
132
246
  # that `duel init` writes passes Duel the pull request's labels
@@ -185,6 +299,11 @@ module Duel
185
299
  # @api private
186
300
  attr_reader :prepares, :in_workers, :hooks, :measures
187
301
 
302
+ # The memory that what the plugins used start for each worker takes at
303
+ # most, as they say (see Duel.plugin), in bytes: 0 where none says.
304
+ # @api private
305
+ attr_reader :worker_memory
306
+
188
307
  # A configuration of the conventions, for the project at +root+ (a
189
308
  # plugin's tests make one to use it in).
190
309
  def initialize(root = Dir.pwd)
@@ -202,6 +321,7 @@ module Duel
202
321
  @in_workers = []
203
322
  @hooks = HOOKS.to_h { |hook| [hook, []] }
204
323
  @used = {}
324
+ @worker_memory = 0
205
325
  @measures = Hash.new do |all, name|
206
326
  all[name] = {} #: Hash[Symbol, untyped]
207
327
  end
@@ -289,6 +409,8 @@ module Duel
289
409
  DSL.add(plugin::DSL) if plugin.const_defined?(:DSL, false)
290
410
  used = @used[name] = Used.new(options)
291
411
  used.value = plugin.configure(self, **options)
412
+ @worker_memory += plugin.worker_memory if plugin.respond_to?(:worker_memory)
413
+ used.value
292
414
  end
293
415
 
294
416
  # The plugins used (see #use), by name, in the order they were first
@@ -298,8 +420,6 @@ module Duel
298
420
  # A plugin as #use configured it: with +options+, returning +value+.
299
421
  # @api private
300
422
  class Used
301
- attr_reader :options
302
-
303
423
  attr_accessor :value
304
424
 
305
425
  def initialize(options)
data/lib/duel/context.rb CHANGED
@@ -339,7 +339,7 @@ module Duel
339
339
  attr_reader :benchmark, :n
340
340
 
341
341
  # @api private
342
- attr_reader :started, :stopped, :timed, :counted, :heap, :details
342
+ attr_reader :timed, :counted, :heap, :details
343
343
 
344
344
  # +metrics+: the worker's metric instances, outermost first (CPU, the
345
345
  # innermost, last). +counting+: whether counted measures are taken
@@ -365,7 +365,6 @@ module Duel
365
365
 
366
366
  # @api private
367
367
  def start
368
- @started = true
369
368
  QUIET_GC.call
370
369
  @heap = -HEAP.call
371
370
  @metrics.each { |m| m.start_sample(self) }
@@ -377,7 +376,6 @@ module Duel
377
376
  @heap += HEAP.call
378
377
  GC.enable
379
378
  @metrics.each { |m| explain(m) }
380
- @stopped = true
381
379
  end
382
380
 
383
381
  # Runs the block with the metrics paused (what they measured so far
@@ -484,11 +482,11 @@ module Duel
484
482
  # A run of a benchmark that measures nothing: to see that it runs.
485
483
  # @api private
486
484
  class Trial
487
- attr_reader :started, :stopped
488
-
489
- def start = @started = true
485
+ def start
486
+ end
490
487
 
491
- def stop = @stopped = true
488
+ def stop
489
+ end
492
490
 
493
491
  def pause = yield
494
492
 
@@ -504,10 +502,13 @@ module Duel
504
502
  # millisecond.
505
503
  # @api private
506
504
  class MemorySample
507
- attr_reader :started, :stopped, :peak
505
+ attr_reader :peak
508
506
 
509
507
  def start
510
- @started = true
508
+ # (Held as the watcher reads, and as it's paused and resumed: on
509
+ # JRuby, whose threads run at once, a reading could land between
510
+ # the two otherwise.)
511
+ @lock = Mutex.new
511
512
  Native::Memory.collect
512
513
  @base = resident
513
514
  @peak = 0
@@ -523,7 +524,6 @@ module Duel
523
524
  @watcher.kill
524
525
  @watcher.join
525
526
  watch
526
- @stopped = true
527
527
  end
528
528
 
529
529
  # Runs the block (an untimed one, or a benchmark's once: see
@@ -559,15 +559,19 @@ module Duel
559
559
  # From now on, nothing is watched, and what's left resident isn't
560
560
  # counted.
561
561
  def unwatched
562
- @paused = true
563
- @unwatched = resident
562
+ @lock.synchronize do
563
+ @paused = true
564
+ @unwatched = resident
565
+ end
564
566
  end
565
567
 
566
568
  # From now on, what's resident is watched again, above what was left
567
569
  # while it wasn't.
568
570
  def watched
569
- @base += resident - @unwatched
570
- @paused = false
571
+ @lock.synchronize do
572
+ @base += resident - @unwatched
573
+ @paused = false
574
+ end
571
575
  end
572
576
 
573
577
  def resident
@@ -576,10 +580,12 @@ module Duel
576
580
  end
577
581
 
578
582
  def watch
579
- return if @paused
580
-
581
- above = resident - @base
582
- @peak = above if above > @peak
583
+ @lock.synchronize do
584
+ unless @paused
585
+ above = resident - @base
586
+ @peak = above if above > @peak
587
+ end
588
+ end
583
589
  end
584
590
  end
585
591
 
@@ -593,26 +599,19 @@ module Duel
593
599
  class ProfileSample
594
600
  INTERVAL = 100 # microseconds between StackProf's samples
595
601
 
596
- # What StackProf samples by on +platform+: CPU time, but on macOS,
597
- # elapsed time. macOS signals that CPU time is up only as the thread
598
- # enters the kernel, so its samples pile up on whatever makes system
599
- # calls, the most CPU-hungry code or not. A benchmark busy on the CPU
600
- # spends elapsed time the same way.
601
- def self.mode(platform = RUBY_PLATFORM) = platform.include?("darwin") ? :wall : :cpu
602
+ # What StackProf samples by on +platform+ (the host's system, as
603
+ # Platform tells it): CPU time, but on macOS, elapsed time. macOS
604
+ # signals that CPU time is up only as the thread enters the kernel, so
605
+ # its samples pile up on whatever makes system calls, the most
606
+ # CPU-hungry code or not. A benchmark busy on the CPU spends elapsed
607
+ # time the same way.
608
+ def self.mode(platform: Platform::OS) = platform.include?("darwin") ? :wall : :cpu
602
609
 
603
610
  MODE = mode
604
611
 
605
- attr_reader :started, :stopped
606
-
607
- def start
608
- @started = true
609
- Sample::QUIET_GC.call
610
- end
612
+ def start = Sample::QUIET_GC.call
611
613
 
612
- def stop
613
- GC.enable
614
- @stopped = true
615
- end
614
+ def stop = GC.enable
616
615
 
617
616
  # Runs the block (an untimed one, or the around_iteration hooks of a
618
617
  # benchmark with them: see #between): what it does is sampled in this
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+
5
+ module Duel
6
+ # The runner's connection to a worker: a Unix socket at a path, or
7
+ # where there are no Unix sockets (JRuby on Windows has none), a port of
8
+ # 127.0.0.1's, its address "127.0.0.1:PORT". (As open to the machine's
9
+ # other processes as the plugins' servers are there.)
10
+ # @api private
11
+ module Control
12
+ HOST = "127.0.0.1"
13
+ UNIX = !defined?(UNIXServer).nil?
14
+ TCP = /\A127\.0\.0\.1:(\d+)\z/
15
+
16
+ # Whether a worker can reach a server on a Unix socket (see
17
+ # Duel.unix_sockets?): not on Windows (+windows+), whose builds of
18
+ # servers (Cygwin's, MSYS2's) have Unix sockets of their own, which
19
+ # Windows's Ruby can't reach, nor where Ruby has none (+unix+: JRuby's,
20
+ # on Windows).
21
+ def self.unix_sockets?(windows:, unix: UNIX) = !windows && unix
22
+
23
+ # A server for a worker to connect to, at +path+ (or a port, unless
24
+ # +unix+), and its address.
25
+ def self.listen(path, unix:)
26
+ return [UNIXServer.new(path), path] if unix
27
+
28
+ server = tcp_server
29
+ [server, "#{HOST}:#{server.addr.fetch(1)}"]
30
+ end
31
+
32
+ # A server on a port of 127.0.0.1's that the system chose.
33
+ # mutant:disable -- a port of nil is 0's alike: any that's free
34
+ def self.tcp_server
35
+ # (RBS's TCPServer.new takes a port alone, but Ruby's takes the host
36
+ # before it.)
37
+ TCPServer.new(HOST, 0) # steep:ignore UnexpectedPositionalArgument
38
+ end
39
+
40
+ # A connection to the server at +address+.
41
+ # mutant:disable -- a host of nil is localhost, which is 127.0.0.1
42
+ # where the tests run
43
+ def self.connect(address)
44
+ port = address[TCP, 1]
45
+ port ? TCPSocket.new(HOST, Integer(port)) : UNIXSocket.new(address)
46
+ end
47
+
48
+ # Removes the server's socket file, once it's done with: a Unix
49
+ # socket's.
50
+ def self.remove(address) = (File.delete(address) unless address.match?(TCP))
51
+ end
52
+ end