duel 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +51 -27
  3. data/CHANGELOG.md +247 -1
  4. data/README.md +30 -14
  5. data/docs/benchmarks.md +28 -24
  6. data/docs/ci.md +328 -362
  7. data/docs/configuration.md +40 -37
  8. data/docs/how-it-works.md +23 -10
  9. data/docs/json.md +21 -25
  10. data/docs/plugins.md +40 -22
  11. data/docs/troubleshooting.md +3 -2
  12. data/lib/duel/across.rb +18 -18
  13. data/lib/duel/badge.rb +174 -0
  14. data/lib/duel/baseline.rb +38 -15
  15. data/lib/duel/cli.rb +175 -107
  16. data/lib/duel/comment.rb +80 -71
  17. data/lib/duel/config.rb +145 -63
  18. data/lib/duel/context.rb +29 -24
  19. data/lib/duel/forge.rb +87 -0
  20. data/lib/duel/init.rb +44 -87
  21. data/lib/duel/json_report.rb +34 -25
  22. data/lib/duel/lane.rb +107 -61
  23. data/lib/duel/lock.rb +1 -1
  24. data/lib/duel/mark_passed.rb +132 -0
  25. data/lib/duel/metric.rb +28 -27
  26. data/lib/duel/native.rb +7 -5
  27. data/lib/duel/platform.rb +4 -0
  28. data/lib/duel/process_sample.rb +9 -6
  29. data/lib/duel/rake_task.rb +4 -5
  30. data/lib/duel/resource.rb +10 -11
  31. data/lib/duel/result.rb +22 -14
  32. data/lib/duel/rubies.rb +6 -2
  33. data/lib/duel/runner.rb +27 -15
  34. data/lib/duel/spawn.rb +29 -4
  35. data/lib/duel/stats.rb +14 -12
  36. data/lib/duel/summary.rb +31 -29
  37. data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
  38. data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
  39. data/lib/duel/templates/duel.rb.erb +7 -7
  40. data/lib/duel/templates/perf-caller.yml.erb +22 -5
  41. data/lib/duel/templates/perf-comment.yml.erb +11 -11
  42. data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
  43. data/lib/duel/templates/perf-push.yml.erb +50 -0
  44. data/lib/duel/templates/perf.yml.erb +113 -335
  45. data/lib/duel/testing.rb +20 -25
  46. data/lib/duel/version.rb +1 -1
  47. data/lib/duel/worker.rb +16 -9
  48. data/lib/duel/worker_process.rb +6 -6
  49. data/lib/duel.rb +88 -33
  50. data/sig/duel/across.rbs +4 -0
  51. data/sig/duel/badge.rbs +51 -0
  52. data/sig/duel/baseline.rbs +7 -2
  53. data/sig/duel/cli.rbs +31 -20
  54. data/sig/duel/comment.rbs +14 -15
  55. data/sig/duel/config.rbs +71 -29
  56. data/sig/duel/context.rbs +4 -4
  57. data/sig/duel/forge.rbs +42 -0
  58. data/sig/duel/init.rbs +8 -26
  59. data/sig/duel/json_report.rbs +7 -6
  60. data/sig/duel/lane.rbs +29 -6
  61. data/sig/duel/mark_passed.rbs +39 -0
  62. data/sig/duel/metric.rbs +10 -10
  63. data/sig/duel/native.rbs +3 -0
  64. data/sig/duel/platform.rbs +1 -0
  65. data/sig/duel/process_sample.rbs +3 -0
  66. data/sig/duel/resource.rbs +3 -3
  67. data/sig/duel/result.rbs +12 -5
  68. data/sig/duel/rubies.rbs +3 -0
  69. data/sig/duel/runner.rbs +23 -16
  70. data/sig/duel/spawn.rbs +7 -0
  71. data/sig/duel/stats.rbs +4 -1
  72. data/sig/duel/summary.rbs +8 -0
  73. data/sig/duel/testing.rbs +2 -2
  74. data/sig/duel/worker.rbs +6 -5
  75. data/sig/duel/worker_process.rbs +3 -3
  76. data/sig/duel.rbs +50 -31
  77. metadata +9 -1
data/lib/duel/context.rb CHANGED
@@ -68,10 +68,10 @@ module Duel
68
68
  body = definition.body
69
69
  @body = bound(body)
70
70
  @given_n = body.arity == 1
71
- @setup = hooks(:setup, hooks).map { |hook| given(hook) }
72
- @teardown = hooks(:teardown, hooks).map { |hook| given(hook) }.reverse
73
- @before_sample = hooks(:before_sample, hooks).map { |hook| given(hook) }
74
- @after = hooks(:after_iterations, hooks)
71
+ @setup = plain_hooks(:setup, hooks)
72
+ @teardown = plain_hooks(:teardown, hooks).reverse
73
+ @before_sample = plain_hooks(:before_sample, hooks)
74
+ @after = plain_hooks(:after_iterations, hooks)
75
75
  # (Each around_iteration hook's Iteration, innermost first, made once:
76
76
  # running one allocates nothing.)
77
77
  iteration = nil #: Iteration?
@@ -188,27 +188,27 @@ module Duel
188
188
  hooks.fetch(name) + [*file, *@definition.hooks[name]].map { |hook| bound(hook) }
189
189
  end
190
190
 
191
- # +hook+, given the benchmark, if it takes it.
192
- def given(hook)
193
- return hook if hook.arity.zero?
194
-
195
- definition = @definition
196
- -> { hook.call(definition) }
191
+ # The hooks +name+d (see #hooks), which Duel gives nothing: one that
192
+ # takes a parameter fails, saying so, not getting nil.
193
+ def plain_hooks(name, hooks)
194
+ hooks(name, hooks).each do |hook|
195
+ raise Error, "#{@definition.name}: a #{name} hook takes no parameter (give it a block without one)" unless hook.arity.zero?
196
+ end
197
197
  end
198
198
 
199
199
  # What pausing and resuming leaves in each of +metrics+' measures, an
200
200
  # iteration: the least of what CALIBRATION samples of STRETCHES empty
201
201
  # iterations left.
202
202
  def paused(metrics)
203
- runs = Array.new(Snapshot::CALIBRATION) do
203
+ samples = Array.new(Snapshot::CALIBRATION) do
204
204
  sample = Sample.new(@definition, metrics, n: STRETCHES)
205
205
  sample.start
206
206
  sample.between { STRETCHES.times { sample.unpause {} } }
207
207
  sample.stop
208
- sample.timed.merge(sample.counted)
208
+ sample
209
209
  end
210
210
  # (Two stretches more: from the start, and to the stop.)
211
- least(runs, STRETCHES + 2)
211
+ least(samples, STRETCHES + 2)
212
212
  end
213
213
 
214
214
  # What the around_iteration hooks add to each of +metrics+' measures
@@ -220,16 +220,19 @@ module Duel
220
220
  def hooked(metrics)
221
221
  return NO_LEAK if metrics.empty?
222
222
 
223
- runs = Array.new(Snapshot::CALIBRATION) do
223
+ samples = Array.new(Snapshot::CALIBRATION) do
224
224
  sample = Sample.new(@definition, metrics, n: HOOKED)
225
225
  run(HOOKED, Hooked.new(sample))
226
- sample.timed.merge(sample.counted)
226
+ sample
227
227
  end
228
- least(runs, HOOKED)
228
+ least(samples, HOOKED)
229
229
  end
230
230
 
231
- # Each measure's least in +runs+, over +stretches+.
232
- def least(runs, stretches) = runs.first.to_h { |name, _| [name, runs.map { |run| run.fetch(name) }.min.fdiv(stretches)] }
231
+ # Each measure's least in +samples+, over +stretches+.
232
+ def least(samples, stretches)
233
+ runs = samples.map { |sample| sample.timed.merge(sample.counted) }
234
+ runs.first.to_h { |name, _| [name, runs.map { |run| run.fetch(name) }.min.fdiv(stretches)] }
235
+ end
233
236
 
234
237
  # Runs +n+ iterations, then its after_iterations hooks, each inside
235
238
  # the around_iteration hooks if it has them, with +sample+ paused
@@ -248,17 +251,19 @@ module Duel
248
251
  end
249
252
  end
250
253
  end
251
- elsif @given_n
254
+ return
255
+ end
256
+
257
+ if @given_n
252
258
  @body.call(n)
253
- @after.each(&:call)
254
259
  else
255
260
  i = 0
256
261
  while i < n
257
262
  @body.call
258
263
  i += 1
259
264
  end
260
- @after.each(&:call)
261
265
  end
266
+ @after.each(&:call)
262
267
  end
263
268
 
264
269
  # +block+ as a method of the context's (of a module of its own, not
@@ -350,7 +355,7 @@ module Duel
350
355
  @benchmark = benchmark
351
356
  @n = n
352
357
  @counting = counting
353
- @metrics = metrics.select { |m| m.kinds.value?(:ratio) || counting }
358
+ @metrics = metrics.select { |m| m.kinds.value?(:timed) || counting }
354
359
  # (Those paused around each iteration, inside around_iteration hooks:
355
360
  # not those costly to read. Without such hooks, or such a metric, the
356
361
  # same Array: nothing more is allocated, or worked out for every
@@ -447,9 +452,9 @@ module Duel
447
452
  return if measured.nil?
448
453
 
449
454
  kind = kinds.fetch(name)
450
- return unless kind == :ratio || @counting
455
+ return unless kind == :timed || @counting
451
456
 
452
- totals = (kind == :ratio) ? @timed : @counted
457
+ totals = (kind == :timed) ? @timed : @counted
453
458
  totals[name] = totals.fetch(name, 0) + measured
454
459
  end
455
460
  end
data/lib/duel/forge.rb ADDED
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "net/http"
5
+ require_relative "spawn"
6
+
7
+ module Duel
8
+ # What the commands that run in CI's jobs of their own (Comment,
9
+ # MarkPassed, Badge) share: GitHub's API, through gh, and GitLab's,
10
+ # over HTTP, each saying why what it asked failed, as the command
11
+ # (#command) it's for. An includer has @env and @err, and names its
12
+ # GitLab token (#gitlab_token).
13
+ # @api private
14
+ module Forge
15
+ private
16
+
17
+ # What gh api prints, given +args+ (and +input+ on its standard
18
+ # input), or nil where it fails (or isn't there), which it says why,
19
+ # unless +quiet+ (where failing says what's there, a branch that
20
+ # isn't, say).
21
+ def gh(*args, quiet: false)
22
+ out, err, status = Spawn.open3(:capture3, "gh", "api", *args)
23
+ return out if status.success?
24
+
25
+ @err.puts "#{command}: gh api #{args.first}: #{err.strip}" unless quiet
26
+ rescue SystemCallError => e
27
+ @err.puts "#{command}: gh: #{e}"
28
+ end
29
+
30
+ # What GitLab's API says to +verb+ (:Get, :Post or :Delete) +path+
31
+ # (under CI_API_V4_URL), given +body+ (as JSON), signed in with
32
+ # #gitlab_token, or nil where it fails, which it says why, unless
33
+ # +quiet+. With +pages+, that and the next page's number
34
+ # (X-Next-Page), if there's one.
35
+ def gitlab(verb, path, body = nil, pages: false, quiet: false)
36
+ uri = URI("#{@env.fetch("CI_API_V4_URL")}#{path}")
37
+ request = Net::HTTP.const_get(verb).new(uri)
38
+ request["PRIVATE-TOKEN"] = @env.fetch(gitlab_token)
39
+ request["Content-Type"] = "application/json"
40
+ request.body = JSON.generate(body) if body
41
+ http = connection(uri)
42
+ http.use_ssl = uri.scheme == "https"
43
+ response = http.request(request)
44
+ unless response.is_a?(Net::HTTPSuccess)
45
+ return if quiet
46
+
47
+ return @err.puts("#{command}: GitLab: #{verb.upcase} #{path}: #{[response.code, response.body.to_s.strip].reject(&:empty?).join(" ")}")
48
+ end
49
+
50
+ text = response.body.to_s
51
+ none = {} #: Hash[String, untyped]
52
+ # (Nothing, as a deletion says.)
53
+ parsed = text.empty? ? none : JSON.parse(text)
54
+ pages ? [parsed, response["X-Next-Page"].to_s[/\A[0-9]+\z/]] : parsed
55
+ rescue SystemCallError, IOError, Timeout::Error, JSON::ParserError, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
56
+ @err.puts "#{command}: GitLab: #{verb.upcase} #{path}: #{e}"
57
+ end
58
+
59
+ # Names the branch (of those +branch+'s name is in, which the block
60
+ # says is there) that keeps Git from making +branch+: a branch named
61
+ # duel blocks duel/passed/main. Returns 1.
62
+ def blocked(branch)
63
+ prefixes = branch.split("/").then { |parts| (1...parts.size).map { |n| parts.first(n).join("/") } }
64
+ prefixes.reverse_each.select { |prefix| yield(prefix) }.each do |prefix|
65
+ @out.puts "#{"::error::" unless gitlab?}The branch #{prefix} keeps Duel from making #{branch}, as Git can't have both. " \
66
+ "Delete it (git push origin --delete #{prefix}), or rename it."
67
+ end
68
+ 1
69
+ end
70
+
71
+ # A connection to +uri+'s host, not yet started.
72
+ # mutant:disable -- a nil host is Net::HTTP's localhost, where the
73
+ # tests' server is, so only another host could tell them apart
74
+ def connection(uri) = Net::HTTP.new(uri.host.to_s, uri.port)
75
+
76
+ # Whether it runs on GitLab CI (GITLAB_CI, set and not empty).
77
+ def gitlab? = !@env["GITLAB_CI"].to_s.empty?
78
+
79
+ def repository = @env.fetch("GITHUB_REPOSITORY")
80
+
81
+ # The project's path in GitLab's API.
82
+ def project = "/projects/#{@env.fetch("CI_PROJECT_ID")}"
83
+
84
+ # +ref+, as a path of GitLab's API takes a branch's name.
85
+ def encoded(ref) = ref.gsub("/", "%2F")
86
+ end
87
+ end
data/lib/duel/init.rb CHANGED
@@ -3,7 +3,6 @@
3
3
  require "erb"
4
4
  require "json"
5
5
  require "fileutils"
6
- require "tmpdir"
7
6
  require "open3"
8
7
  require "optparse"
9
8
  require "pathname"
@@ -20,9 +19,9 @@ module Duel
20
19
  # a job that keeps a badge), or with
21
20
  # --gitlab, GitLab CI jobs to include, adds a Rake
22
21
  # task to the Rakefile, and adds StackProf to the Gemfile (for profiles
23
- # of regressions, on CRuby). With --reusable, its workflows are short
24
- # ones that call Duel's reusable workflows (see Reusable), passing on
25
- # what .duel.rb sets. It overwrites nothing without --force, and
22
+ # of regressions, on CRuby). Its workflows are short ones that call
23
+ # Duel's reusable workflows (see Reusable), passing on what .duel.rb
24
+ # sets, or with --standalone, whole ones, to change as you like. It overwrites nothing without --force, and
26
25
  # writes nothing with --dry-run, or with --check, which says how the
27
26
  # workflows differ from what it writes.
28
27
  # @api private
@@ -50,7 +49,8 @@ module Duel
50
49
  LABEL_COLOR = "FBCA04"
51
50
 
52
51
  # Where Duel's reusable workflows are (see Reusable), which the
53
- # workflows it writes with --reusable call, at this release's tag.
52
+ # workflows it writes call (without --standalone), at this release's
53
+ # tag.
54
54
  REUSABLE = "sferik/duel-ruby/.github/workflows"
55
55
 
56
56
  def initialize(argv, out: $stdout, err: $stderr, input: $stdin, root: Dir.pwd)
@@ -67,7 +67,6 @@ module Duel
67
67
  def run
68
68
  rest = catch(:help) { parser.parse(@argv) } or return 0
69
69
  raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
70
- raise OptionParser::InvalidOption, "--reusable with --gitlab (GitLab CI has no reusable workflows)" if @reusable && @gitlab
71
70
  return rails if File.exist?(File.join(@root, "config", "application.rb"))
72
71
 
73
72
  return check if @check
@@ -91,8 +90,8 @@ module Duel
91
90
  return written.merge(top.join(gitlab_file).to_s => render("duel.gitlab-ci.yml")) if @gitlab
92
91
 
93
92
  workflows = top.join(".github", "workflows")
94
- # (With --reusable, ones that call Duel's.)
95
- perf, comment = @reusable ? %w[perf-caller.yml perf-comment-caller.yml] : %w[perf.yml perf-comment.yml]
93
+ # (Ones that call Duel's, or with --standalone, whole ones.)
94
+ perf, comment = @standalone ? %w[perf.yml perf-comment.yml] : %w[perf-caller.yml perf-comment-caller.yml]
96
95
  written.merge(workflows.join(workflow_name).to_s => render(perf),
97
96
  workflows.join(workflow_name.sub(/\.yml\z/, "-comment.yml")).to_s => render(comment))
98
97
  end
@@ -107,7 +106,7 @@ module Duel
107
106
  o.on("--check", "Say how the workflows differ from what it writes (with the same options), and write nothing") { @check = true }
108
107
  o.on("--badge", "Write a job that keeps a badge for the README (it needs contents: write)") { @badge = true }
109
108
  o.on("--gitlab", "Write GitLab CI jobs (to include from .gitlab-ci.yml), not GitHub Actions workflows") { @gitlab = true }
110
- o.on("--reusable", "Write workflows that call Duel's reusable ones (at this release), not whole workflows") { @reusable = true }
109
+ o.on("--standalone", "Write whole workflows, not ones that call Duel's reusable ones (at this release)") { @standalone = true }
111
110
  o.on("-h", "--help", "Print this help") do
112
111
  @out.puts o
113
112
  throw :help
@@ -316,7 +315,7 @@ module Duel
316
315
  "Commit and push. On a pull request, the workflow compares the code with the branch it targets, and comments with the " \
317
316
  "report when a benchmark fails (c.comment in .duel.rb says when)."
318
317
  end
319
- "\nNext:\n#{steps.each_with_index.map { |step, i| " #{i + 1}. #{step}\n" }.join}"
318
+ "\nNext:\n#{steps.each.with_index(1).map { |step, i| " #{i}. #{step}\n" }.join}"
320
319
  end
321
320
 
322
321
  # The template +name+, rendered. (Read with LF line endings, as Git
@@ -329,33 +328,20 @@ module Duel
329
328
  # mutant:disable -- nil (a mutation's) renders as false does
330
329
  def reusable? = false
331
330
 
332
- # Whether the workflow's jobs write to the repository: to mark a push
333
- # passed, comment on one, or keep the badge.
334
- def writes? = mark_passed? || comment_on_pushes? || badge?
335
-
336
- # What the workflow it writes with --reusable passes on to Duel's
337
- # Perf workflow, as its inputs (see Reusable): what .duel.rb sets that
338
- # isn't the input's default, each as YAML has it.
331
+ # What the workflow it writes (without --standalone) passes on to
332
+ # Duel's Perf workflow, as its inputs (see Reusable): what isn't the
333
+ # input's default, each as YAML has it. (What .duel.rb sets, Duel
334
+ # reads.)
339
335
  def perf_inputs
340
336
  {
341
337
  "ruby-version" => ruby_version,
342
338
  "version-file" => (true if version_file),
343
- "working-directory" => working_directory,
344
- "accept-label" => accept_label,
345
- "release-tags" => (set_release_tags unless set_release_tags == "v*"),
346
- "comment-on-pushes" => (true if comment_on_pushes?),
347
- "mark-passed" => (true if mark_passed?),
348
- "badge" => (true if badge?),
349
- "passed-branch" => (passed_branch if working_directory),
350
- "badge-branch" => (badge_branch if working_directory && badge?),
351
- "marker" => (marker if working_directory)
339
+ "working-directory" => working_directory
352
340
  }.compact.transform_values(&:to_json)
353
341
  end
354
342
 
355
- # The glob release tags match, as .duel.rb sets it (c.release_tags),
356
- # or nil where it doesn't load. (The whole workflows take the
357
- # conventions', and say to set RELEASE_TAGS to match.)
358
- def set_release_tags = settings&.release_tags
343
+ # The same, for Duel's push workflow, besides what the Duel job said.
344
+ def push_inputs = {"working-directory" => working_directory, "badge" => (true if badge?)}.compact.transform_values(&:to_json)
359
345
 
360
346
  # The same, for Duel's Perf comment workflow.
361
347
  def comment_inputs
@@ -369,63 +355,31 @@ module Duel
369
355
  # The project's configuration, as the conventions have it.
370
356
  def config = Config.new(@root)
371
357
 
372
- # The project's settings, which decide the workflow's jobs: its
373
- # .duel.rb's, or where there's none yet or --force replaces it, those
374
- # of the one it writes. Nil where it doesn't load (one
375
- # of its plugins isn't installed, say), and then the workflow has
376
- # both jobs for pushes, as if .duel.rb asked for them, and says so.
358
+ # The project's settings, which decide what the comment workflow runs
359
+ # the Duel job again for (the label that accepts a regression): its
360
+ # .duel.rb's, or nil where there's none yet or --force replaces it
361
+ # (the one it writes sets nothing), or it doesn't load (one of its
362
+ # plugins isn't installed, say, which it says): Duel's own.
377
363
  def settings = instance_variable_defined?(:@settings) ? @settings : (@settings = load_settings)
378
364
 
379
365
  def load_settings
380
366
  path = File.join(@root, ".duel.rb")
381
- return written_settings if @force || !File.exist?(path)
382
-
383
- Duel.load_config(path)
367
+ Duel.load_config(path) if !@force && File.exist?(path)
384
368
  rescue ScriptError, StandardError => e
385
- @err.puts "duel init: .duel.rb didn't load (#{e.class}: #{e}), so the workflow has both jobs for pushes, as if it asked for them"
386
- end
387
-
388
- # The settings of the .duel.rb it writes.
389
- def written_settings
390
- Dir.mktmpdir do |dir|
391
- path = File.join(dir, ".duel.rb")
392
- File.write(path, render("duel.rb"))
393
- Duel.load_config(path)
394
- end
369
+ @err.puts "duel init: .duel.rb didn't load (#{e.class}: #{e}), so the workflows take Duel's defaults"
395
370
  end
396
371
 
397
372
  # The label that accepts a regression (Config#accept_label), for the
398
- # workflows: the Duel job's, to find what was accepted since the last
399
- # release, and the comment's, to run the Duel job again when it's
400
- # added or taken off. Nil without one, or where .duel.rb doesn't load.
401
- def accept_label = settings&.accept_label
402
-
403
- # Whether the workflow comments on a pushed commit: where a regression
404
- # in a push passes (c.fail_on = :pull_requests), the only time Duel
405
- # says to.
406
- def comment_on_pushes?
407
- found = settings
408
- found.nil? || found.fail_on == :pull_requests
409
- end
373
+ # comment workflow, to run the Duel job again when it's added or taken
374
+ # off: .duel.rb's, or where it doesn't load, Duel's own. Nil without
375
+ # one.
376
+ def accept_label = (settings || config).accept_label
410
377
 
411
- # Whether the workflow marks a push passed: where a push is compared
412
- # with the last commit Duel passed (c.push_base = :last_passed).
413
- def mark_passed?
414
- found = settings
415
- found.nil? || found.push_base == :last_passed
416
- end
417
-
418
- # Whether the workflow keeps a badge for the README (--badge): its job
419
- # moves a branch, so it's the workflow's only job, without the jobs
420
- # for pushes, that writes to the repository.
378
+ # Whether the workflow keeps a badge for the README (--badge).
421
379
  def badge? = @badge
422
380
 
423
- # The library the gemspec names (see Config#library), or nil.
424
- def library = config.library
425
-
426
- # The glob release tags match (Config#release_tags), for the workflow
427
- # to find the last release as Duel does.
428
- def release_tags = config.release_tags
381
+ # The library the gemspec names (see Config#gem_library), or nil.
382
+ def library = config.gem_library
429
383
 
430
384
  # The library, without a gemspec: lib/'s only Ruby file, or nil.
431
385
  def guess
@@ -500,8 +454,8 @@ module Duel
500
454
  # project in a subdirectory, duel/badge-<its path>.
501
455
  def badge_branch = suffixed("duel/badge")
502
456
 
503
- # Duel's reusable workflows, which the workflows `duel init --reusable`
504
- # writes call: .github/workflows/duel.yml (the Perf workflow's jobs)
457
+ # Duel's reusable workflows, which the workflows `duel init` writes
458
+ # call: .github/workflows/duel.yml (the Perf workflow's jobs)
505
459
  # and duel-comment.yml (the Perf comment workflow's), in Duel's
506
460
  # repository. They're the templates `duel init` writes from, rendered
507
461
  # with each setting of .duel.rb they need as an input (and a job or a
@@ -513,33 +467,36 @@ module Duel
513
467
  # The files, from the repository's root, to what's in them.
514
468
  def files
515
469
  workflows = File.join(".github", "workflows")
516
- {File.join(workflows, "duel.yml") => render("perf.yml"), File.join(workflows, "duel-comment.yml") => render("perf-comment.yml")}
470
+ {File.join(workflows, "duel.yml") => render("perf.yml"), File.join(workflows, "duel-comment.yml") => render("perf-comment.yml"),
471
+ File.join(workflows, "duel-push.yml") => render("perf-push.yml")}
517
472
  end
518
473
 
519
474
  private
520
475
 
476
+ # The whole Perf workflow's jobs for a push, as the push workflow's:
477
+ # what they take from the Duel job (needs.duel), its inputs.
478
+ def push_jobs = render("perf-push-jobs.yml").gsub(/^ needs: duel\n/, "").gsub("needs.duel.outputs.", "inputs.").gsub("needs.duel.result", "inputs.result")
479
+
521
480
  def reusable? = true
522
481
 
523
482
  # Each an input's (see the templates' workflow_call).
524
483
  def accept_label = "${{ inputs.accept-label }}"
525
484
 
526
- def release_tags = "${{ inputs.release-tags }}"
527
-
528
485
  def ruby_version = "${{ inputs.ruby-version }}"
529
486
 
530
487
  def working_directory = "${{ inputs.working-directory }}"
531
488
 
532
- def passed_branch = "${{ inputs.passed-branch }}"
489
+ # (As the comments name them, for a project at the root: the jobs
490
+ # work them out from working-directory.)
491
+ def passed_branch = "duel/passed"
492
+
493
+ def badge_branch = "duel/badge"
533
494
 
534
- def badge_branch = "${{ inputs.badge-branch }}"
495
+ def workflow_name = "perf.yml"
535
496
 
536
497
  # Each in, run only where its input says to.
537
498
  def version_file = true
538
499
 
539
- def comment_on_pushes? = true
540
-
541
- def mark_passed? = true
542
-
543
500
  def badge? = true
544
501
  end
545
502
  end
@@ -13,48 +13,50 @@ module Duel
13
13
  # isn't a new key.
14
14
  # @api private
15
15
  class JSONReport
16
- SCHEMA = 1
16
+ SCHEMA = 2
17
17
 
18
18
  # A run that didn't compare: +status+ ("nothing_to_compare", or
19
- # "error"), its +exit+ status, +message+, what it said, and what the
20
- # workflow does about its +comment+ (see CLI.comment).
21
- def self.note(status, exit, message, settings, comment)
19
+ # "error"), its +exit+ status, and +message+, what it said.
20
+ def self.note(status, exit, message, settings)
22
21
  none = [] #: Array[value]
23
22
  nothing = {} #: report
24
- head(status, exit, message, settings, comment).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
25
- "benchmarks" => none, "across" => nothing, "errors" => none, "cut_short" => nil)
23
+ head(status, exit, nil, message, settings).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
24
+ "benchmarks" => none, "across" => nothing, "cut_short" => nil)
26
25
  end
27
26
 
28
27
  # What every report starts with.
29
- def self.head(status, exit, message, settings, comment)
30
- {"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message, "comment" => comment,
31
- "settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
32
- "jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
33
- "time_limit" => settings.time_limit, "warm" => settings.warm, "force" => settings.force}}
28
+ def self.head(status, exit, passed_anyway, message, settings)
29
+ {"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "passed_anyway" => passed_anyway,
30
+ "message" => message, "settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
31
+ "jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
32
+ "time_limit" => settings.time_limit, "warm" => settings.warm, "force" => settings.force}}
34
33
  end
35
34
 
36
35
  # +runner+ ran, and made +results+, comparing +baseline+ with
37
36
  # +candidate+ (each {"ref", "commit", "root"} or {"commit",
38
- # "uncommitted_changes", "root"}) as +settings+ say, exits with
39
- # +exit+, and has the workflow do +comment+ (see CLI.comment).
40
- def initialize(runner, results, settings:, exit:, comment:, baseline:, candidate:)
37
+ # "uncommitted_changes", "root"}) as +settings+ say, and whether they
38
+ # +passed+, exits with +exit+, and if they failed, passed anyway as
39
+ # +passed_anyway+ says (see CLI#slower): "accept_label" (a pull
40
+ # request's label accepted it), "fail_on" (only a pull request
41
+ # fails), or nil.
42
+ def initialize(runner, results, settings:, passed:, exit:, passed_anyway:, baseline:, candidate:)
41
43
  @runner = runner
42
44
  @results = results
43
45
  @settings = settings
46
+ @passed = passed
44
47
  @exit = exit
45
- @comment = comment
48
+ @passed_anyway = passed_anyway
46
49
  @versions = {"baseline" => baseline, "candidate" => candidate}
47
50
  end
48
51
 
49
52
  def to_h
50
53
  rubies = @runner.rubies
51
- JSONReport.head(@exit.zero? ? "passed" : "failed", @exit, nil, @settings, @comment).merge(
54
+ JSONReport.head(@passed ? "passed" : "failed", @exit, @passed_anyway, nil, @settings).merge(
52
55
  "seconds" => @runner.seconds,
53
56
  **@versions.to_h { |side, version| [side, version.merge("ruby" => rubies.fetch(side))] },
54
57
  "plugins" => @runner.plugins.map(&:to_s),
55
58
  "benchmarks" => @results.map { |result| benchmark(result) },
56
59
  "across" => across,
57
- "errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} },
58
60
  "cut_short" => @runner.cut_short
59
61
  )
60
62
  end
@@ -64,7 +66,7 @@ module Duel
64
66
  # (Runner#explained says nothing of a benchmark that didn't regress.)
65
67
  def benchmark(result)
66
68
  {"name" => result.name, "verdict" => Summary.verdict(result), "passed" => result.passed?, "regressed" => result.regressed,
67
- "unsettled" => result.unsettled?, "iterations" => result.n, "seconds" => result.seconds,
69
+ "iterations" => result.n, "seconds" => result.seconds,
68
70
  "confirmations" => result.confirmations.size, "timed" => timed(result), "counted" => counted(result),
69
71
  "peak_memory" => peak_memory(result), "details" => result.details,
70
72
  "explanation" => @runner.explained(result), "error" => result.error, "skipped" => result.skipped}
@@ -74,8 +76,8 @@ module Duel
74
76
  # each version's median an iteration.
75
77
  def timed(result)
76
78
  # (Timed, so measured, with an n.)
79
+ n = result.n #: Integer
77
80
  result.timed.to_h do |metric, t|
78
- n = result.n #: Integer
79
81
  [metric, {"ratio" => t.fetch("median"), "interval" => t.fetch("interval"), "limit" => t.fetch("limit"),
80
82
  "pairs" => t.fetch("pairs"), "processes" => t["processes"], "verdict" => t.fetch("verdict"), "baseline" => t.fetch("baseline").fdiv(n),
81
83
  "candidate" => t.fetch("candidate").fdiv(n), "unit" => t.fetch("unit"), "plural" => t.fetch("plural"),
@@ -83,20 +85,27 @@ module Duel
83
85
  end
84
86
  end
85
87
 
86
- # Each counted measure: each version's fewest an iteration, and
87
- # whether that's worse.
88
+ # Each counted measure: each version's fewest an iteration, and its
89
+ # verdict: "more" (worse: more than the metric tolerates), "fewer",
90
+ # or "same" (or more, within what it tolerates).
88
91
  def counted(result)
89
92
  worse = result.worse
90
93
  result.counted.to_h do |metric, c|
91
- n = c.fetch("n")
92
- [metric, {"baseline" => c.fetch("baseline").fdiv(n), "candidate" => c.fetch("candidate").fdiv(n), "unit" => c.fetch("unit"),
93
- "plural" => c.fetch("plural"), "worse" => worse.include?(metric)}]
94
+ base, candidate = Result.per_iteration(c)
95
+ verdict = if worse.include?(metric) then "more"
96
+ elsif candidate < base then "fewer"
97
+ else "same"
98
+ end
99
+ [metric, {"baseline" => base, "candidate" => candidate, "unit" => c.fetch("unit"), "plural" => c.fetch("plural"), "verdict" => verdict}]
94
100
  end
95
101
  end
96
102
 
103
+ # Each version's peak memory, and its verdict: "more" (than Duel
104
+ # tolerates), or "same".
97
105
  def peak_memory(result)
98
106
  result.memory&.then do |memory|
99
- {"baseline" => memory.fetch("baseline"), "candidate" => memory.fetch("candidate"), "unit" => "B", "worse" => result.more_memory?}
107
+ {"baseline" => memory.fetch("baseline"), "candidate" => memory.fetch("candidate"), "unit" => "B",
108
+ "verdict" => result.more_memory? ? "more" : "same"}
100
109
  end
101
110
  end
102
111