duel 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +51 -27
  3. data/CHANGELOG.md +247 -1
  4. data/README.md +30 -14
  5. data/docs/benchmarks.md +28 -24
  6. data/docs/ci.md +328 -362
  7. data/docs/configuration.md +40 -37
  8. data/docs/how-it-works.md +23 -10
  9. data/docs/json.md +21 -25
  10. data/docs/plugins.md +40 -22
  11. data/docs/troubleshooting.md +3 -2
  12. data/lib/duel/across.rb +18 -18
  13. data/lib/duel/badge.rb +174 -0
  14. data/lib/duel/baseline.rb +38 -15
  15. data/lib/duel/cli.rb +175 -107
  16. data/lib/duel/comment.rb +80 -71
  17. data/lib/duel/config.rb +145 -63
  18. data/lib/duel/context.rb +29 -24
  19. data/lib/duel/forge.rb +87 -0
  20. data/lib/duel/init.rb +44 -87
  21. data/lib/duel/json_report.rb +34 -25
  22. data/lib/duel/lane.rb +107 -61
  23. data/lib/duel/lock.rb +1 -1
  24. data/lib/duel/mark_passed.rb +132 -0
  25. data/lib/duel/metric.rb +28 -27
  26. data/lib/duel/native.rb +7 -5
  27. data/lib/duel/platform.rb +4 -0
  28. data/lib/duel/process_sample.rb +9 -6
  29. data/lib/duel/rake_task.rb +4 -5
  30. data/lib/duel/resource.rb +10 -11
  31. data/lib/duel/result.rb +22 -14
  32. data/lib/duel/rubies.rb +6 -2
  33. data/lib/duel/runner.rb +27 -15
  34. data/lib/duel/spawn.rb +29 -4
  35. data/lib/duel/stats.rb +14 -12
  36. data/lib/duel/summary.rb +31 -29
  37. data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
  38. data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
  39. data/lib/duel/templates/duel.rb.erb +7 -7
  40. data/lib/duel/templates/perf-caller.yml.erb +22 -5
  41. data/lib/duel/templates/perf-comment.yml.erb +11 -11
  42. data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
  43. data/lib/duel/templates/perf-push.yml.erb +50 -0
  44. data/lib/duel/templates/perf.yml.erb +113 -335
  45. data/lib/duel/testing.rb +20 -25
  46. data/lib/duel/version.rb +1 -1
  47. data/lib/duel/worker.rb +16 -9
  48. data/lib/duel/worker_process.rb +6 -6
  49. data/lib/duel.rb +88 -33
  50. data/sig/duel/across.rbs +4 -0
  51. data/sig/duel/badge.rbs +51 -0
  52. data/sig/duel/baseline.rbs +7 -2
  53. data/sig/duel/cli.rbs +31 -20
  54. data/sig/duel/comment.rbs +14 -15
  55. data/sig/duel/config.rbs +71 -29
  56. data/sig/duel/context.rbs +4 -4
  57. data/sig/duel/forge.rbs +42 -0
  58. data/sig/duel/init.rbs +8 -26
  59. data/sig/duel/json_report.rbs +7 -6
  60. data/sig/duel/lane.rbs +29 -6
  61. data/sig/duel/mark_passed.rbs +39 -0
  62. data/sig/duel/metric.rbs +10 -10
  63. data/sig/duel/native.rbs +3 -0
  64. data/sig/duel/platform.rbs +1 -0
  65. data/sig/duel/process_sample.rbs +3 -0
  66. data/sig/duel/resource.rbs +3 -3
  67. data/sig/duel/result.rbs +12 -5
  68. data/sig/duel/rubies.rbs +3 -0
  69. data/sig/duel/runner.rbs +23 -16
  70. data/sig/duel/spawn.rbs +7 -0
  71. data/sig/duel/stats.rbs +4 -1
  72. data/sig/duel/summary.rbs +8 -0
  73. data/sig/duel/testing.rbs +2 -2
  74. data/sig/duel/worker.rbs +6 -5
  75. data/sig/duel/worker_process.rbs +3 -3
  76. data/sig/duel.rbs +50 -31
  77. metadata +9 -1
data/lib/duel/across.rb CHANGED
@@ -5,7 +5,7 @@ require_relative "stats"
5
5
  module Duel
6
6
  # Whether the candidate is slower across the board: a slowdown smaller
7
7
  # than any one benchmark can settle (1% everywhere, say) still shows in
8
- # all of them together. For each ratio metric, the candidate is slower
8
+ # all of them together. For each timed metric, the candidate is slower
9
9
  # when a distribution-free confidence interval lies entirely above a
10
10
  # quarter of the metric's own margin (1.005, for a limit of 1.02). It
11
11
  # takes 8 benchmarks at least, at 99%. One that says across: false is
@@ -41,20 +41,14 @@ module Duel
41
41
 
42
42
  # {metric => {"median", "interval", "limit", "benchmarks", "verdict"}},
43
43
  # and where it's judged by process, "processes" (the workers of each
44
- # version), for every ratio metric any benchmark was judged on.
44
+ # version), for every timed metric any benchmark was judged on.
45
45
  def verdicts
46
46
  medians = Hash.new { |h, k| h[k] = [] }
47
47
  # @type var limits: Hash[String, Float]
48
48
  limits = {}
49
- @results.each do |result|
50
- next if result.error || result.skipped
51
-
52
- result.timed.each do |metric, timed|
53
- next unless timed.fetch("across")
54
-
55
- medians[metric] << timed.fetch("median")
56
- limits[metric] ||= Across.limit(timed.fetch("limit"))
57
- end
49
+ each_across do |_, metric, timed|
50
+ medians[metric] << timed.fetch("median")
51
+ limits[metric] ||= Across.limit(timed.fetch("limit"))
58
52
  end
59
53
  return by_process_or_benchmark(medians, limits) if @by_process
60
54
 
@@ -63,6 +57,16 @@ module Duel
63
57
 
64
58
  private
65
59
 
60
+ # Each timed measure that counts across the board, of each benchmark
61
+ # judged: its result, its metric and the measure.
62
+ def each_across
63
+ @results.each do |result|
64
+ next if result.error || result.skipped
65
+
66
+ result.timed.each { |metric, timed| yield result, metric, timed if timed.fetch("across") }
67
+ end
68
+ end
69
+
66
70
  # The verdicts of a run that judged benchmarks over workers: over the
67
71
  # workers' scores, for a metric with MIN_BENCHMARKS judged by process
68
72
  # at least, and else over the benchmarks (+medians+, with +limits+).
@@ -80,13 +84,9 @@ module Duel
80
84
  def workers
81
85
  # @type var found: Hash[String, Array[Result::processes]]
82
86
  found = Hash.new { |h, k| h[k] = [] }
83
- @results.each do |result|
84
- next if result.error || result.skipped
85
-
86
- result.timed.each do |metric, timed|
87
- each = result.processes[metric]
88
- found[metric] << each if each && timed.fetch("across")
89
- end
87
+ each_across do |result, metric, _|
88
+ each = result.processes[metric]
89
+ found[metric] << each if each
90
90
  end
91
91
  found
92
92
  end
data/lib/duel/badge.rb ADDED
@@ -0,0 +1,174 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "optparse"
4
+ require_relative "forge"
5
+
6
+ module Duel
7
+ # `duel badge BRANCH --status STATUS --result RESULT`: what the Update
8
+ # the badge job runs, on GitHub Actions, after a push to the default
9
+ # branch: Duel's badge, for the README, as that push left it, written
10
+ # alone to BRANCH (in GITHUB_REPOSITORY, through gh), which the README
11
+ # shows. Green (perf guarded), red (perf regressed) where Duel failed
12
+ # the push or found a regression (STATUS failed, even one that passed,
13
+ # where Config#fail_on says only a pull request fails), or grey (not
14
+ # measured) where the Duel job failed (RESULT failure) before Duel said
15
+ # either. A run for an older push than the badge's leaves it as it is.
16
+ # It checks nothing out, runs none of the code, and fetches nothing.
17
+ # @api private
18
+ class Badge
19
+ include Forge
20
+
21
+ USAGE = "Usage: duel badge BRANCH --status STATUS --result RESULT\n\n" \
22
+ "Writes Duel's badge, as the push (GITHUB_SHA) left it, to BRANCH: what the Update the badge job runs, with the\n" \
23
+ "Duel job's status and result.\n\n"
24
+
25
+ # Each state's words, and the badge's width, its colors (top and
26
+ # bottom), and its right edge's path, by what's to say.
27
+ STATES = {
28
+ "guarded" => ["perf guarded", 133, "#34D058", "#28A745", "76.939C78.629 0 80 1.343 80"],
29
+ "regressed" => ["perf regressed", 142, "#EA4A5A", "#D73A49", "85.939C87.629 0 89 1.343 89"],
30
+ "unmeasured" => ["not measured", 138, "#959DA5", "#6A737D", "81.939C83.629 0 85 1.343 85"]
31
+ }.freeze #: Hash[String, [String, Integer, String, String, String]]
32
+
33
+ # The badge's image, with %<state>s, %<width>s, %<top>s, %<bottom>s and
34
+ # %<right>s, written here rather than fetched from anywhere.
35
+ SVG = <<~SVG
36
+ <svg xmlns="http://www.w3.org/2000/svg" width="%<width>s" height="20">
37
+ <title>Duel - %<state>s</title>
38
+ <defs>
39
+ <linearGradient id="duel-fill" x1="50%%" y1="0%%" x2="50%%" y2="100%%">
40
+ <stop stop-color="#444D56" offset="0%%"></stop>
41
+ <stop stop-color="#24292E" offset="100%%"></stop>
42
+ </linearGradient>
43
+ <linearGradient id="state-fill" x1="50%%" y1="0%%" x2="50%%" y2="100%%">
44
+ <stop stop-color="%<top>s" offset="0%%"></stop>
45
+ <stop stop-color="%<bottom>s" offset="100%%"></stop>
46
+ </linearGradient>
47
+ </defs>
48
+ <g fill="none" fill-rule="evenodd">
49
+ <g font-family="'DejaVu Sans',Verdana,Geneva,sans-serif" font-size="11">
50
+ <path d="M0,3 C0,1.3431 1.3552,0 3.02702703,0 L53,0 L53,20 L3.02702703,20 C1.3552,20 0,18.6569 0,17 L0,3 Z" fill="url(#duel-fill)" fill-rule="nonzero"></path>
51
+ <text fill="#010101" fill-opacity=".3">
52
+ <tspan x="22.1981982" y="15" aria-hidden="true">Duel</tspan>
53
+ </text>
54
+ <text fill="#FFFFFF">
55
+ <tspan x="22.1981982" y="14">Duel</tspan>
56
+ </text>
57
+ </g>
58
+ <g transform="translate(53)" font-family="'DejaVu Sans',Verdana,Geneva,sans-serif" font-size="11">
59
+ <path d="M0 0h%<right>s 3v14c0 1.657-1.37 3-3.061 3H0V0z" fill="url(#state-fill)" fill-rule="nonzero"></path>
60
+ <text fill="#010101" fill-opacity=".3" aria-hidden="true">
61
+ <tspan x="4" y="15">%<state>s</tspan>
62
+ </text>
63
+ <text fill="#FFFFFF">
64
+ <tspan x="4" y="14">%<state>s</tspan>
65
+ </text>
66
+ </g>
67
+ <g fill="#959DA5">
68
+ <path d="M4.6 3.6L6.9 4.5L14.3 11.9L12.9 13.3L5.5 5.9Z M17.4 3.6L15.1 4.5L7.7 11.9L9.1 13.3L16.5 5.9Z"></path>
69
+ <circle cx="16.6" cy="15.6" r="1.1"></circle>
70
+ <circle cx="5.4" cy="15.6" r="1.1"></circle>
71
+ </g>
72
+ <g stroke="#959DA5" stroke-linecap="round">
73
+ <path d="M12.1 14.5L15.5 11.1 M9.9 14.5L6.5 11.1" stroke-width="1.6"></path>
74
+ <path d="M13.8 12.8L16 15 M8.2 12.8L6 15" stroke-width="1.8"></path>
75
+ </g>
76
+ </g>
77
+ </svg>
78
+ SVG
79
+
80
+ # What its commit says, which names the push: "Duel's badge, for SHA".
81
+ MESSAGE = "Duel's badge, for "
82
+
83
+ def initialize(argv, env:, out:, err:)
84
+ @argv = argv
85
+ @env = env
86
+ @out = out
87
+ @err = err
88
+ end
89
+
90
+ # Returns the exit status: 0 written (or left, for an older push), 1
91
+ # GitHub wouldn't take it, 2 a bad option.
92
+ def run
93
+ branches = catch(:help) { parser.parse(@argv) } or return 0
94
+ raise OptionParser::MissingArgument, "BRANCH" if branches.empty?
95
+ raise OptionParser::NeedlessArgument, branches.drop(1).first if branches.size > 1
96
+
97
+ branch = branches.first #: String
98
+ sha = @env.fetch("GITHUB_SHA")
99
+ return left(branch) if newer?(branch, sha)
100
+
101
+ write(branch, Badge.svg(Badge.state(@status, @result)), sha)
102
+ rescue OptionParser::ParseError => e
103
+ @err.puts "#{command}: #{e}"
104
+ 2
105
+ end
106
+
107
+ # What the badge says, given the Duel job's +status+ (its output:
108
+ # passed, failed or neutral) and +result+ (success or failure, say):
109
+ # one of STATES.
110
+ def self.state(status, result)
111
+ case status
112
+ when "failed" then "regressed"
113
+ when "passed" then "guarded"
114
+ else (result == "failure") ? "unmeasured" : "guarded"
115
+ end
116
+ end
117
+
118
+ # The badge's image, saying +state+ (one of STATES).
119
+ def self.svg(state)
120
+ words, width, top, bottom, right = STATES.fetch(state)
121
+ format(SVG, state: words, width: width, top: top, bottom: bottom, right: right)
122
+ end
123
+
124
+ private
125
+
126
+ def parser
127
+ OptionParser.new do |o|
128
+ o.banner = USAGE
129
+ o.on("--status STATUS", "The Duel job's status (its output: passed, failed or neutral)") { |v| @status = v }
130
+ o.on("--result RESULT", "The Duel job's result (success or failure, say)") { |v| @result = v }
131
+ o.on("-h", "--help", "Print this help") do
132
+ @out.puts o
133
+ throw :help
134
+ end
135
+ end
136
+ end
137
+
138
+ # Whether the badge on +branch+ is a newer push's than +sha+ (its
139
+ # commit names the push): that push's run has the say.
140
+ def newer?(branch, sha)
141
+ message = gh("repos/#{repository}/commits/#{branch}", "--jq", ".commit.message", quiet: true).to_s
142
+ last = message[/\A#{MESSAGE}(\h+)\n?\z/o, 1] or return false
143
+
144
+ gh("repos/#{repository}/compare/#{sha}...#{last}", "--jq", ".status", quiet: true)&.chomp == "ahead"
145
+ end
146
+
147
+ # Writes +svg+, alone, to +branch+ (moved there, or made), in a commit
148
+ # that names the push, +sha+.
149
+ def write(branch, svg, sha)
150
+ # (gh makes the fields' JSON: {"tree": [{"path": "badge.svg", ...}]}.)
151
+ tree = gh("repos/#{repository}/git/trees", "--method", "POST", "-f", "tree[][path]=badge.svg", "-f", "tree[][mode]=100644",
152
+ "-f", "tree[][type]=blob", "-f", "tree[][content]=#{svg}", "--jq", ".sha")&.chomp or return 1
153
+ commit = gh("repos/#{repository}/git/commits", "--method", "POST", "-f", "message=#{MESSAGE}#{sha}", "-f", "tree=#{tree}",
154
+ "--jq", ".sha")&.chomp or return 1
155
+ ref = "repos/#{repository}/git/refs"
156
+ if gh("#{ref}/heads/#{branch}", "--method", "PATCH", "-f", "sha=#{commit}", "-F", "force=true", quiet: true) ||
157
+ gh(ref, "--method", "POST", "-f", "ref=refs/heads/#{branch}", "-f", "sha=#{commit}")
158
+ @out.puts "#{command}: #{branch} shows the badge of #{sha}"
159
+ return 0
160
+ end
161
+ blocked(branch) { |prefix| gh("repos/#{repository}/git/ref/heads/#{prefix}", quiet: true) }
162
+ end
163
+
164
+ def left(branch)
165
+ @out.puts "#{command}: #{branch} shows a newer push's badge, which it's left at"
166
+ 0
167
+ end
168
+
169
+ # (For Forge.)
170
+ def command = "duel badge"
171
+
172
+ def gitlab_token = "DUEL_TOKEN"
173
+ end
174
+ end
data/lib/duel/baseline.rb CHANGED
@@ -50,17 +50,18 @@ module Duel
50
50
  def release
51
51
  return @release if instance_variable_defined?(:@release)
52
52
 
53
- @release = git("describe", "--tags", "--abbrev=0", "--match", @config.release_tags, "HEAD^", allow_failure: true)
53
+ matches = Config.list(@config.release_tags).flat_map { |glob| ["--match", glob] }
54
+ @release = git("describe", "--tags", "--abbrev=0", *matches, "HEAD^", allow_failure: true)
54
55
  end
55
56
 
56
57
  # The last release, or where a pull request merged since then was
57
58
  # accepted with Config#accept_label, the commit that merged the last
58
- # one (Settings#accepted, which the workflow finds): what it slowed
59
- # down was accepted, so it's compared with instead. Nil with no
60
- # release.
59
+ # one (Settings#accepted, or on GitHub Actions, #merged_accepted):
60
+ # what it slowed down was accepted, so it's compared with instead.
61
+ # Nil with no release.
61
62
  def last_release
62
63
  found = release or return
63
- accepted = @settings.accepted
64
+ accepted = @settings.accepted || merged_accepted(found)
64
65
  return found unless accepted && !@config.accept_label.nil?
65
66
  return found unless ancestor?(found, accepted) && !ancestor?(accepted, found) && ancestor?(accepted, "HEAD")
66
67
 
@@ -69,6 +70,26 @@ module Duel
69
70
 
70
71
  def ancestor?(commit, of) = !git("merge-base", "--is-ancestor", commit, of, allow_failure: true).nil?
71
72
 
73
+ # The last commit since +release+ that merged a pull request with
74
+ # Config#accept_label, on GitHub (in Settings#repository: a scheduled
75
+ # run's on GitHub Actions), as gh finds them, or nil: without the
76
+ # label, or gh, or what gh needs (GH_TOKEN), say.
77
+ def merged_accepted(release)
78
+ repository = @settings.repository
79
+ label = @config.accept_label
80
+ return unless repository && label
81
+
82
+ since = Time.at(Integer(git("log", "-1", "--format=%ct", release))).utc.strftime("%Y-%m-%dT%H:%M:%SZ")
83
+ merged, _, status = Spawn.open3(:capture3, "gh", "pr", "list", "--repo", repository, "--state", "merged", "--label", label,
84
+ "--search", "merged:>=#{since}", "--limit", "1000", "--json", "mergeCommit", "--jq", ".[].mergeCommit.oid")
85
+ return unless status.success?
86
+
87
+ merges = merged.split
88
+ git("rev-list", "#{release}..HEAD").split.find { |commit| merges.include?(commit) }
89
+ rescue SystemCallError # (no gh)
90
+ nil
91
+ end
92
+
72
93
  # The branch that marks the last commit Duel passed in a push to the
73
94
  # branch pushed (Settings#pushed), for Config#push_base:
74
95
  # duel/passed/main, or for a project in a subdirectory of its
@@ -99,9 +120,9 @@ module Duel
99
120
  # wildcard), the configuration's file, and with each version on its
100
121
  # own Ruby, the version files at the root.
101
122
  def watched
102
- measured = [benchmarks, config_file]
123
+ measured = [*benchmarks, config_file]
103
124
  measured.concat(version_files) if (@settings.ruby || @config.ruby) == :version_file
104
- (@config.watch + measured).uniq
125
+ (Config.list(@config.watch) + measured).uniq
105
126
  end
106
127
 
107
128
  # The baseline's tree, checked out and built if it's a commit.
@@ -111,15 +132,16 @@ module Duel
111
132
 
112
133
  sha = git("rev-parse", "#{ref}^{commit}")
113
134
  dir = checkout
135
+ tree = within(dir)
114
136
  unless git("-C", dir, "rev-parse", "HEAD", allow_failure: true) == sha
115
137
  FileUtils.rm_rf(dir)
116
138
  git("worktree", "prune") # (git's record of it)
117
139
  git("worktree", "add", dir, sha)
118
- build(within(dir), "the baseline") if Dir.exist?(within(dir))
140
+ build(tree, "the baseline") if Dir.exist?(tree)
119
141
  end
120
- raise Error, "the baseline (#{ref}) has no #{prefix}: compare with --base REF, a commit that has it" unless Dir.exist?(within(dir))
142
+ raise Error, "the baseline (#{ref}) has no #{prefix}: compare with --base REF, a commit that has it" unless Dir.exist?(tree)
121
143
 
122
- within(dir)
144
+ tree
123
145
  end
124
146
 
125
147
  # Where the project is in a checkout of its repository at +dir+: as far
@@ -183,12 +205,13 @@ module Duel
183
205
 
184
206
  private
185
207
 
186
- # The benchmarks' directory (see #watched), or for benchmarks at the
187
- # root, their glob, as git matches one.
208
+ # Each of the benchmarks' globs' directory (see #watched), or for
209
+ # benchmarks at the root, its glob, as git matches one.
188
210
  def benchmarks
189
- glob = @config.benchmarks
190
- dir = glob.split("/").take_while { |part| !part.match?(/[*?\[{]/) }.join("/")
191
- dir.empty? ? ":(glob)#{glob}" : dir
211
+ Config.list(@config.benchmarks).map do |glob|
212
+ dir = glob.split("/").take_while { |part| !part.match?(/[*?\[{]/) }.join("/")
213
+ dir.empty? ? ":(glob)#{glob}" : dir
214
+ end
192
215
  end
193
216
 
194
217
  # The configuration's file, from the root (which is its directory).