duel 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +2 -2
  3. data/CHANGELOG.md +203 -1
  4. data/README.md +41 -1
  5. data/docs/ci.md +170 -76
  6. data/docs/configuration.md +38 -2
  7. data/docs/how-it-works.md +18 -9
  8. data/docs/plugins.md +23 -0
  9. data/lib/duel/across.rb +1 -6
  10. data/lib/duel/baseline.rb +8 -7
  11. data/lib/duel/cli.rb +96 -28
  12. data/lib/duel/comment.rb +296 -0
  13. data/lib/duel/config.rb +124 -4
  14. data/lib/duel/context.rb +33 -34
  15. data/lib/duel/control.rb +52 -0
  16. data/lib/duel/init.rb +255 -20
  17. data/lib/duel/lane.rb +734 -0
  18. data/lib/duel/lock.rb +34 -4
  19. data/lib/duel/memory.rb +14 -5
  20. data/lib/duel/metric.rb +97 -26
  21. data/lib/duel/native.rb +199 -32
  22. data/lib/duel/platform.rb +31 -0
  23. data/lib/duel/process_sample.rb +74 -49
  24. data/lib/duel/resource.rb +71 -42
  25. data/lib/duel/result.rb +165 -0
  26. data/lib/duel/rubies.rb +21 -3
  27. data/lib/duel/runner.rb +33 -1044
  28. data/lib/duel/spawn.rb +327 -0
  29. data/lib/duel/summary.rb +21 -15
  30. data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
  31. data/lib/duel/templates/perf-caller.yml.erb +43 -0
  32. data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
  33. data/lib/duel/templates/perf-comment.yml.erb +61 -44
  34. data/lib/duel/templates/perf.yml.erb +143 -7
  35. data/lib/duel/testing.rb +3 -0
  36. data/lib/duel/version.rb +1 -1
  37. data/lib/duel/worker.rb +13 -4
  38. data/lib/duel/worker_process.rb +142 -0
  39. data/lib/duel.rb +35 -3
  40. data/sig/duel/cli.rbs +19 -0
  41. data/sig/duel/comment.rbs +97 -0
  42. data/sig/duel/config.rbs +57 -2
  43. data/sig/duel/context.rbs +4 -31
  44. data/sig/duel/control.rbs +27 -0
  45. data/sig/duel/init.rbs +95 -1
  46. data/sig/duel/lane.rbs +202 -0
  47. data/sig/duel/lock.rbs +8 -1
  48. data/sig/duel/memory.rbs +1 -1
  49. data/sig/duel/metric.rbs +53 -10
  50. data/sig/duel/native.rbs +109 -9
  51. data/sig/duel/platform.rbs +13 -0
  52. data/sig/duel/process_sample.rbs +7 -9
  53. data/sig/duel/resource.rbs +28 -9
  54. data/sig/duel/result.rbs +164 -0
  55. data/sig/duel/rubies.rbs +4 -0
  56. data/sig/duel/runner.rbs +6 -460
  57. data/sig/duel/spawn.rbs +111 -0
  58. data/sig/duel/summary.rbs +6 -3
  59. data/sig/duel/worker_process.rbs +90 -0
  60. data/sig/duel.rbs +17 -3
  61. metadata +121 -105
data/lib/duel/init.rb CHANGED
@@ -1,12 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "erb"
4
+ require "json"
4
5
  require "fileutils"
5
6
  require "tmpdir"
6
7
  require "open3"
7
8
  require "optparse"
8
9
  require "pathname"
10
+ require "tempfile"
9
11
  require_relative "../duel"
12
+ require_relative "comment"
13
+ require_relative "spawn"
10
14
 
11
15
  module Duel
12
16
  # `duel init`: sets Duel up in a gem or an app that isn't a Rails app
@@ -16,8 +20,11 @@ module Duel
16
20
  # a job that keeps a badge), or with
17
21
  # --gitlab, GitLab CI jobs to include, adds a Rake
18
22
  # task to the Rakefile, and adds StackProf to the Gemfile (for profiles
19
- # of regressions, on CRuby). It overwrites nothing without --force, and
20
- # writes nothing with --dry-run.
23
+ # of regressions, on CRuby). With --reusable, its workflows are short
24
+ # ones that call Duel's reusable workflows (see Reusable), passing on
25
+ # what .duel.rb sets. It overwrites nothing without --force, and
26
+ # writes nothing with --dry-run, or with --check, which says how the
27
+ # workflows differ from what it writes.
21
28
  # @api private
22
29
  class Init
23
30
  USAGE = "Usage: duel init [options]\n\n" \
@@ -38,24 +45,38 @@ module Duel
38
45
  # indentation of its first line.
39
46
  DEVELOPMENT = /^([ \t]*)group[ \t(]+(?::development(?:,[ \t]*:test)?|:test,[ \t]*:development)[ \t)]+do[ \t]*\n/
40
47
 
41
- def initialize(argv, out: $stdout, err: $stderr, root: Dir.pwd)
48
+ # The color of the label it creates (see #create_label): GitHub's
49
+ # yellow, a warning's.
50
+ LABEL_COLOR = "FBCA04"
51
+
52
+ # Where Duel's reusable workflows are (see Reusable), which the
53
+ # workflows it writes with --reusable call, at this release's tag.
54
+ REUSABLE = "sferik/duel-ruby/.github/workflows"
55
+
56
+ def initialize(argv, out: $stdout, err: $stderr, input: $stdin, root: Dir.pwd)
42
57
  @argv = argv
43
58
  @out = out
44
59
  @err = err
60
+ @input = input
45
61
  @root = root
46
62
  end
47
63
 
48
- # Returns the exit status: 0 set up, 1 a Rails app (not set up), 2 a
49
- # bad option.
64
+ # Returns the exit status: 0 set up (or with --check, its workflows
65
+ # are what it writes), 1 a Rails app (not set up, or checked), or with
66
+ # --check, a workflow that differs, 2 a bad option.
50
67
  def run
51
68
  rest = catch(:help) { parser.parse(@argv) } or return 0
52
69
  raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
70
+ raise OptionParser::InvalidOption, "--reusable with --gitlab (GitLab CI has no reusable workflows)" if @reusable && @gitlab
53
71
  return rails if File.exist?(File.join(@root, "config", "application.rb"))
54
72
 
73
+ return check if @check
74
+
55
75
  @out.puts "duel init: a dry run, so nothing is written" if @dry_run
56
76
  files.each { |path, content| write(path, content) }
57
77
  add_rake_task
58
78
  add_stackprof
79
+ create_label
59
80
  @out.print next_steps
60
81
  0
61
82
  rescue OptionParser::ParseError => e
@@ -70,8 +91,10 @@ module Duel
70
91
  return written.merge(top.join(gitlab_file).to_s => render("duel.gitlab-ci.yml")) if @gitlab
71
92
 
72
93
  workflows = top.join(".github", "workflows")
73
- written.merge(workflows.join(workflow_name).to_s => render("perf.yml"),
74
- workflows.join(workflow_name.sub(/\.yml\z/, "-comment.yml")).to_s => render("perf-comment.yml"))
94
+ # (With --reusable, ones that call Duel's.)
95
+ perf, comment = @reusable ? %w[perf-caller.yml perf-comment-caller.yml] : %w[perf.yml perf-comment.yml]
96
+ written.merge(workflows.join(workflow_name).to_s => render(perf),
97
+ workflows.join(workflow_name.sub(/\.yml\z/, "-comment.yml")).to_s => render(comment))
75
98
  end
76
99
 
77
100
  private
@@ -81,8 +104,10 @@ module Duel
81
104
  o.banner = USAGE
82
105
  o.on("--force", "Overwrite files that exist") { @force = true }
83
106
  o.on("--dry-run", "Say what it would do, and write nothing") { @dry_run = true }
107
+ o.on("--check", "Say how the workflows differ from what it writes (with the same options), and write nothing") { @check = true }
84
108
  o.on("--badge", "Write a job that keeps a badge for the README (it needs contents: write)") { @badge = true }
85
109
  o.on("--gitlab", "Write GitLab CI jobs (to include from .gitlab-ci.yml), not GitHub Actions workflows") { @gitlab = true }
110
+ o.on("--reusable", "Write workflows that call Duel's reusable ones (at this release), not whole workflows") { @reusable = true }
86
111
  o.on("-h", "--help", "Print this help") do
87
112
  @out.puts o
88
113
  throw :help
@@ -93,10 +118,124 @@ module Duel
93
118
  # Says to use duel-rails's generator instead.
94
119
  def rails
95
120
  @err.puts "duel init: this is a Rails app (config/application.rb). Add duel-rails to the Gemfile's development group, " \
96
- "and run `bin/rails generate duel:install` instead"
121
+ "and run `bin/rails generate duel:install#{" --check" if @check}` instead"
97
122
  1
98
123
  end
99
124
 
125
+ # Says whether each workflow (or with --gitlab, the GitLab CI jobs) is
126
+ # what it writes, or how it differs, or that it's missing: they're the
127
+ # files a project rarely changes, but Duel's releases do. Returns 0
128
+ # where each is, or else 1.
129
+ def check
130
+ written = files.except(".duel.rb", benchmark_file)
131
+ written.count { |path, content| !same?(path, content) }.zero? ? 0 : 1
132
+ end
133
+
134
+ # Whether the file at +path+ is +content+, saying so, or how it
135
+ # differs (as diff -u has it), or that it's missing. Its lines are
136
+ # compared, whatever they end with: git can check a file out with
137
+ # CRLF line endings, on Windows.
138
+ def same?(path, content)
139
+ full = File.join(@root, path)
140
+ unless File.file?(full)
141
+ said("missing", path)
142
+ return false
143
+ end
144
+
145
+ lines = File.binread(full).gsub("\r\n", "\n")
146
+ # (Nothing, where they're the same.)
147
+ differences = diff(path, lines, content)
148
+ said(differences.empty? ? "same" : "differs", path)
149
+ @out.print differences
150
+ differences.empty?
151
+ end
152
+
153
+ # Creates the label that accepts a regression (Config#accept_label) in
154
+ # the project's GitHub repository, with gh (or with --gitlab, its
155
+ # GitLab project, with glab), if it's missing and the answer to
156
+ # whether to is yes: it asks only at a terminal, and not on a dry run.
157
+ # Where it doesn't, the next steps say to.
158
+ def create_label
159
+ label = accept_label
160
+ return unless label && !@dry_run && @input.tty?
161
+
162
+ repo = (@gitlab ? gitlab_project : cli("gh", "repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner")&.chomp) or return
163
+ names = (@gitlab ? gitlab_labels(repo) : github_labels(repo, label)) or return
164
+ if names.include?(label)
165
+ said("skip", "label #{label} (#{repo} has it)")
166
+ return @label = true
167
+ end
168
+
169
+ @out.print "Create the label #{label.inspect} in #{repo} with #{@gitlab ? "glab" : "gh"}, so a #{request} can accept a regression? [y/N] "
170
+ return unless @input.gets.to_s.strip.match?(/\Ay(?:es)?\z/i)
171
+
172
+ described = ["--repo", repo, "--description", "Accepts a regression that Duel found"]
173
+ created = @gitlab ? cli("glab", "label", "create", "--name", label, "--color", "##{LABEL_COLOR}", *described) :
174
+ cli("gh", "label", "create", label, "--color", LABEL_COLOR, *described)
175
+ return unless created
176
+
177
+ said("create", "label #{label} (#{repo})")
178
+ @label = true
179
+ end
180
+
181
+ # The names of the GitHub repository +repo+'s labels that gh finds by
182
+ # +label+, or nil.
183
+ def github_labels(repo, label) = cli("gh", "label", "list", "--repo", repo, "--search", label, "--limit", "100", "--json", "name",
184
+ "--jq", ".[].name")&.lines(chomp: true)
185
+
186
+ # The project's GitLab project, as glab has it (group/project), or nil.
187
+ def gitlab_project
188
+ project = cli("glab", "repo", "view", "--output", "json") or return
189
+ parsed = JSON.parse(project)
190
+ parsed["path_with_namespace"] if parsed.instance_of?(Hash)
191
+ rescue JSON::ParserError
192
+ nil
193
+ end
194
+
195
+ # The names of the GitLab project +repo+'s labels, every page of them,
196
+ # or nil where glab can't list a page.
197
+ def gitlab_labels(repo)
198
+ names = [] #: Array[String]
199
+ page = 1
200
+ while (listed = cli("glab", "label", "list", "--repo", repo, "--output", "json", "--per-page", "100", "--page", page.to_s))
201
+ labels = JSON.parse(listed)
202
+ names.concat(labels.map { |label| label.fetch("name") })
203
+ return names if labels.size < 100
204
+
205
+ page += 1
206
+ end
207
+ rescue JSON::ParserError
208
+ nil
209
+ end
210
+
211
+ # What +program+ (gh, or glab) prints, run in the project, or nil where
212
+ # it fails (which it says) or isn't there.
213
+ def cli(program, *args)
214
+ out, err, status = Spawn.open3(:capture3, program, *args, chdir: @root)
215
+ return out if status.success?
216
+
217
+ @err.puts "duel init: #{program} #{args.first(2).join(" ")}: #{err.strip}"
218
+ rescue SystemCallError
219
+ nil
220
+ end
221
+
222
+ # How +lines+ (the file at +path+'s) differ from +content+, as diff -u
223
+ # has it.
224
+ # mutant:disable -- the temporary file's name is never shown (diff
225
+ # labels it), and binmode only matters on Windows (where text mode
226
+ # would write CRLF), which mutant doesn't run on
227
+ def diff(path, lines, content)
228
+ Tempfile.create("duel-init") do |file|
229
+ file.binmode
230
+ file.write(lines)
231
+ file.close
232
+ Spawn.open3(:capture2, "diff", "-u", "-L", path, "-L", "#{path} (duel init)", file.path, "-", stdin_data: content).first
233
+ end
234
+ end
235
+
236
+ # What Duel compares: a pull request, or on GitLab, a merge request.
237
+ def request = @gitlab ? "merge request" : "pull request"
238
+
100
239
  # Writes +content+ to +path+, unless it exists (without --force), and
101
240
  # says what it did.
102
241
  def write(path, content)
@@ -108,7 +247,8 @@ module Duel
108
247
  return if @dry_run
109
248
 
110
249
  FileUtils.mkdir_p(File.dirname(full))
111
- File.write(full, content)
250
+ # (Its lines end with LF, as Duel's templates do, on Windows too.)
251
+ File.binwrite(full, content)
112
252
  end
113
253
 
114
254
  # Adds Duel's Rake task to the Rakefile, if there is one without it.
@@ -166,6 +306,8 @@ module Duel
166
306
  end
167
307
  steps << "Set c.library in .duel.rb to what each version's workers require." unless required
168
308
  steps << "Replace the example in #{benchmark_file} with benchmarks of your own code."
309
+ steps << "Create the label #{accept_label.inspect} in your repository, so a #{request} can accept a regression (c.accept_label)." if
310
+ accept_label && !@label
169
311
  steps << "Run `bundle exec duel --force`. It compares the code with itself, and passes when the benchmarks are sound."
170
312
  steps << if @gitlab
171
313
  "Include #{gitlab_file} from .gitlab-ci.yml, and add the tokens it says to, for comments. On a merge request, the job " \
@@ -177,7 +319,52 @@ module Duel
177
319
  "\nNext:\n#{steps.each_with_index.map { |step, i| " #{i + 1}. #{step}\n" }.join}"
178
320
  end
179
321
 
180
- def render(name) = ERB.new(File.read(File.join(TEMPLATES, "#{name}.erb")), trim_mode: "-").result(binding)
322
+ # The template +name+, rendered. (Read with LF line endings, as Git
323
+ # may check it out with CRLF on Windows, where CRuby reads text that
324
+ # way, but JRuby doesn't.)
325
+ def render(name) = ERB.new(File.read(File.join(TEMPLATES, "#{name}.erb")).gsub("\r\n", "\n"), trim_mode: "-").result(binding)
326
+
327
+ # Whether the templates are rendered as Duel's reusable workflows (see
328
+ # Reusable).
329
+ # mutant:disable -- nil (a mutation's) renders as false does
330
+ def reusable? = false
331
+
332
+ # Whether the workflow's jobs write to the repository: to mark a push
333
+ # passed, comment on one, or keep the badge.
334
+ def writes? = mark_passed? || comment_on_pushes? || badge?
335
+
336
+ # What the workflow it writes with --reusable passes on to Duel's
337
+ # Perf workflow, as its inputs (see Reusable): what .duel.rb sets that
338
+ # isn't the input's default, each as YAML has it.
339
+ def perf_inputs
340
+ {
341
+ "ruby-version" => ruby_version,
342
+ "version-file" => (true if version_file),
343
+ "working-directory" => working_directory,
344
+ "accept-label" => accept_label,
345
+ "release-tags" => (set_release_tags unless set_release_tags == "v*"),
346
+ "comment-on-pushes" => (true if comment_on_pushes?),
347
+ "mark-passed" => (true if mark_passed?),
348
+ "badge" => (true if badge?),
349
+ "passed-branch" => (passed_branch if working_directory),
350
+ "badge-branch" => (badge_branch if working_directory && badge?),
351
+ "marker" => (marker if working_directory)
352
+ }.compact.transform_values(&:to_json)
353
+ end
354
+
355
+ # The glob release tags match, as .duel.rb sets it (c.release_tags),
356
+ # or nil where it doesn't load. (The whole workflows take the
357
+ # conventions', and say to set RELEASE_TAGS to match.)
358
+ def set_release_tags = settings&.release_tags
359
+
360
+ # The same, for Duel's Perf comment workflow.
361
+ def comment_inputs
362
+ {
363
+ "accept-label" => accept_label,
364
+ "perf-workflow" => ("Perf (#{working_directory})" if working_directory),
365
+ "marker" => (marker if working_directory)
366
+ }.compact.transform_values(&:to_json)
367
+ end
181
368
 
182
369
  # The project's configuration, as the conventions have it.
183
370
  def config = Config.new(@root)
@@ -262,10 +449,12 @@ module Duel
262
449
  def ruby_version = (RUBY_VERSION[/\A\d+\.\d+/] unless version_file)
263
450
 
264
451
  # The root of the project's git repository: the project's own
265
- # directory, outside one.
452
+ # directory, outside one. (Asked once: the templates ask often.)
266
453
  def repository
267
- top, _, status = Open3.capture3("git", "-C", @root, "rev-parse", "--show-toplevel")
268
- status.success? ? top.chomp : @root
454
+ @repository ||= begin
455
+ top, _, status = Spawn.open3(:capture3, "git", "-C", @root, "rev-parse", "--show-toplevel")
456
+ status.success? ? top.chomp : @root
457
+ end
269
458
  end
270
459
 
271
460
  # The project's directory, from its repository's root ("gems/parser"),
@@ -275,10 +464,13 @@ module Duel
275
464
  path unless path == "."
276
465
  end
277
466
 
278
- # perf.yml, or for a project in a subdirectory of its repository,
279
- # perf-<its path>.yml (perf-gems-parser.yml), so that each of a
280
- # monorepo's projects can have its own.
281
- def workflow_name = working_directory&.then { |dir| "perf-#{dir.tr("/", "-")}.yml" } || "perf.yml"
467
+ # +name+, or for a project in a subdirectory of its repository,
468
+ # +name+-<its path> (perf-gems-parser), so that each of a monorepo's
469
+ # projects has its own.
470
+ def suffixed(name) = [name, working_directory&.tr("/", "-")].compact.join("-")
471
+
472
+ # perf.yml, or for a project in a subdirectory, perf-<its path>.yml.
473
+ def workflow_name = "#{suffixed("perf")}.yml"
282
474
 
283
475
  # What marks the workflow's comment on a pull request.
284
476
  def marker = ["duel", working_directory].compact.join(" ")
@@ -286,7 +478,7 @@ module Duel
286
478
  # Where the GitLab CI jobs are, from the repository's root:
287
479
  # .gitlab/duel.gitlab-ci.yml, or for a project in a subdirectory,
288
480
  # .gitlab/duel-<its path>.gitlab-ci.yml.
289
- def gitlab_file = File.join(".gitlab", "#{["duel", working_directory&.tr("/", "-")].compact.join("-")}.gitlab-ci.yml")
481
+ def gitlab_file = File.join(".gitlab", "#{suffixed("duel")}.gitlab-ci.yml")
290
482
 
291
483
  # The GitLab CI job's name: duel, or for a project in a subdirectory,
292
484
  # duel:<its path>, so each of a monorepo's projects has its own.
@@ -302,10 +494,53 @@ module Duel
302
494
 
303
495
  # What names the branches that mark the last commit Duel passed in a
304
496
  # push, before the branch pushed's name (see Baseline#passed_branch).
305
- def passed_branch = ["duel/passed", working_directory&.tr("/", "-")].compact.join("-")
497
+ def passed_branch = suffixed("duel/passed")
306
498
 
307
499
  # The branch that holds the workflow's badge: duel/badge, or for a
308
500
  # project in a subdirectory, duel/badge-<its path>.
309
- def badge_branch = ["duel/badge", working_directory&.tr("/", "-")].compact.join("-")
501
+ def badge_branch = suffixed("duel/badge")
502
+
503
+ # Duel's reusable workflows, which the workflows `duel init --reusable`
504
+ # writes call: .github/workflows/duel.yml (the Perf workflow's jobs)
505
+ # and duel-comment.yml (the Perf comment workflow's), in Duel's
506
+ # repository. They're the templates `duel init` writes from, rendered
507
+ # with each setting of .duel.rb they need as an input (and a job or a
508
+ # step that would be left out run only where its input says to).
509
+ # `rake workflows` writes them, and a test says they're what's there.
510
+ # @api private
511
+ # (Made with no arguments: Reusable.new([]).)
512
+ class Reusable < Init
513
+ # The files, from the repository's root, to what's in them.
514
+ def files
515
+ workflows = File.join(".github", "workflows")
516
+ {File.join(workflows, "duel.yml") => render("perf.yml"), File.join(workflows, "duel-comment.yml") => render("perf-comment.yml")}
517
+ end
518
+
519
+ private
520
+
521
+ def reusable? = true
522
+
523
+ # Each an input's (see the templates' workflow_call).
524
+ def accept_label = "${{ inputs.accept-label }}"
525
+
526
+ def release_tags = "${{ inputs.release-tags }}"
527
+
528
+ def ruby_version = "${{ inputs.ruby-version }}"
529
+
530
+ def working_directory = "${{ inputs.working-directory }}"
531
+
532
+ def passed_branch = "${{ inputs.passed-branch }}"
533
+
534
+ def badge_branch = "${{ inputs.badge-branch }}"
535
+
536
+ # Each in, run only where its input says to.
537
+ def version_file = true
538
+
539
+ def comment_on_pushes? = true
540
+
541
+ def mark_passed? = true
542
+
543
+ def badge? = true
544
+ end
310
545
  end
311
546
  end