duel 0.1.0

data/lib/duel/init.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "erb"
+require "fileutils"
+require "open3"
+require "optparse"
+require "pathname"
+require_relative "../duel"
+
+module Duel
+  # `duel init`: sets Duel up in a gem or an app that isn't a Rails app
+  # (duel-rails's generator sets a Rails app up). It writes .duel.rb, a
+  # benchmark to start from, and a GitHub Actions workflow, adds a Rake
+  # task to the Rakefile, and adds StackProf to the Gemfile (for profiles
+  # of regressions, on CRuby). It overwrites nothing without --force, and
+  # writes nothing with --dry-run.
+  # @api private
+  class Init
+    USAGE = "Usage: duel init [options]\n\n" \
+      "Sets Duel up in this project: .duel.rb, a benchmark to start from in duel/, a GitHub Actions workflow,\n" \
+      "a Rake task, and StackProf in the Gemfile.\n\n"
+
+    TEMPLATES = File.join(__dir__, "templates")
+
+    # What's added to a Rakefile.
+    RAKE_TASK = %(require "duel/rake_task"\nDuel::RakeTask.new\n)
+
+    # What's added to a Gemfile, for profiles of regressions: StackProf
+    # runs on CRuby only, so it's left out of JRuby's and TruffleRuby's
+    # bundles.
+    STACKPROF = %(gem "stackprof", platforms: :mri)
+
+    # A Gemfile's development group (or development and test's), with the
+    # indentation of its first line.
+    DEVELOPMENT = /^([ \t]*)group[ \t(]+(?::development(?:,[ \t]*:test)?|:test,[ \t]*:development)[ \t)]+do[ \t]*\n/
+
+    def initialize(argv, out: $stdout, err: $stderr, root: Dir.pwd)
+      @argv = argv
+      @out = out
+      @err = err
+      @root = root
+    end
+
+    # Returns the exit status: 0 set up, 1 a Rails app (not set up), 2 a
+    # bad option.
+    def run
+      rest = catch(:help) { parser.parse(@argv) } or return 0
+      raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
+      return rails if File.exist?(File.join(@root, "config", "application.rb"))
+
+      @out.puts "duel init: a dry run, so nothing is written" if @dry_run
+      files.each { |path, content| write(path, content) }
+      add_rake_task
+      add_stackprof
+      @out.print next_steps
+      0
+    rescue OptionParser::ParseError => e
+      @err.puts "duel init: #{e}"
+      2
+    end
+
+    # The files it writes, from the project's root, to what's in them.
+    def files
+      workflow = Pathname(File.realpath(repository)).relative_path_from(File.realpath(@root)).join(".github", "workflows", workflow_name)
+      {".duel.rb" => render("duel.rb"), benchmark_file => render("benchmark_duel.rb"), workflow.to_s => render("perf.yml")}
+    end
+
+    private
+
+    def parser
+      OptionParser.new do |o|
+        o.banner = USAGE
+        o.on("--force", "Overwrite files that exist") { @force = true }
+        o.on("--dry-run", "Say what it would do, and write nothing") { @dry_run = true }
+        o.on("-h", "--help", "Print this help") do
+          @out.puts o
+          throw :help
+        end
+      end
+    end
+
+    # Says to use duel-rails's generator instead.
+    def rails
+      @err.puts "duel init: this is a Rails app (config/application.rb). Add duel-rails to the Gemfile's development group, " \
+        "and run `bin/rails generate duel:install` instead"
+      1
+    end
+
+    # Writes +content+ to +path+, unless it exists (without --force), and
+    # says what it did.
+    def write(path, content)
+      full = File.join(@root, path)
+      exists = File.exist?(full)
+      return said("skip", "#{path} (it exists: --force to overwrite)") if exists && !@force
+
+      said(exists ? "overwrite" : "create", path)
+      return if @dry_run
+
+      FileUtils.mkdir_p(File.dirname(full))
+      File.write(full, content)
+    end
+
+    # Adds Duel's Rake task to the Rakefile, if there is one without it.
+    def add_rake_task
+      rakefile = File.join(@root, "Rakefile")
+      return unless File.exist?(rakefile)
+
+      text = File.read(rakefile)
+      return said("skip", "Rakefile (it has Duel's task)") if text.include?("duel/rake_task")
+
+      said("append", "Rakefile")
+      File.write(rakefile, gap(text) + RAKE_TASK, mode: "a") unless @dry_run
+    end
+
+    # Adds StackProf to the Gemfile, if there is one without it: in its
+    # development group, if it has one, or else at its end.
+    def add_stackprof
+      gemfile = File.join(@root, "Gemfile")
+      return unless File.exist?(gemfile)
+
+      text = File.read(gemfile)
+      return said("skip", "Gemfile (it has stackprof)") if text.include?("stackprof")
+
+      @stackprof = true
+      grouped = in_development_group(text)
+      said(grouped ? "insert" : "append", "Gemfile (#{STACKPROF}#{", in the development group" if grouped})")
+      File.write(gemfile, grouped || at_the_end(text)) unless @dry_run
+    end
+
+    # The Gemfile's +text+ with StackProf last in its development group,
+    # or nil without one (or without the end of one).
+    def in_development_group(text)
+      group = DEVELOPMENT.match(text) or return
+      indent = group[1]
+      ending = /^#{indent}end\b/.match(group.post_match) or return
+      "#{group.pre_match}#{group}#{ending.pre_match}#{indent}  #{STACKPROF}\n#{ending}#{ending.post_match}"
+    end
+
+    # What goes between a file's +text+ and what's added at its end: a
+    # blank line, if there's anything before it.
+    def gap(text) = text.empty? ? "" : "#{"\n" unless text.end_with?("\n")}\n"
+
+    # The Gemfile's +text+ with StackProf at its end.
+    def at_the_end(text) = "#{text}#{gap(text)}# For profiles of regressions (StackProf runs on CRuby only)\n#{STACKPROF}\n"
+
+    def said(what, path) = @out.puts(format("  %-10s%s", what, path))
+
+    # What to do next, numbered, after a blank line.
+    def next_steps
+      steps = [] #: Array[String]
+      if !gemfile?
+        steps << %(Add gem "duel" to the Gemfile's development group, and run `bundle install`.)
+      elsif @stackprof
+        steps << "Run `bundle install`, for StackProf (added to the Gemfile)."
+      end
+      steps << "Set c.library in .duel.rb to what each version's workers require." unless required
+      steps << "Replace the example in #{benchmark_file} with benchmarks of your own code."
+      steps << "Run `bundle exec duel --force`. It compares the code with itself, and passes when the benchmarks are sound."
+      steps << "Commit and push. On a pull request, the workflow compares the code with the main branch, and comments with the report."
+      "\nNext:\n#{steps.each_with_index.map { |step, i| "  #{i + 1}. #{step}\n" }.join}"
+    end
+
+    def render(name) = ERB.new(File.read(File.join(TEMPLATES, "#{name}.erb")), trim_mode: "-").result(binding)
+
+    # The library the gemspec names (see Config#library), or nil.
+    def library = Config.new(@root).library
+
+    # The library, without a gemspec: lib/'s only Ruby file, or nil.
+    def guess
+      found = Dir.glob("*.rb", base: File.join(@root, "lib"))
+      File.basename(found.first, ".rb") if found.size == 1
+    end
+
+    # What each version's workers require: the library, or the guess.
+    def required = library || guess
+
+    def benchmark_file = File.join("duel", "#{(required || "example").tr("/", "_")}_duel.rb")
+
+    def gemfile? = File.exist?(File.join(@root, "Gemfile")) && File.read(File.join(@root, "Gemfile")).match?(/gem ["']duel["']/)
+
+    def ext = Dir.exist?(File.join(@root, "ext"))
+
+    def version_file = File.exist?(File.join(@root, ".ruby-version"))
+
+    # The Ruby the workflow sets up: this one's (its minor version), or
+    # none, where .ruby-version names it.
+    def ruby_version = (RUBY_VERSION[/\A\d+\.\d+/] unless version_file)
+
+    # The root of the project's git repository: the project's own
+    # directory, outside one.
+    def repository
+      top, _, status = Open3.capture3("git", "-C", @root, "rev-parse", "--show-toplevel")
+      status.success? ? top.chomp : @root
+    end
+
+    # The project's directory, from its repository's root ("gems/parser"),
+    # or nil at the root.
+    def working_directory
+      path = Pathname(File.realpath(@root)).relative_path_from(File.realpath(repository)).to_s
+      path unless path == "."
+    end
+
+    # perf.yml, or for a project in a subdirectory of its repository,
+    # perf-<its path>.yml (perf-gems-parser.yml), so that each of a
+    # monorepo's projects can have its own.
+    def workflow_name = working_directory&.then { |dir| "perf-#{dir.tr("/", "-")}.yml" } || "perf.yml"
+
+    # What marks the workflow's comment on a pull request.
+    def marker = ["duel", working_directory].compact.join(" ")
+  end
+end

data/lib/duel/json_report.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "runner"
+require_relative "summary"
+require_relative "version"
+
+module Duel
+  # A run's report as JSON, for a program to read (`--json PATH`, see
+  # CLI): the same results the Markdown report has (see Summary), in plain
+  # JSON types, numbers as numbers. Its schema is documented in
+  # docs/json.md, and SCHEMA is its version, raised for any change that
+  # isn't a new key.
+  # @api private
+  class JSONReport
+    SCHEMA = 1
+
+    # A run that didn't compare: +status+ ("nothing_to_compare", or
+    # "error"), its +exit+ status, and +message+, what it said.
+    def self.note(status, exit, message, settings)
+      none = [] #: Array[value]
+      nothing = {} #: report
+      head(status, exit, message, settings).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
+        "benchmarks" => none, "across" => nothing, "errors" => none)
+    end
+
+    # What every report starts with.
+    def self.head(status, exit, message, settings)
+      {"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message,
+       "settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
+                      "jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
+                      "warm" => settings.warm, "force" => settings.force}}
+    end
+
+    # +runner+ ran, and made +results+, comparing +baseline+ with
+    # +candidate+ (each {"ref", "commit", "root"} or {"commit",
+    # "uncommitted_changes", "root"}) as +settings+ say, and exits with
+    # +exit+.
+    def initialize(runner, results, settings:, exit:, baseline:, candidate:)
+      @runner = runner
+      @results = results
+      @settings = settings
+      @exit = exit
+      @versions = {"baseline" => baseline, "candidate" => candidate}
+    end
+
+    def to_h
+      rubies = @runner.rubies
+      JSONReport.head(@exit.zero? ? "passed" : "failed", @exit, nil, @settings).merge(
+        "seconds" => @runner.seconds,
+        **@versions.to_h { |side, version| [side, version.merge("ruby" => rubies.fetch(side))] },
+        "plugins" => @runner.plugins.map(&:to_s),
+        "benchmarks" => @results.map { |result| benchmark(result) },
+        "across" => across,
+        "errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} }
+      )
+    end
+
+    private
+
+    # (Runner#explained says nothing of a benchmark that didn't regress.)
+    def benchmark(result)
+      {"name" => result.name, "verdict" => Summary.verdict(result), "passed" => result.passed?, "regressed" => result.regressed,
+       "unsettled" => result.unsettled?, "iterations" => result.n, "seconds" => result.seconds,
+       "confirmations" => result.confirmations.size, "timed" => timed(result), "counted" => counted(result),
+       "peak_memory" => peak_memory(result), "details" => result.details,
+       "explanation" => @runner.explained(result), "error" => result.error, "skipped" => result.skipped}
+    end
+
+    # Each timed measure: its ratio's median, interval and verdict, and
+    # each version's median an iteration.
+    def timed(result)
+      # (Timed, so measured, with an n.)
+      result.timed.to_h do |metric, t|
+        n = result.n #: Integer
+        [metric, {"ratio" => t.fetch("median"), "interval" => t.fetch("interval"), "limit" => t.fetch("limit"),
+                  "pairs" => t.fetch("pairs"), "verdict" => t.fetch("verdict"), "baseline" => t.fetch("baseline").fdiv(n),
+                  "candidate" => t.fetch("candidate").fdiv(n), "unit" => t.fetch("unit"), "plural" => t.fetch("plural"),
+                  "across" => t.fetch("across")}]
+      end
+    end
+
+    # Each counted measure: each version's fewest an iteration, and
+    # whether that's worse.
+    def counted(result)
+      worse = result.worse
+      result.counted.to_h do |metric, c|
+        n = c.fetch("n")
+        [metric, {"baseline" => c.fetch("baseline").fdiv(n), "candidate" => c.fetch("candidate").fdiv(n), "unit" => c.fetch("unit"),
+                  "plural" => c.fetch("plural"), "worse" => worse.include?(metric)}]
+      end
+    end
+
+    def peak_memory(result)
+      result.memory&.then do |memory|
+        {"baseline" => memory.fetch("baseline"), "candidate" => memory.fetch("candidate"), "unit" => "B", "worse" => result.more_memory?}
+      end
+    end
+
+    def across
+      @runner.across.transform_values do |v|
+        {"ratio" => v.fetch("median"), "interval" => v.fetch("interval"), "limit" => v.fetch("limit"), "benchmarks" => v.fetch("benchmarks"),
+         "verdict" => v.fetch("verdict"), "confirmed" => v.fetch("confirmed", false)}
+      end
+    end
+  end
+end

data/lib/duel/lock.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "../duel"
+
+module Duel
+  # One run at a time in a project: two would share its tmp/duel and its
+  # baseline's checkout, each checking out and building over the other's.
+  # A run holds an flock on tmp/duel/lock, which ends with its process
+  # however that ends, so a run that crashed leaves nothing to clear. The
+  # file says whose it is, for a run that finds it taken.
+  # @api private
+  module Lock
+    # An exclusive lock, or none if another holds it.
+    EXCLUSIVE = File::LOCK_EX | File::LOCK_NB
+
+    # Runs the block holding +dir+'s lock. Fails, naming the run that
+    # holds it, if another does.
+    def self.hold(dir)
+      FileUtils.mkdir_p(dir)
+      # (Read from the start, and written at the end: there's no more.)
+      file = File.new(File.join(dir, "lock"), "a+")
+      begin
+        unless file.flock(EXCLUSIVE)
+          # (Empty for a moment, while its run writes its pid.)
+          pid = file.read
+          raise Error, "another duel run#{" (pid #{pid})" unless pid.empty?} is using #{dir}: wait for it to end, or stop it"
+        end
+        file.truncate(0)
+        file.write(Process.pid)
+        file.flush
+        yield
+      ensure
+        file.close
+      end
+    end
+  end
+end