duel 0.1.0

data/lib/duel/stats.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Duel
+  # The statistics behind a verdict: a distribution-free confidence
+  # interval for the median of paired ratios (candidate / baseline), and
+  # the decision it supports.
+  # @api private
+  module Stats
+    class << self
+      # The 1-based ranks [lower, upper] of the order statistics that bound
+      # a +confidence+ interval for the median of +n+ samples, or nil when
+      # +n+ is too small for one (fewer than 8 samples, at 99%). It's
+      # distribution-free: the true median lies below the k-th smallest
+      # sample with probability P(Binomial(n, 1/2) <= k - 1), whatever the
+      # samples' distribution, so one outlier can't move it.
+      #
+      # The least k whose P(Binomial(n, 1/2) <= k) exceeds the tail, its
+      # terms each found from the last (C(n, k + 1) = C(n, k) * (n - k) /
+      # (k + 1), exactly), so that a thousand benchmarks across the board
+      # take a thousand steps, not a billion.
+      def median_ranks(n, confidence)
+        tail = (1 - confidence) / 2
+        whole = 2**n
+        k = 0
+        term = cumulative = 1
+        # (It ends: at k = n, cumulative is the whole.)
+        until cumulative.fdiv(whole) > tail
+          term = term * (n - k) / (k + 1)
+          k += 1
+          cumulative += term
+        end
+        [k, n + 1 - k] if k.positive?
+      end
+
+      # [low, high] bounds of the +confidence+ interval for the median of
+      # +samples+, or nil (see #median_ranks).
+      def median_interval(samples, confidence)
+        ranks = median_ranks(samples.size, confidence)
+        return unless ranks
+
+        sorted = samples.sort
+        lower, upper = ranks
+        [sorted.fetch(lower - 1), sorted.fetch(upper - 1)]
+      end
+
+      def median(samples)
+        sorted = samples.sort
+        mid = sorted.size / 2
+        sorted.size.odd? ? sorted.fetch(mid) : (sorted.fetch(mid - 1) + sorted.fetch(mid)) / 2.0
+      end
+
+      # What paired ratios (candidate / baseline, so above 1 is slower) say
+      # against +limit+ (a ratio above 1): see #decide.
+      def verdict(ratios, limit:, confidence:) = decide(median_interval(ratios, confidence), limit)
+
+      # What a confidence +interval+ for the median ratio (or nil, for too
+      # few) says against +limit+: :slower when it lies above the limit,
+      # :faster when it lies below 1 / limit, :same when it lies below the
+      # limit (it isn't slower: nothing more needs proving), and :undecided
+      # while it straddles the limit.
+      def decide(interval, limit)
+        return :undecided unless interval
+
+        low, high = interval
+        return :slower if low > limit
+        return :faster if high < 1 / limit
+        return :same if high < limit
+
+        :undecided
+      end
+    end
+  end
+end

data/lib/duel/summary.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+require_relative "runner"
+
+module Duel
+  # A run's report in Markdown, for a pull request: GitHub Actions' job
+  # summary, or a comment (see CLI). It has the verdict, the two versions
+  # (and their Rubies, where they differ), a table of the benchmarks that
+  # changed, regressions first, each regression explained as the terminal
+  # report explains it, those unsettled and those that failed. The rest
+  # are collapsed, so it stays short with hundreds of benchmarks.
+  # @api private
+  class Summary
+    # The most characters it has (a GitHub comment has at most 65,536):
+    # past it, regressions go unexplained, from the last, and the job's
+    # log has the rest.
+    LIMIT = 60_000
+
+    # The most lines of an error it shows.
+    ERROR_LINES = 12
+
+    # A benchmark's verdict, and where it goes in the table (in that
+    # order).
+    VERDICTS = %w[regressed error unsettled faster changed skipped same].freeze
+
+    # The verdicts that fail a run, which stand out.
+    FAILED = %w[regressed error].freeze
+
+    # +text+ as code, in Markdown: in a span of more backticks than any it
+    # has (and spaced from one at either end).
+    def self.code(text)
+      ticks = "`" * ([0, *text.scan(/`+/).map(&:size)].max + 1)
+      padded = (text.start_with?("`") || text.end_with?("`")) ? " #{text} " : text
+      "#{ticks}#{padded}#{ticks}"
+    end
+
+    # A run that didn't compare (+title+ says why), with what it said.
+    def self.note(title, message) = "### Duel: #{title}\n\n#{block(message)}\n"
+
+    # +text+ in a fenced block of its own: fenced with more backticks than
+    # any run of them it has, three at least.
+    def self.block(text)
+      fence = "`" * [3, *text.scan(/`{3,}/).map { |run| run.size + 1 }].max
+      "#{fence}text\n#{text.chomp}\n#{fence}"
+    end
+
+    # +result+'s verdict (see VERDICTS): the JSON report's too (see
+    # JSONReport).
+    def self.verdict(result)
+      if result.error then "error"
+      elsif result.skipped then "skipped"
+      elsif result.regressed.any? then "regressed"
+      elsif result.unsettled? then "unsettled"
+      elsif result.timed.any? { |_, t| t.fetch("verdict") == "faster" } then "faster"
+      elsif changed_counts(result).any? then "changed"
+      else "same"
+      end
+    end
+
+    # {metric => [baseline, candidate]}, an iteration, of the counted
+    # measures that differ.
+    def self.changed_counts(result)
+      result.counted.to_h do |metric, c|
+        n = c.fetch("n")
+        [metric, [c.fetch("baseline").fdiv(n), c.fetch("candidate").fdiv(n)]] #: [String, [Float, Float]]
+      end.reject { |_, (base, candidate)| base == candidate }
+    end
+
+    # +runner+ ran, and made +results+, comparing +baseline+ with
+    # +candidate+ (each a description, in Markdown) as +settings+ say.
+    def initialize(runner, results, settings:, baseline:, candidate:)
+      @runner = runner
+      @results = results
+      @settings = settings
+      @versions = {"baseline" => baseline, "candidate" => candidate}
+      @regressed = results.select { |r| Summary.verdict(r) == "regressed" }
+    end
+
+    # The Markdown: every regression explained, or as many as LIMIT has
+    # room for.
+    def markdown
+      explained = @regressed.size
+      text = render(explained)
+      text = render(explained -= 1) while text.size > LIMIT && explained.positive?
+      text
+    end
+
+    private
+
+    # The Markdown, with the first +explained+ regressions explained.
+    def render(explained)
+      same, changed = sorted.partition { |r| Summary.verdict(r) == "same" }
+      [heading, versions, overview, table(changed), regressions(explained), across, unsettled, errors, unchanged(same)]
+        .compact.join("\n\n") + "\n"
+    end
+
+    def heading
+      errors = @results.count(&:error)
+      failed = [(Runner.count(@regressed.size, "regression") unless @regressed.empty?),
+        ("slower across the board" unless @runner.regressed_across.empty?),
+        (Runner.count(errors, "error") unless errors.zero?)].compact
+      return "### Duel failed: #{failed.join(", ")}" unless failed.empty?
+
+      "### Duel passed: none of #{Runner.count(@results.size, "benchmark")} slower"
+    end
+
+    # Each version, on its Ruby where theirs differ, and the plugins
+    # used.
+    def versions
+      rubies = @runner.rubies
+      plugins = @runner.plugins
+      lines = @versions.map do |side, described|
+        ruby = " on #{Summary.code(rubies.fetch(side))}" if rubies.values.uniq.size > 1
+        "- **#{side.capitalize}:** #{described}#{ruby}"
+      end
+      lines << "- **Plugins:** #{plugins.map { |plugin| Summary.code(plugin.to_s) }.join(", ")}" unless plugins.empty?
+      lines.join("\n")
+    end
+
+    def overview
+      format("%s in %.0fs. A timing is slower when a %s%% confidence interval for its ratio, the candidate's to the " \
+        "baseline's, is above %s.", Runner.count(@results.size, "benchmark"), @runner.seconds,
+        (@settings.confidence * 100).round(1), @settings.limit)
+    end
+
+    # The results, in the order of their VERDICTS, and the run's order in
+    # each (grouped, not sorted: Ruby's sort isn't stable everywhere).
+    def sorted
+      grouped = @results.group_by { |r| Summary.verdict(r) }
+      VERDICTS.flat_map { |shown| grouped.fetch(shown, []) }
+    end
+
+    def table(changed)
+      return if changed.empty?
+
+      rows = changed.map do |r|
+        shown = Summary.verdict(r)
+        row(Summary.code(name(r)), FAILED.include?(shown) ? "**#{shown}**" : shown, timings(r), counts(r))
+      end
+      ([row("Benchmark", "Verdict", "Timings (ratio [interval])", "Counts an iteration"), "|---|---|---|---|"] + rows).join("\n")
+    end
+
+    # Those that didn't change, collapsed.
+    def unchanged(same)
+      return if same.empty?
+
+      rows = same.map { |r| row(Summary.code(name(r)), timings(r)) }
+      "<details>\n<summary>#{Runner.count(same.size, "benchmark")} unchanged</summary>\n\n" \
+        "#{[row("Benchmark", "Timings (ratio [interval])"), "|---|---|", *rows].join("\n")}\n\n</details>"
+    end
+
+    # A table's row of +cells+.
+    def row(*cells) = "| #{cells.join(" | ")} |"
+
+    # A benchmark's name, for a table's cell.
+    def name(result) = result.name.tr("\n", " ").gsub("|", "\\|")
+
+    # Each timed measure's ratio and its interval, with its verdict where
+    # it isn't the same (or why it was skipped).
+    def timings(result)
+      skipped = result.skipped
+      return skipped.gsub("|", "\\|") if skipped
+
+      result.timed.map do |metric, t|
+        low, high = t.fetch("interval")
+        shown = t.fetch("verdict")
+        format("%s %.3f [%.3f, %.3f]%s", metric, t.fetch("median"), low, high, (" #{shown}" unless shown == "same"))
+      end.join("<br>")
+    end
+
+    # Each counted measure that changed, an iteration, and peak memory
+    # that grew too much.
+    def counts(result)
+      counted = Summary.changed_counts(result).map do |metric, (base, candidate)|
+        format("%s %.4g → %.4g", metric, base, candidate)
+      end
+      if result.more_memory?
+        memory = result.memory #: Hash[String, Integer]
+        counted << "peak_memory #{mb(memory.fetch("baseline"))} → #{mb(memory.fetch("candidate"))}"
+      end
+      counted.join("<br>")
+    end
+
+    def mb(bytes) = format("%.1fMB", bytes / 1_048_576.0)
+
+    # The first +explained+ regressions, explained as the terminal report
+    # explains them (and the rest named).
+    def regressions(explained)
+      return if @regressed.empty?
+
+      shown = @regressed.first(explained).map do |r|
+        what = r.mixed? ? "slower in every measurement, by one of #{r.regressed.join(", ")}" : "worse #{r.regressed.join(", ")} in every measurement"
+        lines = @runner.explained(r)
+        said = "**#{Summary.code(r.name)}**: #{what}"
+        lines.empty? ? said : "#{said}\n\n#{Summary.block(lines.join("\n"))}"
+      end
+      rest = @regressed.size - shown.size
+      shown << "#{Runner.count(rest, shown.empty? ? "regression" : "more regression")} explained in the job's log." if rest.positive?
+      "#### Regressions\n\n#{shown.join("\n\n")}"
+    end
+
+    # The verdicts across the board, of the metrics with enough
+    # benchmarks to tell.
+    def across
+      lines = @runner.across.filter_map do |metric, v|
+        low, high = v.fetch("interval")
+        next unless low
+
+        confirmed = " in both measurements" if v["confirmed"]
+        format("- %s %.3f [%.3f, %.3f] %s%s (limit %.4f, %s)", metric, v.fetch("median"), low, high, v.fetch("verdict"), confirmed,
+          v.fetch("limit"), Runner.count(v.fetch("benchmarks"), "benchmark"))
+      end
+      "#### Across the board\n\n#{lines.join("\n")}" unless lines.empty?
+    end
+
+    def unsettled
+      names = @results.select { |r| Summary.verdict(r) == "unsettled" }.map { |r| Summary.code(r.name) }
+      return if names.empty?
+
+      "#### Unsettled\n\nA timing too close to the limit to settle doesn't fail the run, but one that's unsettled in run " \
+        "after run is noisy: #{names.join(", ")}."
+    end
+
+    def errors
+      failed = @results.select(&:error)
+      return if failed.empty?
+
+      shown = failed.map do |r|
+        error = r.error #: String
+        lines = error.lines
+        cut = "\n(#{Runner.count(lines.size - ERROR_LINES, "more line")} in the job's log)" if lines.size > ERROR_LINES
+        "**#{Summary.code(r.name)}**\n\n#{Summary.block(lines.first(ERROR_LINES).join)}#{cut}"
+      end
+      "#### Errors\n\n#{shown.join("\n\n")}"
+    end
+  end
+end

data/lib/duel/templates/benchmark_duel.rb.erb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# Benchmarks, which Duel runs on two versions of the code, the baseline
+# (where your branch left main) and the candidate (your changes), and
+# fails if the candidate is slower at any of them. Each version's workers
+# load <%= required ? "#{required} and " : "" %>every duel/**/*_duel.rb file. See
+# https://github.com/sferik/duel-ruby/blob/main/docs/benchmarks.md.
+#
+# To write benchmarks that tell a regression from noise:
+#
+# - Benchmark the work that users depend on being fast: the main entry
+#   points, and whatever runs in a loop or once per request. Cover typical
+#   inputs, and the small and large extremes, as separate benchmarks.
+# - Name each one "area: what it does". `duel --filter "^parse:"` picks
+#   out an area.
+# - Time only the work. Build inputs and read fixtures in `once`, which
+#   isn't timed. To prepare each sample's work, take |n| (the
+#   iterations) and wrap the preparation in `untimed`.
+# - Make every iteration alike: fixed inputs, seeded randomness
+#   (Random.new(42)), and caches warmed in `once`. Undo what an
+#   iteration changes with an `around_iteration` hook, or put it back
+#   with a `before_sample` hook.
+# - Aim for microseconds to a few milliseconds an iteration, and split
+#   anything slower.
+# - Use `only: :count` for work that's mostly waiting (on a socket, say),
+#   and `only: :timing` for work on threads of its own.
+# - Use only the API that the main branch has: a benchmark that the
+#   baseline can't run is skipped.
+
+# An example, to replace with benchmarks of your own code.
+duel "example: join words" do
+  once { @words = %w[the quick brown fox jumps over the lazy dog] } # not timed
+  @words.join(" ") # timed
+end

data/lib/duel/templates/duel.rb.erb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Duel's configuration: https://github.com/sferik/duel-ruby/blob/main/docs/configuration.md
+#
+# Duel follows conventions, so set only what differs: benchmarks are in
+# duel/**/*_duel.rb, each version's lib/ is first on the load path, a
+# change is compared when it touches lib/ or ext/, and releases are
+# tagged v*.
+Duel.configure do |c|
+<% if library -%>
+  # The library each version's workers require (from the gemspec, unless
+  # this says otherwise).
+  # c.library = "<%= library %>"
+<% elsif guess -%>
+  # The library each version's workers require, from its lib/.
+  c.library = "<%= guess %>"
+<% else -%>
+  # The library each version's workers require, from its lib/ (there's no
+  # gemspec to name it). Or, to load each version some other way:
+  #   c.load { |root| require File.join(root, "config", "boot") }
+  # c.library = "my_library"
+<% end -%>
+<% if ext -%>
+
+  # Builds each version's C extension, so that each is measured with its
+  # own build.
+  c.build = "bundle exec rake compile"
+<% end -%>
+<% if version_file -%>
+
+  # Each version runs on the Ruby that its own .ruby-version names, so a
+  # branch that upgrades Ruby is compared on each version's Ruby.
+  c.ruby = :version_file
+<% end -%>
+end

data/lib/duel/templates/perf.yml.erb

@@ -0,0 +1,97 @@
+name: Perf
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  schedule:
+    - cron: "0 4 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: write # to comment on a pull request with the report
+
+jobs:
+  duel:
+    name: Duel
+    runs-on: ubuntu-latest
+<% if working_directory -%>
+    # The project is in a subdirectory of the repository: Duel runs there,
+    # and checks out the whole repository for the baseline.
+    defaults:
+      run:
+        working-directory: <%= working_directory %>
+<% end -%>
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0 # the baseline comes from history, tags included
+      # (And whatever else your tests need set up: packages, services.)
+<% if version_file -%>
+      # Each version runs on the Ruby its own .ruby-version names (see
+      # .duel.rb): for a pull request that upgrades Ruby, the main branch's
+      # too, linked where Duel finds it (as chruby would). It's set up
+      # before the project's own Ruby, which runs Duel.
+      - name: Find the baseline's Ruby
+        id: baseline
+        if: github.event_name == 'pull_request'
+        run: echo "ruby=$(git show "origin/$GITHUB_BASE_REF:./.ruby-version" 2>/dev/null)" >> "$GITHUB_OUTPUT"
+      - uses: ruby/setup-ruby@v1
+        id: baseline-ruby
+        if: steps.baseline.outputs.ruby != ''
+        with:
+          ruby-version: ${{ steps.baseline.outputs.ruby }}
+      - name: Link the baseline's Ruby where Duel finds it
+        if: steps.baseline.outputs.ruby != ''
+        run: |
+          prefix="${{ steps.baseline-ruby.outputs.ruby-prefix }}"
+          mkdir -p ~/.rubies
+          ln -sfn "$prefix" ~/.rubies/ruby-"$("$prefix/bin/ruby" -e 'print RUBY_VERSION')"
+<% end -%>
+      - uses: ruby/setup-ruby@v1
+        with:
+<% if working_directory -%>
+          working-directory: <%= working_directory %>
+<% end -%>
+<% if ruby_version -%>
+          ruby-version: "<%= ruby_version %>"
+<% end -%>
+          bundler-cache: true
+      # Against the main branch for a pull request, what was there before
+      # for a push, and the last release on a schedule (slow creep). Duel
+      # adds its report to the job's summary, and writes it to duel.md for
+      # the comment.
+      - name: Compare
+        env:
+          DUEL_BASE: ${{ github.event_name == 'schedule' && 'last-release' || github.event_name == 'push' && github.event.before || '' }}
+          # (Run by hand, it compares anyway: with nothing changed, the code
+          # with itself, which checks the setup.)
+          DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}
+        run: |
+          # (A branch's first push, or one that rewrote history, has no
+          # commit before it here to compare against.)
+          if [ "$GITHUB_EVENT_NAME" = push ] && ! git cat-file -e "$DUEL_BASE^{commit}" 2>/dev/null; then
+            echo "::notice::No commit before this push to compare against"
+            exit 0
+          fi
+          bundle exec duel --markdown "$RUNNER_TEMP/duel.md"
+      # The report, as a comment on the pull request: one comment, updated
+      # on every push. (A pull request from a fork gets no comment, as its
+      # token can't write one: the job's summary has the report.)
+      - name: Comment on the pull request
+        if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR: ${{ github.event.pull_request.number }}
+        run: |
+          [ -f "$RUNNER_TEMP/duel.md" ] || exit 0
+          marker="<!-- <%= marker %> -->"
+          { echo "$marker"; cat "$RUNNER_TEMP/duel.md"; } > "$RUNNER_TEMP/comment.md"
+          ids=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
+            --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id")
+          if [ -n "$ids" ]; then
+            gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/${ids%%$'\n'*}" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
+          else
+            gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
+          fi