derailed_benchmarks 1.4.2 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ed7fac91722a804da9125b03501653c1e1d78993b9c7d874e5e597656cb3c12
-  data.tar.gz: 5c36d946b6e0533d195f9b285d1d6541e4890a2f2eb37f2ca5d26b4c9434b655
+  metadata.gz: 24d0d07556d2d6676ea8b2c06d58583071fd9302c056d36db05d252480889308
+  data.tar.gz: 0c783db16c1612173975c12760b101fc30cbd5765d296b2fd66506992305ac69
 SHA512:
-  metadata.gz: be591fafddeff784a41a817c51f1f0f4c6a7f0b14b80f16b4792c61564af97f22a38f8cc999d5e7431cae3393cbfe427eebb291e2bc2efac7c578a802a870ec7
-  data.tar.gz: 4e0fceb32c0acbc09f74c91959121a9033b5ac188700eb225c7d51a4cee796665640302e5e7eed5508c870c63fc26a4370ed1ed6e88b15757fc8c291a70a3c88
+  metadata.gz: e37aa667a5fe031a384a343de999a34ef9f1e0ba98f8190d6f86d326ee30f968e3555344fbaf8841f99da85354c2b4383fe1b75ab166eb0df9134d20ee48fd95
+  data.tar.gz: 3f581637e835d808b8264786ffd70159fe534c4147e72e65f5649fe7ecb0b92b9edc215c9edd43e2ca51a510793ae475da3e0318262103bb741823746d9f3421

data/.github/workflows/check_changelog.yml

@@ -1,10 +1,13 @@
 name: Check Changelog
-on: [pull_request]
+
+on:
+ pull_request:
+  types: [opened, reopened, edited, synchronize]
 jobs:
- build:
-   runs-on: ubuntu-latest
-   steps:
-   - uses: actions/checkout@v1
-   - name: Check that CHANGELOG is touched
-     run: |
-       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Check that CHANGELOG is touched
+      run: |
+        cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

data/.travis.yml

@@ -1,18 +1,20 @@
 language: ruby
 rvm:
-  - 2.5.5
-  - 2.6.3
-  - ruby-head
+  - 2.2.10
+  - 2.5.8
+  - 2.7.1
 
 gemfile:
   - gemfiles/rails_5_1.gemfile
-  - gemfiles/rails_5_2.gemfile
   - gemfiles/rails_6_0.gemfile
   - gemfiles/rails_git.gemfile
 
 before_install:
-  - gem install bundler
+  - gem install bundler -v 1.17.3
 
-matrix:
+jobs:
   allow_failures:
-    - rvm: ruby-head
+    - rvm: 2.2.10
+      gemfile: gemfiles/rails_6_0.gemfile
+    - rvm: 2.2.10
+      gemfile: gemfiles/rails_git.gemfile

data/CHANGELOG.md

@@ -1,4 +1,31 @@
-## master (unreleased)
+## HEAD
+
+## 1.8.0
+
+- Ruby 2.2 is now officialy supported and tested (https://github.com/schneems/derailed_benchmarks/pull/177)
+
+## 1.7.0
+
+- Add histogram support to `perf:library` (https://github.com/schneems/derailed_benchmarks/pull/169)
+- Fix bug with `Kernel#require` patch when Zeitwerk is enabled (https://github.com/schneems/derailed_benchmarks/pull/170)
+
+## 1.6.0
+
+- Added the `perf:app` command to compare commits within the same application. (https://github.com/schneems/derailed_benchmarks/pull/157)
+- Allow Rails < 7 and 1.0 <= Thor < 2 (https://github.com/schneems/derailed_benchmarks/pull/168)
+
+## 1.5.0
+
+- Test `perf:library` results against 99% confidence interval in addition to 95% (https://github.com/schneems/derailed_benchmarks/pull/165)
+- Change default, `perf:library` tests do not stop automatically any more (https://github.com/schneems/derailed_benchmarks/pull/164)
+
+## 1.4.4
+
+- Fix alignment of deicmals in output (https://github.com/schneems/derailed_benchmarks/pull/161)
+
+## 1.4.3
+
+- perf:library now uses median instead of average (https://github.com/schneems/derailed_benchmarks/pull/160)
 
 ## 1.4.2
 

data/README.md

@@ -10,7 +10,7 @@ A series of things you can use to benchmark a Rails or Ruby app.
 ## Compatibility/Requirements
 
 This gem has been tested and is known to work with Rails 3.2+ using Ruby
-2.1+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
+2.2+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
 
 For some benchmarks, not all, you'll need to verify you have a working version of curl on your OS:
 
@@ -415,13 +415,23 @@ or point it at your local copy:
 gem 'rails', path: "<path/to/your/local/copy/rails>"
 ```
 
-To run your test:
+To run your tests within the context of your current app/repo:
+
+```
+$ bundle exec derailed exec perf:app
+```
+
+This will automatically test the two latest commits of your library/current directory.
+
+If you'd like to test the Rails library instead, make sure that `ENV[DERAILED_PATH_TO_LIBRARY]` is unset.
 
 ```
 $ bundle exec derailed exec perf:library
 ```
 
-This will automatically test the two latest commits of Rails (or the library you've specified). If you would like to compare against different SHAs you can manually specify them:
+This will automatically test the two latest commits of Rails.
+
+If you would also like to compare against different SHAs you can manually specify them:
 
 ```
 $ SHAS_TO_TEST="7b4d80cb373e,13d6aa3a7b70" bundle exec derailed exec perf:library
@@ -431,7 +441,7 @@ Use a comma to seperate your branch names with the `SHAS_TO_TEST` env var, or om
 
 If you only include one SHA, then derailed will grab the latest commit and compare it to that SHA.
 
-These tests might take a along time to run so the output is stored on disk incase you want to see them in the future, they're at `tmp/library_branches/<timestamp>` and labeled with the same names as your commits.
+These tests might take a along time to run so the output is stored on disk incase you want to see them in the future, they're at `tmp/compare_branches/<timestamp>` and labeled with the same names as your commits.
 
 When the test is done it will output which commit "won" and by how much:
 
@@ -453,7 +463,7 @@ You can use this to test changes in other libraries that aren't rails, you just
 
 > To get the best results before running tests you should close all programs on your laptop, turn on a program to prevent your laptop from going to sleep (or increase your sleep timer). Make sure it's plugged into a power outlet and  go grab a cup of coffee. If you do anything on your laptop while this test is running you risk the chance of skewing your results.
 
-By default derailed will stop once statistical signficance has been detected, you can tune this behavior by setting `DERAILED_STOP_VALID_COUNT` env var. Setting this to a positive number, will increase the number of iterations required that are detected to be statistically significant. For example setting it to 100 might result in 120 runs if it takes 20 runs to detect significance. Generally the more runs you have, the more accurate your averages will be. You can disable this all together by setting `DERAILED_STOP_VALID_COUNT=0` which will force derailed to run all iterations.
+As the test is executing, intermediate results will be printed every 50 iterations.
 
 ## Environment Variables
 

data/derailed_benchmarks.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = ">= 2.1.0"
+  gem.required_ruby_version = ">= 2.2.0"
 
   gem.add_dependency "heapy",           "~> 0"
   gem.add_dependency "memory_profiler", "~> 0"
@@ -28,12 +28,13 @@ Gem::Specification.new do |gem|
   gem.add_dependency "benchmark-ips",   "~> 2"
   gem.add_dependency "rack",            ">= 1"
   gem.add_dependency "rake",            "> 10", "< 14"
-  gem.add_dependency "thor",            "~> 0.19"
+  gem.add_dependency "thor",            ">= 0.19", "< 2"
   gem.add_dependency "ruby-statistics", ">= 2.1"
+  gem.add_dependency "mini_histogram",    ">= 0.2.1"
 
   gem.add_development_dependency "capybara",  "~> 2"
   gem.add_development_dependency "m"
-  gem.add_development_dependency "rails",     "> 3", "<= 6"
+  gem.add_development_dependency "rails",     "> 3", "<= 7"
   gem.add_development_dependency "devise",    "> 3", "< 6"
   gem.add_development_dependency "appraisal", "2.2.0"
 end

data/lib/derailed_benchmarks.rb

@@ -43,8 +43,9 @@ end
 require 'derailed_benchmarks/require_tree'
 require 'derailed_benchmarks/auth_helper'
 
-require 'derailed_benchmarks/stats_in_file'
+require 'derailed_benchmarks/stats_for_file'
 require 'derailed_benchmarks/stats_from_dir'
+require 'derailed_benchmarks/git/switch_project'
 
 if DerailedBenchmarks.gem_is_bundled?("devise")
   DerailedBenchmarks.auth = DerailedBenchmarks::AuthHelpers::Devise.new

data/lib/derailed_benchmarks/core_ext/kernel_require.rb

@@ -11,14 +11,20 @@ ENV['CUT_OFF'] ||= "0.3"
 # Monkey patch kernel to ensure that all `require` calls call the same
 # method
 module Kernel
+  REQUIRE_STACK = []
 
-  private
+  module_function
 
-  alias :original_require :require
-  REQUIRE_STACK = []
+  alias_method :original_require, :require
+  alias_method :original_require_relative, :require_relative
 
   def require(file)
-    Kernel.require(file)
+    measure_memory_impact(file) do |file|
+      # "source_annotation_extractor" is deprecated in Rails 6
+      # # if we don't skip the library it leads to a crash
+      # next if file == "rails/source_annotation_extractor" && Rails.version >= '6.0'
+      original_require(file)
+    end
   end
 
   def require_relative(file)
@@ -29,10 +35,7 @@ module Kernel
     end
   end
 
-  class << self
-    alias :original_require          :require
-    alias :original_require_relative :require_relative
-  end
+  private
 
   # The core extension we use to measure require time of all requires
   # When a file is required we create a tree node with its file name.
@@ -46,7 +49,7 @@ module Kernel
   # When a require returns we remove it from the require stack so we don't
   # accidentally push additional children nodes to it. We then store the
   # memory cost of the require in the tree node.
-  def self.measure_memory_impact(file, &block)
+  def measure_memory_impact(file, &block)
     mem    = GetProcessMem.new
     node   = DerailedBenchmarks::RequireTree.new(file)
 
@@ -68,15 +71,6 @@ end
 TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
 REQUIRE_STACK.push(TOP_REQUIRE)
 
-Kernel.define_singleton_method(:require) do |file|
-  measure_memory_impact(file) do |file|
-    # "source_annotation_extractor" is deprecated in Rails 6
-    # # if we don't skip the library it leads to a crash
-    # next if file == "rails/source_annotation_extractor" && Rails.version >= '6.0'
-    original_require(file)
-  end
-end
-
 class Object
   private
 

data/lib/derailed_benchmarks/git/commit.rb

@@ -0,0 +1,36 @@
+module DerailedBenchmarks
+  # Represents a specific commit in a git repo
+  #
+  # Can be used to get information from the commit or to check it out
+  #
+  # commit = GitCommit.new(path: "path/to/repo", ref: "6e642963acec0ff64af51bd6fba8db3c4176ed6e")
+  # commit.short_sha # => "6e64296"
+  # commit.checkout! # Will check out the current commit at the repo in the path
+  class Git::Commit
+    attr_reader :ref, :description, :time, :short_sha, :log
+
+    def initialize(path: , ref: , log_dir: Pathname.new("/dev/null"))
+      @in_git_path = Git::InPath.new(path)
+      @ref = ref
+      @log = log_dir.join("#{file_safe_ref}.bench.txt")
+
+      Dir.chdir(path) do
+        checkout!
+        @description = @in_git_path.description
+        @short_sha = @in_git_path.short_sha
+        @time = @in_git_path.time
+      end
+    end
+
+    alias :desc :description
+    alias :file :log
+
+    def checkout!
+      @in_git_path.checkout!(ref)
+    end
+
+    private def file_safe_ref
+      ref.gsub('/', ':')
+    end
+  end
+end

data/lib/derailed_benchmarks/git/in_path.rb

@@ -0,0 +1,59 @@
+module DerailedBenchmarks
+  # A class for running commands in a git directory
+  #
+  # It's faster to check if we're already in that directory instead
+  # of having to `cd` into each time. https://twitter.com/schneems/status/1305196730170961920
+  #
+  # Example:
+  #
+  #   in_git_path = InGitPath.new(`bundle info heapy --path`.strip)
+  #   in_git_path.checkout!("f0f92b06156f2274021aa42f15326da041ee9009")
+  #   in_git_path.short_sha # => "f0f92b0"
+  class Git::InPath
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    def description
+      run!("git log --oneline --format=%B -n 1 HEAD | head -n 1")
+    end
+
+    def short_sha
+      run!("git rev-parse --short HEAD")
+    end
+
+    def time_stamp_string
+      run!("git log -n 1 --pretty=format:%ci") # https://stackoverflow.com/a/25921837/147390
+    end
+
+    def branch
+      branch = run!("git rev-parse --abbrev-ref HEAD")
+      branch == "HEAD" ? nil : branch
+    end
+
+    def checkout!(ref)
+      run!("git checkout '#{ref}' 2>&1")
+    end
+
+    def time
+      DateTime.parse(time_stamp_string)
+    end
+
+    def run(cmd)
+      if Dir.pwd == path
+        out = `#{cmd}`.strip
+      else
+        out = `cd #{path} && #{cmd}`.strip
+      end
+      out
+    end
+
+    def run!(cmd)
+      out = run(cmd)
+      raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?
+      out
+    end
+  end
+end

data/lib/derailed_benchmarks/git/switch_project.rb

@@ -0,0 +1,128 @@
+module DerailedBenchmarks
+  class Git
+  end
+end
+require_relative "in_path.rb"
+require_relative "commit.rb"
+
+module DerailedBenchmarks
+  # Wraps two or more git commits in a specific location
+  #
+  # Returns an array of GitCommit objects that can be used to manipulate
+  # and checkout the repo
+  #
+  # Example:
+  #
+  #   `git clone https://sharpstone/default_ruby tmp/default_ruby`
+  #
+  #   project = GitSwitchProject.new(path: "tmp/default_ruby")
+  #
+  # By default it will represent the last two commits:
+  #
+  #   project.commits.length # => 2
+  #
+  # You can pass in explicit REFs in an array:
+  #
+  #   ref_array = ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"]
+  #   project = GitSwitchProject.new(path: "tmp/default_ruby", ref_array: ref_array)
+  #
+  #   puts project.commits.map(&:ref) == ref_array # => true
+  #
+  #
+  # It knows the current branch or sha:
+  #
+  #    `cd tmp/ruby && git checkout -b mybranch`
+  #    project.current_branch_or_sha #=> "mybranch"
+  #
+  # It can be used for safely wrapping checkouts to ensure the project returns to it's original branch:
+  #
+  #    project.restore_branch_on_return do
+  #      project.commits.first.checkout!
+  #      project.current_branch_or_sha # => "da748a593"
+  #    end
+  #
+  #    project.current_branch_or_sha # => "mybranch"
+  class Git::SwitchProject
+    attr_reader :commits
+
+    def initialize(path: , ref_array: [], io: STDOUT, log_dir: "/dev/null")
+      @path = Pathname.new(path)
+
+      @in_git_path = Git::InPath.new(@path.expand_path)
+
+      raise "Must be a path with a .git directory '#{@path}'" if !@path.join(".git").exist?
+      @io = io
+      @commits = []
+      log_dir = Pathname(log_dir)
+
+      expand_refs(ref_array).each do |ref|
+        restore_branch_on_return(quiet: true) do
+          @commits << Git::Commit.new(path: @path, ref: ref, log_dir: log_dir)
+        end
+      end
+
+      if (duplicate = @commits.group_by(&:short_sha).detect {|(k, v)| v.length > 1})
+        raise "Duplicate SHA resolved #{duplicate[0].inspect}: #{duplicate[1].map {|c| "'#{c.ref}' => '#{c.short_sha}'"}.join(", ") } at #{@path}"
+      end
+    end
+
+    def current_branch_or_sha
+      branch_or_sha = @in_git_path.branch
+      branch_or_sha ||= @in_git_path.short_sha
+      branch_or_sha
+    end
+
+    def dirty?
+      !clean?
+    end
+
+    # https://stackoverflow.com/a/3879077/147390
+    def clean?
+      @in_git_path.run("git diff-index --quiet HEAD --") && $?.success?
+    end
+
+    private def status(pattern: "*.gemspec")
+      @in_git_path.run("git status #{pattern}")
+    end
+
+    def restore_branch_on_return(quiet: false)
+      if dirty? && status.include?("gemspec")
+        dirty_gemspec = true
+        unless quiet
+          @io.puts "Working tree at #{@path} is dirty, stashing. This will be popped on return"
+          @io.puts "Bundler modifies gemspec files on git install, this is normal"
+          @io.puts "Original status:\n#{status}"
+        end
+        @in_git_path.run!("git stash")
+      end
+      branch_or_sha = self.current_branch_or_sha
+      yield
+    ensure
+      return unless branch_or_sha
+      @io.puts "Resetting git dir of '#{@path.to_s}' to #{branch_or_sha.inspect}" unless quiet
+
+      @in_git_path.checkout!(branch_or_sha)
+      if dirty_gemspec
+        out = @in_git_path.run!("git stash apply 2>&1")
+        @io.puts "Applying stash of '#{@path.to_s}':\n#{out}" unless quiet
+      end
+    end
+
+    # case ref_array.length
+    # when >= 2
+    #   returns original array
+    # when 1
+    #   returns the given ref plus the one before it
+    # when 0
+    #   returns the most recent 2 refs
+    private def expand_refs(ref_array)
+      return ref_array if ref_array.length >= 2
+
+      @in_git_path.checkout!(ref_array.first) if ref_array.first
+
+      branches_string = @in_git_path.run!("git log --format='%H' -n 2")
+      ref_array = branches_string.split($/)
+      return ref_array
+    end
+  end
+end