derailed_benchmarks 1.5.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 596677a7dc15b62f384bbf140c0bbe05eb9c890b73ea66e76c34c115ba0c38c1
-  data.tar.gz: 194a74aac18253ad77bb4fcbb464a70478d8127ea894df86e3b8071d6390e264
+  metadata.gz: a32acaf298b80ee6e02cb5fe12aa972c359cfa7efc191305ac32da9198635d77
+  data.tar.gz: 8f6bddd86d0694ce0c33da08b35d4a464552fc8d4792027a02d6cc19c29b1b4a
 SHA512:
-  metadata.gz: 00bc97b216b94edf416d320749b789793f2c6dd2e35b7b56ddd45daaaaec88c61d972cc2b8f52005f9c6cb81f564797377dcd519639b41af366eafe8f88451be
-  data.tar.gz: b0bd2f96ca277f47f1696d7339d10bc9e0c580607def3e5bb5bd3550302749bc53a4575f353bb34c7d104e1c7be4e712af44221b7094a04b636f093b1dd0035a
+  metadata.gz: e53dc17c06b27e39a6018c20c6277f72c83522d3c66551f133d117cd21f2c60a6c53931be6272738bef929223b6c9eabd44b06d134b9290fd255a2dc6810a106
+  data.tar.gz: da5e3bae0cc24205140a7f85cf6dc2aab0eb90cc375b7eec8f4a1b066f034f578114206f51cc7ee8801dd41eb8aa6fd5046735bb4da559232359e41bc303f6e6

data/.circleci/config.yml

@@ -0,0 +1,75 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@1.1.2
+references:
+  run_tests: &run_tests
+    run:
+      name: Run test suite
+      command: bundle exec rake test
+  # Needed because tests execute raw git commands
+  set_git_config: &set_git_config
+    run:
+      name: Set Git config
+      command: git config --global user.email "you@example.com"; git config --global user.name "Your Name"
+  restore: &restore
+    restore_cache:
+      keys:
+        - v1_bundler_deps-{{ .Environment.CIRCLE_JOB }}
+  save: &save
+    save_cache:
+      paths:
+        - ./vendor/bundle
+      key: v1_bundler_deps-{{ .Environment.CIRCLE_JOB }} # CIRCLE_JOB e.g. "ruby-2.5"
+  bundle: &bundle
+    run:
+      name: install dependencies
+      command: |
+        echo "export BUNDLE_JOBS=4" >> $BASH_ENV
+        echo "export BUNDLE_RETRY=3" >> $BASH_ENV
+        echo "export BUNDLE_PATH=$(pwd)/vendor/bundle" >> $BASH_ENV
+        echo "export BUNDLE_GEMFILE=$(pwd)/gemfiles/$GEMFILE_NAME" >> $BASH_ENV
+        source $BASH_ENV
+
+        bundle install
+        bundle update
+        bundle clean
+  mysteps: &mysteps
+    steps:
+      - checkout
+      - <<: *set_git_config
+      - <<: *restore
+      - <<: *bundle
+      - <<: *run_tests
+      - <<: *save
+
+jobs:
+  test:
+    parameters:
+      ruby_version:
+        type: string
+      gemfile:
+        type: string
+    docker:
+      - image: "ruby:<< parameters.ruby_version >>"
+    environment:
+      GEMFILE_NAME: <<parameters.gemfile>>
+    steps:
+      - checkout
+      - <<: *set_git_config
+      - <<: *restore
+      - <<: *bundle
+      - <<: *run_tests
+      - <<: *save
+
+workflows:
+  all-tests:
+    jobs:
+      - test:
+          matrix:
+            parameters:
+              ruby_version: ["2.5.8", "2.7.2", "3.0.0"]
+              gemfile: ["rails_5_2.gemfile", "rails_6_1.gemfile", "rails_git.gemfile"]
+            exclude:
+              - ruby_version: "3.0.0"
+                gemfile: rails_5_2.gemfile
+          name: test-ruby-<<matrix.ruby_version>>-<<matrix.gemfile>>

data/.github/workflows/check_changelog.yml

@@ -1,10 +1,13 @@
 name: Check Changelog
-on: [pull_request]
+
+on:
+ pull_request:
+  types: [opened, reopened, edited, synchronize]
 jobs:
- build:
-   runs-on: ubuntu-latest
-   steps:
-   - uses: actions/checkout@v1
-   - name: Check that CHANGELOG is touched
-     run: |
-       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Check that CHANGELOG is touched
+      run: |
+        cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

data/CHANGELOG.md

@@ -1,4 +1,29 @@
-## master (unreleased)
+## HEAD
+
+## 2.0.0
+
+- Syntax errors easier to debug with `dead_end` gem (https://github.com/schneems/derailed_benchmarks/pull/182)
+- Minimum ruby version is now 2.5 (https://github.com/schneems/derailed_benchmarks/pull/183)
+- Histograms are now printed side-by-side (https://github.com/schneems/derailed_benchmarks/pull/179)
+
+## 1.8.1
+
+- Derailed now tracks memory use from `load` in addition to `require` (https://github.com/schneems/derailed_benchmarks/pull/178)
+- Correct logging of unsuccessful curl requests to file (https://github.com/schneems/derailed_benchmarks/pull/172)
+
+## 1.8.0
+
+- Ruby 2.2 is now officialy supported and tested (https://github.com/schneems/derailed_benchmarks/pull/177)
+
+## 1.7.0
+
+- Add histogram support to `perf:library` (https://github.com/schneems/derailed_benchmarks/pull/169)
+- Fix bug with `Kernel#require` patch when Zeitwerk is enabled (https://github.com/schneems/derailed_benchmarks/pull/170)
+
+## 1.6.0
+
+- Added the `perf:app` command to compare commits within the same application. (https://github.com/schneems/derailed_benchmarks/pull/157)
+- Allow Rails < 7 and 1.0 <= Thor < 2 (https://github.com/schneems/derailed_benchmarks/pull/168)
 
 ## 1.5.0
 

data/README.md

@@ -9,16 +9,13 @@ A series of things you can use to benchmark a Rails or Ruby app.
 
 ## Compatibility/Requirements
 
-This gem has been tested and is known to work with Rails 3.2+ using Ruby
-2.1+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
-
 For some benchmarks, not all, you'll need to verify you have a working version of curl on your OS:
 
 ```
 $ which curl
 /usr/bin/curl
 $ curl -V
-curl 7.37.1 #...
+curl 7.64.1 #...
 ```
 
 ## Install
@@ -415,13 +412,23 @@ or point it at your local copy:
 gem 'rails', path: "<path/to/your/local/copy/rails>"
 ```
 
-To run your test:
+To run your tests within the context of your current app/repo:
+
+```
+$ bundle exec derailed exec perf:app
+```
+
+This will automatically test the two latest commits of your library/current directory.
+
+If you'd like to test the Rails library instead, make sure that `ENV[DERAILED_PATH_TO_LIBRARY]` is unset.
 
 ```
 $ bundle exec derailed exec perf:library
 ```
 
-This will automatically test the two latest commits of Rails (or the library you've specified). If you would like to compare against different SHAs you can manually specify them:
+This will automatically test the two latest commits of Rails.
+
+If you would also like to compare against different SHAs you can manually specify them:
 
 ```
 $ SHAS_TO_TEST="7b4d80cb373e,13d6aa3a7b70" bundle exec derailed exec perf:library
@@ -438,11 +445,37 @@ When the test is done it will output which commit "won" and by how much:
 ```
 ❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️
 
-[7b4d80cb37] "1.8x Faster Partial Caching - Faster Cache Keys" - (10.9711965 seconds)
-  FASTER by:
-    1.0870x [older/newer]
-    8.0026% [(older - newer) / older * 100]
-[13d6aa3a7b] "Merge pull request #36284 from kamipo/fix_eager_loading_with_string_joins" - (11.9255485 seconds)
+[f1ab117] (11.3844 seconds) "I am the new commit" ref: "winner"
+  FASTER 🚀🚀🚀 by:
+    1.0062x [older/newer]
+    0.6147% [(older - newer) / older * 100]
+[5594a2d] (11.4548 seconds) "Old commit" ref: "loser"
+
+Iterations per sample:
+Samples: 100
+
+Test type: Kolmogorov Smirnov
+Confidence level: 99.0 %
+Is significant? (max > critical): true
+D critical: 0.2145966026289347
+D max: 0.26
+
+Histograms (time ranges are in seconds):
+
+   [f1ab117] description:                                        [5594a2d] description:
+     "I am the new commit"                                         "Old commit"
+                  ┌                                        ┐                    ┌                                        ┐
+   [11.2 , 11.28) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 12                             [11.2 , 11.28) ┤▇▇▇▇ 3
+   [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 22                 [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 19
+   [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 30       [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17
+   [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17                       [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 25
+   [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13                           [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 15
+   [11.58, 11.66) ┤▇▇▇▇▇▇▇ 6                                     [11.58, 11.66) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13
+   [11.65, 11.73) ┤ 0                                            [11.65, 11.73) ┤▇▇▇▇ 3
+   [11.73, 11.81) ┤ 0                                            [11.73, 11.81) ┤▇▇▇▇ 3
+   [11.8 , 11.88) ┤ 0                                            [11.8 , 11.88) ┤▇▇▇ 2
+                  └                                        ┘                    └                                        ┘
+                             # of runs in range                                            # of runs in range
 ```
 
 You can provide this to the Rails team along with the example app you used to benchmark (so they can independently verify if needed).

data/derailed_benchmarks.gemspec

@@ -20,20 +20,22 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = ">= 2.1.0"
+  gem.required_ruby_version = ">= 2.5.0"
 
   gem.add_dependency "heapy",           "~> 0"
-  gem.add_dependency "memory_profiler", "~> 0"
+  gem.add_dependency "memory_profiler", ">= 0", "< 2"
   gem.add_dependency "get_process_mem", "~> 0"
   gem.add_dependency "benchmark-ips",   "~> 2"
   gem.add_dependency "rack",            ">= 1"
   gem.add_dependency "rake",            "> 10", "< 14"
-  gem.add_dependency "thor",            "~> 0.19"
+  gem.add_dependency "thor",            ">= 0.19", "< 2"
   gem.add_dependency "ruby-statistics", ">= 2.1"
+  gem.add_dependency "mini_histogram",  ">= 0.3.0"
+  gem.add_dependency "dead_end",        ">= 0"
 
+  gem.add_development_dependency "webrick",  ">= 0"
   gem.add_development_dependency "capybara",  "~> 2"
   gem.add_development_dependency "m"
-  gem.add_development_dependency "rails",     "> 3", "<= 6"
+  gem.add_development_dependency "rails",     "> 3", "<= 7"
   gem.add_development_dependency "devise",    "> 3", "< 6"
-  gem.add_development_dependency "appraisal", "2.2.0"
 end

data/gemfiles/rails_5_1.gemfile

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+#  BUNDLE_GEMFILE="gemfiles/rails_5_1.gemfile" bundle exec m test/integration/tasks_test.rb:30
+#
 # This file was generated by Appraisal
 
 source "https://rubygems.org"
 
-gem "rails", "~> 5.1.0"
+gem "rails", "~> 5.1.7"
 
 group :development, :test do
   gem "sqlite3", platform: [:ruby, :mswin, :mingw]

data/gemfiles/rails_5_2.gemfile

@@ -1,10 +1,10 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
+#
+# BUNDLE_GEMFILE="gemfiles/rails_5_2.gemfile" bundle exec m test/integration/tasks_test.rb:30
 
 source "https://rubygems.org"
 
-gem "rails", "~> 5.2.0"
+gem "rails", "~> 5.2.4.4"
 
 group :development, :test do
   gem "sqlite3", platform: [:ruby, :mswin, :mingw]

data/gemfiles/rails_6_1.gemfile

@@ -0,0 +1,13 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rails", "~> 6.1.1"
+
+group :development, :test do
+  gem "sqlite3", platform: [:ruby, :mswin, :mingw]
+  gem "activerecord-jdbcsqlite3-adapter", "~> 1.3.13", platform: :jruby
+  gem "test-unit", "~> 3.0"
+end
+
+gemspec path: "../"

data/gemfiles/rails_git.gemfile

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-# $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:30
+# $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:50
 
 source "https://rubygems.org"
 
-gem "rails", github: "rails/rails", ref: "3054e1d584e7eca110c69a1f8423f2e0866abbf9"
+gem "rails", github: "rails/rails", ref: "12bb9d32f56883914abcd98fd72e3c68c444808d"
 
 gem 'devise', github: "plataformatec/devise"
 

data/lib/derailed_benchmarks.rb

@@ -2,9 +2,10 @@
 
 require 'time'
 require 'bundler'
-
 require 'get_process_mem'
 
+require 'dead_end'
+
 module DerailedBenchmarks
   def self.gem_is_bundled?(name)
     specs = ::Bundler.locked_gems.specs.each_with_object({}) {|spec, hash| hash[spec.name] = spec }
@@ -43,8 +44,9 @@ end
 require 'derailed_benchmarks/require_tree'
 require 'derailed_benchmarks/auth_helper'
 
-require 'derailed_benchmarks/stats_in_file'
+require 'derailed_benchmarks/stats_for_file'
 require 'derailed_benchmarks/stats_from_dir'
+require 'derailed_benchmarks/git/switch_project'
 
 if DerailedBenchmarks.gem_is_bundled?("devise")
   DerailedBenchmarks.auth = DerailedBenchmarks::AuthHelpers::Devise.new

data/lib/derailed_benchmarks/core_ext/kernel_require.rb

@@ -11,14 +11,24 @@ ENV['CUT_OFF'] ||= "0.3"
 # Monkey patch kernel to ensure that all `require` calls call the same
 # method
 module Kernel
+  REQUIRE_STACK = []
 
-  private
+  module_function
 
-  alias :original_require :require
-  REQUIRE_STACK = []
+  alias_method :original_require, :require
+  alias_method :original_require_relative, :require_relative
+  alias_method(:original_load, :load)
+
+  def load(file, wrap = false)
+    measure_memory_impact(file) do |file|
+      original_load(file)
+    end
+  end
 
   def require(file)
-    Kernel.require(file)
+    measure_memory_impact(file) do |file|
+      original_require(file)
+    end
   end
 
   def require_relative(file)
@@ -29,10 +39,7 @@ module Kernel
     end
   end
 
-  class << self
-    alias :original_require          :require
-    alias :original_require_relative :require_relative
-  end
+  private
 
   # The core extension we use to measure require time of all requires
   # When a file is required we create a tree node with its file name.
@@ -46,7 +53,7 @@ module Kernel
   # When a require returns we remove it from the require stack so we don't
   # accidentally push additional children nodes to it. We then store the
   # memory cost of the require in the tree node.
-  def self.measure_memory_impact(file, &block)
+  def measure_memory_impact(file, &block)
     mem    = GetProcessMem.new
     node   = DerailedBenchmarks::RequireTree.new(file)
 
@@ -64,31 +71,29 @@ module Kernel
   end
 end
 
-# Top level node that will store all require information for the entire app
-TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
-REQUIRE_STACK.push(TOP_REQUIRE)
-
-Kernel.define_singleton_method(:require) do |file|
-  measure_memory_impact(file) do |file|
-    # "source_annotation_extractor" is deprecated in Rails 6
-    # # if we don't skip the library it leads to a crash
-    # next if file == "rails/source_annotation_extractor" && Rails.version >= '6.0'
-    original_require(file)
-  end
-end
 
+# I honestly have no idea why this Object delegation is needed
+# I keep staring at bootsnap and it doesn't have to do this
+# is there a bug in their implementation they haven't caught or
+# am I doing something different?
 class Object
   private
+  def load(path, wrap = false)
+    Kernel.load(path, wrap)
+  end
 
   def require(path)
     Kernel.require(path)
   end
 end
 
-# Don't forget to assign a cost to the top level
-cost_before_requiring_anything = GetProcessMem.new.mb
-TOP_REQUIRE.cost = cost_before_requiring_anything
+# Top level node that will store all require information for the entire app
+TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
+REQUIRE_STACK.push(TOP_REQUIRE)
+TOP_REQUIRE.cost = GetProcessMem.new.mb
+
 def TOP_REQUIRE.print_sorted_children(*args)
   self.cost = GetProcessMem.new.mb - self.cost
   super
 end
+

data/lib/derailed_benchmarks/git/commit.rb

@@ -0,0 +1,36 @@
+module DerailedBenchmarks
+  # Represents a specific commit in a git repo
+  #
+  # Can be used to get information from the commit or to check it out
+  #
+  # commit = GitCommit.new(path: "path/to/repo", ref: "6e642963acec0ff64af51bd6fba8db3c4176ed6e")
+  # commit.short_sha # => "6e64296"
+  # commit.checkout! # Will check out the current commit at the repo in the path
+  class Git::Commit
+    attr_reader :ref, :description, :time, :short_sha, :log
+
+    def initialize(path: , ref: , log_dir: Pathname.new("/dev/null"))
+      @in_git_path = Git::InPath.new(path)
+      @ref = ref
+      @log = log_dir.join("#{file_safe_ref}.bench.txt")
+
+      Dir.chdir(path) do
+        checkout!
+        @description = @in_git_path.description
+        @short_sha = @in_git_path.short_sha
+        @time = @in_git_path.time
+      end
+    end
+
+    alias :desc :description
+    alias :file :log
+
+    def checkout!
+      @in_git_path.checkout!(ref)
+    end
+
+    private def file_safe_ref
+      ref.gsub('/', ':')
+    end
+  end
+end