derailed_benchmarks 1.7.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 843a6c3b99ee45120c15af62a592e9bbecb2d88f0917fa292faf9f7128ab721a
-  data.tar.gz: f1cdfb9c4145c27a8dac7f2c868077c6359213c3c367f2cf8a0f8909f50c9468
+  metadata.gz: 9519f087d4eeae220039bb6d4566757d627655ac96789809608b9a93af642b1d
+  data.tar.gz: a3f52d5df02db7a7c94043abe86cdf8d2f2bfad72595d36410ac55fc6b036faa
 SHA512:
-  metadata.gz: 1936293cd836af98e6af454a7482534c7481b938f4944813a5261a04a9b766b0a169b866397b3d9745d25316cd8916df3b74fc1fd4f1ff46f6e5753eba92bd96
-  data.tar.gz: 7de041007e174314699cac83eef5b70626e5447eddd705190d68d74cc7d50c0b9d660d032127cf747ea67566a096bbca4aafa05774c413da6f4ca802c8da2c08
+  metadata.gz: 817dd67a37109e785bd8c87bdc12235c41bf8de4fa7803cf241aee6c285fcf760290e52598534498a798efd3e1b0505dd39e2124413b8c65be8b6223a6b01439
+  data.tar.gz: 0c2aeeffa9460fd920ac28945ce081cd8134397999586cc0c4e508833d9bbd93d5924c2438439fc5445ee403c345ab1149406f67fc0c468e598c44562ffd0127

data/.circleci/config.yml

@@ -0,0 +1,75 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@1.1.2
+references:
+  run_tests: &run_tests
+    run:
+      name: Run test suite
+      command: bundle exec rake test
+  # Needed because tests execute raw git commands
+  set_git_config: &set_git_config
+    run:
+      name: Set Git config
+      command: git config --global user.email "you@example.com"; git config --global user.name "Your Name"
+  restore: &restore
+    restore_cache:
+      keys:
+        - v1_bundler_deps-{{ .Environment.CIRCLE_JOB }}
+  save: &save
+    save_cache:
+      paths:
+        - ./vendor/bundle
+      key: v1_bundler_deps-{{ .Environment.CIRCLE_JOB }} # CIRCLE_JOB e.g. "ruby-2.5"
+  bundle: &bundle
+    run:
+      name: install dependencies
+      command: |
+        echo "export BUNDLE_JOBS=4" >> $BASH_ENV
+        echo "export BUNDLE_RETRY=3" >> $BASH_ENV
+        echo "export BUNDLE_PATH=$(pwd)/vendor/bundle" >> $BASH_ENV
+        echo "export BUNDLE_GEMFILE=$(pwd)/gemfiles/$GEMFILE_NAME" >> $BASH_ENV
+        source $BASH_ENV
+
+        bundle install
+        bundle update
+        bundle clean
+  mysteps: &mysteps
+    steps:
+      - checkout
+      - <<: *set_git_config
+      - <<: *restore
+      - <<: *bundle
+      - <<: *run_tests
+      - <<: *save
+
+jobs:
+  test:
+    parameters:
+      ruby_version:
+        type: string
+      gemfile:
+        type: string
+    docker:
+      - image: "ruby:<< parameters.ruby_version >>"
+    environment:
+      GEMFILE_NAME: <<parameters.gemfile>>
+    steps:
+      - checkout
+      - <<: *set_git_config
+      - <<: *restore
+      - <<: *bundle
+      - <<: *run_tests
+      - <<: *save
+
+workflows:
+  all-tests:
+    jobs:
+      - test:
+          matrix:
+            parameters:
+              ruby_version: ["2.5.8", "2.7.2", "3.0.0"]
+              gemfile: ["rails_5_2.gemfile", "rails_6_1.gemfile", "rails_git.gemfile"]
+            exclude:
+              - ruby_version: "3.0.0"
+                gemfile: rails_5_2.gemfile
+          name: test-ruby-<<matrix.ruby_version>>-<<matrix.gemfile>>

data/.github/workflows/check_changelog.yml

@@ -1,10 +1,13 @@
 name: Check Changelog
-on: [pull_request]
+
+on:
+ pull_request:
+  types: [opened, reopened, edited, synchronize]
 jobs:
- build:
-   runs-on: ubuntu-latest
-   steps:
-   - uses: actions/checkout@v1
-   - name: Check that CHANGELOG is touched
-     run: |
-       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Check that CHANGELOG is touched
+      run: |
+        cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

data/CHANGELOG.md

@@ -1,4 +1,27 @@
-## master (unreleased)
+## HEAD
+
+## 2.1.0
+
+- Add `perf:heap_diff` tool (https://github.com/schneems/derailed_benchmarks/pull/193)
+
+## 2.0.1
+
+- `rack-test` dependency added (https://github.com/schneems/derailed_benchmarks/pull/187)
+
+## 2.0.0
+
+- Syntax errors easier to debug with `dead_end` gem (https://github.com/schneems/derailed_benchmarks/pull/182)
+- Minimum ruby version is now 2.5 (https://github.com/schneems/derailed_benchmarks/pull/183)
+- Histograms are now printed side-by-side (https://github.com/schneems/derailed_benchmarks/pull/179)
+
+## 1.8.1
+
+- Derailed now tracks memory use from `load` in addition to `require` (https://github.com/schneems/derailed_benchmarks/pull/178)
+- Correct logging of unsuccessful curl requests to file (https://github.com/schneems/derailed_benchmarks/pull/172)
+
+## 1.8.0
+
+- Ruby 2.2 is now officialy supported and tested (https://github.com/schneems/derailed_benchmarks/pull/177)
 
 ## 1.7.0
 

data/README.md

@@ -9,16 +9,13 @@ A series of things you can use to benchmark a Rails or Ruby app.
 
 ## Compatibility/Requirements
 
-This gem has been tested and is known to work with Rails 3.2+ using Ruby
-2.1+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
-
 For some benchmarks, not all, you'll need to verify you have a working version of curl on your OS:
 
 ```
 $ which curl
 /usr/bin/curl
 $ curl -V
-curl 7.37.1 #...
+curl 7.64.1 #...
 ```
 
 ## Install
@@ -201,12 +198,17 @@ You can run commands against your app by running `$ derailed exec`. There are se
 ```
 $ bundle exec derailed exec --help
   $ derailed exec perf:allocated_objects  # outputs allocated object diff after app is called TEST_COUNT times
+  $ derailed exec perf:app  # runs the performance test against two most recent commits of the current app
   $ derailed exec perf:gc  # outputs GC::Profiler.report data while app is called TEST_COUNT times
+  $ derailed exec perf:heap  # heap analyzer
   $ derailed exec perf:ips  # iterations per second
+  $ derailed exec perf:library  # runs the same test against two different branches for statistical comparison
   $ derailed exec perf:mem  # show memory usage caused by invoking require per gem
-  $ derailed exec perf:objects  # profiles ruby allocation
   $ derailed exec perf:mem_over_time  # outputs memory usage over time
+  $ derailed exec perf:objects  # profiles ruby allocation
+  $ derailed exec perf:stackprof  # stackprof
   $ derailed exec perf:test  # hits the url TEST_COUNT times
+  $ derailed exec perf:heap_diff  # three heaps generation for comparison
 ```
 
 Instead of going over each command we'll look at common problems and which commands are best used to diagnose them. Later on we'll cover all of the environment variables you can use to configure derailed benchmarks in it's own section.
@@ -274,7 +276,7 @@ This is similar to `$ bundle exec derailed bundle:objects` however it includes o
 
 ## I want a Heap Dump
 
-If you're still struggling with runtime memory you can generate a heap dump that can later be analyzed using [heap_inspect](https://github.com/schneems/heapy).
+If you're still struggling with runtime memory you can generate a heap dump that can later be analyzed using [heapy](https://github.com/schneems/heapy).
 
 ```
 $ bundle exec derailed exec perf:heap
@@ -298,6 +300,40 @@ For more help on getting data from a heap dump see
 $ heapy --help
 ```
 
+### I want more heap dumps
+
+When searching for a leak, you can use heap dumps for comparison to see what is
+retained. See [SamSaffron's slides](https://speakerdeck.com/samsaffron/why-ruby-2-dot-1-excites-me?slide=27)
+(or [a more recent inspired blog post](https://blog.skylight.io/hunting-for-leaks-in-ruby/))
+for a clear example. You can generate 3 dumps (one every `TEST_COUNT` calls) using the
+next command:
+
+```
+$ bundle exec derailed exec perf:heap
+Endpoint: "/"
+Running 1000 times
+Heap file generated: "tmp/2021-05-06T15:19:26+02:00-heap-0.ndjson"
+Running 1000 times
+Heap file generated: "tmp/2021-05-06T15:19:26+02:00-heap-1.ndjson"
+Running 1000 times
+Heap file generated: "tmp/2021-05-06T15:19:26+02:00-heap-2.ndjson"
+
+Diff
+====
+Retained STRING 90 objects of size 4790/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/rack-2.2.3/lib/rack/utils.rb:461
+Retained ICLASS 20 objects of size 800/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/sinatra-contrib-2.0.8.1/lib/sinatra/namespace.rb:198
+Retained DATA 20 objects of size 1360/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/2.7.0/monitor.rb:238
+Retained STRING 20 objects of size 800/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/rack-protection-2.0.8.1/lib/rack/protection/xss_header.rb:20
+Retained STRING 10 objects of size 880/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/newrelic_rpm-5.4.0.347/lib/new_relic/agent/transaction.rb:890
+Retained CLASS 10 objects of size 4640/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/sinatra-contrib-2.0.8.1/lib/sinatra/namespace.rb:198
+Retained IMEMO 10 objects of size 480/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/sinatra-2.0.8.1/lib/sinatra/base.rb:1017
+...
+
+Run `$ heapy --help` for more options
+
+Also read https://speakerdeck.com/samsaffron/why-ruby-2-dot-1-excites-me?slide=27 to understand better what you are reading.
+```
+
 ### Memory Is large at boot.
 
 Ruby memory typically goes in one direction, up. If your memory is large when you boot the application it will likely only increase. In addition to debugging memory retained from dependencies obtained while running `$ derailed bundle:mem` you'll likely want to see how your own files contribute to memory use.
@@ -448,11 +484,37 @@ When the test is done it will output which commit "won" and by how much:
 ```
 ❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️
 
-[7b4d80cb37] "1.8x Faster Partial Caching - Faster Cache Keys" - (10.9711965 seconds)
-  FASTER by:
-    1.0870x [older/newer]
-    8.0026% [(older - newer) / older * 100]
-[13d6aa3a7b] "Merge pull request #36284 from kamipo/fix_eager_loading_with_string_joins" - (11.9255485 seconds)
+[f1ab117] (11.3844 seconds) "I am the new commit" ref: "winner"
+  FASTER 🚀🚀🚀 by:
+    1.0062x [older/newer]
+    0.6147% [(older - newer) / older * 100]
+[5594a2d] (11.4548 seconds) "Old commit" ref: "loser"
+
+Iterations per sample:
+Samples: 100
+
+Test type: Kolmogorov Smirnov
+Confidence level: 99.0 %
+Is significant? (max > critical): true
+D critical: 0.2145966026289347
+D max: 0.26
+
+Histograms (time ranges are in seconds):
+
+   [f1ab117] description:                                        [5594a2d] description:
+     "I am the new commit"                                         "Old commit"
+                  ┌                                        ┐                    ┌                                        ┐
+   [11.2 , 11.28) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 12                             [11.2 , 11.28) ┤▇▇▇▇ 3
+   [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 22                 [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 19
+   [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 30       [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17
+   [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17                       [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 25
+   [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13                           [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 15
+   [11.58, 11.66) ┤▇▇▇▇▇▇▇ 6                                     [11.58, 11.66) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13
+   [11.65, 11.73) ┤ 0                                            [11.65, 11.73) ┤▇▇▇▇ 3
+   [11.73, 11.81) ┤ 0                                            [11.73, 11.81) ┤▇▇▇▇ 3
+   [11.8 , 11.88) ┤ 0                                            [11.8 , 11.88) ┤▇▇▇ 2
+                  └                                        ┘                    └                                        ┘
+                             # of runs in range                                            # of runs in range
 ```
 
 You can provide this to the Rails team along with the example app you used to benchmark (so they can independently verify if needed).

data/derailed_benchmarks.gemspec

@@ -20,22 +20,23 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = ">= 2.1.0"
+  gem.required_ruby_version = ">= 2.5.0"
 
   gem.add_dependency "heapy",           "~> 0"
-  gem.add_dependency "memory_profiler", "~> 0"
+  gem.add_dependency "memory_profiler", ">= 0", "< 2"
   gem.add_dependency "get_process_mem", "~> 0"
   gem.add_dependency "benchmark-ips",   "~> 2"
   gem.add_dependency "rack",            ">= 1"
   gem.add_dependency "rake",            "> 10", "< 14"
   gem.add_dependency "thor",            ">= 0.19", "< 2"
   gem.add_dependency "ruby-statistics", ">= 2.1"
-  gem.add_dependency "unicode_plot",    ">= 0.0.4", "< 1.0.0"
-  gem.add_dependency "mini_histogram",    "~> 0"
+  gem.add_dependency "mini_histogram",  ">= 0.3.0"
+  gem.add_dependency "dead_end",        ">= 0"
+  gem.add_dependency "rack-test",       ">= 0"
 
+  gem.add_development_dependency "webrick",  ">= 0"
   gem.add_development_dependency "capybara",  "~> 2"
   gem.add_development_dependency "m"
   gem.add_development_dependency "rails",     "> 3", "<= 7"
   gem.add_development_dependency "devise",    "> 3", "< 6"
-  gem.add_development_dependency "appraisal", "2.2.0"
 end

data/gemfiles/rails_5_1.gemfile

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+#  BUNDLE_GEMFILE="gemfiles/rails_5_1.gemfile" bundle exec m test/integration/tasks_test.rb:30
+#
 # This file was generated by Appraisal
 
 source "https://rubygems.org"
 
-gem "rails", "~> 5.1.0"
+gem "rails", "~> 5.1.7"
 
 group :development, :test do
   gem "sqlite3", platform: [:ruby, :mswin, :mingw]

data/gemfiles/rails_5_2.gemfile

@@ -1,10 +1,10 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
+#
+# BUNDLE_GEMFILE="gemfiles/rails_5_2.gemfile" bundle exec m test/integration/tasks_test.rb:30
 
 source "https://rubygems.org"
 
-gem "rails", "~> 5.2.0"
+gem "rails", "~> 5.2.4.4"
 
 group :development, :test do
   gem "sqlite3", platform: [:ruby, :mswin, :mingw]

data/gemfiles/rails_6_1.gemfile

@@ -0,0 +1,13 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rails", "~> 6.1.1"
+
+group :development, :test do
+  gem "sqlite3", platform: [:ruby, :mswin, :mingw]
+  gem "activerecord-jdbcsqlite3-adapter", "~> 1.3.13", platform: :jruby
+  gem "test-unit", "~> 3.0"
+end
+
+gemspec path: "../"

data/gemfiles/rails_git.gemfile

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-# $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:30
+# $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:50
 
 source "https://rubygems.org"
 
-gem "rails", github: "rails/rails", ref: "3054e1d584e7eca110c69a1f8423f2e0866abbf9"
+gem "rails", github: "rails/rails", ref: "12bb9d32f56883914abcd98fd72e3c68c444808d"
 
 gem 'devise', github: "plataformatec/devise"
 

data/lib/derailed_benchmarks.rb

@@ -2,9 +2,10 @@
 
 require 'time'
 require 'bundler'
-
 require 'get_process_mem'
 
+require 'dead_end'
+
 module DerailedBenchmarks
   def self.gem_is_bundled?(name)
     specs = ::Bundler.locked_gems.specs.each_with_object({}) {|spec, hash| hash[spec.name] = spec }
@@ -43,8 +44,9 @@ end
 require 'derailed_benchmarks/require_tree'
 require 'derailed_benchmarks/auth_helper'
 
-require 'derailed_benchmarks/stats_in_file'
+require 'derailed_benchmarks/stats_for_file'
 require 'derailed_benchmarks/stats_from_dir'
+require 'derailed_benchmarks/git/switch_project'
 
 if DerailedBenchmarks.gem_is_bundled?("devise")
   DerailedBenchmarks.auth = DerailedBenchmarks::AuthHelpers::Devise.new

data/lib/derailed_benchmarks/core_ext/kernel_require.rb

@@ -17,12 +17,16 @@ module Kernel
 
   alias_method :original_require, :require
   alias_method :original_require_relative, :require_relative
+  alias_method(:original_load, :load)
+
+  def load(file, wrap = false)
+    measure_memory_impact(file) do |file|
+      original_load(file)
+    end
+  end
 
   def require(file)
     measure_memory_impact(file) do |file|
-      # "source_annotation_extractor" is deprecated in Rails 6
-      # # if we don't skip the library it leads to a crash
-      # next if file == "rails/source_annotation_extractor" && Rails.version >= '6.0'
       original_require(file)
     end
   end
@@ -67,22 +71,29 @@ module Kernel
   end
 end
 
-# Top level node that will store all require information for the entire app
-TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
-REQUIRE_STACK.push(TOP_REQUIRE)
 
+# I honestly have no idea why this Object delegation is needed
+# I keep staring at bootsnap and it doesn't have to do this
+# is there a bug in their implementation they haven't caught or
+# am I doing something different?
 class Object
   private
+  def load(path, wrap = false)
+    Kernel.load(path, wrap)
+  end
 
   def require(path)
     Kernel.require(path)
   end
 end
 
-# Don't forget to assign a cost to the top level
-cost_before_requiring_anything = GetProcessMem.new.mb
-TOP_REQUIRE.cost = cost_before_requiring_anything
+# Top level node that will store all require information for the entire app
+TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
+REQUIRE_STACK.push(TOP_REQUIRE)
+TOP_REQUIRE.cost = GetProcessMem.new.mb
+
 def TOP_REQUIRE.print_sorted_children(*args)
   self.cost = GetProcessMem.new.mb - self.cost
   super
 end
+

data/lib/derailed_benchmarks/git/commit.rb

@@ -0,0 +1,36 @@
+module DerailedBenchmarks
+  # Represents a specific commit in a git repo
+  #
+  # Can be used to get information from the commit or to check it out
+  #
+  # commit = GitCommit.new(path: "path/to/repo", ref: "6e642963acec0ff64af51bd6fba8db3c4176ed6e")
+  # commit.short_sha # => "6e64296"
+  # commit.checkout! # Will check out the current commit at the repo in the path
+  class Git::Commit
+    attr_reader :ref, :description, :time, :short_sha, :log
+
+    def initialize(path: , ref: , log_dir: Pathname.new("/dev/null"))
+      @in_git_path = Git::InPath.new(path)
+      @ref = ref
+      @log = log_dir.join("#{file_safe_ref}.bench.txt")
+
+      Dir.chdir(path) do
+        checkout!
+        @description = @in_git_path.description
+        @short_sha = @in_git_path.short_sha
+        @time = @in_git_path.time
+      end
+    end
+
+    alias :desc :description
+    alias :file :log
+
+    def checkout!
+      @in_git_path.checkout!(ref)
+    end
+
+    private def file_safe_ref
+      ref.gsub('/', ':')
+    end
+  end
+end