derailed_benchmarks 1.8.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0f29e3b52bfe2c3c549a9908cf835ec63430afc8d8c733f8ccbf73646b0aa07
-  data.tar.gz: c43b9cc6602884bc33c3e6479b75250564ef4b54f895b87a6d0b5f79f4469704
+  metadata.gz: a32acaf298b80ee6e02cb5fe12aa972c359cfa7efc191305ac32da9198635d77
+  data.tar.gz: 8f6bddd86d0694ce0c33da08b35d4a464552fc8d4792027a02d6cc19c29b1b4a
 SHA512:
-  metadata.gz: 23583aa213f4f2ddb8ead5c9325972bd8fc696289303e16faf34023e5f732288d51cd9d5fe9edb0f37a44c0a1f3ebbe32d74c6355ba85b4f9dd1793e8e3359d5
-  data.tar.gz: f4074b2f83fd12c4b81e4f9714bae62a0ed15e012d2a4a821d2c038fc537b9b511475482e4928d689421ba473cfb91543431866d1cd424cf6db101b792bcfdb8
+  metadata.gz: e53dc17c06b27e39a6018c20c6277f72c83522d3c66551f133d117cd21f2c60a6c53931be6272738bef929223b6c9eabd44b06d134b9290fd255a2dc6810a106
+  data.tar.gz: da5e3bae0cc24205140a7f85cf6dc2aab0eb90cc375b7eec8f4a1b066f034f578114206f51cc7ee8801dd41eb8aa6fd5046735bb4da559232359e41bc303f6e6

data/.circleci/config.yml

@@ -0,0 +1,75 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@1.1.2
+references:
+  run_tests: &run_tests
+    run:
+      name: Run test suite
+      command: bundle exec rake test
+  # Needed because tests execute raw git commands
+  set_git_config: &set_git_config
+    run:
+      name: Set Git config
+      command: git config --global user.email "you@example.com"; git config --global user.name "Your Name"
+  restore: &restore
+    restore_cache:
+      keys:
+        - v1_bundler_deps-{{ .Environment.CIRCLE_JOB }}
+  save: &save
+    save_cache:
+      paths:
+        - ./vendor/bundle
+      key: v1_bundler_deps-{{ .Environment.CIRCLE_JOB }} # CIRCLE_JOB e.g. "ruby-2.5"
+  bundle: &bundle
+    run:
+      name: install dependencies
+      command: |
+        echo "export BUNDLE_JOBS=4" >> $BASH_ENV
+        echo "export BUNDLE_RETRY=3" >> $BASH_ENV
+        echo "export BUNDLE_PATH=$(pwd)/vendor/bundle" >> $BASH_ENV
+        echo "export BUNDLE_GEMFILE=$(pwd)/gemfiles/$GEMFILE_NAME" >> $BASH_ENV
+        source $BASH_ENV
+
+        bundle install
+        bundle update
+        bundle clean
+  mysteps: &mysteps
+    steps:
+      - checkout
+      - <<: *set_git_config
+      - <<: *restore
+      - <<: *bundle
+      - <<: *run_tests
+      - <<: *save
+
+jobs:
+  test:
+    parameters:
+      ruby_version:
+        type: string
+      gemfile:
+        type: string
+    docker:
+      - image: "ruby:<< parameters.ruby_version >>"
+    environment:
+      GEMFILE_NAME: <<parameters.gemfile>>
+    steps:
+      - checkout
+      - <<: *set_git_config
+      - <<: *restore
+      - <<: *bundle
+      - <<: *run_tests
+      - <<: *save
+
+workflows:
+  all-tests:
+    jobs:
+      - test:
+          matrix:
+            parameters:
+              ruby_version: ["2.5.8", "2.7.2", "3.0.0"]
+              gemfile: ["rails_5_2.gemfile", "rails_6_1.gemfile", "rails_git.gemfile"]
+            exclude:
+              - ruby_version: "3.0.0"
+                gemfile: rails_5_2.gemfile
+          name: test-ruby-<<matrix.ruby_version>>-<<matrix.gemfile>>

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## HEAD
 
+## 2.0.0
+
+- Syntax errors easier to debug with `dead_end` gem (https://github.com/schneems/derailed_benchmarks/pull/182)
+- Minimum ruby version is now 2.5 (https://github.com/schneems/derailed_benchmarks/pull/183)
+- Histograms are now printed side-by-side (https://github.com/schneems/derailed_benchmarks/pull/179)
+
 ## 1.8.1
 
 - Derailed now tracks memory use from `load` in addition to `require` (https://github.com/schneems/derailed_benchmarks/pull/178)

data/README.md

@@ -9,16 +9,13 @@ A series of things you can use to benchmark a Rails or Ruby app.
 
 ## Compatibility/Requirements
 
-This gem has been tested and is known to work with Rails 3.2+ using Ruby
-2.2+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
-
 For some benchmarks, not all, you'll need to verify you have a working version of curl on your OS:
 
 ```
 $ which curl
 /usr/bin/curl
 $ curl -V
-curl 7.37.1 #...
+curl 7.64.1 #...
 ```
 
 ## Install
@@ -448,11 +445,37 @@ When the test is done it will output which commit "won" and by how much:
 ```
 ❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️
 
-[7b4d80cb37] "1.8x Faster Partial Caching - Faster Cache Keys" - (10.9711965 seconds)
-  FASTER by:
-    1.0870x [older/newer]
-    8.0026% [(older - newer) / older * 100]
-[13d6aa3a7b] "Merge pull request #36284 from kamipo/fix_eager_loading_with_string_joins" - (11.9255485 seconds)
+[f1ab117] (11.3844 seconds) "I am the new commit" ref: "winner"
+  FASTER 🚀🚀🚀 by:
+    1.0062x [older/newer]
+    0.6147% [(older - newer) / older * 100]
+[5594a2d] (11.4548 seconds) "Old commit" ref: "loser"
+
+Iterations per sample:
+Samples: 100
+
+Test type: Kolmogorov Smirnov
+Confidence level: 99.0 %
+Is significant? (max > critical): true
+D critical: 0.2145966026289347
+D max: 0.26
+
+Histograms (time ranges are in seconds):
+
+   [f1ab117] description:                                        [5594a2d] description:
+     "I am the new commit"                                         "Old commit"
+                  ┌                                        ┐                    ┌                                        ┐
+   [11.2 , 11.28) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 12                             [11.2 , 11.28) ┤▇▇▇▇ 3
+   [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 22                 [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 19
+   [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 30       [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17
+   [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17                       [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 25
+   [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13                           [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 15
+   [11.58, 11.66) ┤▇▇▇▇▇▇▇ 6                                     [11.58, 11.66) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13
+   [11.65, 11.73) ┤ 0                                            [11.65, 11.73) ┤▇▇▇▇ 3
+   [11.73, 11.81) ┤ 0                                            [11.73, 11.81) ┤▇▇▇▇ 3
+   [11.8 , 11.88) ┤ 0                                            [11.8 , 11.88) ┤▇▇▇ 2
+                  └                                        ┘                    └                                        ┘
+                             # of runs in range                                            # of runs in range
 ```
 
 You can provide this to the Rails team along with the example app you used to benchmark (so they can independently verify if needed).

data/derailed_benchmarks.gemspec

@@ -20,21 +20,22 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = ">= 2.2.0"
+  gem.required_ruby_version = ">= 2.5.0"
 
   gem.add_dependency "heapy",           "~> 0"
-  gem.add_dependency "memory_profiler", "~> 0"
+  gem.add_dependency "memory_profiler", ">= 0", "< 2"
   gem.add_dependency "get_process_mem", "~> 0"
   gem.add_dependency "benchmark-ips",   "~> 2"
   gem.add_dependency "rack",            ">= 1"
   gem.add_dependency "rake",            "> 10", "< 14"
   gem.add_dependency "thor",            ">= 0.19", "< 2"
   gem.add_dependency "ruby-statistics", ">= 2.1"
-  gem.add_dependency "mini_histogram",    ">= 0.2.1"
+  gem.add_dependency "mini_histogram",  ">= 0.3.0"
+  gem.add_dependency "dead_end",        ">= 0"
 
+  gem.add_development_dependency "webrick",  ">= 0"
   gem.add_development_dependency "capybara",  "~> 2"
   gem.add_development_dependency "m"
   gem.add_development_dependency "rails",     "> 3", "<= 7"
   gem.add_development_dependency "devise",    "> 3", "< 6"
-  gem.add_development_dependency "appraisal", "2.2.0"
 end

data/gemfiles/rails_6_1.gemfile

@@ -0,0 +1,13 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rails", "~> 6.1.1"
+
+group :development, :test do
+  gem "sqlite3", platform: [:ruby, :mswin, :mingw]
+  gem "activerecord-jdbcsqlite3-adapter", "~> 1.3.13", platform: :jruby
+  gem "test-unit", "~> 3.0"
+end
+
+gemspec path: "../"

data/gemfiles/rails_git.gemfile

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-# $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:30
+# $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:50
 
 source "https://rubygems.org"
 
-gem "rails", github: "rails/rails", ref: "3054e1d584e7eca110c69a1f8423f2e0866abbf9"
+gem "rails", github: "rails/rails", ref: "12bb9d32f56883914abcd98fd72e3c68c444808d"
 
 gem 'devise', github: "plataformatec/devise"
 

data/lib/derailed_benchmarks.rb

@@ -2,9 +2,10 @@
 
 require 'time'
 require 'bundler'
-
 require 'get_process_mem'
 
+require 'dead_end'
+
 module DerailedBenchmarks
   def self.gem_is_bundled?(name)
     specs = ::Bundler.locked_gems.specs.each_with_object({}) {|spec, hash| hash[spec.name] = spec }

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -38,7 +38,7 @@ module DerailedBenchmarks
           file = info_hash.fetch(:file)
           desc = info_hash.fetch(:desc)
           time = info_hash.fetch(:time)
-          short_sha = info_hash["short_sha"]
+          short_sha = info_hash[:short_sha]
           @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
         end
       else
@@ -120,22 +120,23 @@ module DerailedBenchmarks
     end
 
     def histogram(io = $stdout)
-      newest_histogram = MiniHistogram.new(newest.values)
-      oldest_histogram = MiniHistogram.new(oldest.values)
-      MiniHistogram.set_average_edges!(newest_histogram, oldest_histogram)
-
-      {newest => newest_histogram, oldest => oldest_histogram}.each do |report, histogram|
-        plot = histogram.plot(
-          title: "\n#{' ' * 18 }Histogram - [#{report.short_sha || report.name}] #{report.desc.inspect}",
-          ylabel: "Time (s)",
+      dual_histogram = MiniHistogram.dual_plot do |a, b|
+        a.values = newest.values
+        a.options = {
+          title: "\n   [#{newest.short_sha || newest.name}] description:\n     #{newest.desc.inspect}",
           xlabel: "# of runs in range"
-        )
-
-        plot.render(io)
-        io.puts
+        }
+        b.values = oldest.values
+        b.options = {
+          title: "\n   [#{oldest.short_sha || oldest.name}] description:\n     #{oldest.desc.inspect}",
+          xlabel: "# of runs in range"
+        }
       end
 
       io.puts
+      io.puts "Histograms (time ranges are in seconds):"
+      io.puts(dual_histogram)
+      io.puts
     end
 
     def banner(io = $stdout)

data/lib/derailed_benchmarks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerailedBenchmarks
-  VERSION = "1.8.1"
+  VERSION = "2.0.0"
 end

data/test/derailed_benchmarks/stats_from_dir_test.rb

@@ -45,13 +45,15 @@ class StatsFromDirTest < ActiveSupport::TestCase
   test "histogram output" do
     dir = fixtures_dir("stats/significant")
     branch_info = {}
-    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
-    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), short_sha: "5594a2d" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), short_sha: "f1ab117" }
     stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
 
     io = StringIO.new
     stats.call.banner(io)
 
+    puts io.string
+
     assert_match(/11\.2 , 11\.28/, io.string)
     assert_match(/11\.8 , 11\.88/, io.string)
   end

data/test/integration/tasks_test.rb

@@ -48,7 +48,8 @@ class TasksTest < ActiveSupport::TestCase
 
     skip unless ENV['USING_RAILS_GIT']
 
-    env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
+    env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
+            "SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
     puts rake "perf:library", { env: env }
   end
 
@@ -58,7 +59,8 @@ class TasksTest < ActiveSupport::TestCase
     skip unless ENV['USING_RAILS_GIT']
 
     error = assert_raises {
-      env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
+      env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
+              "SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
       puts rake "perf:library", { env: env }
     }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: derailed_benchmarks
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Richard Schneeman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-16 00:00:00.000000000 Z
+date: 2021-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: heapy
@@ -28,16 +28,22 @@ dependencies:
   name: memory_profiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: get_process_mem
   requirement: !ruby/object:Gem::Requirement
@@ -140,14 +146,42 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: 0.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: dead_end
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement
@@ -216,20 +250,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '6'
-- !ruby/object:Gem::Dependency
-  name: appraisal
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 2.2.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 2.2.0
 description: " Go faster, off the Rails "
 email:
 - richard.schneeman+rubygems@gmail.com
@@ -238,10 +258,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".github/workflows/check_changelog.yml"
 - ".gitignore"
-- ".travis.yml"
-- Appraisals
 - CHANGELOG.md
 - Gemfile
 - README.md
@@ -252,6 +271,7 @@ files:
 - gemfiles/rails_5_1.gemfile
 - gemfiles/rails_5_2.gemfile
 - gemfiles/rails_6_0.gemfile
+- gemfiles/rails_6_1.gemfile
 - gemfiles/rails_git.gemfile
 - lib/derailed_benchmarks.rb
 - lib/derailed_benchmarks/auth_helper.rb
@@ -316,6 +336,7 @@ files:
 - test/rails_app/config/locales/en.yml
 - test/rails_app/config/locales/es.yml
 - test/rails_app/config/routes.rb
+- test/rails_app/config/storage.yml
 - test/rails_app/db/migrate/20141210070547_devise_create_users.rb
 - test/rails_app/db/schema.rb
 - test/rails_app/perf.rake
@@ -345,14 +366,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Benchmarks designed to performance test your ENTIRE site
@@ -406,6 +427,7 @@ test_files:
 - test/rails_app/config/locales/en.yml
 - test/rails_app/config/locales/es.yml
 - test/rails_app/config/routes.rb
+- test/rails_app/config/storage.yml
 - test/rails_app/db/migrate/20141210070547_devise_create_users.rb
 - test/rails_app/db/schema.rb
 - test/rails_app/perf.rake

data/.travis.yml

@@ -1,20 +0,0 @@
-language: ruby
-rvm:
-  - 2.2.10
-  - 2.5.8
-  - 2.7.1
-
-gemfile:
-  - gemfiles/rails_5_1.gemfile
-  - gemfiles/rails_6_0.gemfile
-  - gemfiles/rails_git.gemfile
-
-before_install:
-  - gem install bundler -v 1.17.3
-
-jobs:
-  allow_failures:
-    - rvm: 2.2.10
-      gemfile: gemfiles/rails_6_0.gemfile
-    - rvm: 2.2.10
-      gemfile: gemfiles/rails_git.gemfile

data/Appraisals

@@ -1,26 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-# appraise "rails-3-2" do
-#   gem "rails", "~> 3.2.0"
-# end
-
-# appraise "rails-4-0" do
-#   gem "rails", "~> 4.0.0"
-# end
-
-# appraise "rails-4-1" do
-#   gem "rails", "~> 4.1.0"
-# end
-
-# appraise "rails-4-2" do
-#   gem "rails", "~> 4.2.0"
-# end
-
-# appraise "rails-5-0" do
-#   gem "rails", "~> 5.0.0"
-# end
-
-appraise "rails-5-1" do
-  gem "rails", "~> 5.1.7"
-end