derailed_benchmarks 1.8.0 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24d0d07556d2d6676ea8b2c06d58583071fd9302c056d36db05d252480889308
-  data.tar.gz: 0c783db16c1612173975c12760b101fc30cbd5765d296b2fd66506992305ac69
+  metadata.gz: 1004e69d7a94d86e514586324fc21ce1871ca9f8690c989eafcf5283fc69e293
+  data.tar.gz: 0bfbb09013d068279ee58e1361ef44f122c944662cc590941299b12d80151a80
 SHA512:
-  metadata.gz: e37aa667a5fe031a384a343de999a34ef9f1e0ba98f8190d6f86d326ee30f968e3555344fbaf8841f99da85354c2b4383fe1b75ab166eb0df9134d20ee48fd95
-  data.tar.gz: 3f581637e835d808b8264786ffd70159fe534c4147e72e65f5649fe7ecb0b92b9edc215c9edd43e2ca51a510793ae475da3e0318262103bb741823746d9f3421
+  metadata.gz: 48dee96d301b69c53f17467bd19918c9ea788568b5d630fe3888e2c0659345dfdef66be0b911ed2624af1cc11c9507a3700760b40f261d846f092b5e49743b79
+  data.tar.gz: d69df370bb2cd4aebc48d1b077eb0d81b9da5dc659209da091d15b49df76af196f32c3b4daa0d402db2fa1aaf968067fc8eb9e3c690a9fce1c2c4458087256aa

data/.circleci/config.yml

@@ -0,0 +1,75 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@1.1.2
+references:
+  run_tests: &run_tests
+    run:
+      name: Run test suite
+      command: bundle exec rake test
+  # Needed because tests execute raw git commands
+  set_git_config: &set_git_config
+    run:
+      name: Set Git config
+      command: git config --global user.email "you@example.com"; git config --global user.name "Your Name"
+  restore: &restore
+    restore_cache:
+      keys:
+        - v1_bundler_deps-{{ .Environment.CIRCLE_JOB }}
+  save: &save
+    save_cache:
+      paths:
+        - ./vendor/bundle
+      key: v1_bundler_deps-{{ .Environment.CIRCLE_JOB }} # CIRCLE_JOB e.g. "ruby-2.5"
+  bundle: &bundle
+    run:
+      name: install dependencies
+      command: |
+        echo "export BUNDLE_JOBS=4" >> $BASH_ENV
+        echo "export BUNDLE_RETRY=3" >> $BASH_ENV
+        echo "export BUNDLE_PATH=$(pwd)/vendor/bundle" >> $BASH_ENV
+        echo "export BUNDLE_GEMFILE=$(pwd)/gemfiles/$GEMFILE_NAME" >> $BASH_ENV
+        source $BASH_ENV
+
+        bundle install
+        bundle update
+        bundle clean
+  mysteps: &mysteps
+    steps:
+      - checkout
+      - <<: *set_git_config
+      - <<: *restore
+      - <<: *bundle
+      - <<: *run_tests
+      - <<: *save
+
+jobs:
+  test:
+    parameters:
+      ruby_version:
+        type: string
+      gemfile:
+        type: string
+    docker:
+      - image: "ruby:<< parameters.ruby_version >>"
+    environment:
+      GEMFILE_NAME: <<parameters.gemfile>>
+    steps:
+      - checkout
+      - <<: *set_git_config
+      - <<: *restore
+      - <<: *bundle
+      - <<: *run_tests
+      - <<: *save
+
+workflows:
+  all-tests:
+    jobs:
+      - test:
+          matrix:
+            parameters:
+              ruby_version: ["2.5.8", "2.7.2", "3.0.0"]
+              gemfile: ["rails_5_2.gemfile", "rails_6_1.gemfile", "rails_git.gemfile"]
+            exclude:
+              - ruby_version: "3.0.0"
+                gemfile: rails_5_2.gemfile
+          name: test-ruby-<<matrix.ruby_version>>-<<matrix.gemfile>>

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 ## HEAD
 
+## 2.1.1
+
+- Fix Thor's deprecation warning by implementing `exit_on_failure?` (https://github.com/schneems/derailed_benchmarks/pull/195)
+
+## 2.1.0
+
+- Add `perf:heap_diff` tool (https://github.com/schneems/derailed_benchmarks/pull/193)
+
+## 2.0.1
+
+- `rack-test` dependency added (https://github.com/schneems/derailed_benchmarks/pull/187)
+
+## 2.0.0
+
+- Syntax errors easier to debug with `dead_end` gem (https://github.com/schneems/derailed_benchmarks/pull/182)
+- Minimum ruby version is now 2.5 (https://github.com/schneems/derailed_benchmarks/pull/183)
+- Histograms are now printed side-by-side (https://github.com/schneems/derailed_benchmarks/pull/179)
+
+## 1.8.1
+
+- Derailed now tracks memory use from `load` in addition to `require` (https://github.com/schneems/derailed_benchmarks/pull/178)
+- Correct logging of unsuccessful curl requests to file (https://github.com/schneems/derailed_benchmarks/pull/172)
+
 ## 1.8.0
 
 - Ruby 2.2 is now officialy supported and tested (https://github.com/schneems/derailed_benchmarks/pull/177)

data/README.md

@@ -9,16 +9,13 @@ A series of things you can use to benchmark a Rails or Ruby app.
 
 ## Compatibility/Requirements
 
-This gem has been tested and is known to work with Rails 3.2+ using Ruby
-2.2+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
-
 For some benchmarks, not all, you'll need to verify you have a working version of curl on your OS:
 
 ```
 $ which curl
 /usr/bin/curl
 $ curl -V
-curl 7.37.1 #...
+curl 7.64.1 #...
 ```
 
 ## Install
@@ -201,12 +198,17 @@ You can run commands against your app by running `$ derailed exec`. There are se
 ```
 $ bundle exec derailed exec --help
   $ derailed exec perf:allocated_objects  # outputs allocated object diff after app is called TEST_COUNT times
+  $ derailed exec perf:app  # runs the performance test against two most recent commits of the current app
   $ derailed exec perf:gc  # outputs GC::Profiler.report data while app is called TEST_COUNT times
+  $ derailed exec perf:heap  # heap analyzer
   $ derailed exec perf:ips  # iterations per second
+  $ derailed exec perf:library  # runs the same test against two different branches for statistical comparison
   $ derailed exec perf:mem  # show memory usage caused by invoking require per gem
-  $ derailed exec perf:objects  # profiles ruby allocation
   $ derailed exec perf:mem_over_time  # outputs memory usage over time
+  $ derailed exec perf:objects  # profiles ruby allocation
+  $ derailed exec perf:stackprof  # stackprof
   $ derailed exec perf:test  # hits the url TEST_COUNT times
+  $ derailed exec perf:heap_diff  # three heaps generation for comparison
 ```
 
 Instead of going over each command we'll look at common problems and which commands are best used to diagnose them. Later on we'll cover all of the environment variables you can use to configure derailed benchmarks in it's own section.
@@ -274,7 +276,7 @@ This is similar to `$ bundle exec derailed bundle:objects` however it includes o
 
 ## I want a Heap Dump
 
-If you're still struggling with runtime memory you can generate a heap dump that can later be analyzed using [heap_inspect](https://github.com/schneems/heapy).
+If you're still struggling with runtime memory you can generate a heap dump that can later be analyzed using [heapy](https://github.com/schneems/heapy).
 
 ```
 $ bundle exec derailed exec perf:heap
@@ -298,6 +300,40 @@ For more help on getting data from a heap dump see
 $ heapy --help
 ```
 
+### I want more heap dumps
+
+When searching for a leak, you can use heap dumps for comparison to see what is
+retained. See [Analyzing memory heaps](https://medium.com/klaxit-techblog/tracking-a-ruby-memory-leak-in-2021-9eb56575f731#875b)
+(inspired from [SamSaffron's original idea](https://speakerdeck.com/samsaffron/why-ruby-2-dot-1-excites-me?slide=27))
+for a clear example. You can generate 3 dumps (one every `TEST_COUNT` calls) using the
+next command:
+
+```
+$ bundle exec derailed exec perf:heap_diff
+Endpoint: "/"
+Running 1000 times
+Heap file generated: "tmp/2021-05-06T15:19:26+02:00-heap-0.ndjson"
+Running 1000 times
+Heap file generated: "tmp/2021-05-06T15:19:26+02:00-heap-1.ndjson"
+Running 1000 times
+Heap file generated: "tmp/2021-05-06T15:19:26+02:00-heap-2.ndjson"
+
+Diff
+====
+Retained STRING 90 objects of size 4790/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/rack-2.2.3/lib/rack/utils.rb:461
+Retained ICLASS 20 objects of size 800/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/sinatra-contrib-2.0.8.1/lib/sinatra/namespace.rb:198
+Retained DATA 20 objects of size 1360/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/2.7.0/monitor.rb:238
+Retained STRING 20 objects of size 800/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/rack-protection-2.0.8.1/lib/rack/protection/xss_header.rb:20
+Retained STRING 10 objects of size 880/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/newrelic_rpm-5.4.0.347/lib/new_relic/agent/transaction.rb:890
+Retained CLASS 10 objects of size 4640/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/sinatra-contrib-2.0.8.1/lib/sinatra/namespace.rb:198
+Retained IMEMO 10 objects of size 480/91280 (in bytes) at: /Users/ulysse/.rbenv/versions/2.7.2/lib/ruby/gems/2.7.0/gems/sinatra-2.0.8.1/lib/sinatra/base.rb:1017
+...
+
+Run `$ heapy --help` for more options
+
+Also read https://medium.com/klaxit-techblog/tracking-a-ruby-memory-leak-in-2021-9eb56575f731#875b to understand better what you are reading.
+```
+
 ### Memory Is large at boot.
 
 Ruby memory typically goes in one direction, up. If your memory is large when you boot the application it will likely only increase. In addition to debugging memory retained from dependencies obtained while running `$ derailed bundle:mem` you'll likely want to see how your own files contribute to memory use.
@@ -448,11 +484,37 @@ When the test is done it will output which commit "won" and by how much:
 ```
 ❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️
 
-[7b4d80cb37] "1.8x Faster Partial Caching - Faster Cache Keys" - (10.9711965 seconds)
-  FASTER by:
-    1.0870x [older/newer]
-    8.0026% [(older - newer) / older * 100]
-[13d6aa3a7b] "Merge pull request #36284 from kamipo/fix_eager_loading_with_string_joins" - (11.9255485 seconds)
+[f1ab117] (11.3844 seconds) "I am the new commit" ref: "winner"
+  FASTER 🚀🚀🚀 by:
+    1.0062x [older/newer]
+    0.6147% [(older - newer) / older * 100]
+[5594a2d] (11.4548 seconds) "Old commit" ref: "loser"
+
+Iterations per sample:
+Samples: 100
+
+Test type: Kolmogorov Smirnov
+Confidence level: 99.0 %
+Is significant? (max > critical): true
+D critical: 0.2145966026289347
+D max: 0.26
+
+Histograms (time ranges are in seconds):
+
+   [f1ab117] description:                                        [5594a2d] description:
+     "I am the new commit"                                         "Old commit"
+                  ┌                                        ┐                    ┌                                        ┐
+   [11.2 , 11.28) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 12                             [11.2 , 11.28) ┤▇▇▇▇ 3
+   [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 22                 [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 19
+   [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 30       [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17
+   [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17                       [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 25
+   [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13                           [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 15
+   [11.58, 11.66) ┤▇▇▇▇▇▇▇ 6                                     [11.58, 11.66) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13
+   [11.65, 11.73) ┤ 0                                            [11.65, 11.73) ┤▇▇▇▇ 3
+   [11.73, 11.81) ┤ 0                                            [11.73, 11.81) ┤▇▇▇▇ 3
+   [11.8 , 11.88) ┤ 0                                            [11.8 , 11.88) ┤▇▇▇ 2
+                  └                                        ┘                    └                                        ┘
+                             # of runs in range                                            # of runs in range
 ```
 
 You can provide this to the Rails team along with the example app you used to benchmark (so they can independently verify if needed).

data/bin/derailed

@@ -20,6 +20,9 @@ Bundler.setup
 require 'thor'
 
 class DerailedBenchmarkCLI < Thor
+  def self.exit_on_failure?
+    true
+  end
 
   desc "exec", "executes given derailed benchmark"
   def exec(task = nil)

data/derailed_benchmarks.gemspec

@@ -20,21 +20,23 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = ">= 2.2.0"
+  gem.required_ruby_version = ">= 2.5.0"
 
   gem.add_dependency "heapy",           "~> 0"
-  gem.add_dependency "memory_profiler", "~> 0"
+  gem.add_dependency "memory_profiler", ">= 0", "< 2"
   gem.add_dependency "get_process_mem", "~> 0"
   gem.add_dependency "benchmark-ips",   "~> 2"
   gem.add_dependency "rack",            ">= 1"
   gem.add_dependency "rake",            "> 10", "< 14"
   gem.add_dependency "thor",            ">= 0.19", "< 2"
   gem.add_dependency "ruby-statistics", ">= 2.1"
-  gem.add_dependency "mini_histogram",    ">= 0.2.1"
+  gem.add_dependency "mini_histogram",  ">= 0.3.0"
+  gem.add_dependency "dead_end",        ">= 0"
+  gem.add_dependency "rack-test",       ">= 0"
 
+  gem.add_development_dependency "webrick",  ">= 0"
   gem.add_development_dependency "capybara",  "~> 2"
   gem.add_development_dependency "m"
   gem.add_development_dependency "rails",     "> 3", "<= 7"
   gem.add_development_dependency "devise",    "> 3", "< 6"
-  gem.add_development_dependency "appraisal", "2.2.0"
 end

data/gemfiles/rails_5_1.gemfile

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+#  BUNDLE_GEMFILE="gemfiles/rails_5_1.gemfile" bundle exec m test/integration/tasks_test.rb:30
+#
 # This file was generated by Appraisal
 
 source "https://rubygems.org"
 
-gem "rails", "~> 5.1.0"
+gem "rails", "~> 5.1.7"
 
 group :development, :test do
   gem "sqlite3", platform: [:ruby, :mswin, :mingw]

data/gemfiles/rails_5_2.gemfile

@@ -1,10 +1,10 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
+#
+# BUNDLE_GEMFILE="gemfiles/rails_5_2.gemfile" bundle exec m test/integration/tasks_test.rb:30
 
 source "https://rubygems.org"
 
-gem "rails", "~> 5.2.0"
+gem "rails", "~> 5.2.4.4"
 
 group :development, :test do
   gem "sqlite3", platform: [:ruby, :mswin, :mingw]

data/gemfiles/rails_6_1.gemfile

@@ -0,0 +1,13 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rails", "~> 6.1.1"
+
+group :development, :test do
+  gem "sqlite3", platform: [:ruby, :mswin, :mingw]
+  gem "activerecord-jdbcsqlite3-adapter", "~> 1.3.13", platform: :jruby
+  gem "test-unit", "~> 3.0"
+end
+
+gemspec path: "../"

data/gemfiles/rails_git.gemfile

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-# $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:30
+# $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:50
 
 source "https://rubygems.org"
 
-gem "rails", github: "rails/rails", ref: "3054e1d584e7eca110c69a1f8423f2e0866abbf9"
+gem "rails", github: "rails/rails", ref: "12bb9d32f56883914abcd98fd72e3c68c444808d"
 
 gem 'devise', github: "plataformatec/devise"
 

data/lib/derailed_benchmarks.rb

@@ -2,9 +2,10 @@
 
 require 'time'
 require 'bundler'
-
 require 'get_process_mem'
 
+require 'dead_end'
+
 module DerailedBenchmarks
   def self.gem_is_bundled?(name)
     specs = ::Bundler.locked_gems.specs.each_with_object({}) {|spec, hash| hash[spec.name] = spec }

data/lib/derailed_benchmarks/core_ext/kernel_require.rb

@@ -17,12 +17,16 @@ module Kernel
 
   alias_method :original_require, :require
   alias_method :original_require_relative, :require_relative
+  alias_method(:original_load, :load)
+
+  def load(file, wrap = false)
+    measure_memory_impact(file) do |file|
+      original_load(file)
+    end
+  end
 
   def require(file)
     measure_memory_impact(file) do |file|
-      # "source_annotation_extractor" is deprecated in Rails 6
-      # # if we don't skip the library it leads to a crash
-      # next if file == "rails/source_annotation_extractor" && Rails.version >= '6.0'
       original_require(file)
     end
   end
@@ -67,22 +71,29 @@ module Kernel
   end
 end
 
-# Top level node that will store all require information for the entire app
-TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
-REQUIRE_STACK.push(TOP_REQUIRE)
 
+# I honestly have no idea why this Object delegation is needed
+# I keep staring at bootsnap and it doesn't have to do this
+# is there a bug in their implementation they haven't caught or
+# am I doing something different?
 class Object
   private
+  def load(path, wrap = false)
+    Kernel.load(path, wrap)
+  end
 
   def require(path)
     Kernel.require(path)
   end
 end
 
-# Don't forget to assign a cost to the top level
-cost_before_requiring_anything = GetProcessMem.new.mb
-TOP_REQUIRE.cost = cost_before_requiring_anything
+# Top level node that will store all require information for the entire app
+TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
+REQUIRE_STACK.push(TOP_REQUIRE)
+TOP_REQUIRE.cost = GetProcessMem.new.mb
+
 def TOP_REQUIRE.print_sorted_children(*args)
   self.cost = GetProcessMem.new.mb - self.cost
   super
 end
+

data/lib/derailed_benchmarks/load_tasks.rb

@@ -110,7 +110,7 @@ namespace :perf do
           STDERR.puts "Bad request to #{cmd.inspect} \n\n***RESPONSE***:\n\n#{ response.inspect }"
 
           FileUtils.mkdir_p("tmp")
-          File.open("tmp/fail.html", "w+") {|f| f.write response.body }
+          File.open("tmp/fail.html", "w+") {|f| f.write response }
 
           `open #{File.expand_path("tmp/fail.html")}` if ENV["DERAILED_DEBUG"]
 

data/lib/derailed_benchmarks/require_tree.rb

@@ -13,6 +13,16 @@ module DerailedBenchmarks
     def initialize(name)
       @name     = name
       @children = {}
+      @cost = 0
+    end
+
+    def self.reset!
+      REQUIRED_BY.clear
+      if defined?(Kernel::REQUIRE_STACK)
+        Kernel::REQUIRE_STACK.clear
+
+        Kernel::REQUIRE_STACK.push(TOP_REQUIRE)
+      end
     end
 
     def <<(tree)

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -38,7 +38,7 @@ module DerailedBenchmarks
           file = info_hash.fetch(:file)
           desc = info_hash.fetch(:desc)
           time = info_hash.fetch(:time)
-          short_sha = info_hash["short_sha"]
+          short_sha = info_hash[:short_sha]
           @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
         end
       else
@@ -120,22 +120,23 @@ module DerailedBenchmarks
     end
 
     def histogram(io = $stdout)
-      newest_histogram = MiniHistogram.new(newest.values)
-      oldest_histogram = MiniHistogram.new(oldest.values)
-      MiniHistogram.set_average_edges!(newest_histogram, oldest_histogram)
-
-      {newest => newest_histogram, oldest => oldest_histogram}.each do |report, histogram|
-        plot = histogram.plot(
-          title: "\n#{' ' * 18 }Histogram - [#{report.short_sha || report.name}] #{report.desc.inspect}",
-          ylabel: "Time (s)",
+      dual_histogram = MiniHistogram.dual_plot do |a, b|
+        a.values = newest.values
+        a.options = {
+          title: "\n   [#{newest.short_sha || newest.name}] description:\n     #{newest.desc.inspect}",
           xlabel: "# of runs in range"
-        )
-
-        plot.render(io)
-        io.puts
+        }
+        b.values = oldest.values
+        b.options = {
+          title: "\n   [#{oldest.short_sha || oldest.name}] description:\n     #{oldest.desc.inspect}",
+          xlabel: "# of runs in range"
+        }
       end
 
       io.puts
+      io.puts "Histograms (time ranges are in seconds):"
+      io.puts(dual_histogram)
+      io.puts
     end
 
     def banner(io = $stdout)

data/lib/derailed_benchmarks/tasks.rb

@@ -247,6 +247,42 @@ namespace :perf do
     puts "Also try uploading #{file_name.inspect} to http://tenderlove.github.io/heap-analyzer/"
   end
 
+  desc "three heaps generation for comparison."
+  task :heap_diff => [:setup] do
+    require 'objspace'
+
+    launch_time = Time.now.iso8601
+    FileUtils.mkdir_p("tmp")
+    ObjectSpace.trace_object_allocations_start
+    3.times do |i|
+      file_name = "tmp/#{launch_time}-heap-#{i}.ndjson"
+      puts "Running #{ TEST_COUNT } times"
+      TEST_COUNT.times {
+        call_app
+      }
+      GC.start
+
+      puts "Heap file generated: #{ file_name.inspect }"
+      ObjectSpace.dump_all(output: File.open(file_name, 'w'))
+    end
+
+    require 'heapy'
+
+    puts ""
+    puts "Diff"
+    puts "===="
+    Heapy::Diff.new(
+      before: "tmp/#{launch_time}-heap-0.ndjson",
+      after: "tmp/#{launch_time}-heap-1.ndjson",
+      retained: "tmp/#{launch_time}-heap-2.ndjson"
+    ).call
+
+    puts ""
+    puts "Run `$ heapy --help` for more options"
+    puts ""
+    puts "Also read https://medium.com/klaxit-techblog/tracking-a-ruby-memory-leak-in-2021-9eb56575f731#875b to understand better what you are reading."
+  end
+
   def run!(cmd)
     out = `#{cmd}`
     raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?

data/lib/derailed_benchmarks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerailedBenchmarks
-  VERSION = "1.8.0"
+  VERSION = "2.1.1"
 end

data/test/derailed_benchmarks/core_ext/kernel_require_test.rb

@@ -3,7 +3,6 @@
 require 'test_helper'
 
 class KernelRequireTest < ActiveSupport::TestCase
-
   setup do
     require 'derailed_benchmarks/core_ext/kernel_require'
     GC.disable
@@ -11,23 +10,83 @@ class KernelRequireTest < ActiveSupport::TestCase
 
   teardown do
     GC.enable
+    DerailedBenchmarks::RequireTree.reset!
+  end
+
+  test "profiles load" do
+    in_fork do
+      require fixtures_dir("require/load_parent.rb")
+
+      parent = assert_node_in_parent("load_parent.rb", TOP_REQUIRE)
+
+      assert_node_in_parent("load_child.rb", parent)
+    end
+  end
+
+  test "profiles autoload" do
+    skip if RUBY_VERSION.start_with?("2.2") # Fails on CI, I can't install Ruby 2.2 locally to debug https://stackoverflow.com/questions/63926460/install-ruby-2-2-on-mac-osx-catalina-with-ruby-install, https://github.com/postmodern/ruby-install/issues/375
+
+    in_fork do
+      require fixtures_dir("require/autoload_parent.rb")
+      parent = assert_node_in_parent("autoload_parent.rb", TOP_REQUIRE)
+
+      assert_node_in_parent("autoload_child.rb", parent)
+    end
   end
 
+  test "core extension profiles useage" do
+    in_fork do
+      require fixtures_dir("require/parent_one.rb")
+      parent    = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
+      assert_node_in_parent("child_one.rb", parent)
+      child_two = assert_node_in_parent("child_two.rb", parent)
+      assert_node_in_parent("relative_child", parent)
+      assert_node_in_parent("relative_child_two", parent)
+      assert_node_in_parent("raise_child.rb", child_two)
+    end
+  end
+
+  # Checks to see that the given file name is present in the
+  # parent tree node and that the memory of that file
+  # is less than the parent (since the parent should include itself
+  # plus its children)
+  #
+  # Returns the child node
   def assert_node_in_parent(file_name, parent)
     file = fixtures_dir(File.join("require", file_name))
     node = parent[file]
-    assert node,                    "Expected:\n#{parent.children}\nto include:\n#{file.inspect}"
-    assert node.cost < parent.cost, "Expected:\n#{node.inspect}\nto cost less than:\n#{parent.inspect}" unless parent == TOP_REQUIRE
+    assert node, "Expected: #{parent.name} to include: #{file.to_s} but it did not.\nChildren: #{parent.children.map(&:name).map(&:to_s)}"
+    unless parent == TOP_REQUIRE
+      assert node.cost < parent.cost, "Expected: #{node.name.inspect} (#{node.cost}) to cost less than: #{parent.name.inspect} (#{parent.cost})"
+    end
     node
   end
 
-  test "core extension profiles useage" do
-    require fixtures_dir("require/parent_one.rb")
-    parent    = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
-    assert_node_in_parent("child_one.rb", parent)
-    child_two = assert_node_in_parent("child_two.rb", parent)
-    assert_node_in_parent("relative_child", parent)
-    assert_node_in_parent("relative_child_two", parent)
-    assert_node_in_parent("raise_child.rb", child_two)
+  # Used to get semi-clean process memory
+  # It would be better to run the requires in a totally different process
+  # but...that would take engineering
+  #
+  # If I was going to do that, I would find a way to serialize RequireTree
+  # into a json structure with file names and costs, run the script
+  # dump the json to a file, then in this process read the file and
+  # run assertions
+  def in_fork
+    Tempfile.create("stdout") do |tmp_file|
+      pid = fork do
+        $stdout.reopen(tmp_file, "w")
+        $stderr.reopen(tmp_file, "w")
+        $stdout.sync = true
+        $stderr.sync = true
+        yield
+        Kernel.exit!(0) # needed for https://github.com/seattlerb/minitest/pull/683
+      end
+      Process.waitpid(pid)
+
+      if $?.success?
+        print File.read(tmp_file)
+      else
+        raise File.read(tmp_file)
+      end
+    end
   end
 end

data/test/derailed_benchmarks/require_tree_test.rb

@@ -9,7 +9,7 @@ class RequireTree < ActiveSupport::TestCase
   end
 
   def teardown
-    DerailedBenchmarks::RequireTree.const_set("REQUIRED_BY",  {})
+    DerailedBenchmarks::RequireTree.reset!
   end
 
   test "default_cost" do

data/test/derailed_benchmarks/stats_from_dir_test.rb

@@ -45,13 +45,15 @@ class StatsFromDirTest < ActiveSupport::TestCase
   test "histogram output" do
     dir = fixtures_dir("stats/significant")
     branch_info = {}
-    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
-    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), short_sha: "5594a2d" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), short_sha: "f1ab117" }
     stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
 
     io = StringIO.new
     stats.call.banner(io)
 
+    puts io.string
+
     assert_match(/11\.2 , 11\.28/, io.string)
     assert_match(/11\.8 , 11\.88/, io.string)
   end

data/test/derailed_test.rb

@@ -11,4 +11,19 @@ class DerailedBenchmarksTest < ActiveSupport::TestCase
     assert DerailedBenchmarks.gem_is_bundled?("rack")
     refute DerailedBenchmarks.gem_is_bundled?("wicked")
   end
+
+  test "readme contains correct output" do
+    readme_path = File.join(__dir__, "..", "README.md")
+    lines = File.foreach(readme_path)
+    lineno = 1
+    expected = lines.lazy.drop_while { |line|
+      lineno += 1
+      line != "$ bundle exec derailed exec --help\n"
+    }.drop(1).take_while { |line| line != "```\n" }.force.join
+    assert_equal(
+      expected,
+      `bundle exec derailed exec --help`,
+      "Please update README.md:#{lineno}"
+    )
+  end
 end

data/test/fixtures/require/autoload_child.rb

@@ -0,0 +1,5 @@
+@retained = String.new("")
+1_000_000.times.map { @retained << "A" }
+
+module AutoLoadChild
+end

data/test/fixtures/require/autoload_parent.rb

@@ -0,0 +1,8 @@
+@retained = String.new("")
+1_000_000.times.map { @retained << "A" }
+
+autoload :AutoLoadChild, File.join(__dir__, 'autoload_child.rb')
+
+if AutoLoadChild
+  # yay
+end

data/test/fixtures/require/load_child.rb

@@ -0,0 +1,3 @@
+@retained = String.new("")
+1_000_000.times.map { @retained << "A" }
+

data/test/fixtures/require/load_parent.rb

@@ -0,0 +1,5 @@
+@retained = String.new("")
+1_000_000.times.map { @retained << "A" }
+
+load File.join(__dir__, "load_child.rb")
+

data/test/integration/tasks_test.rb

@@ -48,7 +48,8 @@ class TasksTest < ActiveSupport::TestCase
 
     skip unless ENV['USING_RAILS_GIT']
 
-    env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
+    env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
+            "SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
     puts rake "perf:library", { env: env }
   end
 
@@ -58,7 +59,8 @@ class TasksTest < ActiveSupport::TestCase
     skip unless ENV['USING_RAILS_GIT']
 
     error = assert_raises {
-      env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
+      env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
+              "SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
       puts rake "perf:library", { env: env }
     }
 
@@ -151,6 +153,10 @@ class TasksTest < ActiveSupport::TestCase
   end
 
   test 'ips' do
-    rake "perf:mem_over_time"
+    rake "perf:ips"
+  end
+
+  test 'heap_diff' do
+    rake "perf:heap_diff", env: { "TEST_COUNT" => 5 }
   end
 end

data/test/test_helper.rb

@@ -51,7 +51,6 @@ class ActiveSupport::IntegrationCase
   end
 end
 
-
 def fixtures_dir(name = "")
   root_path("test/fixtures").join(name)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: derailed_benchmarks
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 2.1.1
 platform: ruby
 authors:
 - Richard Schneeman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-16 00:00:00.000000000 Z
+date: 2021-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: heapy
@@ -28,16 +28,22 @@ dependencies:
   name: memory_profiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: get_process_mem
   requirement: !ruby/object:Gem::Requirement
@@ -140,14 +146,56 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: dead_end
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement
@@ -216,20 +264,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '6'
-- !ruby/object:Gem::Dependency
-  name: appraisal
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 2.2.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 2.2.0
 description: " Go faster, off the Rails "
 email:
 - richard.schneeman+rubygems@gmail.com
@@ -238,10 +272,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".github/workflows/check_changelog.yml"
 - ".gitignore"
-- ".travis.yml"
-- Appraisals
 - CHANGELOG.md
 - Gemfile
 - README.md
@@ -252,6 +285,7 @@ files:
 - gemfiles/rails_5_1.gemfile
 - gemfiles/rails_5_2.gemfile
 - gemfiles/rails_6_0.gemfile
+- gemfiles/rails_6_1.gemfile
 - gemfiles/rails_git.gemfile
 - lib/derailed_benchmarks.rb
 - lib/derailed_benchmarks/auth_helper.rb
@@ -272,8 +306,12 @@ files:
 - test/derailed_benchmarks/require_tree_test.rb
 - test/derailed_benchmarks/stats_from_dir_test.rb
 - test/derailed_test.rb
+- test/fixtures/require/autoload_child.rb
+- test/fixtures/require/autoload_parent.rb
 - test/fixtures/require/child_one.rb
 - test/fixtures/require/child_two.rb
+- test/fixtures/require/load_child.rb
+- test/fixtures/require/load_parent.rb
 - test/fixtures/require/parent_one.rb
 - test/fixtures/require/raise_child.rb
 - test/fixtures/require/relative_child.rb
@@ -312,6 +350,7 @@ files:
 - test/rails_app/config/locales/en.yml
 - test/rails_app/config/locales/es.yml
 - test/rails_app/config/routes.rb
+- test/rails_app/config/storage.yml
 - test/rails_app/db/migrate/20141210070547_devise_create_users.rb
 - test/rails_app/db/schema.rb
 - test/rails_app/perf.rake
@@ -341,14 +380,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Benchmarks designed to performance test your ENTIRE site
@@ -358,8 +397,12 @@ test_files:
 - test/derailed_benchmarks/require_tree_test.rb
 - test/derailed_benchmarks/stats_from_dir_test.rb
 - test/derailed_test.rb
+- test/fixtures/require/autoload_child.rb
+- test/fixtures/require/autoload_parent.rb
 - test/fixtures/require/child_one.rb
 - test/fixtures/require/child_two.rb
+- test/fixtures/require/load_child.rb
+- test/fixtures/require/load_parent.rb
 - test/fixtures/require/parent_one.rb
 - test/fixtures/require/raise_child.rb
 - test/fixtures/require/relative_child.rb
@@ -398,6 +441,7 @@ test_files:
 - test/rails_app/config/locales/en.yml
 - test/rails_app/config/locales/es.yml
 - test/rails_app/config/routes.rb
+- test/rails_app/config/storage.yml
 - test/rails_app/db/migrate/20141210070547_devise_create_users.rb
 - test/rails_app/db/schema.rb
 - test/rails_app/perf.rake

data/.travis.yml

@@ -1,20 +0,0 @@
-language: ruby
-rvm:
-  - 2.2.10
-  - 2.5.8
-  - 2.7.1
-
-gemfile:
-  - gemfiles/rails_5_1.gemfile
-  - gemfiles/rails_6_0.gemfile
-  - gemfiles/rails_git.gemfile
-
-before_install:
-  - gem install bundler -v 1.17.3
-
-jobs:
-  allow_failures:
-    - rvm: 2.2.10
-      gemfile: gemfiles/rails_6_0.gemfile
-    - rvm: 2.2.10
-      gemfile: gemfiles/rails_git.gemfile

data/Appraisals

@@ -1,26 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-appraise "rails-3-2" do
-  gem "rails", "~> 3.2.0"
-end
-
-appraise "rails-4-0" do
-  gem "rails", "~> 4.0.0"
-end
-
-appraise "rails-4-1" do
-  gem "rails", "~> 4.1.0"
-end
-
-appraise "rails-4-2" do
-  gem "rails", "~> 4.2.0"
-end
-
-appraise "rails-5-0" do
-  gem "rails", "~> 5.0.0"
-end
-
-appraise "rails-5-1" do
-  gem "rails", "~> 5.1.0"
-end