derailed_benchmarks 1.4.3 → 1.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 348a836be042f2ebf7785075bc3f552c60ef9cefeb979d5dded1fcac5101111a
-  data.tar.gz: 6c25b2275a5c57ae9abfc6b6c55021869aa8cea9cfcfdda7422df876caf5b10d
+  metadata.gz: c0f29e3b52bfe2c3c549a9908cf835ec63430afc8d8c733f8ccbf73646b0aa07
+  data.tar.gz: c43b9cc6602884bc33c3e6479b75250564ef4b54f895b87a6d0b5f79f4469704
 SHA512:
-  metadata.gz: eb65bb19c23d4c112ae31289c3b610aca98f1541e7a4ee1a6fd1f1b98d1ace4849f061ebb323fc38b2f01ceae64907edd247e36becd849c52fa6998370faef5b
-  data.tar.gz: 0273f740b7458e1feaa59f05681e815c41ad45830fb8b5590b6860769e4f4e96ee556b5249bb90598b96e987a68bf7f3939fba9837a8573b16f0f4ce7dbc182a
+  metadata.gz: 23583aa213f4f2ddb8ead5c9325972bd8fc696289303e16faf34023e5f732288d51cd9d5fe9edb0f37a44c0a1f3ebbe32d74c6355ba85b4f9dd1793e8e3359d5
+  data.tar.gz: f4074b2f83fd12c4b81e4f9714bae62a0ed15e012d2a4a821d2c038fc537b9b511475482e4928d689421ba473cfb91543431866d1cd424cf6db101b792bcfdb8

data/.github/workflows/check_changelog.yml

@@ -1,10 +1,13 @@
 name: Check Changelog
-on: [pull_request]
+
+on:
+ pull_request:
+  types: [opened, reopened, edited, synchronize]
 jobs:
- build:
-   runs-on: ubuntu-latest
-   steps:
-   - uses: actions/checkout@v1
-   - name: Check that CHANGELOG is touched
-     run: |
-       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Check that CHANGELOG is touched
+      run: |
+        cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

data/.travis.yml

@@ -1,18 +1,20 @@
 language: ruby
 rvm:
-  - 2.5.5
-  - 2.6.3
-  - ruby-head
+  - 2.2.10
+  - 2.5.8
+  - 2.7.1
 
 gemfile:
   - gemfiles/rails_5_1.gemfile
-  - gemfiles/rails_5_2.gemfile
   - gemfiles/rails_6_0.gemfile
   - gemfiles/rails_git.gemfile
 
 before_install:
-  - gem install bundler
+  - gem install bundler -v 1.17.3
 
-matrix:
+jobs:
   allow_failures:
-    - rvm: ruby-head
+    - rvm: 2.2.10
+      gemfile: gemfiles/rails_6_0.gemfile
+    - rvm: 2.2.10
+      gemfile: gemfiles/rails_git.gemfile

data/Appraisals

@@ -1,26 +1,26 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-appraise "rails-3-2" do
-  gem "rails", "~> 3.2.0"
-end
+# appraise "rails-3-2" do
+#   gem "rails", "~> 3.2.0"
+# end
 
-appraise "rails-4-0" do
-  gem "rails", "~> 4.0.0"
-end
+# appraise "rails-4-0" do
+#   gem "rails", "~> 4.0.0"
+# end
 
-appraise "rails-4-1" do
-  gem "rails", "~> 4.1.0"
-end
+# appraise "rails-4-1" do
+#   gem "rails", "~> 4.1.0"
+# end
 
-appraise "rails-4-2" do
-  gem "rails", "~> 4.2.0"
-end
+# appraise "rails-4-2" do
+#   gem "rails", "~> 4.2.0"
+# end
 
-appraise "rails-5-0" do
-  gem "rails", "~> 5.0.0"
-end
+# appraise "rails-5-0" do
+#   gem "rails", "~> 5.0.0"
+# end
 
 appraise "rails-5-1" do
-  gem "rails", "~> 5.1.0"
+  gem "rails", "~> 5.1.7"
 end

data/CHANGELOG.md

@@ -1,4 +1,32 @@
-## master (unreleased)
+## HEAD
+
+## 1.8.1
+
+- Derailed now tracks memory use from `load` in addition to `require` (https://github.com/schneems/derailed_benchmarks/pull/178)
+- Correct logging of unsuccessful curl requests to file (https://github.com/schneems/derailed_benchmarks/pull/172)
+
+## 1.8.0
+
+- Ruby 2.2 is now officialy supported and tested (https://github.com/schneems/derailed_benchmarks/pull/177)
+
+## 1.7.0
+
+- Add histogram support to `perf:library` (https://github.com/schneems/derailed_benchmarks/pull/169)
+- Fix bug with `Kernel#require` patch when Zeitwerk is enabled (https://github.com/schneems/derailed_benchmarks/pull/170)
+
+## 1.6.0
+
+- Added the `perf:app` command to compare commits within the same application. (https://github.com/schneems/derailed_benchmarks/pull/157)
+- Allow Rails < 7 and 1.0 <= Thor < 2 (https://github.com/schneems/derailed_benchmarks/pull/168)
+
+## 1.5.0
+
+- Test `perf:library` results against 99% confidence interval in addition to 95% (https://github.com/schneems/derailed_benchmarks/pull/165)
+- Change default, `perf:library` tests do not stop automatically any more (https://github.com/schneems/derailed_benchmarks/pull/164)
+
+## 1.4.4
+
+- Fix alignment of deicmals in output (https://github.com/schneems/derailed_benchmarks/pull/161)
 
 ## 1.4.3
 

data/README.md

@@ -10,7 +10,7 @@ A series of things you can use to benchmark a Rails or Ruby app.
 ## Compatibility/Requirements
 
 This gem has been tested and is known to work with Rails 3.2+ using Ruby
-2.1+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
+2.2+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
 
 For some benchmarks, not all, you'll need to verify you have a working version of curl on your OS:
 
@@ -415,13 +415,23 @@ or point it at your local copy:
 gem 'rails', path: "<path/to/your/local/copy/rails>"
 ```
 
-To run your test:
+To run your tests within the context of your current app/repo:
+
+```
+$ bundle exec derailed exec perf:app
+```
+
+This will automatically test the two latest commits of your library/current directory.
+
+If you'd like to test the Rails library instead, make sure that `ENV[DERAILED_PATH_TO_LIBRARY]` is unset.
 
 ```
 $ bundle exec derailed exec perf:library
 ```
 
-This will automatically test the two latest commits of Rails (or the library you've specified). If you would like to compare against different SHAs you can manually specify them:
+This will automatically test the two latest commits of Rails.
+
+If you would also like to compare against different SHAs you can manually specify them:
 
 ```
 $ SHAS_TO_TEST="7b4d80cb373e,13d6aa3a7b70" bundle exec derailed exec perf:library
@@ -453,7 +463,7 @@ You can use this to test changes in other libraries that aren't rails, you just
 
 > To get the best results before running tests you should close all programs on your laptop, turn on a program to prevent your laptop from going to sleep (or increase your sleep timer). Make sure it's plugged into a power outlet and  go grab a cup of coffee. If you do anything on your laptop while this test is running you risk the chance of skewing your results.
 
-By default derailed will stop once statistical signficance has been detected, you can tune this behavior by setting `DERAILED_STOP_VALID_COUNT` env var. Setting this to a positive number, will increase the number of iterations required that are detected to be statistically significant. For example setting it to 100 might result in 120 runs if it takes 20 runs to detect significance. Generally the more runs you have, the more accurate your averages will be. You can disable this all together by setting `DERAILED_STOP_VALID_COUNT=0` which will force derailed to run all iterations.
+As the test is executing, intermediate results will be printed every 50 iterations.
 
 ## Environment Variables
 

data/derailed_benchmarks.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = ">= 2.1.0"
+  gem.required_ruby_version = ">= 2.2.0"
 
   gem.add_dependency "heapy",           "~> 0"
   gem.add_dependency "memory_profiler", "~> 0"
@@ -28,12 +28,13 @@ Gem::Specification.new do |gem|
   gem.add_dependency "benchmark-ips",   "~> 2"
   gem.add_dependency "rack",            ">= 1"
   gem.add_dependency "rake",            "> 10", "< 14"
-  gem.add_dependency "thor",            "~> 0.19"
+  gem.add_dependency "thor",            ">= 0.19", "< 2"
   gem.add_dependency "ruby-statistics", ">= 2.1"
+  gem.add_dependency "mini_histogram",    ">= 0.2.1"
 
   gem.add_development_dependency "capybara",  "~> 2"
   gem.add_development_dependency "m"
-  gem.add_development_dependency "rails",     "> 3", "<= 6"
+  gem.add_development_dependency "rails",     "> 3", "<= 7"
   gem.add_development_dependency "devise",    "> 3", "< 6"
   gem.add_development_dependency "appraisal", "2.2.0"
 end

data/gemfiles/rails_5_1.gemfile

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+#  BUNDLE_GEMFILE="gemfiles/rails_5_1.gemfile" bundle exec m test/integration/tasks_test.rb:30
+#
 # This file was generated by Appraisal
 
 source "https://rubygems.org"
 
-gem "rails", "~> 5.1.0"
+gem "rails", "~> 5.1.7"
 
 group :development, :test do
   gem "sqlite3", platform: [:ruby, :mswin, :mingw]

data/gemfiles/rails_5_2.gemfile

@@ -1,10 +1,10 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
+#
+# BUNDLE_GEMFILE="gemfiles/rails_5_2.gemfile" bundle exec m test/integration/tasks_test.rb:30
 
 source "https://rubygems.org"
 
-gem "rails", "~> 5.2.0"
+gem "rails", "~> 5.2.4.4"
 
 group :development, :test do
   gem "sqlite3", platform: [:ruby, :mswin, :mingw]

data/lib/derailed_benchmarks.rb

@@ -43,8 +43,9 @@ end
 require 'derailed_benchmarks/require_tree'
 require 'derailed_benchmarks/auth_helper'
 
-require 'derailed_benchmarks/stats_in_file'
+require 'derailed_benchmarks/stats_for_file'
 require 'derailed_benchmarks/stats_from_dir'
+require 'derailed_benchmarks/git/switch_project'
 
 if DerailedBenchmarks.gem_is_bundled?("devise")
   DerailedBenchmarks.auth = DerailedBenchmarks::AuthHelpers::Devise.new

data/lib/derailed_benchmarks/core_ext/kernel_require.rb

@@ -11,14 +11,24 @@ ENV['CUT_OFF'] ||= "0.3"
 # Monkey patch kernel to ensure that all `require` calls call the same
 # method
 module Kernel
+  REQUIRE_STACK = []
 
-  private
+  module_function
 
-  alias :original_require :require
-  REQUIRE_STACK = []
+  alias_method :original_require, :require
+  alias_method :original_require_relative, :require_relative
+  alias_method(:original_load, :load)
+
+  def load(file, wrap = false)
+    measure_memory_impact(file) do |file|
+      original_load(file)
+    end
+  end
 
   def require(file)
-    Kernel.require(file)
+    measure_memory_impact(file) do |file|
+      original_require(file)
+    end
   end
 
   def require_relative(file)
@@ -29,10 +39,7 @@ module Kernel
     end
   end
 
-  class << self
-    alias :original_require          :require
-    alias :original_require_relative :require_relative
-  end
+  private
 
   # The core extension we use to measure require time of all requires
   # When a file is required we create a tree node with its file name.
@@ -46,7 +53,7 @@ module Kernel
   # When a require returns we remove it from the require stack so we don't
   # accidentally push additional children nodes to it. We then store the
   # memory cost of the require in the tree node.
-  def self.measure_memory_impact(file, &block)
+  def measure_memory_impact(file, &block)
     mem    = GetProcessMem.new
     node   = DerailedBenchmarks::RequireTree.new(file)
 
@@ -64,31 +71,29 @@ module Kernel
   end
 end
 
-# Top level node that will store all require information for the entire app
-TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
-REQUIRE_STACK.push(TOP_REQUIRE)
-
-Kernel.define_singleton_method(:require) do |file|
-  measure_memory_impact(file) do |file|
-    # "source_annotation_extractor" is deprecated in Rails 6
-    # # if we don't skip the library it leads to a crash
-    # next if file == "rails/source_annotation_extractor" && Rails.version >= '6.0'
-    original_require(file)
-  end
-end
 
+# I honestly have no idea why this Object delegation is needed
+# I keep staring at bootsnap and it doesn't have to do this
+# is there a bug in their implementation they haven't caught or
+# am I doing something different?
 class Object
   private
+  def load(path, wrap = false)
+    Kernel.load(path, wrap)
+  end
 
   def require(path)
     Kernel.require(path)
   end
 end
 
-# Don't forget to assign a cost to the top level
-cost_before_requiring_anything = GetProcessMem.new.mb
-TOP_REQUIRE.cost = cost_before_requiring_anything
+# Top level node that will store all require information for the entire app
+TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
+REQUIRE_STACK.push(TOP_REQUIRE)
+TOP_REQUIRE.cost = GetProcessMem.new.mb
+
 def TOP_REQUIRE.print_sorted_children(*args)
   self.cost = GetProcessMem.new.mb - self.cost
   super
 end
+

data/lib/derailed_benchmarks/git/commit.rb

@@ -0,0 +1,36 @@
+module DerailedBenchmarks
+  # Represents a specific commit in a git repo
+  #
+  # Can be used to get information from the commit or to check it out
+  #
+  # commit = GitCommit.new(path: "path/to/repo", ref: "6e642963acec0ff64af51bd6fba8db3c4176ed6e")
+  # commit.short_sha # => "6e64296"
+  # commit.checkout! # Will check out the current commit at the repo in the path
+  class Git::Commit
+    attr_reader :ref, :description, :time, :short_sha, :log
+
+    def initialize(path: , ref: , log_dir: Pathname.new("/dev/null"))
+      @in_git_path = Git::InPath.new(path)
+      @ref = ref
+      @log = log_dir.join("#{file_safe_ref}.bench.txt")
+
+      Dir.chdir(path) do
+        checkout!
+        @description = @in_git_path.description
+        @short_sha = @in_git_path.short_sha
+        @time = @in_git_path.time
+      end
+    end
+
+    alias :desc :description
+    alias :file :log
+
+    def checkout!
+      @in_git_path.checkout!(ref)
+    end
+
+    private def file_safe_ref
+      ref.gsub('/', ':')
+    end
+  end
+end

data/lib/derailed_benchmarks/git/in_path.rb

@@ -0,0 +1,59 @@
+module DerailedBenchmarks
+  # A class for running commands in a git directory
+  #
+  # It's faster to check if we're already in that directory instead
+  # of having to `cd` into each time. https://twitter.com/schneems/status/1305196730170961920
+  #
+  # Example:
+  #
+  #   in_git_path = InGitPath.new(`bundle info heapy --path`.strip)
+  #   in_git_path.checkout!("f0f92b06156f2274021aa42f15326da041ee9009")
+  #   in_git_path.short_sha # => "f0f92b0"
+  class Git::InPath
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    def description
+      run!("git log --oneline --format=%B -n 1 HEAD | head -n 1")
+    end
+
+    def short_sha
+      run!("git rev-parse --short HEAD")
+    end
+
+    def time_stamp_string
+      run!("git log -n 1 --pretty=format:%ci") # https://stackoverflow.com/a/25921837/147390
+    end
+
+    def branch
+      branch = run!("git rev-parse --abbrev-ref HEAD")
+      branch == "HEAD" ? nil : branch
+    end
+
+    def checkout!(ref)
+      run!("git checkout '#{ref}' 2>&1")
+    end
+
+    def time
+      DateTime.parse(time_stamp_string)
+    end
+
+    def run(cmd)
+      if Dir.pwd == path
+        out = `#{cmd}`.strip
+      else
+        out = `cd #{path} && #{cmd}`.strip
+      end
+      out
+    end
+
+    def run!(cmd)
+      out = run(cmd)
+      raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?
+      out
+    end
+  end
+end

data/lib/derailed_benchmarks/git/switch_project.rb

@@ -0,0 +1,128 @@
+module DerailedBenchmarks
+  class Git
+  end
+end
+require_relative "in_path.rb"
+require_relative "commit.rb"
+
+module DerailedBenchmarks
+  # Wraps two or more git commits in a specific location
+  #
+  # Returns an array of GitCommit objects that can be used to manipulate
+  # and checkout the repo
+  #
+  # Example:
+  #
+  #   `git clone https://sharpstone/default_ruby tmp/default_ruby`
+  #
+  #   project = GitSwitchProject.new(path: "tmp/default_ruby")
+  #
+  # By default it will represent the last two commits:
+  #
+  #   project.commits.length # => 2
+  #
+  # You can pass in explicit REFs in an array:
+  #
+  #   ref_array = ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"]
+  #   project = GitSwitchProject.new(path: "tmp/default_ruby", ref_array: ref_array)
+  #
+  #   puts project.commits.map(&:ref) == ref_array # => true
+  #
+  #
+  # It knows the current branch or sha:
+  #
+  #    `cd tmp/ruby && git checkout -b mybranch`
+  #    project.current_branch_or_sha #=> "mybranch"
+  #
+  # It can be used for safely wrapping checkouts to ensure the project returns to it's original branch:
+  #
+  #    project.restore_branch_on_return do
+  #      project.commits.first.checkout!
+  #      project.current_branch_or_sha # => "da748a593"
+  #    end
+  #
+  #    project.current_branch_or_sha # => "mybranch"
+  class Git::SwitchProject
+    attr_reader :commits
+
+    def initialize(path: , ref_array: [], io: STDOUT, log_dir: "/dev/null")
+      @path = Pathname.new(path)
+
+      @in_git_path = Git::InPath.new(@path.expand_path)
+
+      raise "Must be a path with a .git directory '#{@path}'" if !@path.join(".git").exist?
+      @io = io
+      @commits = []
+      log_dir = Pathname(log_dir)
+
+      expand_refs(ref_array).each do |ref|
+        restore_branch_on_return(quiet: true) do
+          @commits << Git::Commit.new(path: @path, ref: ref, log_dir: log_dir)
+        end
+      end
+
+      if (duplicate = @commits.group_by(&:short_sha).detect {|(k, v)| v.length > 1})
+        raise "Duplicate SHA resolved #{duplicate[0].inspect}: #{duplicate[1].map {|c| "'#{c.ref}' => '#{c.short_sha}'"}.join(", ") } at #{@path}"
+      end
+    end
+
+    def current_branch_or_sha
+      branch_or_sha = @in_git_path.branch
+      branch_or_sha ||= @in_git_path.short_sha
+      branch_or_sha
+    end
+
+    def dirty?
+      !clean?
+    end
+
+    # https://stackoverflow.com/a/3879077/147390
+    def clean?
+      @in_git_path.run("git diff-index --quiet HEAD --") && $?.success?
+    end
+
+    private def status(pattern: "*.gemspec")
+      @in_git_path.run("git status #{pattern}")
+    end
+
+    def restore_branch_on_return(quiet: false)
+      if dirty? && status.include?("gemspec")
+        dirty_gemspec = true
+        unless quiet
+          @io.puts "Working tree at #{@path} is dirty, stashing. This will be popped on return"
+          @io.puts "Bundler modifies gemspec files on git install, this is normal"
+          @io.puts "Original status:\n#{status}"
+        end
+        @in_git_path.run!("git stash")
+      end
+      branch_or_sha = self.current_branch_or_sha
+      yield
+    ensure
+      return unless branch_or_sha
+      @io.puts "Resetting git dir of '#{@path.to_s}' to #{branch_or_sha.inspect}" unless quiet
+
+      @in_git_path.checkout!(branch_or_sha)
+      if dirty_gemspec
+        out = @in_git_path.run!("git stash apply 2>&1")
+        @io.puts "Applying stash of '#{@path.to_s}':\n#{out}" unless quiet
+      end
+    end
+
+    # case ref_array.length
+    # when >= 2
+    #   returns original array
+    # when 1
+    #   returns the given ref plus the one before it
+    # when 0
+    #   returns the most recent 2 refs
+    private def expand_refs(ref_array)
+      return ref_array if ref_array.length >= 2
+
+      @in_git_path.checkout!(ref_array.first) if ref_array.first
+
+      branches_string = @in_git_path.run!("git log --format='%H' -n 2")
+      ref_array = branches_string.split($/)
+      return ref_array
+    end
+  end
+end