derailed_benchmarks 1.6.0 → 2.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +75 -0
  3. data/.github/workflows/check_changelog.yml +11 -8
  4. data/CHANGELOG.md +25 -1
  5. data/README.md +32 -9
  6. data/derailed_benchmarks.gemspec +6 -3
  7. data/gemfiles/rails_5_1.gemfile +3 -1
  8. data/gemfiles/rails_5_2.gemfile +3 -3
  9. data/gemfiles/rails_6_1.gemfile +13 -0
  10. data/gemfiles/rails_git.gemfile +2 -2
  11. data/lib/derailed_benchmarks.rb +4 -2
  12. data/lib/derailed_benchmarks/core_ext/kernel_require.rb +29 -24
  13. data/lib/derailed_benchmarks/git/commit.rb +36 -0
  14. data/lib/derailed_benchmarks/git/in_path.rb +59 -0
  15. data/lib/derailed_benchmarks/git/switch_project.rb +128 -0
  16. data/lib/derailed_benchmarks/git_switch_project.rb +1 -0
  17. data/lib/derailed_benchmarks/load_tasks.rb +11 -4
  18. data/lib/derailed_benchmarks/require_tree.rb +11 -1
  19. data/lib/derailed_benchmarks/{stats_in_file.rb → stats_for_file.rb} +8 -2
  20. data/lib/derailed_benchmarks/stats_from_dir.rb +53 -9
  21. data/lib/derailed_benchmarks/tasks.rb +26 -63
  22. data/lib/derailed_benchmarks/version.rb +1 -1
  23. data/test/derailed_benchmarks/core_ext/kernel_require_test.rb +70 -11
  24. data/test/derailed_benchmarks/git_switch_project_test.rb +83 -0
  25. data/test/derailed_benchmarks/require_tree_test.rb +1 -1
  26. data/test/derailed_benchmarks/stats_from_dir_test.rb +43 -10
  27. data/test/fixtures/require/autoload_child.rb +5 -0
  28. data/test/fixtures/require/autoload_parent.rb +8 -0
  29. data/test/fixtures/require/child_one.rb +1 -1
  30. data/test/fixtures/require/child_two.rb +1 -1
  31. data/test/fixtures/require/load_child.rb +3 -0
  32. data/test/fixtures/require/load_parent.rb +5 -0
  33. data/test/fixtures/require/parent_one.rb +1 -1
  34. data/test/integration/tasks_test.rb +31 -5
  35. data/test/rails_app/config/application.rb +2 -0
  36. data/test/rails_app/config/storage.yml +0 -0
  37. data/test/test_helper.rb +6 -1
  38. metadata +90 -26
  39. data/.travis.yml +0 -18
  40. data/Appraisals +0 -26
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 534c94c162b5f9959a2013292d3f58fdf3bad5f17276a6db240a75596f1b041e
4
- data.tar.gz: e7db4cf3bb3ea0f965dd25af1dcfe1a85dbfb80d6c3508e295ac68432b4e9868
3
+ metadata.gz: cd9db4d2438152b5594f932fa35b162859a60024b4d74d00f3d4c7d5ada3f09f
4
+ data.tar.gz: f77ae05368c875fee001102adbd9652c1ef8001d2160064260c8b59dd455b00b
5
5
  SHA512:
6
- metadata.gz: 4f39f3b7df3d063c703df439aba0dd0be34650bbbf490d9af11c28dac0a9449eb347bfd9201124bedfea3b53b6f731280fafa5a9f6e40e7c311d1584b5061e13
7
- data.tar.gz: 151966ff16e1e07bf217ce88e3c4868089eb2ef6428ddd6e93d8d3761d97902a4c4ee8cca32389e80e5dcaf658f21c121e9a9bffcd2a52d80985e1560021767d
6
+ metadata.gz: f97fc2da5011cad2e69a6269776ceb7f4c769c3a7bd8bc5813848fc367f6f11ae6fdef1306139f3b8a3d5ead9cc93f4ce6d199688643111a794964392dd2d5d2
7
+ data.tar.gz: 39597ebde572f3ea60dcf0224c5986a6480055e5a50b4998f277c2a7fe70503b36d288e0ea146f02bf99a15edafb7c06d6d3be1ed1ba1cee81c3dfc2c0c165f7
@@ -0,0 +1,75 @@
1
+ version: 2.1
2
+ orbs:
3
+ ruby: circleci/ruby@1.1.2
4
+ references:
5
+ run_tests: &run_tests
6
+ run:
7
+ name: Run test suite
8
+ command: bundle exec rake test
9
+ # Needed because tests execute raw git commands
10
+ set_git_config: &set_git_config
11
+ run:
12
+ name: Set Git config
13
+ command: git config --global user.email "you@example.com"; git config --global user.name "Your Name"
14
+ restore: &restore
15
+ restore_cache:
16
+ keys:
17
+ - v1_bundler_deps-{{ .Environment.CIRCLE_JOB }}
18
+ save: &save
19
+ save_cache:
20
+ paths:
21
+ - ./vendor/bundle
22
+ key: v1_bundler_deps-{{ .Environment.CIRCLE_JOB }} # CIRCLE_JOB e.g. "ruby-2.5"
23
+ bundle: &bundle
24
+ run:
25
+ name: install dependencies
26
+ command: |
27
+ echo "export BUNDLE_JOBS=4" >> $BASH_ENV
28
+ echo "export BUNDLE_RETRY=3" >> $BASH_ENV
29
+ echo "export BUNDLE_PATH=$(pwd)/vendor/bundle" >> $BASH_ENV
30
+ echo "export BUNDLE_GEMFILE=$(pwd)/gemfiles/$GEMFILE_NAME" >> $BASH_ENV
31
+ source $BASH_ENV
32
+
33
+ bundle install
34
+ bundle update
35
+ bundle clean
36
+ mysteps: &mysteps
37
+ steps:
38
+ - checkout
39
+ - <<: *set_git_config
40
+ - <<: *restore
41
+ - <<: *bundle
42
+ - <<: *run_tests
43
+ - <<: *save
44
+
45
+ jobs:
46
+ test:
47
+ parameters:
48
+ ruby_version:
49
+ type: string
50
+ gemfile:
51
+ type: string
52
+ docker:
53
+ - image: "ruby:<< parameters.ruby_version >>"
54
+ environment:
55
+ GEMFILE_NAME: <<parameters.gemfile>>
56
+ steps:
57
+ - checkout
58
+ - <<: *set_git_config
59
+ - <<: *restore
60
+ - <<: *bundle
61
+ - <<: *run_tests
62
+ - <<: *save
63
+
64
+ workflows:
65
+ all-tests:
66
+ jobs:
67
+ - test:
68
+ matrix:
69
+ parameters:
70
+ ruby_version: ["2.5.8", "2.7.2", "3.0.0"]
71
+ gemfile: ["rails_5_2.gemfile", "rails_6_1.gemfile", "rails_git.gemfile"]
72
+ exclude:
73
+ - ruby_version: "3.0.0"
74
+ gemfile: rails_5_2.gemfile
75
+ name: test-ruby-<<matrix.ruby_version>>-<<matrix.gemfile>>
@@ -1,10 +1,13 @@
1
1
  name: Check Changelog
2
- on: [pull_request]
2
+
3
+ on:
4
+ pull_request:
5
+ types: [opened, reopened, edited, synchronize]
3
6
  jobs:
4
- build:
5
- runs-on: ubuntu-latest
6
- steps:
7
- - uses: actions/checkout@v1
8
- - name: Check that CHANGELOG is touched
9
- run: |
10
- cat $GITHUB_EVENT_PATH | jq .pull_request.title | grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' || git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
7
+ build:
8
+ runs-on: ubuntu-latest
9
+ steps:
10
+ - uses: actions/checkout@v1
11
+ - name: Check that CHANGELOG is touched
12
+ run: |
13
+ cat $GITHUB_EVENT_PATH | jq .pull_request.title | grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' || git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
data/CHANGELOG.md CHANGED
@@ -1,4 +1,28 @@
1
- ## master (unreleased)
1
+ ## HEAD
2
+
3
+ ## 2.0.1
4
+
5
+ - `rack-test` dependency added (https://github.com/schneems/derailed_benchmarks/pull/187)
6
+
7
+ ## 2.0.0
8
+
9
+ - Syntax errors easier to debug with `dead_end` gem (https://github.com/schneems/derailed_benchmarks/pull/182)
10
+ - Minimum ruby version is now 2.5 (https://github.com/schneems/derailed_benchmarks/pull/183)
11
+ - Histograms are now printed side-by-side (https://github.com/schneems/derailed_benchmarks/pull/179)
12
+
13
+ ## 1.8.1
14
+
15
+ - Derailed now tracks memory use from `load` in addition to `require` (https://github.com/schneems/derailed_benchmarks/pull/178)
16
+ - Correct logging of unsuccessful curl requests to file (https://github.com/schneems/derailed_benchmarks/pull/172)
17
+
18
+ ## 1.8.0
19
+
20
+ - Ruby 2.2 is now officialy supported and tested (https://github.com/schneems/derailed_benchmarks/pull/177)
21
+
22
+ ## 1.7.0
23
+
24
+ - Add histogram support to `perf:library` (https://github.com/schneems/derailed_benchmarks/pull/169)
25
+ - Fix bug with `Kernel#require` patch when Zeitwerk is enabled (https://github.com/schneems/derailed_benchmarks/pull/170)
2
26
 
3
27
  ## 1.6.0
4
28
 
data/README.md CHANGED
@@ -9,16 +9,13 @@ A series of things you can use to benchmark a Rails or Ruby app.
9
9
 
10
10
  ## Compatibility/Requirements
11
11
 
12
- This gem has been tested and is known to work with Rails 3.2+ using Ruby
13
- 2.1+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
14
-
15
12
  For some benchmarks, not all, you'll need to verify you have a working version of curl on your OS:
16
13
 
17
14
  ```
18
15
  $ which curl
19
16
  /usr/bin/curl
20
17
  $ curl -V
21
- curl 7.37.1 #...
18
+ curl 7.64.1 #...
22
19
  ```
23
20
 
24
21
  ## Install
@@ -448,11 +445,37 @@ When the test is done it will output which commit "won" and by how much:
448
445
  ```
449
446
  ❤️ ❤️ ❤️ (Statistically Significant) ❤️ ❤️ ❤️
450
447
 
451
- [7b4d80cb37] "1.8x Faster Partial Caching - Faster Cache Keys" - (10.9711965 seconds)
452
- FASTER by:
453
- 1.0870x [older/newer]
454
- 8.0026% [(older - newer) / older * 100]
455
- [13d6aa3a7b] "Merge pull request #36284 from kamipo/fix_eager_loading_with_string_joins" - (11.9255485 seconds)
448
+ [f1ab117] (11.3844 seconds) "I am the new commit" ref: "winner"
449
+ FASTER 🚀🚀🚀 by:
450
+ 1.0062x [older/newer]
451
+ 0.6147% [(older - newer) / older * 100]
452
+ [5594a2d] (11.4548 seconds) "Old commit" ref: "loser"
453
+
454
+ Iterations per sample:
455
+ Samples: 100
456
+
457
+ Test type: Kolmogorov Smirnov
458
+ Confidence level: 99.0 %
459
+ Is significant? (max > critical): true
460
+ D critical: 0.2145966026289347
461
+ D max: 0.26
462
+
463
+ Histograms (time ranges are in seconds):
464
+
465
+ [f1ab117] description: [5594a2d] description:
466
+ "I am the new commit" "Old commit"
467
+ ┌ ┐ ┌ ┐
468
+ [11.2 , 11.28) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 12 [11.2 , 11.28) ┤▇▇▇▇ 3
469
+ [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 22 [11.28, 11.36) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 19
470
+ [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 30 [11.35, 11.43) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17
471
+ [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 17 [11.43, 11.51) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 25
472
+ [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13 [11.5 , 11.58) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 15
473
+ [11.58, 11.66) ┤▇▇▇▇▇▇▇ 6 [11.58, 11.66) ┤▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 13
474
+ [11.65, 11.73) ┤ 0 [11.65, 11.73) ┤▇▇▇▇ 3
475
+ [11.73, 11.81) ┤ 0 [11.73, 11.81) ┤▇▇▇▇ 3
476
+ [11.8 , 11.88) ┤ 0 [11.8 , 11.88) ┤▇▇▇ 2
477
+ └ ┘ └ ┘
478
+ # of runs in range # of runs in range
456
479
  ```
457
480
 
458
481
  You can provide this to the Rails team along with the example app you used to benchmark (so they can independently verify if needed).
@@ -20,20 +20,23 @@ Gem::Specification.new do |gem|
20
20
  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
21
21
  gem.require_paths = ["lib"]
22
22
 
23
- gem.required_ruby_version = ">= 2.1.0"
23
+ gem.required_ruby_version = ">= 2.5.0"
24
24
 
25
25
  gem.add_dependency "heapy", "~> 0"
26
- gem.add_dependency "memory_profiler", "~> 0"
26
+ gem.add_dependency "memory_profiler", ">= 0", "< 2"
27
27
  gem.add_dependency "get_process_mem", "~> 0"
28
28
  gem.add_dependency "benchmark-ips", "~> 2"
29
29
  gem.add_dependency "rack", ">= 1"
30
30
  gem.add_dependency "rake", "> 10", "< 14"
31
31
  gem.add_dependency "thor", ">= 0.19", "< 2"
32
32
  gem.add_dependency "ruby-statistics", ">= 2.1"
33
+ gem.add_dependency "mini_histogram", ">= 0.3.0"
34
+ gem.add_dependency "dead_end", ">= 0"
35
+ gem.add_dependency "rack-test", ">= 0"
33
36
 
37
+ gem.add_development_dependency "webrick", ">= 0"
34
38
  gem.add_development_dependency "capybara", "~> 2"
35
39
  gem.add_development_dependency "m"
36
40
  gem.add_development_dependency "rails", "> 3", "<= 7"
37
41
  gem.add_development_dependency "devise", "> 3", "< 6"
38
- gem.add_development_dependency "appraisal", "2.2.0"
39
42
  end
@@ -1,10 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # BUNDLE_GEMFILE="gemfiles/rails_5_1.gemfile" bundle exec m test/integration/tasks_test.rb:30
4
+ #
3
5
  # This file was generated by Appraisal
4
6
 
5
7
  source "https://rubygems.org"
6
8
 
7
- gem "rails", "~> 5.1.0"
9
+ gem "rails", "~> 5.1.7"
8
10
 
9
11
  group :development, :test do
10
12
  gem "sqlite3", platform: [:ruby, :mswin, :mingw]
@@ -1,10 +1,10 @@
1
- # frozen_string_literal: true
2
-
3
1
  # This file was generated by Appraisal
2
+ #
3
+ # BUNDLE_GEMFILE="gemfiles/rails_5_2.gemfile" bundle exec m test/integration/tasks_test.rb:30
4
4
 
5
5
  source "https://rubygems.org"
6
6
 
7
- gem "rails", "~> 5.2.0"
7
+ gem "rails", "~> 5.2.4.4"
8
8
 
9
9
  group :development, :test do
10
10
  gem "sqlite3", platform: [:ruby, :mswin, :mingw]
@@ -0,0 +1,13 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "~> 6.1.1"
6
+
7
+ group :development, :test do
8
+ gem "sqlite3", platform: [:ruby, :mswin, :mingw]
9
+ gem "activerecord-jdbcsqlite3-adapter", "~> 1.3.13", platform: :jruby
10
+ gem "test-unit", "~> 3.0"
11
+ end
12
+
13
+ gemspec path: "../"
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:30
3
+ # $ BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:50
4
4
 
5
5
  source "https://rubygems.org"
6
6
 
7
- gem "rails", github: "rails/rails", ref: "3054e1d584e7eca110c69a1f8423f2e0866abbf9"
7
+ gem "rails", github: "rails/rails", ref: "12bb9d32f56883914abcd98fd72e3c68c444808d"
8
8
 
9
9
  gem 'devise', github: "plataformatec/devise"
10
10
 
@@ -2,9 +2,10 @@
2
2
 
3
3
  require 'time'
4
4
  require 'bundler'
5
-
6
5
  require 'get_process_mem'
7
6
 
7
+ require 'dead_end'
8
+
8
9
  module DerailedBenchmarks
9
10
  def self.gem_is_bundled?(name)
10
11
  specs = ::Bundler.locked_gems.specs.each_with_object({}) {|spec, hash| hash[spec.name] = spec }
@@ -43,8 +44,9 @@ end
43
44
  require 'derailed_benchmarks/require_tree'
44
45
  require 'derailed_benchmarks/auth_helper'
45
46
 
46
- require 'derailed_benchmarks/stats_in_file'
47
+ require 'derailed_benchmarks/stats_for_file'
47
48
  require 'derailed_benchmarks/stats_from_dir'
49
+ require 'derailed_benchmarks/git/switch_project'
48
50
 
49
51
  if DerailedBenchmarks.gem_is_bundled?("devise")
50
52
  DerailedBenchmarks.auth = DerailedBenchmarks::AuthHelpers::Devise.new
@@ -11,14 +11,24 @@ ENV['CUT_OFF'] ||= "0.3"
11
11
  # Monkey patch kernel to ensure that all `require` calls call the same
12
12
  # method
13
13
  module Kernel
14
+ REQUIRE_STACK = []
14
15
 
15
- private
16
+ module_function
16
17
 
17
- alias :original_require :require
18
- REQUIRE_STACK = []
18
+ alias_method :original_require, :require
19
+ alias_method :original_require_relative, :require_relative
20
+ alias_method(:original_load, :load)
21
+
22
+ def load(file, wrap = false)
23
+ measure_memory_impact(file) do |file|
24
+ original_load(file)
25
+ end
26
+ end
19
27
 
20
28
  def require(file)
21
- Kernel.require(file)
29
+ measure_memory_impact(file) do |file|
30
+ original_require(file)
31
+ end
22
32
  end
23
33
 
24
34
  def require_relative(file)
@@ -29,10 +39,7 @@ module Kernel
29
39
  end
30
40
  end
31
41
 
32
- class << self
33
- alias :original_require :require
34
- alias :original_require_relative :require_relative
35
- end
42
+ private
36
43
 
37
44
  # The core extension we use to measure require time of all requires
38
45
  # When a file is required we create a tree node with its file name.
@@ -46,7 +53,7 @@ module Kernel
46
53
  # When a require returns we remove it from the require stack so we don't
47
54
  # accidentally push additional children nodes to it. We then store the
48
55
  # memory cost of the require in the tree node.
49
- def self.measure_memory_impact(file, &block)
56
+ def measure_memory_impact(file, &block)
50
57
  mem = GetProcessMem.new
51
58
  node = DerailedBenchmarks::RequireTree.new(file)
52
59
 
@@ -64,31 +71,29 @@ module Kernel
64
71
  end
65
72
  end
66
73
 
67
- # Top level node that will store all require information for the entire app
68
- TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
69
- REQUIRE_STACK.push(TOP_REQUIRE)
70
-
71
- Kernel.define_singleton_method(:require) do |file|
72
- measure_memory_impact(file) do |file|
73
- # "source_annotation_extractor" is deprecated in Rails 6
74
- # # if we don't skip the library it leads to a crash
75
- # next if file == "rails/source_annotation_extractor" && Rails.version >= '6.0'
76
- original_require(file)
77
- end
78
- end
79
74
 
75
+ # I honestly have no idea why this Object delegation is needed
76
+ # I keep staring at bootsnap and it doesn't have to do this
77
+ # is there a bug in their implementation they haven't caught or
78
+ # am I doing something different?
80
79
  class Object
81
80
  private
81
+ def load(path, wrap = false)
82
+ Kernel.load(path, wrap)
83
+ end
82
84
 
83
85
  def require(path)
84
86
  Kernel.require(path)
85
87
  end
86
88
  end
87
89
 
88
- # Don't forget to assign a cost to the top level
89
- cost_before_requiring_anything = GetProcessMem.new.mb
90
- TOP_REQUIRE.cost = cost_before_requiring_anything
90
+ # Top level node that will store all require information for the entire app
91
+ TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
92
+ REQUIRE_STACK.push(TOP_REQUIRE)
93
+ TOP_REQUIRE.cost = GetProcessMem.new.mb
94
+
91
95
  def TOP_REQUIRE.print_sorted_children(*args)
92
96
  self.cost = GetProcessMem.new.mb - self.cost
93
97
  super
94
98
  end
99
+
@@ -0,0 +1,36 @@
1
+ module DerailedBenchmarks
2
+ # Represents a specific commit in a git repo
3
+ #
4
+ # Can be used to get information from the commit or to check it out
5
+ #
6
+ # commit = GitCommit.new(path: "path/to/repo", ref: "6e642963acec0ff64af51bd6fba8db3c4176ed6e")
7
+ # commit.short_sha # => "6e64296"
8
+ # commit.checkout! # Will check out the current commit at the repo in the path
9
+ class Git::Commit
10
+ attr_reader :ref, :description, :time, :short_sha, :log
11
+
12
+ def initialize(path: , ref: , log_dir: Pathname.new("/dev/null"))
13
+ @in_git_path = Git::InPath.new(path)
14
+ @ref = ref
15
+ @log = log_dir.join("#{file_safe_ref}.bench.txt")
16
+
17
+ Dir.chdir(path) do
18
+ checkout!
19
+ @description = @in_git_path.description
20
+ @short_sha = @in_git_path.short_sha
21
+ @time = @in_git_path.time
22
+ end
23
+ end
24
+
25
+ alias :desc :description
26
+ alias :file :log
27
+
28
+ def checkout!
29
+ @in_git_path.checkout!(ref)
30
+ end
31
+
32
+ private def file_safe_ref
33
+ ref.gsub('/', ':')
34
+ end
35
+ end
36
+ end