derailed_benchmarks 1.7.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +75 -0
  3. data/.github/workflows/check_changelog.yml +11 -8
  4. data/CHANGELOG.md +24 -1
  5. data/README.md +73 -11
  6. data/derailed_benchmarks.gemspec +6 -5
  7. data/gemfiles/rails_5_1.gemfile +3 -1
  8. data/gemfiles/rails_5_2.gemfile +3 -3
  9. data/gemfiles/rails_6_1.gemfile +13 -0
  10. data/gemfiles/rails_git.gemfile +2 -2
  11. data/lib/derailed_benchmarks.rb +4 -2
  12. data/lib/derailed_benchmarks/core_ext/kernel_require.rb +20 -9
  13. data/lib/derailed_benchmarks/git/commit.rb +36 -0
  14. data/lib/derailed_benchmarks/git/in_path.rb +59 -0
  15. data/lib/derailed_benchmarks/git/switch_project.rb +128 -0
  16. data/lib/derailed_benchmarks/git_switch_project.rb +1 -0
  17. data/lib/derailed_benchmarks/load_tasks.rb +1 -1
  18. data/lib/derailed_benchmarks/require_tree.rb +11 -1
  19. data/lib/derailed_benchmarks/{stats_in_file.rb → stats_for_file.rb} +8 -2
  20. data/lib/derailed_benchmarks/stats_from_dir.rb +40 -21
  21. data/lib/derailed_benchmarks/tasks.rb +63 -66
  22. data/lib/derailed_benchmarks/version.rb +1 -1
  23. data/test/derailed_benchmarks/core_ext/kernel_require_test.rb +70 -11
  24. data/test/derailed_benchmarks/git_switch_project_test.rb +83 -0
  25. data/test/derailed_benchmarks/require_tree_test.rb +1 -1
  26. data/test/derailed_benchmarks/stats_from_dir_test.rb +29 -12
  27. data/test/derailed_test.rb +15 -0
  28. data/test/fixtures/require/autoload_child.rb +5 -0
  29. data/test/fixtures/require/autoload_parent.rb +8 -0
  30. data/test/fixtures/require/child_one.rb +1 -1
  31. data/test/fixtures/require/child_two.rb +1 -1
  32. data/test/fixtures/require/load_child.rb +3 -0
  33. data/test/fixtures/require/load_parent.rb +5 -0
  34. data/test/fixtures/require/parent_one.rb +1 -1
  35. data/test/integration/tasks_test.rb +36 -6
  36. data/test/rails_app/config/storage.yml +0 -0
  37. data/test/test_helper.rb +6 -1
  38. metadata +67 -37
  39. data/.travis.yml +0 -18
  40. data/Appraisals +0 -26
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module DerailedBenchmarks
4
- VERSION = "1.7.0"
4
+ VERSION = "2.1.0"
5
5
  end
@@ -3,7 +3,6 @@
3
3
  require 'test_helper'
4
4
 
5
5
  class KernelRequireTest < ActiveSupport::TestCase
6
-
7
6
  setup do
8
7
  require 'derailed_benchmarks/core_ext/kernel_require'
9
8
  GC.disable
@@ -11,23 +10,83 @@ class KernelRequireTest < ActiveSupport::TestCase
11
10
 
12
11
  teardown do
13
12
  GC.enable
13
+ DerailedBenchmarks::RequireTree.reset!
14
+ end
15
+
16
+ test "profiles load" do
17
+ in_fork do
18
+ require fixtures_dir("require/load_parent.rb")
19
+
20
+ parent = assert_node_in_parent("load_parent.rb", TOP_REQUIRE)
21
+
22
+ assert_node_in_parent("load_child.rb", parent)
23
+ end
24
+ end
25
+
26
+ test "profiles autoload" do
27
+ skip if RUBY_VERSION.start_with?("2.2") # Fails on CI, I can't install Ruby 2.2 locally to debug https://stackoverflow.com/questions/63926460/install-ruby-2-2-on-mac-osx-catalina-with-ruby-install, https://github.com/postmodern/ruby-install/issues/375
28
+
29
+ in_fork do
30
+ require fixtures_dir("require/autoload_parent.rb")
31
+ parent = assert_node_in_parent("autoload_parent.rb", TOP_REQUIRE)
32
+
33
+ assert_node_in_parent("autoload_child.rb", parent)
34
+ end
14
35
  end
15
36
 
37
+ test "core extension profiles useage" do
38
+ in_fork do
39
+ require fixtures_dir("require/parent_one.rb")
40
+ parent = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
41
+ assert_node_in_parent("child_one.rb", parent)
42
+ child_two = assert_node_in_parent("child_two.rb", parent)
43
+ assert_node_in_parent("relative_child", parent)
44
+ assert_node_in_parent("relative_child_two", parent)
45
+ assert_node_in_parent("raise_child.rb", child_two)
46
+ end
47
+ end
48
+
49
+ # Checks to see that the given file name is present in the
50
+ # parent tree node and that the memory of that file
51
+ # is less than the parent (since the parent should include itself
52
+ # plus its children)
53
+ #
54
+ # Returns the child node
16
55
  def assert_node_in_parent(file_name, parent)
17
56
  file = fixtures_dir(File.join("require", file_name))
18
57
  node = parent[file]
19
- assert node, "Expected:\n#{parent.children}\nto include:\n#{file.inspect}"
20
- assert node.cost < parent.cost, "Expected:\n#{node.inspect}\nto cost less than:\n#{parent.inspect}" unless parent == TOP_REQUIRE
58
+ assert node, "Expected: #{parent.name} to include: #{file.to_s} but it did not.\nChildren: #{parent.children.map(&:name).map(&:to_s)}"
59
+ unless parent == TOP_REQUIRE
60
+ assert node.cost < parent.cost, "Expected: #{node.name.inspect} (#{node.cost}) to cost less than: #{parent.name.inspect} (#{parent.cost})"
61
+ end
21
62
  node
22
63
  end
23
64
 
24
- test "core extension profiles useage" do
25
- require fixtures_dir("require/parent_one.rb")
26
- parent = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
27
- assert_node_in_parent("child_one.rb", parent)
28
- child_two = assert_node_in_parent("child_two.rb", parent)
29
- assert_node_in_parent("relative_child", parent)
30
- assert_node_in_parent("relative_child_two", parent)
31
- assert_node_in_parent("raise_child.rb", child_two)
65
+ # Used to get semi-clean process memory
66
+ # It would be better to run the requires in a totally different process
67
+ # but...that would take engineering
68
+ #
69
+ # If I was going to do that, I would find a way to serialize RequireTree
70
+ # into a json structure with file names and costs, run the script
71
+ # dump the json to a file, then in this process read the file and
72
+ # run assertions
73
+ def in_fork
74
+ Tempfile.create("stdout") do |tmp_file|
75
+ pid = fork do
76
+ $stdout.reopen(tmp_file, "w")
77
+ $stderr.reopen(tmp_file, "w")
78
+ $stdout.sync = true
79
+ $stderr.sync = true
80
+ yield
81
+ Kernel.exit!(0) # needed for https://github.com/seattlerb/minitest/pull/683
82
+ end
83
+ Process.waitpid(pid)
84
+
85
+ if $?.success?
86
+ print File.read(tmp_file)
87
+ else
88
+ raise File.read(tmp_file)
89
+ end
90
+ end
32
91
  end
33
92
  end
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'test_helper'
4
+
5
+ class GitSwitchProjectTest < ActiveSupport::TestCase
6
+ test "tells me when it's not pointing at a git project" do
7
+ exception = assert_raises {
8
+ DerailedBenchmarks::Git::SwitchProject.new(path: "/dev/null")
9
+ }
10
+ assert_includes(exception.message, '.git directory')
11
+ end
12
+
13
+ test "dirty gemspec cleaning" do
14
+ Dir.mktmpdir do |dir|
15
+ run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
16
+ run!("cd #{dir} && echo lol > foo.gemspec && git add .")
17
+
18
+ io = StringIO.new
19
+ project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, io: io)
20
+
21
+ assert project.dirty?
22
+ refute project.clean?
23
+
24
+ project.restore_branch_on_return do
25
+ project.commits.map(&:checkout!)
26
+ end
27
+
28
+ assert_includes io.string, "Bundler modifies gemspec files"
29
+ assert_includes io.string, "Applying stash"
30
+ end
31
+ end
32
+
33
+ test "works on a git repo" do
34
+ Dir.mktmpdir do |dir|
35
+ run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
36
+
37
+ # finds shas when none given
38
+ project = DerailedBenchmarks::Git::SwitchProject.new(path: dir)
39
+
40
+ assert_equal ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "da748a59340be8b950e7bbbfb32077eb67d70c3c"], project.commits.map(&:ref)
41
+ first_commit = project.commits.first
42
+
43
+ assert_equal "CI test support", first_commit.description
44
+ assert_equal "6e64296", first_commit.short_sha
45
+ assert_equal "/dev/null/6e642963acec0ff64af51bd6fba8db3c4176ed6e.bench.txt", first_commit.log.to_s
46
+ assert_equal DateTime.parse("Tue, 14 Apr 2020 13:26:03 -0500"), first_commit.time
47
+
48
+ assert_equal "mybranch", project.current_branch_or_sha
49
+
50
+ # Finds shas when 1 is given
51
+ project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c"])
52
+
53
+ assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "5c09f748957d2098182762004adee27d1ff83160"], project.commits.map(&:ref)
54
+
55
+
56
+ # Returns correct refs if given
57
+ project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"])
58
+
59
+ assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"], project.commits.map(&:ref)
60
+
61
+ first_commit = project.commits.first
62
+
63
+ first_commit.checkout!
64
+
65
+ assert_equal first_commit.short_sha, project.current_branch_or_sha
66
+
67
+ # Test restore_branch_on_return
68
+ project.restore_branch_on_return(quiet: true) do
69
+ project.commits.last.checkout!
70
+
71
+ assert_equal project.commits.last.short_sha, project.current_branch_or_sha
72
+ end
73
+
74
+ assert_equal project.commits.first.short_sha, project.current_branch_or_sha
75
+
76
+ exception = assert_raise {
77
+ DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "mybranch"])
78
+ }
79
+
80
+ assert_includes(exception.message, 'Duplicate SHA resolved "6e64296"')
81
+ end
82
+ end
83
+ end
@@ -9,7 +9,7 @@ class RequireTree < ActiveSupport::TestCase
9
9
  end
10
10
 
11
11
  def teardown
12
- DerailedBenchmarks::RequireTree.const_set("REQUIRED_BY", {})
12
+ DerailedBenchmarks::RequireTree.reset!
13
13
  end
14
14
 
15
15
  test "default_cost" do
@@ -3,6 +3,19 @@
3
3
  require 'test_helper'
4
4
 
5
5
  class StatsFromDirTest < ActiveSupport::TestCase
6
+ test "empty files" do
7
+ Dir.mktmpdir do |dir|
8
+ dir = Pathname.new(dir)
9
+ branch_info = {}
10
+ branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser"), name: "loser" }
11
+ branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner"), name: "winner" }
12
+ stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
13
+ io = StringIO.new
14
+ stats.call.banner(io: io) # Doesn't error
15
+ assert_equal "", io.read
16
+ end
17
+ end
18
+
6
19
  test "that it works" do
7
20
  dir = fixtures_dir("stats/significant")
8
21
  branch_info = {}
@@ -32,12 +45,13 @@ class StatsFromDirTest < ActiveSupport::TestCase
32
45
  test "histogram output" do
33
46
  dir = fixtures_dir("stats/significant")
34
47
  branch_info = {}
35
- branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
36
- branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
48
+ branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), short_sha: "5594a2d" }
49
+ branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), short_sha: "f1ab117" }
37
50
  stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
38
51
 
39
52
  io = StringIO.new
40
53
  stats.call.banner(io)
54
+
41
55
  puts io.string
42
56
 
43
57
  assert_match(/11\.2 , 11\.28/, io.string)
@@ -64,10 +78,13 @@ class StatsFromDirTest < ActiveSupport::TestCase
64
78
 
65
79
  test "banner faster" do
66
80
  dir = fixtures_dir("stats/significant")
67
- branch_info = {}
68
- branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
69
- branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
70
- stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
81
+ Branch_info = {}
82
+
83
+ require 'ostruct'
84
+ commits = []
85
+ commits << OpenStruct.new({ desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), ref: "loser", short_sha: "aaaaa" })
86
+ commits << OpenStruct.new({ desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), ref: "winner", short_sha: "bbbbb" })
87
+ stats = DerailedBenchmarks::StatsFromDir.new(commits).call
71
88
  newest = stats.newest
72
89
  oldest = stats.oldest
73
90
 
@@ -88,12 +105,12 @@ class StatsFromDirTest < ActiveSupport::TestCase
88
105
  11.0
89
106
  end
90
107
 
91
- expected = <<~EOM
92
- [winner] "I am the new commit" - (10.5 seconds)
108
+ expected = <<-EOM
109
+ [bbbbb] (10.5000 seconds) "I am the new commit" ref: "winner"
93
110
  FASTER 🚀🚀🚀 by:
94
111
  1.0476x [older/newer]
95
112
  4.5455% [(older - newer) / older * 100]
96
- [loser] "Old commit" - (11.0 seconds)
113
+ [aaaaa] (11.0000 seconds) "Old commit" ref: "loser"
97
114
  EOM
98
115
 
99
116
  actual = StringIO.new
@@ -119,12 +136,12 @@ EOM
119
136
  11.0
120
137
  end
121
138
 
122
- expected = <<~EOM
123
- [loser] "I am the new commit" - (11.0 seconds)
139
+ expected = <<-EOM
140
+ [loser] (11.0000 seconds) "I am the new commit" ref: "loser"
124
141
  SLOWER 🐢🐢🐢 by:
125
142
  0.9545x [older/newer]
126
143
  -4.7619% [(older - newer) / older * 100]
127
- [winner] "Old commit" - (10.5 seconds)
144
+ [winner] (10.5000 seconds) "Old commit" ref: "winner"
128
145
  EOM
129
146
 
130
147
  actual = StringIO.new
@@ -11,4 +11,19 @@ class DerailedBenchmarksTest < ActiveSupport::TestCase
11
11
  assert DerailedBenchmarks.gem_is_bundled?("rack")
12
12
  refute DerailedBenchmarks.gem_is_bundled?("wicked")
13
13
  end
14
+
15
+ test "readme contains correct output" do
16
+ readme_path = File.join(__dir__, "..", "README.md")
17
+ lines = File.foreach(readme_path)
18
+ lineno = 1
19
+ expected = lines.lazy.drop_while { |line|
20
+ lineno += 1
21
+ line != "$ bundle exec derailed exec --help\n"
22
+ }.drop(1).take_while { |line| line != "```\n" }.force.join
23
+ assert_equal(
24
+ expected,
25
+ `bundle exec derailed exec --help`,
26
+ "Please update README.md:#{lineno}"
27
+ )
28
+ end
14
29
  end
@@ -0,0 +1,5 @@
1
+ @retained = String.new("")
2
+ 1_000_000.times.map { @retained << "A" }
3
+
4
+ module AutoLoadChild
5
+ end
@@ -0,0 +1,8 @@
1
+ @retained = String.new("")
2
+ 1_000_000.times.map { @retained << "A" }
3
+
4
+ autoload :AutoLoadChild, File.join(__dir__, 'autoload_child.rb')
5
+
6
+ if AutoLoadChild
7
+ # yay
8
+ end
@@ -1,4 +1,4 @@
1
1
  class ChildOne
2
- @retained = +""
2
+ @retained = String.new("")
3
3
  50_000.times.map { @retained << "A" }
4
4
  end
@@ -1,5 +1,5 @@
1
1
  class ChildTwo
2
- @retained = +""
2
+ @retained = String.new("")
3
3
  200_000.times.map { @retained << "A" }
4
4
  end
5
5
 
@@ -0,0 +1,3 @@
1
+ @retained = String.new("")
2
+ 1_000_000.times.map { @retained << "A" }
3
+
@@ -0,0 +1,5 @@
1
+ @retained = String.new("")
2
+ 1_000_000.times.map { @retained << "A" }
3
+
4
+ load File.join(__dir__, "load_child.rb")
5
+
@@ -1,5 +1,5 @@
1
1
  class ParentOne
2
- @retained = +""
2
+ @retained = String.new("")
3
3
  1_000_000.times.map { @retained << "A" }
4
4
  end
5
5
  require File.expand_path('../child_one.rb', __FILE__)
@@ -26,21 +26,47 @@ class TasksTest < ActiveSupport::TestCase
26
26
  env_string = env.map {|key, value| "#{key.shellescape}=#{value.to_s.shellescape}" }.join(" ")
27
27
  cmd = "env #{env_string} bundle exec rake -f perf.rake #{cmd} --trace"
28
28
  puts "Running: #{cmd}"
29
- result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd}` }
30
- if assert_success
31
- assert $?.success?, "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
29
+ result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd} 2>&1` }
30
+ if assert_success && !$?.success?
31
+ puts result
32
+ raise "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
32
33
  end
33
34
 
34
35
  result
35
36
  end
36
37
 
37
- test 'library_branches' do
38
+ test 'non-rails library with branch specified' do
39
+ skip unless ENV['USING_RAILS_WICKED_BRANCH']
40
+
41
+ gem_path = run!("bundle info wicked --path")
42
+ env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2, "DERAILED_PATH_TO_LIBRARY" => gem_path}
43
+ puts rake "perf:library", { env: env }
44
+ end
45
+
46
+ test 'rails perf:library from git' do
47
+ # BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
48
+
38
49
  skip unless ENV['USING_RAILS_GIT']
39
50
 
40
- env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
51
+ env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
52
+ "SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
41
53
  puts rake "perf:library", { env: env }
42
54
  end
43
55
 
56
+ test "rails perf:library with bad script" do
57
+ # BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
58
+
59
+ skip unless ENV['USING_RAILS_GIT']
60
+
61
+ error = assert_raises {
62
+ env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
63
+ "SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
64
+ puts rake "perf:library", { env: env }
65
+ }
66
+
67
+ assert error.message =~ /nopenopenop:( command)? not found/, "Expected #{error.message} to include /nopenopenop: (command)? not found/ but it did not"
68
+ end
69
+
44
70
  test 'hitting authenticated devise apps' do
45
71
  env = { "PATH_TO_HIT" => "authenticated", "USE_AUTH" => "true", "TEST_COUNT" => "2" }
46
72
  result = rake 'perf:test', env: env
@@ -127,6 +153,10 @@ class TasksTest < ActiveSupport::TestCase
127
153
  end
128
154
 
129
155
  test 'ips' do
130
- rake "perf:mem_over_time"
156
+ rake "perf:ips"
157
+ end
158
+
159
+ test 'heap_diff' do
160
+ rake "perf:heap_diff", env: { "TEST_COUNT" => 5 }
131
161
  end
132
162
  end
File without changes
data/test/test_helper.rb CHANGED
@@ -51,7 +51,6 @@ class ActiveSupport::IntegrationCase
51
51
  end
52
52
  end
53
53
 
54
-
55
54
  def fixtures_dir(name = "")
56
55
  root_path("test/fixtures").join(name)
57
56
  end
@@ -63,3 +62,9 @@ end
63
62
  def rails_app_path(name = "")
64
63
  root_path("test/rails_app").join(name)
65
64
  end
65
+
66
+ def run!(cmd)
67
+ output = `#{cmd}`
68
+ raise "Cmd #{cmd} failed:\n#{output}" unless $?.success?
69
+ output
70
+ end