derailed_benchmarks 1.7.0 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +75 -0
- data/.github/workflows/check_changelog.yml +11 -8
- data/CHANGELOG.md +24 -1
- data/README.md +73 -11
- data/derailed_benchmarks.gemspec +6 -5
- data/gemfiles/rails_5_1.gemfile +3 -1
- data/gemfiles/rails_5_2.gemfile +3 -3
- data/gemfiles/rails_6_1.gemfile +13 -0
- data/gemfiles/rails_git.gemfile +2 -2
- data/lib/derailed_benchmarks.rb +4 -2
- data/lib/derailed_benchmarks/core_ext/kernel_require.rb +20 -9
- data/lib/derailed_benchmarks/git/commit.rb +36 -0
- data/lib/derailed_benchmarks/git/in_path.rb +59 -0
- data/lib/derailed_benchmarks/git/switch_project.rb +128 -0
- data/lib/derailed_benchmarks/git_switch_project.rb +1 -0
- data/lib/derailed_benchmarks/load_tasks.rb +1 -1
- data/lib/derailed_benchmarks/require_tree.rb +11 -1
- data/lib/derailed_benchmarks/{stats_in_file.rb → stats_for_file.rb} +8 -2
- data/lib/derailed_benchmarks/stats_from_dir.rb +40 -21
- data/lib/derailed_benchmarks/tasks.rb +63 -66
- data/lib/derailed_benchmarks/version.rb +1 -1
- data/test/derailed_benchmarks/core_ext/kernel_require_test.rb +70 -11
- data/test/derailed_benchmarks/git_switch_project_test.rb +83 -0
- data/test/derailed_benchmarks/require_tree_test.rb +1 -1
- data/test/derailed_benchmarks/stats_from_dir_test.rb +29 -12
- data/test/derailed_test.rb +15 -0
- data/test/fixtures/require/autoload_child.rb +5 -0
- data/test/fixtures/require/autoload_parent.rb +8 -0
- data/test/fixtures/require/child_one.rb +1 -1
- data/test/fixtures/require/child_two.rb +1 -1
- data/test/fixtures/require/load_child.rb +3 -0
- data/test/fixtures/require/load_parent.rb +5 -0
- data/test/fixtures/require/parent_one.rb +1 -1
- data/test/integration/tasks_test.rb +36 -6
- data/test/rails_app/config/storage.yml +0 -0
- data/test/test_helper.rb +6 -1
- metadata +67 -37
- data/.travis.yml +0 -18
- data/Appraisals +0 -26
@@ -3,7 +3,6 @@
|
|
3
3
|
require 'test_helper'
|
4
4
|
|
5
5
|
class KernelRequireTest < ActiveSupport::TestCase
|
6
|
-
|
7
6
|
setup do
|
8
7
|
require 'derailed_benchmarks/core_ext/kernel_require'
|
9
8
|
GC.disable
|
@@ -11,23 +10,83 @@ class KernelRequireTest < ActiveSupport::TestCase
|
|
11
10
|
|
12
11
|
teardown do
|
13
12
|
GC.enable
|
13
|
+
DerailedBenchmarks::RequireTree.reset!
|
14
|
+
end
|
15
|
+
|
16
|
+
test "profiles load" do
|
17
|
+
in_fork do
|
18
|
+
require fixtures_dir("require/load_parent.rb")
|
19
|
+
|
20
|
+
parent = assert_node_in_parent("load_parent.rb", TOP_REQUIRE)
|
21
|
+
|
22
|
+
assert_node_in_parent("load_child.rb", parent)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
test "profiles autoload" do
|
27
|
+
skip if RUBY_VERSION.start_with?("2.2") # Fails on CI, I can't install Ruby 2.2 locally to debug https://stackoverflow.com/questions/63926460/install-ruby-2-2-on-mac-osx-catalina-with-ruby-install, https://github.com/postmodern/ruby-install/issues/375
|
28
|
+
|
29
|
+
in_fork do
|
30
|
+
require fixtures_dir("require/autoload_parent.rb")
|
31
|
+
parent = assert_node_in_parent("autoload_parent.rb", TOP_REQUIRE)
|
32
|
+
|
33
|
+
assert_node_in_parent("autoload_child.rb", parent)
|
34
|
+
end
|
14
35
|
end
|
15
36
|
|
37
|
+
test "core extension profiles useage" do
|
38
|
+
in_fork do
|
39
|
+
require fixtures_dir("require/parent_one.rb")
|
40
|
+
parent = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
|
41
|
+
assert_node_in_parent("child_one.rb", parent)
|
42
|
+
child_two = assert_node_in_parent("child_two.rb", parent)
|
43
|
+
assert_node_in_parent("relative_child", parent)
|
44
|
+
assert_node_in_parent("relative_child_two", parent)
|
45
|
+
assert_node_in_parent("raise_child.rb", child_two)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# Checks to see that the given file name is present in the
|
50
|
+
# parent tree node and that the memory of that file
|
51
|
+
# is less than the parent (since the parent should include itself
|
52
|
+
# plus its children)
|
53
|
+
#
|
54
|
+
# Returns the child node
|
16
55
|
def assert_node_in_parent(file_name, parent)
|
17
56
|
file = fixtures_dir(File.join("require", file_name))
|
18
57
|
node = parent[file]
|
19
|
-
assert node,
|
20
|
-
|
58
|
+
assert node, "Expected: #{parent.name} to include: #{file.to_s} but it did not.\nChildren: #{parent.children.map(&:name).map(&:to_s)}"
|
59
|
+
unless parent == TOP_REQUIRE
|
60
|
+
assert node.cost < parent.cost, "Expected: #{node.name.inspect} (#{node.cost}) to cost less than: #{parent.name.inspect} (#{parent.cost})"
|
61
|
+
end
|
21
62
|
node
|
22
63
|
end
|
23
64
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
65
|
+
# Used to get semi-clean process memory
|
66
|
+
# It would be better to run the requires in a totally different process
|
67
|
+
# but...that would take engineering
|
68
|
+
#
|
69
|
+
# If I was going to do that, I would find a way to serialize RequireTree
|
70
|
+
# into a json structure with file names and costs, run the script
|
71
|
+
# dump the json to a file, then in this process read the file and
|
72
|
+
# run assertions
|
73
|
+
def in_fork
|
74
|
+
Tempfile.create("stdout") do |tmp_file|
|
75
|
+
pid = fork do
|
76
|
+
$stdout.reopen(tmp_file, "w")
|
77
|
+
$stderr.reopen(tmp_file, "w")
|
78
|
+
$stdout.sync = true
|
79
|
+
$stderr.sync = true
|
80
|
+
yield
|
81
|
+
Kernel.exit!(0) # needed for https://github.com/seattlerb/minitest/pull/683
|
82
|
+
end
|
83
|
+
Process.waitpid(pid)
|
84
|
+
|
85
|
+
if $?.success?
|
86
|
+
print File.read(tmp_file)
|
87
|
+
else
|
88
|
+
raise File.read(tmp_file)
|
89
|
+
end
|
90
|
+
end
|
32
91
|
end
|
33
92
|
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'test_helper'
|
4
|
+
|
5
|
+
class GitSwitchProjectTest < ActiveSupport::TestCase
|
6
|
+
test "tells me when it's not pointing at a git project" do
|
7
|
+
exception = assert_raises {
|
8
|
+
DerailedBenchmarks::Git::SwitchProject.new(path: "/dev/null")
|
9
|
+
}
|
10
|
+
assert_includes(exception.message, '.git directory')
|
11
|
+
end
|
12
|
+
|
13
|
+
test "dirty gemspec cleaning" do
|
14
|
+
Dir.mktmpdir do |dir|
|
15
|
+
run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
|
16
|
+
run!("cd #{dir} && echo lol > foo.gemspec && git add .")
|
17
|
+
|
18
|
+
io = StringIO.new
|
19
|
+
project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, io: io)
|
20
|
+
|
21
|
+
assert project.dirty?
|
22
|
+
refute project.clean?
|
23
|
+
|
24
|
+
project.restore_branch_on_return do
|
25
|
+
project.commits.map(&:checkout!)
|
26
|
+
end
|
27
|
+
|
28
|
+
assert_includes io.string, "Bundler modifies gemspec files"
|
29
|
+
assert_includes io.string, "Applying stash"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
test "works on a git repo" do
|
34
|
+
Dir.mktmpdir do |dir|
|
35
|
+
run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
|
36
|
+
|
37
|
+
# finds shas when none given
|
38
|
+
project = DerailedBenchmarks::Git::SwitchProject.new(path: dir)
|
39
|
+
|
40
|
+
assert_equal ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "da748a59340be8b950e7bbbfb32077eb67d70c3c"], project.commits.map(&:ref)
|
41
|
+
first_commit = project.commits.first
|
42
|
+
|
43
|
+
assert_equal "CI test support", first_commit.description
|
44
|
+
assert_equal "6e64296", first_commit.short_sha
|
45
|
+
assert_equal "/dev/null/6e642963acec0ff64af51bd6fba8db3c4176ed6e.bench.txt", first_commit.log.to_s
|
46
|
+
assert_equal DateTime.parse("Tue, 14 Apr 2020 13:26:03 -0500"), first_commit.time
|
47
|
+
|
48
|
+
assert_equal "mybranch", project.current_branch_or_sha
|
49
|
+
|
50
|
+
# Finds shas when 1 is given
|
51
|
+
project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c"])
|
52
|
+
|
53
|
+
assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "5c09f748957d2098182762004adee27d1ff83160"], project.commits.map(&:ref)
|
54
|
+
|
55
|
+
|
56
|
+
# Returns correct refs if given
|
57
|
+
project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"])
|
58
|
+
|
59
|
+
assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"], project.commits.map(&:ref)
|
60
|
+
|
61
|
+
first_commit = project.commits.first
|
62
|
+
|
63
|
+
first_commit.checkout!
|
64
|
+
|
65
|
+
assert_equal first_commit.short_sha, project.current_branch_or_sha
|
66
|
+
|
67
|
+
# Test restore_branch_on_return
|
68
|
+
project.restore_branch_on_return(quiet: true) do
|
69
|
+
project.commits.last.checkout!
|
70
|
+
|
71
|
+
assert_equal project.commits.last.short_sha, project.current_branch_or_sha
|
72
|
+
end
|
73
|
+
|
74
|
+
assert_equal project.commits.first.short_sha, project.current_branch_or_sha
|
75
|
+
|
76
|
+
exception = assert_raise {
|
77
|
+
DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "mybranch"])
|
78
|
+
}
|
79
|
+
|
80
|
+
assert_includes(exception.message, 'Duplicate SHA resolved "6e64296"')
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -3,6 +3,19 @@
|
|
3
3
|
require 'test_helper'
|
4
4
|
|
5
5
|
class StatsFromDirTest < ActiveSupport::TestCase
|
6
|
+
test "empty files" do
|
7
|
+
Dir.mktmpdir do |dir|
|
8
|
+
dir = Pathname.new(dir)
|
9
|
+
branch_info = {}
|
10
|
+
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser"), name: "loser" }
|
11
|
+
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner"), name: "winner" }
|
12
|
+
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
|
13
|
+
io = StringIO.new
|
14
|
+
stats.call.banner(io: io) # Doesn't error
|
15
|
+
assert_equal "", io.read
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
6
19
|
test "that it works" do
|
7
20
|
dir = fixtures_dir("stats/significant")
|
8
21
|
branch_info = {}
|
@@ -32,12 +45,13 @@ class StatsFromDirTest < ActiveSupport::TestCase
|
|
32
45
|
test "histogram output" do
|
33
46
|
dir = fixtures_dir("stats/significant")
|
34
47
|
branch_info = {}
|
35
|
-
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"),
|
36
|
-
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"),
|
48
|
+
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), short_sha: "5594a2d" }
|
49
|
+
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), short_sha: "f1ab117" }
|
37
50
|
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
|
38
51
|
|
39
52
|
io = StringIO.new
|
40
53
|
stats.call.banner(io)
|
54
|
+
|
41
55
|
puts io.string
|
42
56
|
|
43
57
|
assert_match(/11\.2 , 11\.28/, io.string)
|
@@ -64,10 +78,13 @@ class StatsFromDirTest < ActiveSupport::TestCase
|
|
64
78
|
|
65
79
|
test "banner faster" do
|
66
80
|
dir = fixtures_dir("stats/significant")
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
81
|
+
Branch_info = {}
|
82
|
+
|
83
|
+
require 'ostruct'
|
84
|
+
commits = []
|
85
|
+
commits << OpenStruct.new({ desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), ref: "loser", short_sha: "aaaaa" })
|
86
|
+
commits << OpenStruct.new({ desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), ref: "winner", short_sha: "bbbbb" })
|
87
|
+
stats = DerailedBenchmarks::StatsFromDir.new(commits).call
|
71
88
|
newest = stats.newest
|
72
89
|
oldest = stats.oldest
|
73
90
|
|
@@ -88,12 +105,12 @@ class StatsFromDirTest < ActiveSupport::TestCase
|
|
88
105
|
11.0
|
89
106
|
end
|
90
107
|
|
91
|
-
expected =
|
92
|
-
[
|
108
|
+
expected = <<-EOM
|
109
|
+
[bbbbb] (10.5000 seconds) "I am the new commit" ref: "winner"
|
93
110
|
FASTER 🚀🚀🚀 by:
|
94
111
|
1.0476x [older/newer]
|
95
112
|
4.5455% [(older - newer) / older * 100]
|
96
|
-
[
|
113
|
+
[aaaaa] (11.0000 seconds) "Old commit" ref: "loser"
|
97
114
|
EOM
|
98
115
|
|
99
116
|
actual = StringIO.new
|
@@ -119,12 +136,12 @@ EOM
|
|
119
136
|
11.0
|
120
137
|
end
|
121
138
|
|
122
|
-
expected =
|
123
|
-
[loser] "I am the new commit"
|
139
|
+
expected = <<-EOM
|
140
|
+
[loser] (11.0000 seconds) "I am the new commit" ref: "loser"
|
124
141
|
SLOWER 🐢🐢🐢 by:
|
125
142
|
0.9545x [older/newer]
|
126
143
|
-4.7619% [(older - newer) / older * 100]
|
127
|
-
[winner] "Old commit"
|
144
|
+
[winner] (10.5000 seconds) "Old commit" ref: "winner"
|
128
145
|
EOM
|
129
146
|
|
130
147
|
actual = StringIO.new
|
data/test/derailed_test.rb
CHANGED
@@ -11,4 +11,19 @@ class DerailedBenchmarksTest < ActiveSupport::TestCase
|
|
11
11
|
assert DerailedBenchmarks.gem_is_bundled?("rack")
|
12
12
|
refute DerailedBenchmarks.gem_is_bundled?("wicked")
|
13
13
|
end
|
14
|
+
|
15
|
+
test "readme contains correct output" do
|
16
|
+
readme_path = File.join(__dir__, "..", "README.md")
|
17
|
+
lines = File.foreach(readme_path)
|
18
|
+
lineno = 1
|
19
|
+
expected = lines.lazy.drop_while { |line|
|
20
|
+
lineno += 1
|
21
|
+
line != "$ bundle exec derailed exec --help\n"
|
22
|
+
}.drop(1).take_while { |line| line != "```\n" }.force.join
|
23
|
+
assert_equal(
|
24
|
+
expected,
|
25
|
+
`bundle exec derailed exec --help`,
|
26
|
+
"Please update README.md:#{lineno}"
|
27
|
+
)
|
28
|
+
end
|
14
29
|
end
|
@@ -26,21 +26,47 @@ class TasksTest < ActiveSupport::TestCase
|
|
26
26
|
env_string = env.map {|key, value| "#{key.shellescape}=#{value.to_s.shellescape}" }.join(" ")
|
27
27
|
cmd = "env #{env_string} bundle exec rake -f perf.rake #{cmd} --trace"
|
28
28
|
puts "Running: #{cmd}"
|
29
|
-
result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd}` }
|
30
|
-
if assert_success
|
31
|
-
|
29
|
+
result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd} 2>&1` }
|
30
|
+
if assert_success && !$?.success?
|
31
|
+
puts result
|
32
|
+
raise "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
|
32
33
|
end
|
33
34
|
|
34
35
|
result
|
35
36
|
end
|
36
37
|
|
37
|
-
test '
|
38
|
+
test 'non-rails library with branch specified' do
|
39
|
+
skip unless ENV['USING_RAILS_WICKED_BRANCH']
|
40
|
+
|
41
|
+
gem_path = run!("bundle info wicked --path")
|
42
|
+
env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2, "DERAILED_PATH_TO_LIBRARY" => gem_path}
|
43
|
+
puts rake "perf:library", { env: env }
|
44
|
+
end
|
45
|
+
|
46
|
+
test 'rails perf:library from git' do
|
47
|
+
# BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
|
48
|
+
|
38
49
|
skip unless ENV['USING_RAILS_GIT']
|
39
50
|
|
40
|
-
env = { "TEST_COUNT" =>
|
51
|
+
env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
|
52
|
+
"SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
|
41
53
|
puts rake "perf:library", { env: env }
|
42
54
|
end
|
43
55
|
|
56
|
+
test "rails perf:library with bad script" do
|
57
|
+
# BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
|
58
|
+
|
59
|
+
skip unless ENV['USING_RAILS_GIT']
|
60
|
+
|
61
|
+
error = assert_raises {
|
62
|
+
env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
|
63
|
+
"SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
|
64
|
+
puts rake "perf:library", { env: env }
|
65
|
+
}
|
66
|
+
|
67
|
+
assert error.message =~ /nopenopenop:( command)? not found/, "Expected #{error.message} to include /nopenopenop: (command)? not found/ but it did not"
|
68
|
+
end
|
69
|
+
|
44
70
|
test 'hitting authenticated devise apps' do
|
45
71
|
env = { "PATH_TO_HIT" => "authenticated", "USE_AUTH" => "true", "TEST_COUNT" => "2" }
|
46
72
|
result = rake 'perf:test', env: env
|
@@ -127,6 +153,10 @@ class TasksTest < ActiveSupport::TestCase
|
|
127
153
|
end
|
128
154
|
|
129
155
|
test 'ips' do
|
130
|
-
rake "perf:
|
156
|
+
rake "perf:ips"
|
157
|
+
end
|
158
|
+
|
159
|
+
test 'heap_diff' do
|
160
|
+
rake "perf:heap_diff", env: { "TEST_COUNT" => 5 }
|
131
161
|
end
|
132
162
|
end
|
File without changes
|
data/test/test_helper.rb
CHANGED
@@ -51,7 +51,6 @@ class ActiveSupport::IntegrationCase
|
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
-
|
55
54
|
def fixtures_dir(name = "")
|
56
55
|
root_path("test/fixtures").join(name)
|
57
56
|
end
|
@@ -63,3 +62,9 @@ end
|
|
63
62
|
def rails_app_path(name = "")
|
64
63
|
root_path("test/rails_app").join(name)
|
65
64
|
end
|
65
|
+
|
66
|
+
def run!(cmd)
|
67
|
+
output = `#{cmd}`
|
68
|
+
raise "Cmd #{cmd} failed:\n#{output}" unless $?.success?
|
69
|
+
output
|
70
|
+
end
|