derailed_benchmarks 1.4.2 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/check_changelog.yml +11 -8
- data/.travis.yml +9 -7
- data/CHANGELOG.md +28 -1
- data/README.md +15 -5
- data/derailed_benchmarks.gemspec +4 -3
- data/lib/derailed_benchmarks.rb +2 -1
- data/lib/derailed_benchmarks/core_ext/kernel_require.rb +12 -18
- data/lib/derailed_benchmarks/git/commit.rb +36 -0
- data/lib/derailed_benchmarks/git/in_path.rb +59 -0
- data/lib/derailed_benchmarks/git/switch_project.rb +128 -0
- data/lib/derailed_benchmarks/git_switch_project.rb +1 -0
- data/lib/derailed_benchmarks/load_tasks.rb +10 -3
- data/lib/derailed_benchmarks/require_tree.rb +1 -1
- data/lib/derailed_benchmarks/{stats_in_file.rb → stats_for_file.rb} +15 -2
- data/lib/derailed_benchmarks/stats_from_dir.rb +84 -17
- data/lib/derailed_benchmarks/tasks.rb +43 -64
- data/lib/derailed_benchmarks/version.rb +1 -1
- data/test/derailed_benchmarks/git_switch_project_test.rb +83 -0
- data/test/derailed_benchmarks/stats_from_dir_test.rb +67 -16
- data/test/fixtures/require/child_one.rb +1 -1
- data/test/fixtures/require/child_two.rb +1 -1
- data/test/fixtures/require/parent_one.rb +1 -1
- data/test/integration/tasks_test.rb +43 -5
- data/test/rails_app/app/assets/config/manifest.js +0 -0
- data/test/rails_app/config/application.rb +2 -0
- data/test/test_helper.rb +6 -0
- metadata +40 -12
@@ -3,6 +3,19 @@
|
|
3
3
|
require 'test_helper'
|
4
4
|
|
5
5
|
class StatsFromDirTest < ActiveSupport::TestCase
|
6
|
+
test "empty files" do
|
7
|
+
Dir.mktmpdir do |dir|
|
8
|
+
dir = Pathname.new(dir)
|
9
|
+
branch_info = {}
|
10
|
+
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser"), name: "loser" }
|
11
|
+
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner"), name: "winner" }
|
12
|
+
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
|
13
|
+
io = StringIO.new
|
14
|
+
stats.call.banner(io: io) # Doesn't error
|
15
|
+
assert_equal "", io.read
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
6
19
|
test "that it works" do
|
7
20
|
dir = fixtures_dir("stats/significant")
|
8
21
|
branch_info = {}
|
@@ -19,19 +32,57 @@ class StatsFromDirTest < ActiveSupport::TestCase
|
|
19
32
|
assert_equal "loser", oldest.name
|
20
33
|
|
21
34
|
assert_in_delta 0.26, stats.d_max, 0.01
|
22
|
-
assert_in_delta 0.
|
35
|
+
assert_in_delta 0.2145966026289347, stats.d_critical, 0.00001
|
23
36
|
assert_equal true, stats.significant?
|
24
37
|
|
25
|
-
|
26
|
-
assert_equal "
|
27
|
-
|
38
|
+
format = DerailedBenchmarks::StatsFromDir::FORMAT
|
39
|
+
assert_equal "1.0062", format % stats.x_faster
|
40
|
+
assert_equal "0.6147", format % stats.percent_faster
|
28
41
|
|
29
|
-
|
42
|
+
assert_equal "11.3844", format % newest.median
|
43
|
+
end
|
44
|
+
|
45
|
+
test "histogram output" do
|
30
46
|
dir = fixtures_dir("stats/significant")
|
31
47
|
branch_info = {}
|
32
48
|
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
|
33
49
|
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
|
34
50
|
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
|
51
|
+
|
52
|
+
io = StringIO.new
|
53
|
+
stats.call.banner(io)
|
54
|
+
|
55
|
+
assert_match(/11\.2 , 11\.28/, io.string)
|
56
|
+
assert_match(/11\.8 , 11\.88/, io.string)
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
test "alignment" do
|
61
|
+
dir = fixtures_dir("stats/significant")
|
62
|
+
branch_info = {}
|
63
|
+
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
|
64
|
+
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
|
65
|
+
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
|
66
|
+
def stats.percent_faster
|
67
|
+
-0.1
|
68
|
+
end
|
69
|
+
|
70
|
+
def stats.x_faster
|
71
|
+
0.9922
|
72
|
+
end
|
73
|
+
|
74
|
+
assert_equal 1, stats.align.length
|
75
|
+
end
|
76
|
+
|
77
|
+
test "banner faster" do
|
78
|
+
dir = fixtures_dir("stats/significant")
|
79
|
+
Branch_info = {}
|
80
|
+
|
81
|
+
require 'ostruct'
|
82
|
+
commits = []
|
83
|
+
commits << OpenStruct.new({ desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), ref: "loser", short_sha: "aaaaa" })
|
84
|
+
commits << OpenStruct.new({ desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), ref: "winner", short_sha: "bbbbb" })
|
85
|
+
stats = DerailedBenchmarks::StatsFromDir.new(commits).call
|
35
86
|
newest = stats.newest
|
36
87
|
oldest = stats.oldest
|
37
88
|
|
@@ -44,20 +95,20 @@ class StatsFromDirTest < ActiveSupport::TestCase
|
|
44
95
|
"0.001"
|
45
96
|
end
|
46
97
|
|
47
|
-
def newest.
|
98
|
+
def newest.median
|
48
99
|
10.5
|
49
100
|
end
|
50
101
|
|
51
|
-
def oldest.
|
102
|
+
def oldest.median
|
52
103
|
11.0
|
53
104
|
end
|
54
105
|
|
55
106
|
expected = <<-EOM
|
56
|
-
[
|
57
|
-
FASTER by:
|
107
|
+
[bbbbb] (10.5000 seconds) "I am the new commit" ref: "winner"
|
108
|
+
FASTER 🚀🚀🚀 by:
|
58
109
|
1.0476x [older/newer]
|
59
110
|
4.5455% [(older - newer) / older * 100]
|
60
|
-
[
|
111
|
+
[aaaaa] (11.0000 seconds) "Old commit" ref: "loser"
|
61
112
|
EOM
|
62
113
|
|
63
114
|
actual = StringIO.new
|
@@ -75,20 +126,20 @@ EOM
|
|
75
126
|
newest = stats.newest
|
76
127
|
oldest = stats.oldest
|
77
128
|
|
78
|
-
def oldest.
|
129
|
+
def oldest.median
|
79
130
|
10.5
|
80
131
|
end
|
81
132
|
|
82
|
-
def newest.
|
133
|
+
def newest.median
|
83
134
|
11.0
|
84
135
|
end
|
85
136
|
|
86
137
|
expected = <<-EOM
|
87
|
-
[loser] "I am the new commit"
|
88
|
-
SLOWER by:
|
89
|
-
|
138
|
+
[loser] (11.0000 seconds) "I am the new commit" ref: "loser"
|
139
|
+
SLOWER 🐢🐢🐢 by:
|
140
|
+
0.9545x [older/newer]
|
90
141
|
-4.7619% [(older - newer) / older * 100]
|
91
|
-
[winner] "Old commit"
|
142
|
+
[winner] (10.5000 seconds) "Old commit" ref: "winner"
|
92
143
|
EOM
|
93
144
|
|
94
145
|
actual = StringIO.new
|
@@ -13,27 +13,58 @@ class TasksTest < ActiveSupport::TestCase
|
|
13
13
|
FileUtils.remove_entry_secure(rails_app_path('tmp'))
|
14
14
|
end
|
15
15
|
|
16
|
+
def run!(cmd)
|
17
|
+
puts "Running: #{cmd}"
|
18
|
+
out = `#{cmd}`
|
19
|
+
raise "Could not run #{cmd}, output: #{out}" unless $?.success?
|
20
|
+
out
|
21
|
+
end
|
22
|
+
|
16
23
|
def rake(cmd, options = {})
|
17
24
|
assert_success = options.key?(:assert_success) ? options[:assert_success] : true
|
18
25
|
env = options[:env] || {}
|
19
26
|
env_string = env.map {|key, value| "#{key.shellescape}=#{value.to_s.shellescape}" }.join(" ")
|
20
27
|
cmd = "env #{env_string} bundle exec rake -f perf.rake #{cmd} --trace"
|
21
28
|
puts "Running: #{cmd}"
|
22
|
-
result = `cd '#{rails_app_path}' && #{cmd}`
|
23
|
-
if assert_success
|
24
|
-
|
29
|
+
result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd} 2>&1` }
|
30
|
+
if assert_success && !$?.success?
|
31
|
+
puts result
|
32
|
+
raise "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
|
25
33
|
end
|
26
34
|
|
27
35
|
result
|
28
36
|
end
|
29
37
|
|
30
|
-
test '
|
38
|
+
test 'non-rails library with branch specified' do
|
39
|
+
skip unless ENV['USING_RAILS_WICKED_BRANCH']
|
40
|
+
|
41
|
+
gem_path = run!("bundle info wicked --path")
|
42
|
+
env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2, "DERAILED_PATH_TO_LIBRARY" => gem_path}
|
43
|
+
puts rake "perf:library", { env: env }
|
44
|
+
end
|
45
|
+
|
46
|
+
test 'rails perf:library from git' do
|
47
|
+
# BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
|
48
|
+
|
31
49
|
skip unless ENV['USING_RAILS_GIT']
|
32
50
|
|
33
|
-
env = { "TEST_COUNT" =>
|
51
|
+
env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
|
34
52
|
puts rake "perf:library", { env: env }
|
35
53
|
end
|
36
54
|
|
55
|
+
test "rails perf:library with bad script" do
|
56
|
+
# BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
|
57
|
+
|
58
|
+
skip unless ENV['USING_RAILS_GIT']
|
59
|
+
|
60
|
+
error = assert_raises {
|
61
|
+
env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
|
62
|
+
puts rake "perf:library", { env: env }
|
63
|
+
}
|
64
|
+
|
65
|
+
assert error.message =~ /nopenopenop:( command)? not found/, "Expected #{error.message} to include /nopenopenop: (command)? not found/ but it did not"
|
66
|
+
end
|
67
|
+
|
37
68
|
test 'hitting authenticated devise apps' do
|
38
69
|
env = { "PATH_TO_HIT" => "authenticated", "USE_AUTH" => "true", "TEST_COUNT" => "2" }
|
39
70
|
result = rake 'perf:test', env: env
|
@@ -55,6 +86,13 @@ class TasksTest < ActiveSupport::TestCase
|
|
55
86
|
rake "perf:test"
|
56
87
|
end
|
57
88
|
|
89
|
+
test 'app' do
|
90
|
+
skip unless ENV['USING_RAILS_GIT']
|
91
|
+
run!("cd #{rails_app_path} && git init . && git add . && git commit -m first && git commit --allow-empty -m second")
|
92
|
+
env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2 }
|
93
|
+
puts rake "perf:app", { env: env }
|
94
|
+
end
|
95
|
+
|
58
96
|
test 'TEST_COUNT' do
|
59
97
|
result = rake "perf:test", env: { "TEST_COUNT" => 1 }
|
60
98
|
assert_match "1 derailed requests", result
|
File without changes
|
@@ -13,6 +13,8 @@ require 'devise'
|
|
13
13
|
|
14
14
|
module Dummy
|
15
15
|
class Application < Rails::Application
|
16
|
+
config.load_defaults Rails.version.to_f
|
17
|
+
|
16
18
|
config.action_mailer.default_url_options = { host: 'localhost:3000' }
|
17
19
|
|
18
20
|
# Settings in config/environments/* take precedence over those specified here.
|
data/test/test_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: derailed_benchmarks
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Richard Schneeman
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-09-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: heapy
|
@@ -104,16 +104,22 @@ dependencies:
|
|
104
104
|
name: thor
|
105
105
|
requirement: !ruby/object:Gem::Requirement
|
106
106
|
requirements:
|
107
|
-
- - "
|
107
|
+
- - ">="
|
108
108
|
- !ruby/object:Gem::Version
|
109
109
|
version: '0.19'
|
110
|
+
- - "<"
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: '2'
|
110
113
|
type: :runtime
|
111
114
|
prerelease: false
|
112
115
|
version_requirements: !ruby/object:Gem::Requirement
|
113
116
|
requirements:
|
114
|
-
- - "
|
117
|
+
- - ">="
|
115
118
|
- !ruby/object:Gem::Version
|
116
119
|
version: '0.19'
|
120
|
+
- - "<"
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '2'
|
117
123
|
- !ruby/object:Gem::Dependency
|
118
124
|
name: ruby-statistics
|
119
125
|
requirement: !ruby/object:Gem::Requirement
|
@@ -128,6 +134,20 @@ dependencies:
|
|
128
134
|
- - ">="
|
129
135
|
- !ruby/object:Gem::Version
|
130
136
|
version: '2.1'
|
137
|
+
- !ruby/object:Gem::Dependency
|
138
|
+
name: mini_histogram
|
139
|
+
requirement: !ruby/object:Gem::Requirement
|
140
|
+
requirements:
|
141
|
+
- - ">="
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: 0.2.1
|
144
|
+
type: :runtime
|
145
|
+
prerelease: false
|
146
|
+
version_requirements: !ruby/object:Gem::Requirement
|
147
|
+
requirements:
|
148
|
+
- - ">="
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: 0.2.1
|
131
151
|
- !ruby/object:Gem::Dependency
|
132
152
|
name: capybara
|
133
153
|
requirement: !ruby/object:Gem::Requirement
|
@@ -165,7 +185,7 @@ dependencies:
|
|
165
185
|
version: '3'
|
166
186
|
- - "<="
|
167
187
|
- !ruby/object:Gem::Version
|
168
|
-
version: '
|
188
|
+
version: '7'
|
169
189
|
type: :development
|
170
190
|
prerelease: false
|
171
191
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -175,7 +195,7 @@ dependencies:
|
|
175
195
|
version: '3'
|
176
196
|
- - "<="
|
177
197
|
- !ruby/object:Gem::Version
|
178
|
-
version: '
|
198
|
+
version: '7'
|
179
199
|
- !ruby/object:Gem::Dependency
|
180
200
|
name: devise
|
181
201
|
requirement: !ruby/object:Gem::Requirement
|
@@ -237,13 +257,18 @@ files:
|
|
237
257
|
- lib/derailed_benchmarks/auth_helper.rb
|
238
258
|
- lib/derailed_benchmarks/auth_helpers/devise.rb
|
239
259
|
- lib/derailed_benchmarks/core_ext/kernel_require.rb
|
260
|
+
- lib/derailed_benchmarks/git/commit.rb
|
261
|
+
- lib/derailed_benchmarks/git/in_path.rb
|
262
|
+
- lib/derailed_benchmarks/git/switch_project.rb
|
263
|
+
- lib/derailed_benchmarks/git_switch_project.rb
|
240
264
|
- lib/derailed_benchmarks/load_tasks.rb
|
241
265
|
- lib/derailed_benchmarks/require_tree.rb
|
266
|
+
- lib/derailed_benchmarks/stats_for_file.rb
|
242
267
|
- lib/derailed_benchmarks/stats_from_dir.rb
|
243
|
-
- lib/derailed_benchmarks/stats_in_file.rb
|
244
268
|
- lib/derailed_benchmarks/tasks.rb
|
245
269
|
- lib/derailed_benchmarks/version.rb
|
246
270
|
- test/derailed_benchmarks/core_ext/kernel_require_test.rb
|
271
|
+
- test/derailed_benchmarks/git_switch_project_test.rb
|
247
272
|
- test/derailed_benchmarks/require_tree_test.rb
|
248
273
|
- test/derailed_benchmarks/stats_from_dir_test.rb
|
249
274
|
- test/derailed_test.rb
|
@@ -257,6 +282,7 @@ files:
|
|
257
282
|
- test/fixtures/stats/significant/winner.bench.txt
|
258
283
|
- test/integration/tasks_test.rb
|
259
284
|
- test/rails_app/Rakefile
|
285
|
+
- test/rails_app/app/assets/config/manifest.js
|
260
286
|
- test/rails_app/app/assets/javascripts/authenticated.js
|
261
287
|
- test/rails_app/app/assets/stylesheets/authenticated.css
|
262
288
|
- test/rails_app/app/controllers/application_controller.rb
|
@@ -307,7 +333,7 @@ homepage: https://github.com/schneems/derailed_benchmarks
|
|
307
333
|
licenses:
|
308
334
|
- MIT
|
309
335
|
metadata: {}
|
310
|
-
post_install_message:
|
336
|
+
post_install_message:
|
311
337
|
rdoc_options: []
|
312
338
|
require_paths:
|
313
339
|
- lib
|
@@ -315,19 +341,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
315
341
|
requirements:
|
316
342
|
- - ">="
|
317
343
|
- !ruby/object:Gem::Version
|
318
|
-
version: 2.
|
344
|
+
version: 2.2.0
|
319
345
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
320
346
|
requirements:
|
321
347
|
- - ">="
|
322
348
|
- !ruby/object:Gem::Version
|
323
349
|
version: '0'
|
324
350
|
requirements: []
|
325
|
-
rubygems_version: 3.
|
326
|
-
signing_key:
|
351
|
+
rubygems_version: 3.1.2
|
352
|
+
signing_key:
|
327
353
|
specification_version: 4
|
328
354
|
summary: Benchmarks designed to performance test your ENTIRE site
|
329
355
|
test_files:
|
330
356
|
- test/derailed_benchmarks/core_ext/kernel_require_test.rb
|
357
|
+
- test/derailed_benchmarks/git_switch_project_test.rb
|
331
358
|
- test/derailed_benchmarks/require_tree_test.rb
|
332
359
|
- test/derailed_benchmarks/stats_from_dir_test.rb
|
333
360
|
- test/derailed_test.rb
|
@@ -341,6 +368,7 @@ test_files:
|
|
341
368
|
- test/fixtures/stats/significant/winner.bench.txt
|
342
369
|
- test/integration/tasks_test.rb
|
343
370
|
- test/rails_app/Rakefile
|
371
|
+
- test/rails_app/app/assets/config/manifest.js
|
344
372
|
- test/rails_app/app/assets/javascripts/authenticated.js
|
345
373
|
- test/rails_app/app/assets/stylesheets/authenticated.css
|
346
374
|
- test/rails_app/app/controllers/application_controller.rb
|