derailed_benchmarks 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/derailed_benchmarks.gemspec +2 -0
- data/lib/derailed_benchmarks/core_ext/kernel_require.rb +12 -18
- data/lib/derailed_benchmarks/load_tasks.rb +10 -3
- data/lib/derailed_benchmarks/stats_from_dir.rb +26 -1
- data/lib/derailed_benchmarks/tasks.rb +4 -2
- data/lib/derailed_benchmarks/version.rb +1 -1
- data/test/derailed_benchmarks/stats_from_dir_test.rb +16 -0
- data/test/integration/tasks_test.rb +1 -1
- data/test/rails_app/config/application.rb +2 -0
- metadata +36 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 843a6c3b99ee45120c15af62a592e9bbecb2d88f0917fa292faf9f7128ab721a
|
4
|
+
data.tar.gz: f1cdfb9c4145c27a8dac7f2c868077c6359213c3c367f2cf8a0f8909f50c9468
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1936293cd836af98e6af454a7482534c7481b938f4944813a5261a04a9b766b0a169b866397b3d9745d25316cd8916df3b74fc1fd4f1ff46f6e5753eba92bd96
|
7
|
+
data.tar.gz: 7de041007e174314699cac83eef5b70626e5447eddd705190d68d74cc7d50c0b9d660d032127cf747ea67566a096bbca4aafa05774c413da6f4ca802c8da2c08
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,10 @@
|
|
1
1
|
## master (unreleased)
|
2
2
|
|
3
|
+
## 1.7.0
|
4
|
+
|
5
|
+
- Add histogram support to `perf:library` (https://github.com/schneems/derailed_benchmarks/pull/169)
|
6
|
+
- Fix bug with `Kernel#require` patch when Zeitwerk is enabled (https://github.com/schneems/derailed_benchmarks/pull/170)
|
7
|
+
|
3
8
|
## 1.6.0
|
4
9
|
|
5
10
|
- Added the `perf:app` command to compare commits within the same application. (https://github.com/schneems/derailed_benchmarks/pull/157)
|
data/derailed_benchmarks.gemspec
CHANGED
@@ -30,6 +30,8 @@ Gem::Specification.new do |gem|
|
|
30
30
|
gem.add_dependency "rake", "> 10", "< 14"
|
31
31
|
gem.add_dependency "thor", ">= 0.19", "< 2"
|
32
32
|
gem.add_dependency "ruby-statistics", ">= 2.1"
|
33
|
+
gem.add_dependency "unicode_plot", ">= 0.0.4", "< 1.0.0"
|
34
|
+
gem.add_dependency "mini_histogram", "~> 0"
|
33
35
|
|
34
36
|
gem.add_development_dependency "capybara", "~> 2"
|
35
37
|
gem.add_development_dependency "m"
|
@@ -11,14 +11,20 @@ ENV['CUT_OFF'] ||= "0.3"
|
|
11
11
|
# Monkey patch kernel to ensure that all `require` calls call the same
|
12
12
|
# method
|
13
13
|
module Kernel
|
14
|
+
REQUIRE_STACK = []
|
14
15
|
|
15
|
-
|
16
|
+
module_function
|
16
17
|
|
17
|
-
|
18
|
-
|
18
|
+
alias_method :original_require, :require
|
19
|
+
alias_method :original_require_relative, :require_relative
|
19
20
|
|
20
21
|
def require(file)
|
21
|
-
|
22
|
+
measure_memory_impact(file) do |file|
|
23
|
+
# "source_annotation_extractor" is deprecated in Rails 6
|
24
|
+
# # if we don't skip the library it leads to a crash
|
25
|
+
# next if file == "rails/source_annotation_extractor" && Rails.version >= '6.0'
|
26
|
+
original_require(file)
|
27
|
+
end
|
22
28
|
end
|
23
29
|
|
24
30
|
def require_relative(file)
|
@@ -29,10 +35,7 @@ module Kernel
|
|
29
35
|
end
|
30
36
|
end
|
31
37
|
|
32
|
-
|
33
|
-
alias :original_require :require
|
34
|
-
alias :original_require_relative :require_relative
|
35
|
-
end
|
38
|
+
private
|
36
39
|
|
37
40
|
# The core extension we use to measure require time of all requires
|
38
41
|
# When a file is required we create a tree node with its file name.
|
@@ -46,7 +49,7 @@ module Kernel
|
|
46
49
|
# When a require returns we remove it from the require stack so we don't
|
47
50
|
# accidentally push additional children nodes to it. We then store the
|
48
51
|
# memory cost of the require in the tree node.
|
49
|
-
def
|
52
|
+
def measure_memory_impact(file, &block)
|
50
53
|
mem = GetProcessMem.new
|
51
54
|
node = DerailedBenchmarks::RequireTree.new(file)
|
52
55
|
|
@@ -68,15 +71,6 @@ end
|
|
68
71
|
TOP_REQUIRE = DerailedBenchmarks::RequireTree.new("TOP")
|
69
72
|
REQUIRE_STACK.push(TOP_REQUIRE)
|
70
73
|
|
71
|
-
Kernel.define_singleton_method(:require) do |file|
|
72
|
-
measure_memory_impact(file) do |file|
|
73
|
-
# "source_annotation_extractor" is deprecated in Rails 6
|
74
|
-
# # if we don't skip the library it leads to a crash
|
75
|
-
# next if file == "rails/source_annotation_extractor" && Rails.version >= '6.0'
|
76
|
-
original_require(file)
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
80
74
|
class Object
|
81
75
|
private
|
82
76
|
|
@@ -30,7 +30,7 @@ namespace :perf do
|
|
30
30
|
DERAILED_APP.initialize! unless DERAILED_APP.instance_variable_get(:@initialized)
|
31
31
|
end
|
32
32
|
|
33
|
-
if
|
33
|
+
if !ENV["DERAILED_SKIP_ACTIVE_RECORD"] && defined? ActiveRecord
|
34
34
|
if defined? ActiveRecord::Tasks::DatabaseTasks
|
35
35
|
ActiveRecord::Tasks::DatabaseTasks.create_current
|
36
36
|
else # Rails 3.2
|
@@ -39,7 +39,14 @@ namespace :perf do
|
|
39
39
|
|
40
40
|
ActiveRecord::Migrator.migrations_paths = DERAILED_APP.paths['db/migrate'].to_a
|
41
41
|
ActiveRecord::Migration.verbose = true
|
42
|
-
|
42
|
+
|
43
|
+
if Rails.version.start_with? '6'
|
44
|
+
ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths, ActiveRecord::SchemaMigration).migrate
|
45
|
+
elsif Rails.version.start_with? '5.2'
|
46
|
+
ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths).migrate
|
47
|
+
else
|
48
|
+
ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, nil)
|
49
|
+
end
|
43
50
|
end
|
44
51
|
|
45
52
|
DERAILED_APP.config.consider_all_requests_local = true
|
@@ -135,4 +142,4 @@ namespace :perf do
|
|
135
142
|
WARM_COUNT.times { call_app }
|
136
143
|
end
|
137
144
|
end
|
138
|
-
end
|
145
|
+
end
|
@@ -2,6 +2,9 @@
|
|
2
2
|
|
3
3
|
require 'bigdecimal'
|
4
4
|
require 'statistics'
|
5
|
+
require 'unicode_plot'
|
6
|
+
require 'stringio'
|
7
|
+
require 'mini_histogram'
|
5
8
|
|
6
9
|
module DerailedBenchmarks
|
7
10
|
# A class used to read several benchmark files
|
@@ -100,7 +103,26 @@ module DerailedBenchmarks
|
|
100
103
|
" " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
|
101
104
|
end
|
102
105
|
|
103
|
-
def
|
106
|
+
def histogram(io = $stdout)
|
107
|
+
newest_histogram = MiniHistogram.new(newest.values)
|
108
|
+
oldest_histogram = MiniHistogram.new(oldest.values)
|
109
|
+
MiniHistogram.set_average_edges!(newest_histogram, oldest_histogram)
|
110
|
+
|
111
|
+
{newest => newest_histogram, oldest => oldest_histogram}.each do |report, histogram|
|
112
|
+
plot = UnicodePlot.histogram(
|
113
|
+
histogram,
|
114
|
+
title: "\n#{' ' * 18 }Histogram - [#{report.name}] #{report.desc.inspect}",
|
115
|
+
ylabel: "Time (s)",
|
116
|
+
xlabel: "# of runs in range"
|
117
|
+
)
|
118
|
+
plot.render(io)
|
119
|
+
io.puts
|
120
|
+
end
|
121
|
+
|
122
|
+
io.puts
|
123
|
+
end
|
124
|
+
|
125
|
+
def banner(io = $stdout)
|
104
126
|
io.puts
|
105
127
|
if significant?
|
106
128
|
io.puts "❤️ ❤️ ❤️ (Statistically Significant) ❤️ ❤️ ❤️"
|
@@ -122,6 +144,9 @@ module DerailedBenchmarks
|
|
122
144
|
io.puts "Is significant? (max > critical): #{significant?}"
|
123
145
|
io.puts "D critical: #{d_critical}"
|
124
146
|
io.puts "D max: #{d_max}"
|
147
|
+
|
148
|
+
histogram(io)
|
149
|
+
|
125
150
|
io.puts
|
126
151
|
end
|
127
152
|
end
|
@@ -88,7 +88,8 @@ namespace :perf do
|
|
88
88
|
|
89
89
|
if (i % 50).zero?
|
90
90
|
puts "Intermediate result"
|
91
|
-
stats.call
|
91
|
+
stats.call
|
92
|
+
stats.banner
|
92
93
|
puts "Continuing execution"
|
93
94
|
end
|
94
95
|
end
|
@@ -102,7 +103,8 @@ namespace :perf do
|
|
102
103
|
end
|
103
104
|
|
104
105
|
if stats
|
105
|
-
stats.call
|
106
|
+
stats.call
|
107
|
+
stats.banner
|
106
108
|
|
107
109
|
result_file = out_dir + "results.txt"
|
108
110
|
File.open(result_file, "w") do |f|
|
@@ -29,6 +29,22 @@ class StatsFromDirTest < ActiveSupport::TestCase
|
|
29
29
|
assert_equal "11.3844", format % newest.median
|
30
30
|
end
|
31
31
|
|
32
|
+
test "histogram output" do
|
33
|
+
dir = fixtures_dir("stats/significant")
|
34
|
+
branch_info = {}
|
35
|
+
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
|
36
|
+
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
|
37
|
+
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
|
38
|
+
|
39
|
+
io = StringIO.new
|
40
|
+
stats.call.banner(io)
|
41
|
+
puts io.string
|
42
|
+
|
43
|
+
assert_match(/11\.2 , 11\.28/, io.string)
|
44
|
+
assert_match(/11\.8 , 11\.88/, io.string)
|
45
|
+
end
|
46
|
+
|
47
|
+
|
32
48
|
test "alignment" do
|
33
49
|
dir = fixtures_dir("stats/significant")
|
34
50
|
branch_info = {}
|
@@ -26,7 +26,7 @@ class TasksTest < ActiveSupport::TestCase
|
|
26
26
|
env_string = env.map {|key, value| "#{key.shellescape}=#{value.to_s.shellescape}" }.join(" ")
|
27
27
|
cmd = "env #{env_string} bundle exec rake -f perf.rake #{cmd} --trace"
|
28
28
|
puts "Running: #{cmd}"
|
29
|
-
result = `cd '#{rails_app_path}' && #{cmd}`
|
29
|
+
result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd}` }
|
30
30
|
if assert_success
|
31
31
|
assert $?.success?, "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
|
32
32
|
end
|
@@ -13,6 +13,8 @@ require 'devise'
|
|
13
13
|
|
14
14
|
module Dummy
|
15
15
|
class Application < Rails::Application
|
16
|
+
config.load_defaults Rails.version.to_f
|
17
|
+
|
16
18
|
config.action_mailer.default_url_options = { host: 'localhost:3000' }
|
17
19
|
|
18
20
|
# Settings in config/environments/* take precedence over those specified here.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: derailed_benchmarks
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Richard Schneeman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-03-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: heapy
|
@@ -134,6 +134,40 @@ dependencies:
|
|
134
134
|
- - ">="
|
135
135
|
- !ruby/object:Gem::Version
|
136
136
|
version: '2.1'
|
137
|
+
- !ruby/object:Gem::Dependency
|
138
|
+
name: unicode_plot
|
139
|
+
requirement: !ruby/object:Gem::Requirement
|
140
|
+
requirements:
|
141
|
+
- - ">="
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: 0.0.4
|
144
|
+
- - "<"
|
145
|
+
- !ruby/object:Gem::Version
|
146
|
+
version: 1.0.0
|
147
|
+
type: :runtime
|
148
|
+
prerelease: false
|
149
|
+
version_requirements: !ruby/object:Gem::Requirement
|
150
|
+
requirements:
|
151
|
+
- - ">="
|
152
|
+
- !ruby/object:Gem::Version
|
153
|
+
version: 0.0.4
|
154
|
+
- - "<"
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: 1.0.0
|
157
|
+
- !ruby/object:Gem::Dependency
|
158
|
+
name: mini_histogram
|
159
|
+
requirement: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - "~>"
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
type: :runtime
|
165
|
+
prerelease: false
|
166
|
+
version_requirements: !ruby/object:Gem::Requirement
|
167
|
+
requirements:
|
168
|
+
- - "~>"
|
169
|
+
- !ruby/object:Gem::Version
|
170
|
+
version: '0'
|
137
171
|
- !ruby/object:Gem::Dependency
|
138
172
|
name: capybara
|
139
173
|
requirement: !ruby/object:Gem::Requirement
|