derailed_benchmarks 1.7.0 → 1.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/check_changelog.yml +11 -8
- data/.travis.yml +9 -7
- data/CHANGELOG.md +5 -1
- data/README.md +1 -1
- data/derailed_benchmarks.gemspec +2 -3
- data/lib/derailed_benchmarks.rb +2 -1
- data/lib/derailed_benchmarks/git/commit.rb +36 -0
- data/lib/derailed_benchmarks/git/in_path.rb +59 -0
- data/lib/derailed_benchmarks/git/switch_project.rb +128 -0
- data/lib/derailed_benchmarks/git_switch_project.rb +1 -0
- data/lib/derailed_benchmarks/require_tree.rb +1 -1
- data/lib/derailed_benchmarks/{stats_in_file.rb → stats_for_file.rb} +8 -2
- data/lib/derailed_benchmarks/stats_from_dir.rb +30 -12
- data/lib/derailed_benchmarks/tasks.rb +27 -66
- data/lib/derailed_benchmarks/version.rb +1 -1
- data/test/derailed_benchmarks/git_switch_project_test.rb +83 -0
- data/test/derailed_benchmarks/stats_from_dir_test.rb +26 -11
- data/test/fixtures/require/child_one.rb +1 -1
- data/test/fixtures/require/child_two.rb +1 -1
- data/test/fixtures/require/parent_one.rb +1 -1
- data/test/integration/tasks_test.rb +29 -5
- data/test/test_helper.rb +6 -0
- metadata +16 -30
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24d0d07556d2d6676ea8b2c06d58583071fd9302c056d36db05d252480889308
|
4
|
+
data.tar.gz: 0c783db16c1612173975c12760b101fc30cbd5765d296b2fd66506992305ac69
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e37aa667a5fe031a384a343de999a34ef9f1e0ba98f8190d6f86d326ee30f968e3555344fbaf8841f99da85354c2b4383fe1b75ab166eb0df9134d20ee48fd95
|
7
|
+
data.tar.gz: 3f581637e835d808b8264786ffd70159fe534c4147e72e65f5649fe7ecb0b92b9edc215c9edd43e2ca51a510793ae475da3e0318262103bb741823746d9f3421
|
@@ -1,10 +1,13 @@
|
|
1
1
|
name: Check Changelog
|
2
|
-
|
2
|
+
|
3
|
+
on:
|
4
|
+
pull_request:
|
5
|
+
types: [opened, reopened, edited, synchronize]
|
3
6
|
jobs:
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
7
|
+
build:
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
steps:
|
10
|
+
- uses: actions/checkout@v1
|
11
|
+
- name: Check that CHANGELOG is touched
|
12
|
+
run: |
|
13
|
+
cat $GITHUB_EVENT_PATH | jq .pull_request.title | grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' || git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
|
data/.travis.yml
CHANGED
@@ -1,18 +1,20 @@
|
|
1
1
|
language: ruby
|
2
2
|
rvm:
|
3
|
-
- 2.
|
4
|
-
- 2.
|
5
|
-
-
|
3
|
+
- 2.2.10
|
4
|
+
- 2.5.8
|
5
|
+
- 2.7.1
|
6
6
|
|
7
7
|
gemfile:
|
8
8
|
- gemfiles/rails_5_1.gemfile
|
9
|
-
- gemfiles/rails_5_2.gemfile
|
10
9
|
- gemfiles/rails_6_0.gemfile
|
11
10
|
- gemfiles/rails_git.gemfile
|
12
11
|
|
13
12
|
before_install:
|
14
|
-
- gem install bundler
|
13
|
+
- gem install bundler -v 1.17.3
|
15
14
|
|
16
|
-
|
15
|
+
jobs:
|
17
16
|
allow_failures:
|
18
|
-
- rvm:
|
17
|
+
- rvm: 2.2.10
|
18
|
+
gemfile: gemfiles/rails_6_0.gemfile
|
19
|
+
- rvm: 2.2.10
|
20
|
+
gemfile: gemfiles/rails_git.gemfile
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -10,7 +10,7 @@ A series of things you can use to benchmark a Rails or Ruby app.
|
|
10
10
|
## Compatibility/Requirements
|
11
11
|
|
12
12
|
This gem has been tested and is known to work with Rails 3.2+ using Ruby
|
13
|
-
2.
|
13
|
+
2.2+. Some commands __may__ work on older versions of Ruby, but not all commands are supported.
|
14
14
|
|
15
15
|
For some benchmarks, not all, you'll need to verify you have a working version of curl on your OS:
|
16
16
|
|
data/derailed_benchmarks.gemspec
CHANGED
@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
|
|
20
20
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
21
21
|
gem.require_paths = ["lib"]
|
22
22
|
|
23
|
-
gem.required_ruby_version = ">= 2.
|
23
|
+
gem.required_ruby_version = ">= 2.2.0"
|
24
24
|
|
25
25
|
gem.add_dependency "heapy", "~> 0"
|
26
26
|
gem.add_dependency "memory_profiler", "~> 0"
|
@@ -30,8 +30,7 @@ Gem::Specification.new do |gem|
|
|
30
30
|
gem.add_dependency "rake", "> 10", "< 14"
|
31
31
|
gem.add_dependency "thor", ">= 0.19", "< 2"
|
32
32
|
gem.add_dependency "ruby-statistics", ">= 2.1"
|
33
|
-
gem.add_dependency "
|
34
|
-
gem.add_dependency "mini_histogram", "~> 0"
|
33
|
+
gem.add_dependency "mini_histogram", ">= 0.2.1"
|
35
34
|
|
36
35
|
gem.add_development_dependency "capybara", "~> 2"
|
37
36
|
gem.add_development_dependency "m"
|
data/lib/derailed_benchmarks.rb
CHANGED
@@ -43,8 +43,9 @@ end
|
|
43
43
|
require 'derailed_benchmarks/require_tree'
|
44
44
|
require 'derailed_benchmarks/auth_helper'
|
45
45
|
|
46
|
-
require 'derailed_benchmarks/
|
46
|
+
require 'derailed_benchmarks/stats_for_file'
|
47
47
|
require 'derailed_benchmarks/stats_from_dir'
|
48
|
+
require 'derailed_benchmarks/git/switch_project'
|
48
49
|
|
49
50
|
if DerailedBenchmarks.gem_is_bundled?("devise")
|
50
51
|
DerailedBenchmarks.auth = DerailedBenchmarks::AuthHelpers::Devise.new
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module DerailedBenchmarks
|
2
|
+
# Represents a specific commit in a git repo
|
3
|
+
#
|
4
|
+
# Can be used to get information from the commit or to check it out
|
5
|
+
#
|
6
|
+
# commit = GitCommit.new(path: "path/to/repo", ref: "6e642963acec0ff64af51bd6fba8db3c4176ed6e")
|
7
|
+
# commit.short_sha # => "6e64296"
|
8
|
+
# commit.checkout! # Will check out the current commit at the repo in the path
|
9
|
+
class Git::Commit
|
10
|
+
attr_reader :ref, :description, :time, :short_sha, :log
|
11
|
+
|
12
|
+
def initialize(path: , ref: , log_dir: Pathname.new("/dev/null"))
|
13
|
+
@in_git_path = Git::InPath.new(path)
|
14
|
+
@ref = ref
|
15
|
+
@log = log_dir.join("#{file_safe_ref}.bench.txt")
|
16
|
+
|
17
|
+
Dir.chdir(path) do
|
18
|
+
checkout!
|
19
|
+
@description = @in_git_path.description
|
20
|
+
@short_sha = @in_git_path.short_sha
|
21
|
+
@time = @in_git_path.time
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
alias :desc :description
|
26
|
+
alias :file :log
|
27
|
+
|
28
|
+
def checkout!
|
29
|
+
@in_git_path.checkout!(ref)
|
30
|
+
end
|
31
|
+
|
32
|
+
private def file_safe_ref
|
33
|
+
ref.gsub('/', ':')
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module DerailedBenchmarks
|
2
|
+
# A class for running commands in a git directory
|
3
|
+
#
|
4
|
+
# It's faster to check if we're already in that directory instead
|
5
|
+
# of having to `cd` into each time. https://twitter.com/schneems/status/1305196730170961920
|
6
|
+
#
|
7
|
+
# Example:
|
8
|
+
#
|
9
|
+
# in_git_path = InGitPath.new(`bundle info heapy --path`.strip)
|
10
|
+
# in_git_path.checkout!("f0f92b06156f2274021aa42f15326da041ee9009")
|
11
|
+
# in_git_path.short_sha # => "f0f92b0"
|
12
|
+
class Git::InPath
|
13
|
+
attr_reader :path
|
14
|
+
|
15
|
+
def initialize(path)
|
16
|
+
@path = path
|
17
|
+
end
|
18
|
+
|
19
|
+
def description
|
20
|
+
run!("git log --oneline --format=%B -n 1 HEAD | head -n 1")
|
21
|
+
end
|
22
|
+
|
23
|
+
def short_sha
|
24
|
+
run!("git rev-parse --short HEAD")
|
25
|
+
end
|
26
|
+
|
27
|
+
def time_stamp_string
|
28
|
+
run!("git log -n 1 --pretty=format:%ci") # https://stackoverflow.com/a/25921837/147390
|
29
|
+
end
|
30
|
+
|
31
|
+
def branch
|
32
|
+
branch = run!("git rev-parse --abbrev-ref HEAD")
|
33
|
+
branch == "HEAD" ? nil : branch
|
34
|
+
end
|
35
|
+
|
36
|
+
def checkout!(ref)
|
37
|
+
run!("git checkout '#{ref}' 2>&1")
|
38
|
+
end
|
39
|
+
|
40
|
+
def time
|
41
|
+
DateTime.parse(time_stamp_string)
|
42
|
+
end
|
43
|
+
|
44
|
+
def run(cmd)
|
45
|
+
if Dir.pwd == path
|
46
|
+
out = `#{cmd}`.strip
|
47
|
+
else
|
48
|
+
out = `cd #{path} && #{cmd}`.strip
|
49
|
+
end
|
50
|
+
out
|
51
|
+
end
|
52
|
+
|
53
|
+
def run!(cmd)
|
54
|
+
out = run(cmd)
|
55
|
+
raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?
|
56
|
+
out
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
module DerailedBenchmarks
|
2
|
+
class Git
|
3
|
+
end
|
4
|
+
end
|
5
|
+
require_relative "in_path.rb"
|
6
|
+
require_relative "commit.rb"
|
7
|
+
|
8
|
+
module DerailedBenchmarks
|
9
|
+
# Wraps two or more git commits in a specific location
|
10
|
+
#
|
11
|
+
# Returns an array of GitCommit objects that can be used to manipulate
|
12
|
+
# and checkout the repo
|
13
|
+
#
|
14
|
+
# Example:
|
15
|
+
#
|
16
|
+
# `git clone https://sharpstone/default_ruby tmp/default_ruby`
|
17
|
+
#
|
18
|
+
# project = GitSwitchProject.new(path: "tmp/default_ruby")
|
19
|
+
#
|
20
|
+
# By default it will represent the last two commits:
|
21
|
+
#
|
22
|
+
# project.commits.length # => 2
|
23
|
+
#
|
24
|
+
# You can pass in explicit REFs in an array:
|
25
|
+
#
|
26
|
+
# ref_array = ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"]
|
27
|
+
# project = GitSwitchProject.new(path: "tmp/default_ruby", ref_array: ref_array)
|
28
|
+
#
|
29
|
+
# puts project.commits.map(&:ref) == ref_array # => true
|
30
|
+
#
|
31
|
+
#
|
32
|
+
# It knows the current branch or sha:
|
33
|
+
#
|
34
|
+
# `cd tmp/ruby && git checkout -b mybranch`
|
35
|
+
# project.current_branch_or_sha #=> "mybranch"
|
36
|
+
#
|
37
|
+
# It can be used for safely wrapping checkouts to ensure the project returns to it's original branch:
|
38
|
+
#
|
39
|
+
# project.restore_branch_on_return do
|
40
|
+
# project.commits.first.checkout!
|
41
|
+
# project.current_branch_or_sha # => "da748a593"
|
42
|
+
# end
|
43
|
+
#
|
44
|
+
# project.current_branch_or_sha # => "mybranch"
|
45
|
+
class Git::SwitchProject
|
46
|
+
attr_reader :commits
|
47
|
+
|
48
|
+
def initialize(path: , ref_array: [], io: STDOUT, log_dir: "/dev/null")
|
49
|
+
@path = Pathname.new(path)
|
50
|
+
|
51
|
+
@in_git_path = Git::InPath.new(@path.expand_path)
|
52
|
+
|
53
|
+
raise "Must be a path with a .git directory '#{@path}'" if !@path.join(".git").exist?
|
54
|
+
@io = io
|
55
|
+
@commits = []
|
56
|
+
log_dir = Pathname(log_dir)
|
57
|
+
|
58
|
+
expand_refs(ref_array).each do |ref|
|
59
|
+
restore_branch_on_return(quiet: true) do
|
60
|
+
@commits << Git::Commit.new(path: @path, ref: ref, log_dir: log_dir)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
if (duplicate = @commits.group_by(&:short_sha).detect {|(k, v)| v.length > 1})
|
65
|
+
raise "Duplicate SHA resolved #{duplicate[0].inspect}: #{duplicate[1].map {|c| "'#{c.ref}' => '#{c.short_sha}'"}.join(", ") } at #{@path}"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def current_branch_or_sha
|
70
|
+
branch_or_sha = @in_git_path.branch
|
71
|
+
branch_or_sha ||= @in_git_path.short_sha
|
72
|
+
branch_or_sha
|
73
|
+
end
|
74
|
+
|
75
|
+
def dirty?
|
76
|
+
!clean?
|
77
|
+
end
|
78
|
+
|
79
|
+
# https://stackoverflow.com/a/3879077/147390
|
80
|
+
def clean?
|
81
|
+
@in_git_path.run("git diff-index --quiet HEAD --") && $?.success?
|
82
|
+
end
|
83
|
+
|
84
|
+
private def status(pattern: "*.gemspec")
|
85
|
+
@in_git_path.run("git status #{pattern}")
|
86
|
+
end
|
87
|
+
|
88
|
+
def restore_branch_on_return(quiet: false)
|
89
|
+
if dirty? && status.include?("gemspec")
|
90
|
+
dirty_gemspec = true
|
91
|
+
unless quiet
|
92
|
+
@io.puts "Working tree at #{@path} is dirty, stashing. This will be popped on return"
|
93
|
+
@io.puts "Bundler modifies gemspec files on git install, this is normal"
|
94
|
+
@io.puts "Original status:\n#{status}"
|
95
|
+
end
|
96
|
+
@in_git_path.run!("git stash")
|
97
|
+
end
|
98
|
+
branch_or_sha = self.current_branch_or_sha
|
99
|
+
yield
|
100
|
+
ensure
|
101
|
+
return unless branch_or_sha
|
102
|
+
@io.puts "Resetting git dir of '#{@path.to_s}' to #{branch_or_sha.inspect}" unless quiet
|
103
|
+
|
104
|
+
@in_git_path.checkout!(branch_or_sha)
|
105
|
+
if dirty_gemspec
|
106
|
+
out = @in_git_path.run!("git stash apply 2>&1")
|
107
|
+
@io.puts "Applying stash of '#{@path.to_s}':\n#{out}" unless quiet
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
# case ref_array.length
|
112
|
+
# when >= 2
|
113
|
+
# returns original array
|
114
|
+
# when 1
|
115
|
+
# returns the given ref plus the one before it
|
116
|
+
# when 0
|
117
|
+
# returns the most recent 2 refs
|
118
|
+
private def expand_refs(ref_array)
|
119
|
+
return ref_array if ref_array.length >= 2
|
120
|
+
|
121
|
+
@in_git_path.checkout!(ref_array.first) if ref_array.first
|
122
|
+
|
123
|
+
branches_string = @in_git_path.run!("git log --format='%H' -n 2")
|
124
|
+
ref_array = branches_string.split($/)
|
125
|
+
return ref_array
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
|
@@ -40,7 +40,7 @@ module DerailedBenchmarks
|
|
40
40
|
end
|
41
41
|
|
42
42
|
def to_string
|
43
|
-
str =
|
43
|
+
str = String.new("#{name}: #{cost.round(4)} MiB")
|
44
44
|
if parent && REQUIRED_BY[self.name.to_s]
|
45
45
|
names = REQUIRED_BY[self.name.to_s].uniq - [parent.name.to_s]
|
46
46
|
if names.any?
|
@@ -16,24 +16,30 @@ module DerailedBenchmarks
|
|
16
16
|
# x.average # => 10.5
|
17
17
|
# x.name # => "muhfile"
|
18
18
|
class StatsForFile
|
19
|
-
attr_reader :name, :values, :desc, :time
|
19
|
+
attr_reader :name, :values, :desc, :time, :short_sha
|
20
20
|
|
21
|
-
def initialize(file:, name:, desc: "", time: )
|
21
|
+
def initialize(file:, name:, desc: "", time: , short_sha: nil)
|
22
22
|
@file = Pathname.new(file)
|
23
23
|
FileUtils.touch(@file)
|
24
24
|
|
25
25
|
@name = name
|
26
26
|
@desc = desc
|
27
27
|
@time = time
|
28
|
+
@short_sha = short_sha
|
28
29
|
end
|
29
30
|
|
30
31
|
def call
|
31
32
|
load_file!
|
33
|
+
return if values.empty?
|
32
34
|
|
33
35
|
@median = (values[(values.length - 1) / 2] + values[values.length/ 2]) / 2.0
|
34
36
|
@average = values.inject(:+) / values.length
|
35
37
|
end
|
36
38
|
|
39
|
+
def empty?
|
40
|
+
values.empty?
|
41
|
+
end
|
42
|
+
|
37
43
|
def median
|
38
44
|
@median.to_f
|
39
45
|
end
|
@@ -2,9 +2,9 @@
|
|
2
2
|
|
3
3
|
require 'bigdecimal'
|
4
4
|
require 'statistics'
|
5
|
-
require 'unicode_plot'
|
6
5
|
require 'stringio'
|
7
6
|
require 'mini_histogram'
|
7
|
+
require 'mini_histogram/plot'
|
8
8
|
|
9
9
|
module DerailedBenchmarks
|
10
10
|
# A class used to read several benchmark files
|
@@ -29,14 +29,28 @@ module DerailedBenchmarks
|
|
29
29
|
FORMAT = "%0.4f"
|
30
30
|
attr_reader :stats, :oldest, :newest
|
31
31
|
|
32
|
-
def initialize(
|
32
|
+
def initialize(input)
|
33
33
|
@files = []
|
34
34
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
35
|
+
if input.is_a?(Hash)
|
36
|
+
hash = input
|
37
|
+
hash.each do |branch, info_hash|
|
38
|
+
file = info_hash.fetch(:file)
|
39
|
+
desc = info_hash.fetch(:desc)
|
40
|
+
time = info_hash.fetch(:time)
|
41
|
+
short_sha = info_hash["short_sha"]
|
42
|
+
@files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
|
43
|
+
end
|
44
|
+
else
|
45
|
+
input.each do |commit|
|
46
|
+
@files << StatsForFile.new(
|
47
|
+
file: commit.file,
|
48
|
+
desc: commit.desc,
|
49
|
+
time: commit.time,
|
50
|
+
name: commit.ref,
|
51
|
+
short_sha: commit.short_sha
|
52
|
+
)
|
53
|
+
end
|
40
54
|
end
|
41
55
|
@files.sort_by! { |f| f.time }
|
42
56
|
@oldest = @files.first
|
@@ -46,6 +60,8 @@ module DerailedBenchmarks
|
|
46
60
|
def call
|
47
61
|
@files.each(&:call)
|
48
62
|
|
63
|
+
return self if @files.detect(&:empty?)
|
64
|
+
|
49
65
|
stats_95 = statistical_test(confidence: 95)
|
50
66
|
|
51
67
|
# If default check is good, see if we also pass a more rigorous test
|
@@ -109,12 +125,12 @@ module DerailedBenchmarks
|
|
109
125
|
MiniHistogram.set_average_edges!(newest_histogram, oldest_histogram)
|
110
126
|
|
111
127
|
{newest => newest_histogram, oldest => oldest_histogram}.each do |report, histogram|
|
112
|
-
plot =
|
113
|
-
|
114
|
-
title: "\n#{' ' * 18 }Histogram - [#{report.name}] #{report.desc.inspect}",
|
128
|
+
plot = histogram.plot(
|
129
|
+
title: "\n#{' ' * 18 }Histogram - [#{report.short_sha || report.name}] #{report.desc.inspect}",
|
115
130
|
ylabel: "Time (s)",
|
116
131
|
xlabel: "# of runs in range"
|
117
132
|
)
|
133
|
+
|
118
134
|
plot.render(io)
|
119
135
|
io.puts
|
120
136
|
end
|
@@ -123,6 +139,8 @@ module DerailedBenchmarks
|
|
123
139
|
end
|
124
140
|
|
125
141
|
def banner(io = $stdout)
|
142
|
+
return if @files.detect(&:empty?)
|
143
|
+
|
126
144
|
io.puts
|
127
145
|
if significant?
|
128
146
|
io.puts "❤️ ❤️ ❤️ (Statistically Significant) ❤️ ❤️ ❤️"
|
@@ -130,11 +148,11 @@ module DerailedBenchmarks
|
|
130
148
|
io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
|
131
149
|
end
|
132
150
|
io.puts
|
133
|
-
io.puts "[#{newest.name}] #{newest.desc.inspect}
|
151
|
+
io.puts "[#{newest.short_sha || newest.name}] (#{FORMAT % newest.median} seconds) #{newest.desc.inspect} ref: #{newest.name.inspect}"
|
134
152
|
io.puts " #{change_direction} by:"
|
135
153
|
io.puts " #{align}#{FORMAT % x_faster}x [older/newer]"
|
136
154
|
io.puts " #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
|
137
|
-
io.puts "[#{oldest.name}] #{oldest.desc.inspect}
|
155
|
+
io.puts "[#{oldest.short_sha || oldest.name}] (#{FORMAT % oldest.median} seconds) #{oldest.desc.inspect} ref: #{oldest.name.inspect}"
|
138
156
|
io.puts
|
139
157
|
io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
|
140
158
|
io.puts "Samples: #{newest.values.length}"
|
@@ -17,96 +17,57 @@ namespace :perf do
|
|
17
17
|
script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test"
|
18
18
|
|
19
19
|
if ENV["DERAILED_PATH_TO_LIBRARY"]
|
20
|
-
library_dir = ENV["DERAILED_PATH_TO_LIBRARY"]
|
20
|
+
library_dir = ENV["DERAILED_PATH_TO_LIBRARY"].chomp
|
21
21
|
else
|
22
22
|
library_dir = DerailedBenchmarks.rails_path_on_disk
|
23
23
|
end
|
24
|
-
|
25
|
-
raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git"))
|
26
|
-
|
27
|
-
# Use either the explicit SHAs when present or grab last two SHAs from commit history
|
28
|
-
# if only one SHA is given, then use it and the last SHA from commit history
|
29
|
-
branch_names = []
|
30
|
-
branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"]
|
31
|
-
if branch_names.length < 2
|
32
|
-
Dir.chdir(library_dir) do
|
33
|
-
run!("git checkout '#{branch_names.first}'") unless branch_names.empty?
|
34
|
-
|
35
|
-
branches = run!('git log --format="%H" -n 2').chomp.split($/)
|
36
|
-
if branch_names.empty?
|
37
|
-
branch_names = branches
|
38
|
-
else
|
39
|
-
branches.shift
|
40
|
-
branch_names << branches.shift
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
current_library_branch = ""
|
46
|
-
Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }
|
24
|
+
library_dir = Pathname.new(library_dir)
|
47
25
|
|
48
26
|
out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
|
49
27
|
out_dir.mkpath
|
50
28
|
|
51
|
-
|
52
|
-
branch_info = {}
|
53
|
-
branch_to_sha = {}
|
29
|
+
ref_string = ENV["SHAS_TO_TEST"] || ENV["REFS_TO_TEST"] || ""
|
54
30
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
short_sha = run!("git rev-parse --short HEAD").strip
|
61
|
-
branch_to_sha[branch] = short_sha
|
31
|
+
project = DerailedBenchmarks::Git::SwitchProject.new(
|
32
|
+
path: library_dir,
|
33
|
+
ref_array: ref_string.split(","),
|
34
|
+
log_dir: out_dir
|
35
|
+
)
|
62
36
|
|
63
|
-
|
64
|
-
end
|
65
|
-
run!("#{script}")
|
66
|
-
end
|
37
|
+
stats = DerailedBenchmarks::StatsFromDir.new(project.commits)
|
67
38
|
|
39
|
+
# Advertise branch names early to make sure people know what they're testing
|
68
40
|
puts
|
69
41
|
puts
|
70
|
-
|
71
|
-
short_sha
|
72
|
-
desc = branch_info[short_sha][:desc]
|
73
|
-
puts "Testing #{i + 1}: #{short_sha}: #{desc}"
|
42
|
+
project.commits.each_with_index do |commit, i|
|
43
|
+
puts "Testing #{i + 1}: #{commit.short_sha}: #{commit.description}"
|
74
44
|
end
|
75
45
|
puts
|
76
46
|
puts
|
77
47
|
|
78
|
-
|
79
|
-
|
80
|
-
|
48
|
+
project.restore_branch_on_return do
|
49
|
+
DERAILED_SCRIPT_COUNT.times do |i|
|
50
|
+
puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
|
51
|
+
project.commits.each do |commit|
|
52
|
+
commit.checkout!
|
81
53
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
|
86
|
-
run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
|
87
|
-
end
|
54
|
+
output = run!("#{script} 2>&1")
|
55
|
+
commit.log.open("a") {|f| f.puts output.lines.last }
|
56
|
+
end
|
88
57
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
58
|
+
if (i % 50).zero?
|
59
|
+
puts "Intermediate result"
|
60
|
+
stats.call.banner
|
61
|
+
puts "Continuing execution"
|
62
|
+
end
|
94
63
|
end
|
95
64
|
end
|
96
65
|
|
97
66
|
ensure
|
98
|
-
if library_dir && current_library_branch
|
99
|
-
puts "Resetting git dir of '#{library_dir.to_s}' to #{current_library_branch.inspect}"
|
100
|
-
Dir.chdir(library_dir) do
|
101
|
-
run!("git checkout '#{current_library_branch}'")
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
67
|
if stats
|
106
|
-
stats.call
|
107
|
-
stats.banner
|
68
|
+
stats.call.banner
|
108
69
|
|
109
|
-
result_file = out_dir
|
70
|
+
result_file = out_dir.join("results.txt")
|
110
71
|
File.open(result_file, "w") do |f|
|
111
72
|
stats.banner(f)
|
112
73
|
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'test_helper'
|
4
|
+
|
5
|
+
class GitSwitchProjectTest < ActiveSupport::TestCase
|
6
|
+
test "tells me when it's not pointing at a git project" do
|
7
|
+
exception = assert_raises {
|
8
|
+
DerailedBenchmarks::Git::SwitchProject.new(path: "/dev/null")
|
9
|
+
}
|
10
|
+
assert_includes(exception.message, '.git directory')
|
11
|
+
end
|
12
|
+
|
13
|
+
test "dirty gemspec cleaning" do
|
14
|
+
Dir.mktmpdir do |dir|
|
15
|
+
run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
|
16
|
+
run!("cd #{dir} && echo lol > foo.gemspec && git add .")
|
17
|
+
|
18
|
+
io = StringIO.new
|
19
|
+
project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, io: io)
|
20
|
+
|
21
|
+
assert project.dirty?
|
22
|
+
refute project.clean?
|
23
|
+
|
24
|
+
project.restore_branch_on_return do
|
25
|
+
project.commits.map(&:checkout!)
|
26
|
+
end
|
27
|
+
|
28
|
+
assert_includes io.string, "Bundler modifies gemspec files"
|
29
|
+
assert_includes io.string, "Applying stash"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
test "works on a git repo" do
|
34
|
+
Dir.mktmpdir do |dir|
|
35
|
+
run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
|
36
|
+
|
37
|
+
# finds shas when none given
|
38
|
+
project = DerailedBenchmarks::Git::SwitchProject.new(path: dir)
|
39
|
+
|
40
|
+
assert_equal ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "da748a59340be8b950e7bbbfb32077eb67d70c3c"], project.commits.map(&:ref)
|
41
|
+
first_commit = project.commits.first
|
42
|
+
|
43
|
+
assert_equal "CI test support", first_commit.description
|
44
|
+
assert_equal "6e64296", first_commit.short_sha
|
45
|
+
assert_equal "/dev/null/6e642963acec0ff64af51bd6fba8db3c4176ed6e.bench.txt", first_commit.log.to_s
|
46
|
+
assert_equal DateTime.parse("Tue, 14 Apr 2020 13:26:03 -0500"), first_commit.time
|
47
|
+
|
48
|
+
assert_equal "mybranch", project.current_branch_or_sha
|
49
|
+
|
50
|
+
# Finds shas when 1 is given
|
51
|
+
project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c"])
|
52
|
+
|
53
|
+
assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "5c09f748957d2098182762004adee27d1ff83160"], project.commits.map(&:ref)
|
54
|
+
|
55
|
+
|
56
|
+
# Returns correct refs if given
|
57
|
+
project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"])
|
58
|
+
|
59
|
+
assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"], project.commits.map(&:ref)
|
60
|
+
|
61
|
+
first_commit = project.commits.first
|
62
|
+
|
63
|
+
first_commit.checkout!
|
64
|
+
|
65
|
+
assert_equal first_commit.short_sha, project.current_branch_or_sha
|
66
|
+
|
67
|
+
# Test restore_branch_on_return
|
68
|
+
project.restore_branch_on_return(quiet: true) do
|
69
|
+
project.commits.last.checkout!
|
70
|
+
|
71
|
+
assert_equal project.commits.last.short_sha, project.current_branch_or_sha
|
72
|
+
end
|
73
|
+
|
74
|
+
assert_equal project.commits.first.short_sha, project.current_branch_or_sha
|
75
|
+
|
76
|
+
exception = assert_raise {
|
77
|
+
DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "mybranch"])
|
78
|
+
}
|
79
|
+
|
80
|
+
assert_includes(exception.message, 'Duplicate SHA resolved "6e64296"')
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -3,6 +3,19 @@
|
|
3
3
|
require 'test_helper'
|
4
4
|
|
5
5
|
class StatsFromDirTest < ActiveSupport::TestCase
|
6
|
+
test "empty files" do
|
7
|
+
Dir.mktmpdir do |dir|
|
8
|
+
dir = Pathname.new(dir)
|
9
|
+
branch_info = {}
|
10
|
+
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser"), name: "loser" }
|
11
|
+
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner"), name: "winner" }
|
12
|
+
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
|
13
|
+
io = StringIO.new
|
14
|
+
stats.call.banner(io: io) # Doesn't error
|
15
|
+
assert_equal "", io.read
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
6
19
|
test "that it works" do
|
7
20
|
dir = fixtures_dir("stats/significant")
|
8
21
|
branch_info = {}
|
@@ -38,7 +51,6 @@ class StatsFromDirTest < ActiveSupport::TestCase
|
|
38
51
|
|
39
52
|
io = StringIO.new
|
40
53
|
stats.call.banner(io)
|
41
|
-
puts io.string
|
42
54
|
|
43
55
|
assert_match(/11\.2 , 11\.28/, io.string)
|
44
56
|
assert_match(/11\.8 , 11\.88/, io.string)
|
@@ -64,10 +76,13 @@ class StatsFromDirTest < ActiveSupport::TestCase
|
|
64
76
|
|
65
77
|
test "banner faster" do
|
66
78
|
dir = fixtures_dir("stats/significant")
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
79
|
+
Branch_info = {}
|
80
|
+
|
81
|
+
require 'ostruct'
|
82
|
+
commits = []
|
83
|
+
commits << OpenStruct.new({ desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), ref: "loser", short_sha: "aaaaa" })
|
84
|
+
commits << OpenStruct.new({ desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), ref: "winner", short_sha: "bbbbb" })
|
85
|
+
stats = DerailedBenchmarks::StatsFromDir.new(commits).call
|
71
86
|
newest = stats.newest
|
72
87
|
oldest = stats.oldest
|
73
88
|
|
@@ -88,12 +103,12 @@ class StatsFromDirTest < ActiveSupport::TestCase
|
|
88
103
|
11.0
|
89
104
|
end
|
90
105
|
|
91
|
-
expected =
|
92
|
-
[
|
106
|
+
expected = <<-EOM
|
107
|
+
[bbbbb] (10.5000 seconds) "I am the new commit" ref: "winner"
|
93
108
|
FASTER 🚀🚀🚀 by:
|
94
109
|
1.0476x [older/newer]
|
95
110
|
4.5455% [(older - newer) / older * 100]
|
96
|
-
[
|
111
|
+
[aaaaa] (11.0000 seconds) "Old commit" ref: "loser"
|
97
112
|
EOM
|
98
113
|
|
99
114
|
actual = StringIO.new
|
@@ -119,12 +134,12 @@ EOM
|
|
119
134
|
11.0
|
120
135
|
end
|
121
136
|
|
122
|
-
expected =
|
123
|
-
[loser] "I am the new commit"
|
137
|
+
expected = <<-EOM
|
138
|
+
[loser] (11.0000 seconds) "I am the new commit" ref: "loser"
|
124
139
|
SLOWER 🐢🐢🐢 by:
|
125
140
|
0.9545x [older/newer]
|
126
141
|
-4.7619% [(older - newer) / older * 100]
|
127
|
-
[winner] "Old commit"
|
142
|
+
[winner] (10.5000 seconds) "Old commit" ref: "winner"
|
128
143
|
EOM
|
129
144
|
|
130
145
|
actual = StringIO.new
|
@@ -26,21 +26,45 @@ class TasksTest < ActiveSupport::TestCase
|
|
26
26
|
env_string = env.map {|key, value| "#{key.shellescape}=#{value.to_s.shellescape}" }.join(" ")
|
27
27
|
cmd = "env #{env_string} bundle exec rake -f perf.rake #{cmd} --trace"
|
28
28
|
puts "Running: #{cmd}"
|
29
|
-
result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd}` }
|
30
|
-
if assert_success
|
31
|
-
|
29
|
+
result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd} 2>&1` }
|
30
|
+
if assert_success && !$?.success?
|
31
|
+
puts result
|
32
|
+
raise "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
|
32
33
|
end
|
33
34
|
|
34
35
|
result
|
35
36
|
end
|
36
37
|
|
37
|
-
test '
|
38
|
+
test 'non-rails library with branch specified' do
|
39
|
+
skip unless ENV['USING_RAILS_WICKED_BRANCH']
|
40
|
+
|
41
|
+
gem_path = run!("bundle info wicked --path")
|
42
|
+
env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2, "DERAILED_PATH_TO_LIBRARY" => gem_path}
|
43
|
+
puts rake "perf:library", { env: env }
|
44
|
+
end
|
45
|
+
|
46
|
+
test 'rails perf:library from git' do
|
47
|
+
# BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
|
48
|
+
|
38
49
|
skip unless ENV['USING_RAILS_GIT']
|
39
50
|
|
40
|
-
env = { "TEST_COUNT" =>
|
51
|
+
env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
|
41
52
|
puts rake "perf:library", { env: env }
|
42
53
|
end
|
43
54
|
|
55
|
+
test "rails perf:library with bad script" do
|
56
|
+
# BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
|
57
|
+
|
58
|
+
skip unless ENV['USING_RAILS_GIT']
|
59
|
+
|
60
|
+
error = assert_raises {
|
61
|
+
env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
|
62
|
+
puts rake "perf:library", { env: env }
|
63
|
+
}
|
64
|
+
|
65
|
+
assert error.message =~ /nopenopenop:( command)? not found/, "Expected #{error.message} to include /nopenopenop: (command)? not found/ but it did not"
|
66
|
+
end
|
67
|
+
|
44
68
|
test 'hitting authenticated devise apps' do
|
45
69
|
env = { "PATH_TO_HIT" => "authenticated", "USE_AUTH" => "true", "TEST_COUNT" => "2" }
|
46
70
|
result = rake 'perf:test', env: env
|
data/test/test_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: derailed_benchmarks
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Richard Schneeman
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-09-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: heapy
|
@@ -135,39 +135,19 @@ dependencies:
|
|
135
135
|
- !ruby/object:Gem::Version
|
136
136
|
version: '2.1'
|
137
137
|
- !ruby/object:Gem::Dependency
|
138
|
-
name:
|
138
|
+
name: mini_histogram
|
139
139
|
requirement: !ruby/object:Gem::Requirement
|
140
140
|
requirements:
|
141
141
|
- - ">="
|
142
142
|
- !ruby/object:Gem::Version
|
143
|
-
version: 0.
|
144
|
-
- - "<"
|
145
|
-
- !ruby/object:Gem::Version
|
146
|
-
version: 1.0.0
|
143
|
+
version: 0.2.1
|
147
144
|
type: :runtime
|
148
145
|
prerelease: false
|
149
146
|
version_requirements: !ruby/object:Gem::Requirement
|
150
147
|
requirements:
|
151
148
|
- - ">="
|
152
149
|
- !ruby/object:Gem::Version
|
153
|
-
version: 0.
|
154
|
-
- - "<"
|
155
|
-
- !ruby/object:Gem::Version
|
156
|
-
version: 1.0.0
|
157
|
-
- !ruby/object:Gem::Dependency
|
158
|
-
name: mini_histogram
|
159
|
-
requirement: !ruby/object:Gem::Requirement
|
160
|
-
requirements:
|
161
|
-
- - "~>"
|
162
|
-
- !ruby/object:Gem::Version
|
163
|
-
version: '0'
|
164
|
-
type: :runtime
|
165
|
-
prerelease: false
|
166
|
-
version_requirements: !ruby/object:Gem::Requirement
|
167
|
-
requirements:
|
168
|
-
- - "~>"
|
169
|
-
- !ruby/object:Gem::Version
|
170
|
-
version: '0'
|
150
|
+
version: 0.2.1
|
171
151
|
- !ruby/object:Gem::Dependency
|
172
152
|
name: capybara
|
173
153
|
requirement: !ruby/object:Gem::Requirement
|
@@ -277,13 +257,18 @@ files:
|
|
277
257
|
- lib/derailed_benchmarks/auth_helper.rb
|
278
258
|
- lib/derailed_benchmarks/auth_helpers/devise.rb
|
279
259
|
- lib/derailed_benchmarks/core_ext/kernel_require.rb
|
260
|
+
- lib/derailed_benchmarks/git/commit.rb
|
261
|
+
- lib/derailed_benchmarks/git/in_path.rb
|
262
|
+
- lib/derailed_benchmarks/git/switch_project.rb
|
263
|
+
- lib/derailed_benchmarks/git_switch_project.rb
|
280
264
|
- lib/derailed_benchmarks/load_tasks.rb
|
281
265
|
- lib/derailed_benchmarks/require_tree.rb
|
266
|
+
- lib/derailed_benchmarks/stats_for_file.rb
|
282
267
|
- lib/derailed_benchmarks/stats_from_dir.rb
|
283
|
-
- lib/derailed_benchmarks/stats_in_file.rb
|
284
268
|
- lib/derailed_benchmarks/tasks.rb
|
285
269
|
- lib/derailed_benchmarks/version.rb
|
286
270
|
- test/derailed_benchmarks/core_ext/kernel_require_test.rb
|
271
|
+
- test/derailed_benchmarks/git_switch_project_test.rb
|
287
272
|
- test/derailed_benchmarks/require_tree_test.rb
|
288
273
|
- test/derailed_benchmarks/stats_from_dir_test.rb
|
289
274
|
- test/derailed_test.rb
|
@@ -348,7 +333,7 @@ homepage: https://github.com/schneems/derailed_benchmarks
|
|
348
333
|
licenses:
|
349
334
|
- MIT
|
350
335
|
metadata: {}
|
351
|
-
post_install_message:
|
336
|
+
post_install_message:
|
352
337
|
rdoc_options: []
|
353
338
|
require_paths:
|
354
339
|
- lib
|
@@ -356,7 +341,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
356
341
|
requirements:
|
357
342
|
- - ">="
|
358
343
|
- !ruby/object:Gem::Version
|
359
|
-
version: 2.
|
344
|
+
version: 2.2.0
|
360
345
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
361
346
|
requirements:
|
362
347
|
- - ">="
|
@@ -364,11 +349,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
364
349
|
version: '0'
|
365
350
|
requirements: []
|
366
351
|
rubygems_version: 3.1.2
|
367
|
-
signing_key:
|
352
|
+
signing_key:
|
368
353
|
specification_version: 4
|
369
354
|
summary: Benchmarks designed to performance test your ENTIRE site
|
370
355
|
test_files:
|
371
356
|
- test/derailed_benchmarks/core_ext/kernel_require_test.rb
|
357
|
+
- test/derailed_benchmarks/git_switch_project_test.rb
|
372
358
|
- test/derailed_benchmarks/require_tree_test.rb
|
373
359
|
- test/derailed_benchmarks/stats_from_dir_test.rb
|
374
360
|
- test/derailed_test.rb
|