better-benchmark 0.8.3 → 0.8.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/bin/bbench-compare +18 -0
- data/lib/better-benchmark.rb +3 -2
- data/lib/better-benchmark/comparer.rb +60 -0
- metadata +8 -9
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ZTc2OWI4OTQ3ZDg3M2FiNmQ3YTE3NmFmZmRhMmVlMWYyZGYzNWY5Yg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
Y2Y3NzViNzA4MGE2OWVmYzkyMmU0MDdkZGQxOGQzYTBjM2YzZWE0Zg==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
N2RjNzBlNDAyMWRkYjY0NzdhYmJhYTJjMmZiNzE2ZTJhZmE1NTBjNmM2YTRm
|
10
|
+
MDEzZDY1NDYyMTU2ZDQ0YWIyZDBjNDQ0M2I2NGQ1MDdhNWY3YWU2NjNmNzBm
|
11
|
+
M2ZhMDJmMmQ5ODE1NzE1ODhjZjg3Y2ZkNDVjNjcwN2Q0MDMyZDc=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
NGRkMzIxMDc4ZWJmOWFmNzY4OWJmYjVhNGNkZjU3NjFhZTM4NDJjYjFhOWVh
|
14
|
+
M2Y5MjEwNWU5OWI0ZWYwYmMxYTMzNTAwNWVhM2IzM2JkNTNmN2Q0M2M1Mjdk
|
15
|
+
YzI2ZGE4ZjNkMzk1MDFlMzQ0Y2RiZGI1ZTBhNjUyN2VhYzRlODI=
|
data/bin/bbench-compare
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'better-benchmark'
|
4
|
+
|
5
|
+
c = Benchmark::Comparer.new( ARGV.dup )
|
6
|
+
results = c.run
|
7
|
+
|
8
|
+
num_insignificant = results.reduce(0) { |n,key_and_result|
|
9
|
+
key, result = key_and_result
|
10
|
+
if ! result[:significant]
|
11
|
+
n += 1
|
12
|
+
end
|
13
|
+
n
|
14
|
+
}
|
15
|
+
|
16
|
+
if num_insignificant > 0
|
17
|
+
$stderr.puts "* #{num_insignificant} out of #{results.keys.count} results are statistically insignificant"
|
18
|
+
end
|
data/lib/better-benchmark.rb
CHANGED
@@ -3,10 +3,11 @@ require 'rsruby'
|
|
3
3
|
|
4
4
|
require 'better-benchmark/comparison-partial'
|
5
5
|
require 'better-benchmark/bencher'
|
6
|
+
require 'better-benchmark/comparer'
|
6
7
|
|
7
8
|
module Benchmark
|
8
9
|
|
9
|
-
BETTER_BENCHMARK_VERSION = '0.8.
|
10
|
+
BETTER_BENCHMARK_VERSION = '0.8.6'
|
10
11
|
DEFAULT_REQUIRED_SIGNIFICANCE = 0.01
|
11
12
|
|
12
13
|
def self.write_realtime( data_dir, &block )
|
@@ -100,4 +101,4 @@ module Benchmark
|
|
100
101
|
]
|
101
102
|
)
|
102
103
|
end
|
103
|
-
end
|
104
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
module Benchmark
|
4
|
+
class Comparer
|
5
|
+
def print_usage
|
6
|
+
puts "bbench-compare [-p <max p-value>] <timings1.csv> <timings2.csv>"
|
7
|
+
end
|
8
|
+
|
9
|
+
# @param [Array] argv
|
10
|
+
# The command line arguments passed to the bencher script
|
11
|
+
# Expected column layout of the CSV files is:
|
12
|
+
# <identifier of thing tested>,<time in fractional seconds>
|
13
|
+
def initialize( argv )
|
14
|
+
@iterations = 10
|
15
|
+
@executable = 'ruby'
|
16
|
+
|
17
|
+
while argv.any?
|
18
|
+
arg = argv.shift
|
19
|
+
case arg
|
20
|
+
when '-p'
|
21
|
+
@max_p = argv.shift
|
22
|
+
else
|
23
|
+
if @file1.nil?
|
24
|
+
@file1 = arg
|
25
|
+
elsif @file2.nil?
|
26
|
+
@file2 = arg
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
if @file1.nil? || @file2.nil?
|
32
|
+
print_usage
|
33
|
+
exit 2
|
34
|
+
end
|
35
|
+
|
36
|
+
@timings_before = Hash.new { |h,k| h[k] = Array.new }
|
37
|
+
@timings_after = Hash.new { |h,k| h[k] = Array.new }
|
38
|
+
end
|
39
|
+
|
40
|
+
def run
|
41
|
+
CSV.foreach(@file1) do |row|
|
42
|
+
@timings_before[ row[0] ] << row[1].to_f
|
43
|
+
end
|
44
|
+
CSV.foreach(@file2) do |row|
|
45
|
+
@timings_after[ row[0] ] << row[1].to_f
|
46
|
+
end
|
47
|
+
|
48
|
+
run_results = Hash.new
|
49
|
+
@timings_before.each_key do |thing_tested|
|
50
|
+
results = Benchmark.compare_times( @timings_before[thing_tested], @timings_after[thing_tested], @max_p )
|
51
|
+
improvement = ( results[:results2][:mean] - results[:results1][:mean] ) / results[:results1][:mean]
|
52
|
+
|
53
|
+
run_results[thing_tested] = { improvement: improvement, significant: results[:significant] }
|
54
|
+
puts( "%s\t%+.1f%%\t%s" % [thing_tested, improvement * 100.0, results[:significant] ? '' : '*' ] )
|
55
|
+
end
|
56
|
+
|
57
|
+
run_results
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
metadata
CHANGED
@@ -1,20 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: better-benchmark
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
5
|
-
prerelease:
|
4
|
+
version: 0.8.6
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Pistos
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2013-05-13 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: rsruby
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
17
|
- - ! '>='
|
20
18
|
- !ruby/object:Gem::Version
|
@@ -22,7 +20,6 @@ dependencies:
|
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
24
|
- - ! '>='
|
28
25
|
- !ruby/object:Gem::Version
|
@@ -31,6 +28,7 @@ description: Statistically correct benchmarking for Ruby.
|
|
31
28
|
email: betterbenchmark dot pistos at purepistos dot net
|
32
29
|
executables:
|
33
30
|
- bbench
|
31
|
+
- bbench-compare
|
34
32
|
extensions: []
|
35
33
|
extra_rdoc_files:
|
36
34
|
- README.md
|
@@ -42,22 +40,23 @@ files:
|
|
42
40
|
- run-example
|
43
41
|
- lib/better-benchmark.rb
|
44
42
|
- lib/better-benchmark/bencher.rb
|
43
|
+
- lib/better-benchmark/comparer.rb
|
45
44
|
- lib/better-benchmark/comparison-partial.rb
|
46
45
|
- bin/bbench
|
46
|
+
- bin/bbench-compare
|
47
47
|
homepage: http://github.com/Pistos/better-benchmark
|
48
48
|
licenses: []
|
49
|
+
metadata: {}
|
49
50
|
post_install_message:
|
50
51
|
rdoc_options: []
|
51
52
|
require_paths:
|
52
53
|
- lib
|
53
54
|
required_ruby_version: !ruby/object:Gem::Requirement
|
54
|
-
none: false
|
55
55
|
requirements:
|
56
56
|
- - ! '>='
|
57
57
|
- !ruby/object:Gem::Version
|
58
58
|
version: '0'
|
59
59
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
60
|
-
none: false
|
61
60
|
requirements:
|
62
61
|
- - ! '>='
|
63
62
|
- !ruby/object:Gem::Version
|
@@ -65,8 +64,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
65
64
|
requirements:
|
66
65
|
- ! 'The R project: http://www.r-project.org/'
|
67
66
|
rubyforge_project: better-benchmark
|
68
|
-
rubygems_version:
|
67
|
+
rubygems_version: 2.0.3
|
69
68
|
signing_key:
|
70
|
-
specification_version:
|
69
|
+
specification_version: 4
|
71
70
|
summary: Statistically correct benchmarking for Ruby.
|
72
71
|
test_files: []
|