better-benchmark 0.8.3 → 0.8.6

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ ZTc2OWI4OTQ3ZDg3M2FiNmQ3YTE3NmFmZmRhMmVlMWYyZGYzNWY5Yg==
5
+ data.tar.gz: !binary |-
6
+ Y2Y3NzViNzA4MGE2OWVmYzkyMmU0MDdkZGQxOGQzYTBjM2YzZWE0Zg==
7
+ !binary "U0hBNTEy":
8
+ metadata.gz: !binary |-
9
+ N2RjNzBlNDAyMWRkYjY0NzdhYmJhYTJjMmZiNzE2ZTJhZmE1NTBjNmM2YTRm
10
+ MDEzZDY1NDYyMTU2ZDQ0YWIyZDBjNDQ0M2I2NGQ1MDdhNWY3YWU2NjNmNzBm
11
+ M2ZhMDJmMmQ5ODE1NzE1ODhjZjg3Y2ZkNDVjNjcwN2Q0MDMyZDc=
12
+ data.tar.gz: !binary |-
13
+ NGRkMzIxMDc4ZWJmOWFmNzY4OWJmYjVhNGNkZjU3NjFhZTM4NDJjYjFhOWVh
14
+ M2Y5MjEwNWU5OWI0ZWYwYmMxYTMzNTAwNWVhM2IzM2JkNTNmN2Q0M2M1Mjdk
15
+ YzI2ZGE4ZjNkMzk1MDFlMzQ0Y2RiZGI1ZTBhNjUyN2VhYzRlODI=
@@ -0,0 +1,18 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'better-benchmark'
4
+
5
+ c = Benchmark::Comparer.new( ARGV.dup )
6
+ results = c.run
7
+
8
+ num_insignificant = results.reduce(0) { |n,key_and_result|
9
+ key, result = key_and_result
10
+ if ! result[:significant]
11
+ n += 1
12
+ end
13
+ n
14
+ }
15
+
16
+ if num_insignificant > 0
17
+ $stderr.puts "* #{num_insignificant} out of #{results.keys.count} results are statistically insignificant"
18
+ end
@@ -3,10 +3,11 @@ require 'rsruby'
3
3
 
4
4
  require 'better-benchmark/comparison-partial'
5
5
  require 'better-benchmark/bencher'
6
+ require 'better-benchmark/comparer'
6
7
 
7
8
  module Benchmark
8
9
 
9
- BETTER_BENCHMARK_VERSION = '0.8.1'
10
+ BETTER_BENCHMARK_VERSION = '0.8.6'
10
11
  DEFAULT_REQUIRED_SIGNIFICANCE = 0.01
11
12
 
12
13
  def self.write_realtime( data_dir, &block )
@@ -100,4 +101,4 @@ module Benchmark
100
101
  ]
101
102
  )
102
103
  end
103
- end
104
+ end
@@ -0,0 +1,60 @@
1
+ require 'csv'
2
+
3
+ module Benchmark
4
+ class Comparer
5
+ def print_usage
6
+ puts "bbench-compare [-p <max p-value>] <timings1.csv> <timings2.csv>"
7
+ end
8
+
9
+ # @param [Array] argv
10
+ # The command line arguments passed to the bencher script
11
+ # Expected column layout of the CSV files is:
12
+ # <identifier of thing tested>,<time in fractional seconds>
13
+ def initialize( argv )
14
+ @iterations = 10
15
+ @executable = 'ruby'
16
+
17
+ while argv.any?
18
+ arg = argv.shift
19
+ case arg
20
+ when '-p'
21
+ @max_p = argv.shift
22
+ else
23
+ if @file1.nil?
24
+ @file1 = arg
25
+ elsif @file2.nil?
26
+ @file2 = arg
27
+ end
28
+ end
29
+ end
30
+
31
+ if @file1.nil? || @file2.nil?
32
+ print_usage
33
+ exit 2
34
+ end
35
+
36
+ @timings_before = Hash.new { |h,k| h[k] = Array.new }
37
+ @timings_after = Hash.new { |h,k| h[k] = Array.new }
38
+ end
39
+
40
+ def run
41
+ CSV.foreach(@file1) do |row|
42
+ @timings_before[ row[0] ] << row[1].to_f
43
+ end
44
+ CSV.foreach(@file2) do |row|
45
+ @timings_after[ row[0] ] << row[1].to_f
46
+ end
47
+
48
+ run_results = Hash.new
49
+ @timings_before.each_key do |thing_tested|
50
+ results = Benchmark.compare_times( @timings_before[thing_tested], @timings_after[thing_tested], @max_p )
51
+ improvement = ( results[:results2][:mean] - results[:results1][:mean] ) / results[:results1][:mean]
52
+
53
+ run_results[thing_tested] = { improvement: improvement, significant: results[:significant] }
54
+ puts( "%s\t%+.1f%%\t%s" % [thing_tested, improvement * 100.0, results[:significant] ? '' : '*' ] )
55
+ end
56
+
57
+ run_results
58
+ end
59
+ end
60
+ end
metadata CHANGED
@@ -1,20 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: better-benchmark
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.3
5
- prerelease:
4
+ version: 0.8.6
6
5
  platform: ruby
7
6
  authors:
8
7
  - Pistos
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2012-08-29 00:00:00.000000000 Z
11
+ date: 2013-05-13 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: rsruby
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
17
  - - ! '>='
20
18
  - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
24
  - - ! '>='
28
25
  - !ruby/object:Gem::Version
@@ -31,6 +28,7 @@ description: Statistically correct benchmarking for Ruby.
31
28
  email: betterbenchmark dot pistos at purepistos dot net
32
29
  executables:
33
30
  - bbench
31
+ - bbench-compare
34
32
  extensions: []
35
33
  extra_rdoc_files:
36
34
  - README.md
@@ -42,22 +40,23 @@ files:
42
40
  - run-example
43
41
  - lib/better-benchmark.rb
44
42
  - lib/better-benchmark/bencher.rb
43
+ - lib/better-benchmark/comparer.rb
45
44
  - lib/better-benchmark/comparison-partial.rb
46
45
  - bin/bbench
46
+ - bin/bbench-compare
47
47
  homepage: http://github.com/Pistos/better-benchmark
48
48
  licenses: []
49
+ metadata: {}
49
50
  post_install_message:
50
51
  rdoc_options: []
51
52
  require_paths:
52
53
  - lib
53
54
  required_ruby_version: !ruby/object:Gem::Requirement
54
- none: false
55
55
  requirements:
56
56
  - - ! '>='
57
57
  - !ruby/object:Gem::Version
58
58
  version: '0'
59
59
  required_rubygems_version: !ruby/object:Gem::Requirement
60
- none: false
61
60
  requirements:
62
61
  - - ! '>='
63
62
  - !ruby/object:Gem::Version
@@ -65,8 +64,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
65
64
  requirements:
66
65
  - ! 'The R project: http://www.r-project.org/'
67
66
  rubyforge_project: better-benchmark
68
- rubygems_version: 1.8.18
67
+ rubygems_version: 2.0.3
69
68
  signing_key:
70
- specification_version: 3
69
+ specification_version: 4
71
70
  summary: Statistically correct benchmarking for Ruby.
72
71
  test_files: []