better-benchmark 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. data/LICENCE +21 -0
  2. data/README +34 -0
  3. data/example.rb +27 -0
  4. data/lib/better-benchmark.rb +114 -0
  5. data/run-example +2 -0
  6. metadata +58 -0
data/LICENCE ADDED
@@ -0,0 +1,21 @@
1
+ The MIT Licence
2
+
3
+ Copyright (c) 2008-2009 Pistos
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README ADDED
@@ -0,0 +1,34 @@
1
+ ## Dependencies
2
+
3
+ The R Project: http://www.r-project.org/
4
+ rsruby: http://web.kuicr.kyoto-u.ac.jp/~alexg/rsruby/
5
+
6
+ ## Usage
7
+
8
+ result = Benchmark.compare_realtime {
9
+ do_something_one_way
10
+ }.with {
11
+ do_it_another_way
12
+ }
13
+ Benchmark.report_on result
14
+
15
+ See also example.rb for a more comprehensive example.
16
+
17
+ ## Example Output
18
+
19
+ ....................
20
+ Set 1 mean: 0.484 s
21
+ Set 1 std dev: 0.098
22
+ Set 2 mean: 0.469 s
23
+ Set 2 std dev: 0.088
24
+ p.value: 0.601661885634415
25
+ W: 220.0
26
+ The difference (-3.2%) IS NOT statistically significant.
27
+
28
+ ## Help, etc.
29
+
30
+ irc.freenode.net#mathetes or http://mibbit.com/?server=irc.freenode.net&channel=%23mathetes
31
+
32
+ ## Repository
33
+
34
+ git clone git://github.com/Pistos/better-benchmark.git
data/example.rb ADDED
@@ -0,0 +1,27 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'better-benchmark'
4
+
5
+ # Provide two blocks of code to compare. For example, two blocks that
6
+ # accomplish the same thing, but differ in implementation. For optimal
7
+ # results, the amount of time to execute a single iteration should be large
8
+ # enough to adequately diminish the significance of any startup and stoppage
9
+ # time of one iteration. The number of benchmark iterations should not be
10
+ # too large; better to increase the amount of work done per iteration.
11
+
12
+ result = Benchmark.compare_realtime(
13
+ :iterations => 20,
14
+ :inner_iterations => 500_000,
15
+ :verbose => true
16
+ ) { |iteration|
17
+ if 1 < 2
18
+ x = 'foo'
19
+ else
20
+ x = 'bar'
21
+ end
22
+ }.with { |iteration|
23
+ x = ( 1 < 2 ? 'foo' : 'bar' )
24
+ }
25
+
26
+ Benchmark.report_on result
27
+
@@ -0,0 +1,114 @@
1
+ require 'benchmark'
2
+ require 'rsruby'
3
+
4
+ module Benchmark
5
+
6
+ BETTER_BENCHMARK_VERSION = '0.7.0'
7
+
8
+ class ComparisonPartial
9
+ def initialize( block, options )
10
+ @block1 = block
11
+ @options = options
12
+ end
13
+
14
+ def with( &block2 )
15
+ times1 = []
16
+ times2 = []
17
+
18
+ (1..@options[ :iterations ]).each do |iteration|
19
+ if @options[ :verbose ]
20
+ $stdout.print "."; $stdout.flush
21
+ end
22
+
23
+ times1 << Benchmark.realtime do
24
+ @options[ :inner_iterations ].times do |i|
25
+ @block1.call( iteration )
26
+ end
27
+ end
28
+ times2 << Benchmark.realtime do
29
+ @options[ :inner_iterations ].times do |i|
30
+ block2.call( iteration )
31
+ end
32
+ end
33
+ end
34
+
35
+ r = RSRuby.instance
36
+ wilcox_result = r.wilcox_test( times1, times2 )
37
+
38
+ {
39
+ :results1 => {
40
+ :times => times1,
41
+ :mean => r.mean( times1 ),
42
+ :stddev => r.sd( times1 ),
43
+ },
44
+ :results2 => {
45
+ :times => times2,
46
+ :mean => r.mean( times2 ),
47
+ :stddev => r.sd( times2 ),
48
+ },
49
+ :p => wilcox_result[ 'p.value' ],
50
+ :W => wilcox_result[ 'statistic' ][ 'W' ],
51
+ :significant => (
52
+ wilcox_result[ 'p.value' ] < @options[ :required_significance ]
53
+ ),
54
+ }
55
+ end
56
+ alias to with
57
+ end
58
+
59
+ # Options:
60
+ # :iterations
61
+ # The number of times to execute the pair of blocks.
62
+ # :inner_iterations
63
+ # Used to increase the time taken per iteration.
64
+ # :required_significance
65
+ # Maximum allowed p value in order to declare the results statistically significant.
66
+ # :verbose
67
+ # Whether to print a dot for each iteration (as a sort of progress meter).
68
+ #
69
+ # To use better-benchmark properly, it is important to set :iterations and
70
+ # :inner_iterations properly. There are a few things to bear in mind:
71
+ #
72
+ # (1) Do not set :iterations too high. It should normally be in the range
73
+ # of 10-20, but can be lower. Over 25 should be considered too high.
74
+ # (2) Execution time for one run of the blocks under test should not be too
75
+ # small (or else random variance will muddle the results). Aim for at least
76
+ # 1.0 seconds per iteration.
77
+ # (3) Minimize the proportion of any warmup time (and cooldown time) of one
78
+ # block run.
79
+ #
80
+ # In order to achieve these goals, you will need to tweak :inner_iterations
81
+ # based on your situation. The exact number you should use will depend on
82
+ # the strength of the hardware (CPU, RAM, disk), and the amount of work done
83
+ # by the blocks. For code blocks that execute extremely rapidly, you may
84
+ # need hundreds of thousands of :inner_iterations.
85
+ def self.compare_realtime( options = {}, &block1 )
86
+ options[ :iterations ] ||= 20
87
+ options[ :inner_iterations ] ||= 1
88
+ options[ :required_significance ] ||= 0.01
89
+
90
+ if options[ :iterations ] > 30
91
+ warn "The number of iterations is set to #{options[ :iterations ]}. " +
92
+ "Using too many iterations may make the test results less reliable. " +
93
+ "It is recommended to increase the number of :inner_iterations instead."
94
+ end
95
+
96
+ ComparisonPartial.new( block1, options )
97
+ end
98
+
99
+ def self.report_on( result )
100
+ puts
101
+ puts( "Set 1 mean: %.3f s" % [ result[ :results1 ][ :mean ] ] )
102
+ puts( "Set 1 std dev: %.3f" % [ result[ :results1 ][ :stddev ] ] )
103
+ puts( "Set 2 mean: %.3f s" % [ result[ :results2 ][ :mean ] ] )
104
+ puts( "Set 2 std dev: %.3f" % [ result[ :results2 ][ :stddev ] ] )
105
+ puts "p.value: #{result[ :p ]}"
106
+ puts "W: #{result[ :W ]}"
107
+ puts(
108
+ "The difference (%+.1f%%) %s statistically significant." % [
109
+ ( ( result[ :results2 ][ :mean ] - result[ :results1 ][ :mean ] ) / result[ :results1 ][ :mean ] ) * 100,
110
+ result[ :significant ] ? 'IS' : 'IS NOT'
111
+ ]
112
+ )
113
+ end
114
+ end
data/run-example ADDED
@@ -0,0 +1,2 @@
1
+ export R_HOME=/usr/lib/R
2
+ ruby -Ilib example.rb
metadata ADDED
@@ -0,0 +1,58 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: better-benchmark
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.7.0
5
+ platform: ruby
6
+ authors:
7
+ - Pistos
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2009-02-11 00:00:00 -05:00
13
+ default_executable:
14
+ dependencies: []
15
+
16
+ description: Statistically correct benchmarking for Ruby.
17
+ email: pistos at purepistos dot net
18
+ executables: []
19
+
20
+ extensions: []
21
+
22
+ extra_rdoc_files:
23
+ - README
24
+ - LICENCE
25
+ files:
26
+ - README
27
+ - LICENCE
28
+ - example.rb
29
+ - run-example
30
+ - lib/better-benchmark.rb
31
+ has_rdoc: false
32
+ homepage: http://github.com/Pistos/better-benchmark/tree
33
+ post_install_message:
34
+ rdoc_options: []
35
+
36
+ require_paths:
37
+ - lib
38
+ required_ruby_version: !ruby/object:Gem::Requirement
39
+ requirements:
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: "0"
43
+ version:
44
+ required_rubygems_version: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ version: "0"
49
+ version:
50
+ requirements:
51
+ - "The R project: http://www.r-project.org/"
52
+ rubyforge_project: better-benchmark
53
+ rubygems_version: 1.3.1
54
+ signing_key:
55
+ specification_version: 2
56
+ summary: Statistically correct benchmarking for Ruby.
57
+ test_files: []
58
+