better-benchmark 0.7.0

data/LICENCE

@@ -0,0 +1,21 @@
+The MIT Licence
+
+Copyright (c) 2008-2009 Pistos
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README

@@ -0,0 +1,34 @@
+## Dependencies
+
+The R Project: http://www.r-project.org/
+rsruby: http://web.kuicr.kyoto-u.ac.jp/~alexg/rsruby/
+
+## Usage
+
+result = Benchmark.compare_realtime {
+  do_something_one_way
+}.with {
+  do_it_another_way
+}
+Benchmark.report_on result
+
+See also example.rb for a more comprehensive example.
+
+## Example Output
+
+....................
+Set 1 mean: 0.484 s
+Set 1 std dev: 0.098
+Set 2 mean: 0.469 s
+Set 2 std dev: 0.088
+p.value: 0.601661885634415
+W: 220.0
+The difference (-3.2%) IS NOT statistically significant.
+
+## Help, etc.
+
+irc.freenode.net#mathetes or http://mibbit.com/?server=irc.freenode.net&channel=%23mathetes
+
+## Repository
+
+git clone git://github.com/Pistos/better-benchmark.git

data/example.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'better-benchmark'
+
+# Provide two blocks of code to compare.  For example, two blocks that
+# accomplish the same thing, but differ in implementation.  For optimal
+# results, the amount of time to execute a single iteration should be large
+# enough to adequately diminish the significance of any startup and stoppage
+# time of one iteration.  The number of benchmark iterations should not be
+# too large; better to increase the amount of work done per iteration.
+
+result = Benchmark.compare_realtime(
+  :iterations => 20,
+  :inner_iterations => 500_000,
+  :verbose => true
+) { |iteration|
+  if 1 < 2
+    x = 'foo'
+  else
+    x = 'bar'
+  end
+}.with { |iteration|
+  x = ( 1 < 2 ? 'foo' : 'bar' )
+}
+
+Benchmark.report_on result
+

data/lib/better-benchmark.rb

@@ -0,0 +1,114 @@
+require 'benchmark'
+require 'rsruby'
+
+module Benchmark
+
+  BETTER_BENCHMARK_VERSION = '0.7.0'
+
+  class ComparisonPartial
+    def initialize( block, options )
+      @block1 = block
+      @options = options
+    end
+
+    def with( &block2 )
+      times1 = []
+      times2 = []
+
+      (1..@options[ :iterations ]).each do |iteration|
+        if @options[ :verbose ]
+          $stdout.print "."; $stdout.flush
+        end
+
+        times1 << Benchmark.realtime do
+          @options[ :inner_iterations ].times do |i|
+            @block1.call( iteration )
+          end
+        end
+        times2 << Benchmark.realtime do
+          @options[ :inner_iterations ].times do |i|
+            block2.call( iteration )
+          end
+        end
+      end
+
+      r = RSRuby.instance
+      wilcox_result = r.wilcox_test( times1, times2 )
+
+      {
+        :results1 => {
+          :times => times1,
+          :mean => r.mean( times1 ),
+          :stddev => r.sd( times1 ),
+        },
+        :results2 => {
+          :times => times2,
+          :mean => r.mean( times2 ),
+          :stddev => r.sd( times2 ),
+        },
+        :p => wilcox_result[ 'p.value' ],
+        :W => wilcox_result[ 'statistic' ][ 'W' ],
+        :significant => (
+          wilcox_result[ 'p.value' ] < @options[ :required_significance ]
+        ),
+      }
+    end
+    alias to with
+  end
+
+  # Options:
+  #   :iterations
+  #     The number of times to execute the pair of blocks.
+  #   :inner_iterations
+  #     Used to increase the time taken per iteration.
+  #   :required_significance
+  #     Maximum allowed p value in order to declare the results statistically significant.
+  #   :verbose
+  #     Whether to print a dot for each iteration (as a sort of progress meter).
+  #
+  # To use better-benchmark properly, it is important to set :iterations and
+  # :inner_iterations properly.  There are a few things to bear in mind:
+  #
+  # (1) Do not set :iterations too high.  It should normally be in the range
+  # of 10-20, but can be lower.  Over 25 should be considered too high.
+  # (2) Execution time for one run of the blocks under test should not be too
+  # small (or else random variance will muddle the results).  Aim for at least
+  # 1.0 seconds per iteration.
+  # (3) Minimize the proportion of any warmup time (and cooldown time) of one
+  # block run.
+  #
+  # In order to achieve these goals, you will need to tweak :inner_iterations
+  # based on your situation.  The exact number you should use will depend on
+  # the strength of the hardware (CPU, RAM, disk), and the amount of work done
+  # by the blocks.  For code blocks that execute extremely rapidly, you may
+  # need hundreds of thousands of :inner_iterations.
+  def self.compare_realtime( options = {}, &block1 )
+    options[ :iterations ] ||= 20
+    options[ :inner_iterations ] ||= 1
+    options[ :required_significance ] ||= 0.01
+
+    if options[ :iterations ] > 30
+      warn "The number of iterations is set to #{options[ :iterations ]}.  " +
+        "Using too many iterations may make the test results less reliable.  " +
+        "It is recommended to increase the number of :inner_iterations instead."
+    end
+
+    ComparisonPartial.new( block1, options )
+  end
+
+  def self.report_on( result )
+    puts
+    puts( "Set 1 mean: %.3f s" % [ result[ :results1 ][ :mean ] ] )
+    puts( "Set 1 std dev: %.3f" % [ result[ :results1 ][ :stddev ] ] )
+    puts( "Set 2 mean: %.3f s" % [ result[ :results2 ][ :mean ] ] )
+    puts( "Set 2 std dev: %.3f" % [ result[ :results2 ][ :stddev ] ] )
+    puts "p.value: #{result[ :p ]}"
+    puts "W: #{result[ :W ]}"
+    puts(
+      "The difference (%+.1f%%) %s statistically significant." % [
+        ( ( result[ :results2 ][ :mean ] - result[ :results1 ][ :mean ] ) / result[ :results1 ][ :mean ] ) * 100,
+        result[ :significant ] ? 'IS' : 'IS NOT'
+      ]
+    )
+  end
+end

data/run-example

@@ -0,0 +1,2 @@
+export R_HOME=/usr/lib/R
+ruby -Ilib example.rb

metadata

@@ -0,0 +1,58 @@
+--- !ruby/object:Gem::Specification 
+name: better-benchmark
+version: !ruby/object:Gem::Version 
+  version: 0.7.0
+platform: ruby
+authors: 
+- Pistos
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-02-11 00:00:00 -05:00
+default_executable: 
+dependencies: []
+
+description: Statistically correct benchmarking for Ruby.
+email: pistos at purepistos dot net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+- LICENCE
+files: 
+- README
+- LICENCE
+- example.rb
+- run-example
+- lib/better-benchmark.rb
+has_rdoc: false
+homepage: http://github.com/Pistos/better-benchmark/tree
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: 
+- "The R project: http://www.r-project.org/"
+rubyforge_project: better-benchmark
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: Statistically correct benchmarking for Ruby.
+test_files: []
+