better-benchmark 0.8.3 → 0.8.6

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZTc2OWI4OTQ3ZDg3M2FiNmQ3YTE3NmFmZmRhMmVlMWYyZGYzNWY5Yg==
+  data.tar.gz: !binary |-
+    Y2Y3NzViNzA4MGE2OWVmYzkyMmU0MDdkZGQxOGQzYTBjM2YzZWE0Zg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    N2RjNzBlNDAyMWRkYjY0NzdhYmJhYTJjMmZiNzE2ZTJhZmE1NTBjNmM2YTRm
+    MDEzZDY1NDYyMTU2ZDQ0YWIyZDBjNDQ0M2I2NGQ1MDdhNWY3YWU2NjNmNzBm
+    M2ZhMDJmMmQ5ODE1NzE1ODhjZjg3Y2ZkNDVjNjcwN2Q0MDMyZDc=
+  data.tar.gz: !binary |-
+    NGRkMzIxMDc4ZWJmOWFmNzY4OWJmYjVhNGNkZjU3NjFhZTM4NDJjYjFhOWVh
+    M2Y5MjEwNWU5OWI0ZWYwYmMxYTMzNTAwNWVhM2IzM2JkNTNmN2Q0M2M1Mjdk
+    YzI2ZGE4ZjNkMzk1MDFlMzQ0Y2RiZGI1ZTBhNjUyN2VhYzRlODI=

data/bin/bbench-compare

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'better-benchmark'
+
+c = Benchmark::Comparer.new( ARGV.dup )
+results = c.run
+
+num_insignificant = results.reduce(0) { |n,key_and_result|
+  key, result = key_and_result
+  if ! result[:significant]
+    n += 1
+  end
+  n
+}
+
+if num_insignificant > 0
+  $stderr.puts "* #{num_insignificant} out of #{results.keys.count} results are statistically insignificant"
+end

data/lib/better-benchmark.rb

@@ -3,10 +3,11 @@ require 'rsruby'
 
 require 'better-benchmark/comparison-partial'
 require 'better-benchmark/bencher'
+require 'better-benchmark/comparer'
 
 module Benchmark
 
-  BETTER_BENCHMARK_VERSION = '0.8.1'
+  BETTER_BENCHMARK_VERSION = '0.8.6'
   DEFAULT_REQUIRED_SIGNIFICANCE = 0.01
 
   def self.write_realtime( data_dir, &block )
@@ -100,4 +101,4 @@ module Benchmark
       ]
     )
   end
-end
+end

data/lib/better-benchmark/comparer.rb

@@ -0,0 +1,60 @@
+require 'csv'
+
+module Benchmark
+  class Comparer
+    def print_usage
+      puts "bbench-compare [-p <max p-value>] <timings1.csv> <timings2.csv>"
+    end
+
+    # @param [Array] argv
+    #   The command line arguments passed to the bencher script
+    # Expected column layout of the CSV files is:
+    # <identifier of thing tested>,<time in fractional seconds>
+    def initialize( argv )
+      @iterations = 10
+      @executable = 'ruby'
+
+      while argv.any?
+        arg = argv.shift
+        case arg
+        when '-p'
+          @max_p = argv.shift
+        else
+          if @file1.nil?
+            @file1 = arg
+          elsif @file2.nil?
+            @file2 = arg
+          end
+        end
+      end
+
+      if @file1.nil? || @file2.nil?
+        print_usage
+        exit 2
+      end
+
+      @timings_before = Hash.new { |h,k| h[k] = Array.new }
+      @timings_after = Hash.new { |h,k| h[k] = Array.new }
+    end
+
+    def run
+      CSV.foreach(@file1) do |row|
+        @timings_before[ row[0] ] << row[1].to_f
+      end
+      CSV.foreach(@file2) do |row|
+        @timings_after[ row[0] ] << row[1].to_f
+      end
+
+      run_results = Hash.new
+      @timings_before.each_key do |thing_tested|
+        results = Benchmark.compare_times( @timings_before[thing_tested], @timings_after[thing_tested], @max_p )
+        improvement = ( results[:results2][:mean] - results[:results1][:mean] ) / results[:results1][:mean]
+
+        run_results[thing_tested] = { improvement: improvement, significant: results[:significant] }
+        puts( "%s\t%+.1f%%\t%s" % [thing_tested, improvement * 100.0, results[:significant] ? '' : '*' ] )
+      end
+
+      run_results
+    end
+  end
+end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: better-benchmark
 version: !ruby/object:Gem::Version
-  version: 0.8.3
-  prerelease: 
+  version: 0.8.6
 platform: ruby
 authors:
 - Pistos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-29 00:00:00.000000000 Z
+date: 2013-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rsruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -31,6 +28,7 @@ description: Statistically correct benchmarking for Ruby.
 email: betterbenchmark dot pistos at purepistos dot net
 executables:
 - bbench
+- bbench-compare
 extensions: []
 extra_rdoc_files:
 - README.md
@@ -42,22 +40,23 @@ files:
 - run-example
 - lib/better-benchmark.rb
 - lib/better-benchmark/bencher.rb
+- lib/better-benchmark/comparer.rb
 - lib/better-benchmark/comparison-partial.rb
 - bin/bbench
+- bin/bbench-compare
 homepage: http://github.com/Pistos/better-benchmark
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
@@ -65,8 +64,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements:
 - ! 'The R project: http://www.r-project.org/'
 rubyforge_project: better-benchmark
-rubygems_version: 1.8.18
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Statistically correct benchmarking for Ruby.
 test_files: []