bullshit 0.1.0 → 0.1.1

data/CHANGES

@@ -0,0 +1,8 @@
+2009-10-17 (0.1.1)
+  * Added the display of a speedup matrix in the comparison.
+  * Added a bs_compare executble in order to compare data files.
+  * A lot of smaller changes, esp. make comparing more robust by skipping
+    unavailable data files.
+  * throw_raise example added.
+2009-08-31 (0.1.0)
+  * Initial Release Version

data/README

@@ -0,0 +1,25 @@
+= Bullshit - Benchmarking in Ruby
+
+== Description
+
+== Usage
+
+== Example
+
+TODO
+
+== Download
+
+The homepage of this library is located at
+
+* http://flori.github.com/bullshit
+
+== Author
+
+Florian Frank mailto:flori@ping.de
+
+== License
+
+This is free software; you can redistribute it and/or modify it under the
+terms of the GNU General Public License Version 2 as published by the Free
+Software Foundation: www.gnu.org/copyleft/gpl.html

data/Rakefile

@@ -33,41 +33,30 @@ task :doc do
 end
 
 if defined? Gem
-  spec_src = <<GEM
-# -*- encoding: utf-8 -*-
-Gem::Specification.new do |s|
-  s.name = '#{PKG_NAME}'
-  s.version = '#{PKG_VERSION}'
-  s.summary = "Benchmarking is Bullshit"
-  s.description = ""
+  spec = Gem::Specification.new do |s|
+    s.name = PKG_NAME
+    s.version = PKG_VERSION
+    s.summary = "Benchmarking is Bullshit"
+    s.description = ""
 
-  s.add_dependency('dslkit', '>= 0.2.5')
+    s.add_dependency('dslkit', '>= 0.2.5')
 
-  s.files = #{PKG_FILES.to_a.sort.inspect}
+    s.files = PKG_FILES
 
-  s.require_path = 'lib'
+    s.require_path = 'lib'
+    s.executables = 'bs_compare'
 
-  s.has_rdoc = true
-  s.rdoc_options <<
-    '--title' <<  'Bullshit -- Benchmarking in Ruby' <<
-    '--line-numbers'
-  s.test_files = #{Dir['tests/*.rb'].to_a.sort.inspect}
+    s.has_rdoc = true
+    s.rdoc_options <<
+      '--title' <<  'Bullshit -- Benchmarking in Ruby' << '--main' << 'README'
+    s.test_files = Dir['tests/*.rb']
 
-  s.author = "Florian Frank"
-  s.email = "flori@ping.de"
-  s.homepage = "http://#{PKG_NAME}.rubyforge.org"
-  s.rubyforge_project = "#{PKG_NAME}"
-end
-GEM
-
-  desc 'Create a gemspec file'
-  task :gemspec do
-    File.open("#{PKG_NAME}.gemspec", 'w') do |f|
-      f.puts spec_src
-    end
+    s.author = "Florian Frank"
+    s.email = "flori@ping.de"
+    s.homepage = "http://flori.github.com/#{PKG_NAME}"
+    s.rubyforge_project = PKG_NAME
   end
 
-  spec = eval(spec_src)
   Rake::GemPackageTask.new(spec) do |pkg|
     pkg.need_tar = true
     pkg.package_files += PKG_FILES
@@ -91,6 +80,6 @@ EOT
   end
 end
 
-task :default => [ :version, :gemspec, :test ]
+task :default => [ :version, :test ]
 
-task :release => [ :clobber, :version, :gemspec, :package ]
+task :release => [ :clobber, :version, :package ]

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/bin/bs_compare

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+require 'bullshit'
+Bullshit::Case.autorun false
+
+bs_cases_methods = []
+for filename in ARGV
+  if filename =~ /\.rb\Z/
+    warn "Loading #{filename}."
+    load filename
+    next
+  end
+  basename = File.basename(filename)
+  basename =~ /\A([^-]+)#([^-]+)\.dat\Z/ or next
+  bs_cases_methods << [ $1, $2, filename ]
+end
+Bullshit.compare do
+  bs_cases_methods.each do |c, m, f|
+    benchmark Object.const_get(c), m.to_sym, :load => f
+  end
+end

data/examples/compare.rb

@@ -1,9 +1,9 @@
 #!/usr/bin/env ruby
 
 require 'bullshit'
-require File.join(File.dirname(__FILE__), 'fibonacci')
+load File.join(File.dirname(__FILE__), 'fibonacci.rb')
 
-FibonacciBenchmark.run # to demonstrate loading
+FibonacciBenchmark.run false # to demonstrate loading, do not compare yet
 
 Bullshit.compare do
   benchmark FibonacciBenchmark, :fib_iter, :load => yes

data/examples/throw_raise.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'bullshit'
+
+class ThrowRaise < Bullshit::RepeatCase
+  compare_time            real
+  warmup                  yes
+  iterations              100
+
+  N = 2_000
+
+  autocorrelation         yes
+
+  output_dir              'data'
+  data_file               yes
+
+  #output_filename         "#{benchmark_name}.log"
+
+  def benchmark_throw
+    N.times do
+      catch(:foo) do
+        throw :foo
+      end
+    end
+  end
+
+  def benchmark_raise
+    N.times do
+      begin
+        raise 'foo'
+      rescue
+      end
+    end
+  end
+end

data/lib/bullshit.rb

@@ -1,32 +1,3 @@
-# = Bullshit - Benchmarking in Ruby
-#
-# == Description
-#
-# == Usage
-#
-# == Author
-#
-# Florian Frank mailto:flori@ping.de
-#
-# == License
-#
-# This is free software; you can redistribute it and/or modify it under the
-# terms of the GNU General Public License Version 2 as published by the Free
-# Software Foundation: www.gnu.org/copyleft/gpl.html
-#
-# == Download
-#
-# The latest version of this library can be downloaded at
-#
-# * http://rubyforge.org/frs/?group_id=8323
-#
-# The homepage of this library is located at
-#
-# * http://bullshit.rubyforge.org
-# 
-# == Example
-#
-
 require 'dslkit'
 require 'enumerator'
 
@@ -463,6 +434,11 @@ module Bullshit
       define_method(time) { analysis[time].sum }
     end
 
+    # Returns the sum of measurement times for +time+.
+    def sum(time)
+      __send__ time
+    end
+
     # Seconds per call (mean)
     def call_time_mean
       mean(self.case.compare_time)
@@ -1301,11 +1277,11 @@ module Bullshit
     # This method tries to detect autocorrelation with the Ljung-Box
     # statistic. If enough lags can be considered it returns a hash with
     # results, otherwise nil is returned. The keys are
-    #   :lags: the number of lags,
-    #   :alpha_level: the alpha level for the test,
-    #   :q: the value of the ljung_box_statistic,
-    #   :p: the p-value computed, if p is higher than alpha no correlation was detected,
-    #   :detected: true if a correlation was found.
+    # :lags:: the number of lags,
+    # :alpha_level:: the alpha level for the test,
+    # :q:: the value of the ljung_box_statistic,
+    # :p:: the p-value computed, if p is higher than alpha no correlation was detected,
+    # :detected:: true if a correlation was found.
     def detect_autocorrelation(lags = 20, alpha_level = 0.05)
       if q = ljung_box_statistic(lags)
         p = ChiSquareDistribution.new(lags).probability(q)
@@ -1446,7 +1422,10 @@ module Bullshit
         end
       end
       self
+    rescue Errno::ENOENT
     end
+
+    alias to_s long_name
   end
 
   module CommonConstants
@@ -1489,6 +1468,8 @@ module Bullshit
 
       dsl_accessor :batch_size, 1
 
+      dsl_accessor :comparison, true
+
       class TruncateData
         extend DSLKit::DSLAccessor
         extend DSLKit::Constant
@@ -1507,8 +1488,8 @@ module Bullshit
         def initialize(enable, &block)
           if block
             enable.nil? or raise ArgumentError, "block form doesn't take an argument"
-            instance_eval(&block)
             enabled true
+            instance_eval(&block)
           else
             enabled enable.nil? ? false : enable
           end
@@ -1556,8 +1537,8 @@ module Bullshit
         def initialize(enable = nil, &block)
           if block
             enable.nil? or raise ArgumentError, "block form doesn't take an argument"
-            instance_eval(&block)
             enabled true
+            instance_eval(&block)
           else
             enabled enable.nil? ? true : enable
           end
@@ -1658,8 +1639,10 @@ module Bullshit
     # measure their running time.
     def initialize
       @clocks = []
-      @comparison = Comparison.new
-      @comparison.output self.class.output
+      if self.class.comparison
+        @comparison = Comparison.new
+        @comparison.output self.class.output
+      end
     end
 
     # Return the name of the benchmark case as a string.
@@ -1769,7 +1752,9 @@ module Bullshit
           self.class.output.puts evaluation(clock)
         end
         @clocks << clock
-        @comparison.benchmark(self, bc_method.short_name, :run => false)
+        if @comparison
+          @comparison.benchmark(self, bc_method.short_name, :run => false)
+        end
       end
       @clocks
     end
@@ -1779,12 +1764,12 @@ module Bullshit
 
     # Setup, run all benchmark cases (warmup and the real run) and output
     # results, run method speed comparisons, and teardown.
-    def run(comparison = true)
+    def run(do_compare = true)
       old_sync, self.class.output.sync = self.class.output.sync, true
       $DEBUG and warn "Calling setup."
       setup
       run_once
-      comparison and @comparison.display
+      do_compare and @comparison and @comparison.display
       self
     rescue => e
       warn "Caught #{e.class}: #{e}\n\n#{e.backtrace.map { |x| "\t#{x}\n" }}"
@@ -1796,8 +1781,8 @@ module Bullshit
     end
 
     # Creates an instance of this class and run it.
-    def self.run
-      new.run
+    def self.run(do_compare = true)
+      new.run do_compare
     end
 
     # Write all output files after run.
@@ -1880,7 +1865,7 @@ module Bullshit
   module EvaluationModule
     def statistics_table(clock)
       result = ' ' * NAME_COLUMN_SIZE << ('%17s ' * 4) % times << "\n"
-      result << evaluation_line('sum', times.map { |t| clock.__send__(t) })
+      result << evaluation_line('sum', times.map { |t| clock.sum(t) })
       result << evaluation_line('min', times.map { |t| clock.min(t) })
       result << evaluation_line('std-', times.map { |t| clock.arithmetic_mean(t) - clock.sample_standard_deviation(t) })
       result << evaluation_line('mean', times.map { |t| clock.arithmetic_mean(t) })
@@ -2127,7 +2112,7 @@ module Bullshit
     def benchmark(bc_class, method, opts = {})
       opts = { :run => true, :combine => true }.merge opts
       if Case === bc_class
-        bullshit_case, bc_class = bc_class, bullshit_case.class
+        bullshit_case, bc_class = bc_class, bc_class.class
         @cases[bc_class] ||= []
         if opts[:combine]
           if @cases[bc_class].empty?
@@ -2155,15 +2140,20 @@ module Bullshit
       if comment = opts[:comment]
         bc_method.comment = comment
       end
-      @benchmark_methods << bc_method
       if file_path = opts[:load]
-        if file_path != true
-          bc_method.load file_path
+        success = if file_path != true
+          bc_method.load(file_path)
         else
           bc_method.load
         end
+        if success
+          @benchmark_methods << bc_method
+        else
+          warn "Loading of #{bc_method} failed. Skipping to next."
+        end
       else
         opts[:run] and bullshit_case.run false
+        @benchmark_methods << bc_method
       end
       nil
     end
@@ -2193,43 +2183,68 @@ module Bullshit
       output.puts Time.now.strftime(' %FT%T %Z ').center(COLUMNS, '=')
       for comparator in [ :call_time_mean, :call_time_median ]
         output.puts
-        methods = compare_methods(comparator)
-        methods.size < 2 and return
-        max = methods.last.clock.__send__(comparator)
+        cmethods = compare_methods(comparator)
+        cmethods.size < 2 and return
+        max = cmethods.last.clock.__send__(comparator)
         output.puts "Comparing times (#{comparator}):"
-        methods.each_with_index do |m, i|
-          covers = []
-          for x in methods
-            if m != x and m.cover?(x)
-              j = 0
-              if methods.find { |y| j += 1; x == y }
-                my_case = m.case.class
-                iterations = m.clock.analysis[my_case.compare_time].suggested_sample_size(
-                  x.clock.analysis[my_case.compare_time], my_case.covering.alpha_level, my_case.covering.beta_level)
-                if iterations.nan? or iterations.infinite?
-                  covers << "#{j} (?)"
-                else
-                  min_iterations = iterations.ceil
-                  min_iterations >= 1E6 and min_iterations = "%f" % (1 / 0.0)
-                  covers << "#{j} (>=#{min_iterations})"
-                end
-              end
-            end
-          end
-          covers *= ', '
+        cmethods.each_with_index do |m, i|
           output.printf\
             "% 2u #{prefix_string(m)}\n   %17.9f"\
-            " (%#{::Bullshit::Clock::TIMES_MAX}s) -> %8.3fx %s\n"\
-            "   %17.9f\n",
+            " (%#{::Bullshit::Clock::TIMES_MAX}s) %s\n"\
+            "   %17.9f %8.2f %17.9f\n",
             i + 1, m.clock.calls(comparator), m.case.class.compare_time,
-            max / m.clock.__send__(comparator), covers, m.clock.__send__(comparator)
+            compute_covers(cmethods, m), m.clock.__send__(comparator),
+            m.clock.sample_standard_deviation_percentage(m.case.class.compare_time),
+            m.clock.sum(m.case.class.compare_time)
         end
-        output.puts "   %17s (%#{::Bullshit::Clock::TIMES_MAX}s) -> %8s  %s\n"\
-                    "   %17s\n"\
-                    % %w[calls/sec time speed covers secs/call]
+        output.puts "   %17s (%#{::Bullshit::Clock::TIMES_MAX}s) %s\n"\
+                    "   %17s %8s %17s\n"\
+                    % %w[calls/sec time covers secs/call std% sum]
+        display_speedup_matrix(cmethods, comparator)
       end
       output.puts '=' * COLUMNS
     end
+
+    private
+
+    def display_speedup_matrix(cmethods, comparator)
+      output.print "\n", " " * 3
+      cmethods.size.times do |i|
+        output.printf "%7d ", i + 1
+      end
+      output.puts
+      cmethods.each_with_index do |x, i|
+        output.printf "%2d ", i + 1
+        cmethods.each do |y|
+          ratio = x.clock.calls(comparator).to_f / y.clock.calls(comparator)
+          ratio /= 0.0 if ratio >= 1000
+          output.printf "%7.2f ", ratio
+        end
+        output.puts
+      end
+    end
+
+    def compute_covers(cmethods, m)
+      covers = []
+      for x in cmethods
+        if m != x and m.cover?(x)
+          j = 0
+          if cmethods.find { |y| j += 1; x == y }
+            my_case = m.case.class
+            iterations = m.clock.analysis[my_case.compare_time].suggested_sample_size(
+              x.clock.analysis[my_case.compare_time], my_case.covering.alpha_level, my_case.covering.beta_level)
+            if iterations.nan? or iterations.infinite?
+              covers << "#{j} (?)"
+            else
+              min_iterations = iterations.ceil
+              min_iterations >= 1E6 and min_iterations = "%f" % (1 / 0.0)
+              covers << "#{j} (>=#{min_iterations})"
+            end
+          end
+        end
+      end
+      covers * ', '
+    end
   end
 
   # Create a Comparison instance configured by +block+ and compute and display

data/lib/bullshit/version.rb

@@ -1,6 +1,6 @@
 module Bullshit
   # Bullshit version
-  VERSION         = '0.1.0'
+  VERSION         = '0.1.1'
   VERSION_ARRAY   = VERSION.split(/\./).map { |x| x.to_i } # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/make_doc.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
 
 puts "Creating documentation."
-system "rdoc -S -d #{Dir['lib/**/*.rb'] * ' '}"
+system "rdoc --main README --title 'Bullshit -- Benchmarking in Ruby'"\
+  " -d #{Dir['lib/**/*.rb'] * ' '}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: bullshit
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors: 
 - Florian Frank
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-31 00:00:00 +02:00
+date: 2009-11-18 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -24,45 +24,46 @@ dependencies:
     version: 
 description: ""
 email: flori@ping.de
-executables: []
-
+executables: 
+- bs_compare
 extensions: []
 
 extra_rdoc_files: []
 
 files: 
-- COPYING
-- Rakefile
+- CHANGES
+- bin/bs_compare
 - VERSION
-- bullshit.gemspec
-- data
-- examples
-- examples/compare.rb
-- examples/fibonacci.rb
+- README
+- make_doc.rb
+- Rakefile
+- examples/throw_raise.rb
 - examples/iteration.rb
 - examples/josephus.rb
+- examples/fibonacci.rb
+- examples/compare.rb
 - examples/sorting.rb
-- install.rb
-- lib
-- lib/bullshit
-- lib/bullshit.rb
 - lib/bullshit/version.rb
-- make_doc.rb
-- tests
+- lib/bullshit.rb
+- tests/test_continued_fraction.rb
+- tests/test_window.rb
+- tests/test_functions.rb
 - tests/test_analysis.rb
 - tests/test_bullshit.rb
-- tests/test_continued_fraction.rb
 - tests/test_distribution.rb
-- tests/test_functions.rb
 - tests/test_newton_bisection.rb
-- tests/test_window.rb
+- COPYING
+- install.rb
 has_rdoc: true
-homepage: http://bullshit.rubyforge.org
+homepage: http://flori.github.com/bullshit
+licenses: []
+
 post_install_message: 
 rdoc_options: 
 - --title
 - Bullshit -- Benchmarking in Ruby
-- --line-numbers
+- --main
+- README
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -80,15 +81,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: bullshit
-rubygems_version: 1.3.1
+rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: Benchmarking is Bullshit
 test_files: 
+- tests/test_continued_fraction.rb
+- tests/test_window.rb
+- tests/test_functions.rb
 - tests/test_analysis.rb
 - tests/test_bullshit.rb
-- tests/test_continued_fraction.rb
 - tests/test_distribution.rb
-- tests/test_functions.rb
 - tests/test_newton_bisection.rb
-- tests/test_window.rb

data/bullshit.gemspec

@@ -1,24 +0,0 @@
-# -*- encoding: utf-8 -*-
-Gem::Specification.new do |s|
-  s.name = 'bullshit'
-  s.version = '0.1.0'
-  s.summary = "Benchmarking is Bullshit"
-  s.description = ""
-
-  s.add_dependency('dslkit', '>= 0.2.5')
-
-  s.files = ["COPYING", "Rakefile", "VERSION", "bullshit.gemspec", "data", "examples", "examples/compare.rb", "examples/fibonacci.rb", "examples/iteration.rb", "examples/josephus.rb", "examples/sorting.rb", "install.rb", "lib", "lib/bullshit", "lib/bullshit.rb", "lib/bullshit/version.rb", "make_doc.rb", "tests", "tests/test_analysis.rb", "tests/test_bullshit.rb", "tests/test_continued_fraction.rb", "tests/test_distribution.rb", "tests/test_functions.rb", "tests/test_newton_bisection.rb", "tests/test_window.rb"]
-
-  s.require_path = 'lib'
-
-  s.has_rdoc = true
-  s.rdoc_options <<
-    '--title' <<  'Bullshit -- Benchmarking in Ruby' <<
-    '--line-numbers'
-  s.test_files = ["tests/test_analysis.rb", "tests/test_bullshit.rb", "tests/test_continued_fraction.rb", "tests/test_distribution.rb", "tests/test_functions.rb", "tests/test_newton_bisection.rb", "tests/test_window.rb"]
-
-  s.author = "Florian Frank"
-  s.email = "flori@ping.de"
-  s.homepage = "http://bullshit.rubyforge.org"
-  s.rubyforge_project = "bullshit"
-end