automatthew-stevedore 0.1.0 → 0.2.0

data/README

@@ -0,0 +1,57 @@
+Stevedore is a tool for performing and comparing sets of benchmarks based on similar setup and teardown needs.  Instances use their class's setup/teardown where available, and you can set or override these methods per instance.
+
+Steve (and his subclasses, naturally) track any instances created and can run a comparison using Steve.compare_instances.  If the R language is available, you can do a power analysis using the data from a trial run to suggest sample sizes and number of runs needed to meet certain figures of statistical reliability.
+  
+Steve uses the rsruby gem, if available, for basic stats and power analysis.  As the rsruby gem uses a C extension, this could be problematic for non-MRI Rubys.  In the absence of rsruby, stevedore uses bmarini-mathstats for the basic stats and attempts to shell out to R for power analysis (i.e. power.t.test)
+  
+
+  class StringConcat < Stevedore
+    before_sample do
+      ...
+    end
+    after_sample do
+      ...
+    end
+  end
+
+  plus = StringConcat.new "Combining strings with +" do
+    measure do
+      10000.times do
+        "supercalifragilisticexpiali" + "docious"
+      end
+    end
+  end
+
+  append = StringConcat.new "Combining strings with <<" do
+    measure do
+      10000.times do
+        "supercalifragilisticexpiali" << "docious"
+      end
+    end
+  end
+
+  interpolate = StringConcat.new "Combining strings with interpolation" do
+    measure do
+      10000.times do
+        "supercalifragilisticexpiali#{'docious'}"
+      end
+    end
+  end
+
+  run_count, sample_size = 3, 8
+
+  puts "Measuring #{run_count} runs of #{sample_size} for each test."
+  StringConcat.run([append, interpolate], run_count, sample_size)
+  puts
+  StringConcat.report([append, interpolate]) => putses the following:
+  
+  
+  Combining strings
+
+                                Mean       Stddev      Minimum       Median          Max
+  --------------------------------------------------------------------------------------
+  interpolating strings     0.003064     0.002386     0.001356     0.001682     0.008544
+  appending strings         0.009267     0.002933     0.005337     0.010756     0.013299
+  Ratio of means:  3.02487110476993
+
+

data/Rakefile

@@ -1,6 +1,4 @@
-require 'rubygems'
-
-Version = '0.1.0'
+Version = '0.2.0'
 
 task :default => [ :test ]
 
@@ -9,9 +7,9 @@ begin
   require 'echoe'
   Echoe.new('stevedore', Version) do |p|
     p.project = 'stevedore'
-    p.summary = "Benchmarking with a whiff of statistical awareness"
+    p.summary = "Benchmarking framework with some statistickal stuff"
     p.author = "Matthew King"
-    p.email = "automatthew@gmail.com"
+    p.email = "self@automatthew.com"
     p.ignore_pattern = /^(\.git).+/
     p.test_pattern = "test/*.rb"
   end

data/examples/concat.rb

@@ -0,0 +1,49 @@
+require 'rubygems'
+$:.unshift "#{File.dirname(__FILE__)}/../lib"
+require 'stevedore'
+
+class StringConcat < Steve
+  subject "Combining strings"
+end
+
+plus = StringConcat.new "plus" do
+  
+  measure do
+    10000.times do
+      "supercalifragilisticexpiali" + "docious"
+    end
+  end
+  
+end
+
+append = StringConcat.new "appending strings" do
+  
+  measure do
+    10000.times do
+      "supercalifragilisticexpiali" << "docious"
+    end
+  end
+  
+end
+
+interpolate = StringConcat.new "interpolating strings" do
+  
+  measure do
+    10000.times do
+      "supercalifragilisticexpiali#{'docious'}"
+    end
+  end
+  
+end
+
+# append.load("append")
+# interpolate.load("interpolate")
+
+run_count, sample_size = 3, 8
+
+puts "Measuring #{run_count} runs of #{sample_size} for each test."
+StringConcat.run([append, interpolate], run_count, sample_size)
+puts
+StringConcat.report([append, interpolate])
+append.dump
+interpolate.dump

data/examples/instance_eval.rb

@@ -25,7 +25,5 @@ multi_eval = Stevedore.new "instance_eval &block multiple times" do
 end
 
 Stevedore.compare_instances(5, 10)
-puts
-puts
-Stevedore.recommend_test_size(2, 4)
+
 

data/examples/stats_libs.rb

@@ -5,15 +5,20 @@ require 'rsruby'
 $:.unshift "#{File.dirname(__FILE__)}/../lib"
 require 'stevedore'
 
-array = YAML.load_file("data.yml")
+srand(2009)
+array = (1..2000).to_a.map { |i| rand(800.0) }
 
-# rstats_sd = Stevedore.new "rstats standard deviation" do
-#   
-#   measure do
-#     array.standard_deviation
-#   end
-#   
-# end
+rsruby = Stevedore.new "rsruby standard deviation" do
+  
+  before do
+    @r = RSRuby.instance
+  end
+  
+  measure do
+    @r.sd(array)
+  end
+  
+end
 
 mathstats = Stevedore.new "mathstats standard deviation" do
   
@@ -23,19 +28,16 @@ mathstats = Stevedore.new "mathstats standard deviation" do
   
 end
 
-rsruby = Stevedore.new "rsruby standard deviation" do
-  
+steve_stats = Stevedore.new "steve's own stats" do
   before do
-    @r = RSRuby.instance
+    array.extend Stevedore::Stats
   end
-  
   measure do
-    @r.sd(array)
+    array.standard_deviation
   end
-  
 end
 
 
-Stevedore.compare_instances(5, 10)
+Stevedore.compare_instances(5, 25)
 
 

data/examples/variance.rb

@@ -0,0 +1,64 @@
+require 'rubygems'
+$:.unshift "#{File.dirname(__FILE__)}/../lib"
+require 'stevedore'
+
+
+def variance_two_pass(array, mean)
+  n        = array.size
+  denom    =  n - 1 
+  variance = array.inject(0) {|memo, element| memo + (element - mean)**2 } / denom
+end
+
+def my_var(array, mean)
+  n        = array.size
+  denom    =  n - 1
+  s = 0
+  array.each { |element| s = s + (element - mean)**2; }
+  s / denom
+end
+
+def for_var(array, mean)
+  n        = array.size
+  denom    =  n - 1
+  s = 0
+  for element in array
+    s = s + (element - mean)**2
+  end
+  s / denom
+end
+
+
+
+
+class Variances < Steve
+  subject = "Calculating a statistical variance"
+  before do
+    srand(2009)
+    @array = (1..2000).to_a.map { |i| rand(800.0) }
+    s = 0; @array.each { |element| s = s + element }
+    @sum = s
+    @mean = @sum / @array.size.to_f
+  end
+end
+
+
+mathstats = Variances.new "bmarini-mathstats" do
+  measure do
+    @var = variance_two_pass(@array, @mean)
+  end
+end
+
+no_inject = Variances.new "not using inject" do
+  measure do
+    @var = my_var(@array, @mean)
+  end
+end
+
+for_loop = Variances.new "using for loop" do
+  measure do
+    @var = for_var(@array, @mean)
+  end
+end
+
+Variances.run_all( 8, 500)
+Variances.report_all

data/lib/stevedore/class.rb

@@ -17,7 +17,11 @@ class Stevedore
     val ? @delta = val : @delta ||= 0.001
   end
   
+  def self.subject(val=nil)
+    @subject ||= val
+  end
   
+  # Heritable blocks for setup and measurement
   def self.before(&block)
     block ? @before = block : @before
   end
@@ -42,28 +46,52 @@ class Stevedore
     block ? @after_measure = block : @after_measure
   end
   
-  def self.compare_instances(run_count, sample_size)
-    puts "Measuring #{run_count} runs of #{sample_size} for each instance.\n\n"
-    @instances.each do |instance|
-      puts "'#{instance.name}'"
+  def self.run(instances, run_count, sample_size)
+    instances.each do |instance|
       instance.go(run_count, sample_size)
     end
+  end
+  
+  def self.run_all(run_count, sample_size)
+    run(@instances, run_count, sample_size)
+  end
+  
+  def self.report_all
+    report(@instances)
+  end
+  
+  def self.report(instances)
+    puts @subject if @subject
     name_size = @instances.map { |i| i.name.size }.max
-    puts "\n%-#{name_size}s %12s %12s %12s %12s %12s" % %w{ Name Mean Stddev Minimum Median Max  }
+    puts "\n%-#{name_size}s %12s %12s %12s %12s %12s" % ["", "Mean", "Stddev", "Minimum", "Median", "Max"]
     puts "-" * (name_size + 5 * 13)
-    @instances.each do |instance|
+    instances.sort_by { |i| i.mean }.each do |instance|
       puts "%-#{name_size}s %12f %12f %12f %12f %12f" % 
         [ instance.name, instance.mean, instance.standard_deviation, instance.min, instance.median, instance.max]
     end
+    means = instances.map { |i| i.mean }.sort
+    baseline = means.shift
+    diffs = means.map { |m| m / baseline }
+    puts "Ratio of means:  #{diffs.join(', ')}"
     puts
   end
   
+  def self.compare_instances(run_count, sample_size)
+    puts
+    puts "Benchmark: #{@subject}" if @subject
+    puts
+    puts "Measuring #{run_count} runs of #{sample_size} for each test."
+    self.run_all(run_count, sample_size)
+    puts
+    self.report_all
+  end
+  
   # Run a small set of samples and use a power test to determine
   # the optimal run count and sample size.
   def self.recommend_test_size(run_count, sample_size)
     puts "\nRunning trials (#{run_count} runs of #{sample_size}) for each instance.\n\n"
     @instances.each do |instance|
-      print "'#{instance.name}'"
+      puts "'#{instance.name}'"
       instance.go(run_count, sample_size)
       puts "  Mean: %6f" % instance.mean
       puts "  Stddev: %6f" % instance.standard_deviation
@@ -83,4 +111,15 @@ class Stevedore
     Stevedore::Stats.power_test(args)["n"].to_i
   end
   
+  def self.calculated_power(n, d, std)
+    args = { :n => n,  :delta => d, :sig_level => sig_level, :sd => std }
+    puts args.inspect
+    Stevedore::Stats.power_test(args)["power"]
+  end
+  
+  def self.calculated_delta(n, std)
+    args = { :n => n, :power => power, :sig_level => sig_level, :sd => std }
+    Stevedore::Stats.power_test(args)["delta"]
+  end
+  
 end

data/lib/stevedore/instance.rb

@@ -1,3 +1,4 @@
+require 'fileutils'
 class Stevedore
   
   attr_accessor :name, :description, :samples
@@ -9,7 +10,7 @@ class Stevedore
     @before_measure, @after_measure = klass.before_measure, klass.after_measure
     @before_sample, @after_sample = klass.before_sample, klass.after_sample
     @name, @description = name, description
-    @samples = []
+    @samples = Sample.new
     instance_eval( &block ) if block
   end
   
@@ -18,10 +19,33 @@ class Stevedore
   end
   
   def reset
-    @samples = []
-    @flattened_samples = []
+    @samples = Sample.new
+    @flat = nil
   end
   
+  def dump_path(name)
+    "stevedata/#{name.gsub(/\W+/, '_')}.yaml"
+  end
+  
+  # Dump sample data to file in YAML format
+  def dump(path=nil)
+    filename = ( path || dump_path(@name))
+    FileUtils.mkdir_p(File.dirname(filename))
+    data = {}
+    @samples.each_with_index { |s, i| data[i] = s.to_a }
+    File.open filename, "w" do |f|
+      ::YAML.dump(data, f)
+    end
+  end
+  
+  def load(name)
+    filename = dump_path(name)
+    File.open(filename, 'r') do |f|
+      @samples = ::YAML.load(f)
+    end
+  end
+  
+  
   # Blocks for setup and measurement
   
   def before(&block); @before = block; end
@@ -36,62 +60,52 @@ class Stevedore
   
   def after_measure(&block); @after_measure = block; end
   
+  # Here you do the work you actually want to measure.
   def measure(&block); @measure = block; end
   
   def go(run_count, sample_size)
     reset
     instance_eval( &@before ) if @before
     
-    run_count.times do |i|
-      sample = []
+    run_count.times do
+      sample = Sample.new
       instance_eval( &@before_sample ) if @before_sample
       
+      print "."; $stdout.flush
       sample_size.times do
         instance_eval( &@before_measure ) if @before_measure
-        sample << Benchmark.realtime do          
-          instance_eval( &@measure )
-        end
+        
+        sample << Benchmark.realtime { instance_eval( &@measure ) }
+        
         instance_eval( &@after_measure ) if @after_measure
       end
-      
       instance_eval( &@after_sample ) if @after_sample
       @samples << sample
     end
-    
+    puts; $stdout.flush
     instance_eval( &@after ) if @after
   end
   
   def report
     puts self.name
-    puts "#{run_count} sample runs, #{sample_size} measurements each"
     puts "  Mean: #{self.mean}"
     puts "  Standard deviation: #{self.standard_deviation}"
     puts
   end
   
-  def mean
-    self.flattened_samples.mean
-  end
+  def mean; @samples.mean; end
   
-  def median
-    self.flattened_samples.median
-  end
+  def median; @samples.median; end
   
-  def min
-    self.flattened_samples.min
-  end
+  def min; @samples.min; end
   
-  def max
-    self.flattened_samples.max
-  end
+  def max; @samples.max; end
   
-  def standard_deviation
-    self.flattened_samples.standard_deviation
-  end
+  def standard_deviation; @samples.standard_deviation; end
   
-  def sample_means
-    @samples.map { |s| s.mean }
-  end
+  def sample_means; @samples.map { |s| s.mean }; end
+  
+  def sample_sds; @samples.map { |s| s.standard_deviation }; end
   
   
   

data/lib/stevedore/rsruby.rb

@@ -3,34 +3,21 @@ class Stevedore
   
   module RSRuby
     
-    def rsruby
+    def self.rsruby
       @rsruby ||= ::RSRuby.instance
     end
     
-    def mean
-      rsruby.mean(self)
+    def rsruby
+      @rsruby ||= ::RSRuby.instance
     end
     
-    def standard_deviation
-      rsruby.sd(self)
-    end
+    def mean; rsruby.mean(self); end
     
-    def median
-      rsruby.median(self)
-    end
+    def standard_deviation; rsruby.sd(self); end
     
-    def self.rsruby
-      @rsruby ||= ::RSRuby.instance
-    end
+    def median; rsruby.median(self); end
     
-    def self.power_test(args)
-      delta, power, sig_level, sd = args[:delta], args[:power], args[:sig_level], args[:sd]
-      if sd < 0.00007
-        warn "Stddev is very small, which makes power.t.test sad. \nSetting stddev to 0.00007 so we can get this done."
-        sd = 0.00007
-      end
-      rsruby.power_t_test( :delta => delta, :power => power, :sig_level => sig_level, :sd => sd)
-    end
+    def self.power_test(args); rsruby.power_t_test( args ); end
         
   end
   

data/lib/stevedore/sample.rb

@@ -0,0 +1,38 @@
+begin
+  require 'stevedore/rsruby'
+  Stats = Stevedore::RSRuby
+rescue LoadError
+  require 'stevedore/stats'
+end
+
+class Stevedore
+  
+  class Sample < Array
+    include Stats
+    
+    attr_reader :flat
+    
+    alias :old_to_a :to_a
+    alias :old_mean :mean
+    alias :old_sd :standard_deviation
+    alias :old_median :median
+    alias :old_min :min
+    alias :old_max :max
+    
+    def to_a
+      self.map { |a| a.is_a?( Sample ) ? a.to_a : a  }
+    end
+    
+    def mean; (@flat ||= self.flatten).old_mean; end
+
+    def standard_deviation; (@flat ||= self.flatten).old_sd; end
+    
+    def median; (@flat ||= self.flatten).old_median; end
+    
+    def min; (@flat ||= self.flatten).old_min; end
+    
+    def max; (@flat ||= self.flatten).old_max; end
+    
+  end
+  
+end

data/lib/stevedore/shell_r.rb

@@ -34,7 +34,5 @@ class Stevedore
       Hash[*r]
     end
 
-
-
   end
 end

data/lib/stevedore/stats.rb

@@ -1,30 +1,41 @@
 class Stevedore
   
-  begin
-    require 'stevedore/rsruby'
-    Stats = Stevedore::RSRuby
-    puts "Stevedore will use RSRuby"
-  rescue
-    require 'mathstats'
-    Stats = Mathstats
-    module Stats
-      unless defined?(median)
-        def median
-          sorted = self.sort; i = sorted.size % 2
-          case i
-           when 0 then sorted[i/2 - 1, 2].mean
-           when 1 then sorted[i/2].to_f
-          end if sorted.size > 0
-        end
-      end
-      def self.power_test(*args)
-        Stevedore::ShellR.new.power_test(*args)
-      end
+  module Stats
+    
+    def sum
+      s = 0; self.each { |element| s = s + element }; s
     end
-    puts "Stevedore will use Mathstats"
+    
+    def mean
+      self.sum.to_f / self.size
+    end
+    
+    def variance
+      n = self.size
+      denom = n - 1
+      m = self.mean
+      s = 0
+      self.each { |element| s = s + (element - m)**2; }
+      s / denom
+    end
+    
+    def standard_deviation
+      Math.sqrt( self.variance )
+    end
+    
+    def median
+      sorted = self.sort; i = sorted.size % 2
+      case i
+       when 0 then sorted[i/2 - 1, 2].mean
+       when 1 then sorted[i/2].to_f
+      end if sorted.size > 0
+    end
+    
+    def self.power_test(*args)
+      Stevedore::ShellR.new.power_test(*args)
+    end
+        
   end
-
-  Array.send :include, Stats
   
 end
 

data/lib/stevedore.rb

@@ -1,11 +1,9 @@
 require 'benchmark'
-require 'stevedore/stats'
+require 'yaml'
+require 'stevedore/sample'
 require 'stevedore/shell_r'
 require 'stevedore/class'
 require 'stevedore/instance'
 
-class Stevedore
-  
-end
-
-Steve = Stevedore
+class Stevedore; end
+Steve = Stevedore

data/stevedore.gemspec

@@ -2,22 +2,22 @@
 
 Gem::Specification.new do |s|
   s.name = %q{stevedore}
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matthew King"]
-  s.date = %q{2009-01-01}
-  s.description = %q{Benchmarking with a whiff of statistical awareness}
-  s.email = %q{automatthew@gmail.com}
-  s.extra_rdoc_files = ["lib/stevedore/class.rb", "lib/stevedore/instance.rb", "lib/stevedore/rsruby.rb", "lib/stevedore/shell_r.rb", "lib/stevedore/stats.rb", "lib/stevedore.rb", "README"]
-  s.files = ["examples/data.yml", "examples/instance_eval.rb", "examples/stats_libs.rb", "lib/stevedore/class.rb", "lib/stevedore/instance.rb", "lib/stevedore/rsruby.rb", "lib/stevedore/shell_r.rb", "lib/stevedore/stats.rb", "lib/stevedore.rb", "Manifest", "Rakefile", "README", "stevedore.gemspec", "test/helper.rb", "test/test_R.rb", "test/test_sample.rb"]
+  s.date = %q{2009-02-09}
+  s.description = %q{Benchmarking framework with some statistickal stuff}
+  s.email = %q{self@automatthew.com}
+  s.extra_rdoc_files = ["lib/stevedore/class.rb", "lib/stevedore/instance.rb", "lib/stevedore/rsruby.rb", "lib/stevedore/sample.rb", "lib/stevedore/shell_r.rb", "lib/stevedore/stats.rb", "lib/stevedore.rb", "README"]
+  s.files = ["examples/aliasing.rb", "examples/concat.rb", "examples/data.yml", "examples/instance_eval.rb", "examples/stats_libs.rb", "examples/stevedata/appending_strings.yaml", "examples/stevedata/interpolating_strings.yaml", "examples/variance.rb", "lib/stevedore/class.rb", "lib/stevedore/instance.rb", "lib/stevedore/rsruby.rb", "lib/stevedore/sample.rb", "lib/stevedore/shell_r.rb", "lib/stevedore/stats.rb", "lib/stevedore.rb", "Manifest", "Rakefile", "README", "stevedore.gemspec", "test/helper.rb", "test/test_R.rb", "test/test_sample.rb"]
   s.has_rdoc = true
   s.homepage = %q{}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Stevedore", "--main", "README"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{stevedore}
   s.rubygems_version = %q{1.3.1}
-  s.summary = %q{Benchmarking with a whiff of statistical awareness}
+  s.summary = %q{Benchmarking framework with some statistickal stuff}
   s.test_files = ["test/helper.rb", "test/test_R.rb", "test/test_sample.rb"]
 
   if s.respond_to? :specification_version then

data/test/helper.rb

@@ -1,4 +1,4 @@
-%w{ rubygems bacon facon }.each { |dep| require dep  }
+%w{ bacon facon }.each { |dep| require dep  }
 
 Bacon.extend Bacon::TestUnitOutput
 Bacon.summary_on_exit

data/test/test_sample.rb

@@ -6,7 +6,7 @@ describe "Stevedore::Sample" do
   
   before do
     @sample = Sample.new((1..40).to_a)
-    @nested_sample = Sample.new [@sample, @sample, @sample ]
+    @nested_sample = Sample.new [Sample.new((1..40).to_a), Sample.new((1..40).to_a), Sample.new((1..40).to_a) ]
   end
   
   it "a flattened Sample is still a Sample" do
@@ -24,4 +24,10 @@ describe "Stevedore::Sample" do
     @nested_sample.standard_deviation.to_i.should == 11
   end
   
+  it "can become simple arrays" do
+    @nested_sample.to_a.each do |a|
+      a.class.should == Array
+    end
+  end
+  
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: automatthew-stevedore
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Matthew King
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-01 00:00:00 -08:00
+date: 2009-02-09 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -21,8 +21,8 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "0"
     version: 
-description: Benchmarking with a whiff of statistical awareness
-email: automatthew@gmail.com
+description: Benchmarking framework with some statistickal stuff
+email: self@automatthew.com
 executables: []
 
 extensions: []
@@ -31,17 +31,24 @@ extra_rdoc_files:
 - lib/stevedore/class.rb
 - lib/stevedore/instance.rb
 - lib/stevedore/rsruby.rb
+- lib/stevedore/sample.rb
 - lib/stevedore/shell_r.rb
 - lib/stevedore/stats.rb
 - lib/stevedore.rb
 - README
 files: 
+- examples/aliasing.rb
+- examples/concat.rb
 - examples/data.yml
 - examples/instance_eval.rb
 - examples/stats_libs.rb
+- examples/stevedata/appending_strings.yaml
+- examples/stevedata/interpolating_strings.yaml
+- examples/variance.rb
 - lib/stevedore/class.rb
 - lib/stevedore/instance.rb
 - lib/stevedore/rsruby.rb
+- lib/stevedore/sample.rb
 - lib/stevedore/shell_r.rb
 - lib/stevedore/stats.rb
 - lib/stevedore.rb
@@ -82,7 +89,7 @@ rubyforge_project: stevedore
 rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
-summary: Benchmarking with a whiff of statistical awareness
+summary: Benchmarking framework with some statistickal stuff
 test_files: 
 - test/helper.rb
 - test/test_R.rb