automatthew-stevedore 0.1.0

data/Rakefile

@@ -0,0 +1,20 @@
+require 'rubygems'
+
+Version = '0.1.0'
+
+task :default => [ :test ]
+
+begin
+  gem 'echoe', '~>3.0'
+  require 'echoe'
+  Echoe.new('stevedore', Version) do |p|
+    p.project = 'stevedore'
+    p.summary = "Benchmarking with a whiff of statistical awareness"
+    p.author = "Matthew King"
+    p.email = "automatthew@gmail.com"
+    p.ignore_pattern = /^(\.git).+/
+    p.test_pattern = "test/*.rb"
+  end
+rescue
+  "(ignored echoe gemification, as you don't have the Right Stuff)"
+end

data/examples/data.yml

@@ -0,0 +1,51 @@
+--- 
+- 0.00554585456848145
+- 0.010728120803833
+- 0.00568890571594238
+- 0.00575709342956543
+- 0.00926589965820312
+- 0.00586318969726562
+- 0.00583505630493164
+- 0.00585389137268066
+- 0.00891590118408203
+- 0.00550603866577148
+- 0.00578689575195312
+- 0.00865912437438965
+- 0.00515890121459961
+- 0.00583219528198242
+- 0.00541400909423828
+- 0.00829195976257324
+- 0.00518512725830078
+- 0.005615234375
+- 0.00883793830871582
+- 0.00561618804931641
+- 0.00561308860778809
+- 0.00577306747436523
+- 0.00857686996459961
+- 0.0057990550994873
+- 0.00524306297302246
+- 0.00851821899414062
+- 0.00578618049621582
+- 0.00547695159912109
+- 0.00515198707580566
+- 0.00598001480102539
+- 0.00586414337158203
+- 0.00577092170715332
+- 0.0083918571472168
+- 0.00514411926269531
+- 0.0051429271697998
+- 0.00795292854309082
+- 0.00558185577392578
+- 0.00593113899230957
+- 0.00522780418395996
+- 0.00866293907165527
+- 0.00590395927429199
+- 0.00517797470092773
+- 0.00850200653076172
+- 0.0058290958404541
+- 0.00525999069213867
+- 0.00512909889221191
+- 0.00773906707763672
+- 0.00583696365356445
+- 0.00583600997924805
+- 0.00795888900756836

data/examples/instance_eval.rb

@@ -0,0 +1,31 @@
+%w{ rubygems  }.each { |dep| require dep  }
+
+$:.unshift "#{File.dirname(__FILE__)}/../lib"
+require 'stevedore'
+
+thing = lambda { true }
+n = 10000
+
+one_eval = Stevedore.new "One instance eval with several block.call" do
+  
+  measure do
+    instance_eval do
+      n.times { thing.call }
+    end
+  end
+  
+end
+
+multi_eval = Stevedore.new "instance_eval &block multiple times" do
+  
+  measure do
+    n.times { instance_eval &thing }
+  end
+  
+end
+
+Stevedore.compare_instances(5, 10)
+puts
+puts
+Stevedore.recommend_test_size(2, 4)
+

data/examples/stats_libs.rb

@@ -0,0 +1,41 @@
+%w{ rubygems  }.each { |dep| require dep  }
+require 'mathstats'
+require 'rsruby'
+
+$:.unshift "#{File.dirname(__FILE__)}/../lib"
+require 'stevedore'
+
+array = YAML.load_file("data.yml")
+
+# rstats_sd = Stevedore.new "rstats standard deviation" do
+#   
+#   measure do
+#     array.standard_deviation
+#   end
+#   
+# end
+
+mathstats = Stevedore.new "mathstats standard deviation" do
+  
+  measure do
+    Mathstats::Lib.standard_deviation(array)
+  end
+  
+end
+
+rsruby = Stevedore.new "rsruby standard deviation" do
+  
+  before do
+    @r = RSRuby.instance
+  end
+  
+  measure do
+    @r.sd(array)
+  end
+  
+end
+
+
+Stevedore.compare_instances(5, 10)
+
+

data/lib/stevedore.rb

@@ -0,0 +1,11 @@
+require 'benchmark'
+require 'stevedore/stats'
+require 'stevedore/shell_r'
+require 'stevedore/class'
+require 'stevedore/instance'
+
+class Stevedore
+  
+end
+
+Steve = Stevedore

data/lib/stevedore/class.rb

@@ -0,0 +1,86 @@
+class Stevedore
+  
+  def self.instances; @instances ||= []; end
+  
+  def self.reset; @instances = []; end
+  
+  
+  # Moderately stringent defaults for power analysis
+  
+  def self.power(val=nil)
+    val ? @power = val : @power ||= 0.9
+  end
+  def self.sig_level(val=nil)
+    val ? @sig_level = val : @sig_level ||= 0.01
+  end
+  def self.delta(val=nil)
+    val ? @delta = val : @delta ||= 0.001
+  end
+  
+  
+  def self.before(&block)
+    block ? @before = block : @before
+  end
+  
+  def self.after(&block)
+    block ? @after = block : @after
+  end
+  
+  def self.before_sample(&block)
+    block ? @before_sample = block : @before_sample
+  end
+  
+  def self.after_sample(&block)
+    block ? @after_sample = block : @after_sample
+  end
+  
+  def self.before_measure(&block)
+    block ? @before_measure = block : @before_measure
+  end  
+  
+  def self.after_measure(&block)
+    block ? @after_measure = block : @after_measure
+  end
+  
+  def self.compare_instances(run_count, sample_size)
+    puts "Measuring #{run_count} runs of #{sample_size} for each instance.\n\n"
+    @instances.each do |instance|
+      puts "'#{instance.name}'"
+      instance.go(run_count, sample_size)
+    end
+    name_size = @instances.map { |i| i.name.size }.max
+    puts "\n%-#{name_size}s %12s %12s %12s %12s %12s" % %w{ Name Mean Stddev Minimum Median Max  }
+    puts "-" * (name_size + 5 * 13)
+    @instances.each do |instance|
+      puts "%-#{name_size}s %12f %12f %12f %12f %12f" % 
+        [ instance.name, instance.mean, instance.standard_deviation, instance.min, instance.median, instance.max]
+    end
+    puts
+  end
+  
+  # Run a small set of samples and use a power test to determine
+  # the optimal run count and sample size.
+  def self.recommend_test_size(run_count, sample_size)
+    puts "\nRunning trials (#{run_count} runs of #{sample_size}) for each instance.\n\n"
+    @instances.each do |instance|
+      print "'#{instance.name}'"
+      instance.go(run_count, sample_size)
+      puts "  Mean: %6f" % instance.mean
+      puts "  Stddev: %6f" % instance.standard_deviation
+    end
+    puts
+    worst = @instances.sort_by { |i| i.standard_deviation }.last
+    puts "'#{worst.name}' has the greatest standard deviation,"
+    puts "so we'll use it in the power test to determine optimal run size"
+    rec_size = optimal_n(worst.standard_deviation).to_i
+    rec_runs = optimal_n(worst.sample_means.standard_deviation).to_i
+    puts "Recommendation: #{rec_runs} sample runs of #{rec_size} measurements.\n\n"
+    [rec_runs, rec_size]
+  end
+  
+  def self.optimal_n(stddev);
+    args = { :power => power, :delta => delta, :sig_level => sig_level, :sd => stddev }
+    Stevedore::Stats.power_test(args)["n"].to_i
+  end
+  
+end

data/lib/stevedore/instance.rb

@@ -0,0 +1,98 @@
+class Stevedore
+  
+  attr_accessor :name, :description, :samples
+  
+  def initialize(name, description='', &block)
+    klass = self.class
+    klass.instances << self
+    @before, @after = klass.before, klass.after
+    @before_measure, @after_measure = klass.before_measure, klass.after_measure
+    @before_sample, @after_sample = klass.before_sample, klass.after_sample
+    @name, @description = name, description
+    @samples = []
+    instance_eval( &block ) if block
+  end
+  
+  def flattened_samples
+     @flat ||= @samples.flatten
+  end
+  
+  def reset
+    @samples = []
+    @flattened_samples = []
+  end
+  
+  # Blocks for setup and measurement
+  
+  def before(&block); @before = block; end
+  
+  def after(&block); @after = block; end
+  
+  def before_sample(&block); @before_sample = block; end
+  
+  def after_sample(&block); @after_sample = block; end
+  
+  def before_measure(&block); @before_measure = block; end
+  
+  def after_measure(&block); @after_measure = block; end
+  
+  def measure(&block); @measure = block; end
+  
+  def go(run_count, sample_size)
+    reset
+    instance_eval( &@before ) if @before
+    
+    run_count.times do |i|
+      sample = []
+      instance_eval( &@before_sample ) if @before_sample
+      
+      sample_size.times do
+        instance_eval( &@before_measure ) if @before_measure
+        sample << Benchmark.realtime do          
+          instance_eval( &@measure )
+        end
+        instance_eval( &@after_measure ) if @after_measure
+      end
+      
+      instance_eval( &@after_sample ) if @after_sample
+      @samples << sample
+    end
+    
+    instance_eval( &@after ) if @after
+  end
+  
+  def report
+    puts self.name
+    puts "#{run_count} sample runs, #{sample_size} measurements each"
+    puts "  Mean: #{self.mean}"
+    puts "  Standard deviation: #{self.standard_deviation}"
+    puts
+  end
+  
+  def mean
+    self.flattened_samples.mean
+  end
+  
+  def median
+    self.flattened_samples.median
+  end
+  
+  def min
+    self.flattened_samples.min
+  end
+  
+  def max
+    self.flattened_samples.max
+  end
+  
+  def standard_deviation
+    self.flattened_samples.standard_deviation
+  end
+  
+  def sample_means
+    @samples.map { |s| s.mean }
+  end
+  
+  
+  
+end

data/lib/stevedore/rsruby.rb

@@ -0,0 +1,37 @@
+require 'rsruby'
+class Stevedore
+  
+  module RSRuby
+    
+    def rsruby
+      @rsruby ||= ::RSRuby.instance
+    end
+    
+    def mean
+      rsruby.mean(self)
+    end
+    
+    def standard_deviation
+      rsruby.sd(self)
+    end
+    
+    def median
+      rsruby.median(self)
+    end
+    
+    def self.rsruby
+      @rsruby ||= ::RSRuby.instance
+    end
+    
+    def self.power_test(args)
+      delta, power, sig_level, sd = args[:delta], args[:power], args[:sig_level], args[:sd]
+      if sd < 0.00007
+        warn "Stddev is very small, which makes power.t.test sad. \nSetting stddev to 0.00007 so we can get this done."
+        sd = 0.00007
+      end
+      rsruby.power_t_test( :delta => delta, :power => power, :sig_level => sig_level, :sd => sd)
+    end
+        
+  end
+  
+end

data/lib/stevedore/shell_r.rb

@@ -0,0 +1,40 @@
+class Stevedore
+  class ShellR
+
+    def self.path(val=nil)
+      val ? @path = val : @path ||= `which R`.chomp
+    end
+
+    def path
+      self.class.path
+    end
+
+    def command(expression)
+      %Q{#{path} --slave -e '#{expression}'}
+    end
+
+    def execute(expression)
+      command = command(expression)
+      r = `#{command}`
+    end
+
+    def power_test(options)
+      power, delta, sig_level, sd = options[:power], options[:delta], options[:sig_level], options[:sd]
+      if options[:sd] < 0.00007
+        warn "Stddev is very small, which makes power.t.test sad. \nSetting stddev to 0.00007 so we can get this done."
+        options[:sd] = 0.00007
+      end
+      expression = %Q{power.t.test(power=#{power}, delta=#{delta}, sd=#{sd}, sig.level=#{sig_level})[]}
+      parse(execute(expression))
+    end
+
+    def parse(r_output)
+      attrs = r_output.scan( /\$([\w\-\.]+)\s+\[\d+\]\s+([\d\.\-e\+]+)/m )
+      r = attrs.map { |k,v| [k, v.to_f]  }.flatten
+      Hash[*r]
+    end
+
+
+
+  end
+end

data/lib/stevedore/stats.rb

@@ -0,0 +1,30 @@
+class Stevedore
+  
+  begin
+    require 'stevedore/rsruby'
+    Stats = Stevedore::RSRuby
+    puts "Stevedore will use RSRuby"
+  rescue
+    require 'mathstats'
+    Stats = Mathstats
+    module Stats
+      unless defined?(median)
+        def median
+          sorted = self.sort; i = sorted.size % 2
+          case i
+           when 0 then sorted[i/2 - 1, 2].mean
+           when 1 then sorted[i/2].to_f
+          end if sorted.size > 0
+        end
+      end
+      def self.power_test(*args)
+        Stevedore::ShellR.new.power_test(*args)
+      end
+    end
+    puts "Stevedore will use Mathstats"
+  end
+
+  Array.send :include, Stats
+  
+end
+

data/stevedore.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{stevedore}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Matthew King"]
+  s.date = %q{2009-01-01}
+  s.description = %q{Benchmarking with a whiff of statistical awareness}
+  s.email = %q{automatthew@gmail.com}
+  s.extra_rdoc_files = ["lib/stevedore/class.rb", "lib/stevedore/instance.rb", "lib/stevedore/rsruby.rb", "lib/stevedore/shell_r.rb", "lib/stevedore/stats.rb", "lib/stevedore.rb", "README"]
+  s.files = ["examples/data.yml", "examples/instance_eval.rb", "examples/stats_libs.rb", "lib/stevedore/class.rb", "lib/stevedore/instance.rb", "lib/stevedore/rsruby.rb", "lib/stevedore/shell_r.rb", "lib/stevedore/stats.rb", "lib/stevedore.rb", "Manifest", "Rakefile", "README", "stevedore.gemspec", "test/helper.rb", "test/test_R.rb", "test/test_sample.rb"]
+  s.has_rdoc = true
+  s.homepage = %q{}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Stevedore", "--main", "README"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{stevedore}
+  s.rubygems_version = %q{1.3.1}
+  s.summary = %q{Benchmarking with a whiff of statistical awareness}
+  s.test_files = ["test/helper.rb", "test/test_R.rb", "test/test_sample.rb"]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<echoe>, [">= 0"])
+    else
+      s.add_dependency(%q<echoe>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<echoe>, [">= 0"])
+  end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+%w{ rubygems bacon facon }.each { |dep| require dep  }
+
+Bacon.extend Bacon::TestUnitOutput
+Bacon.summary_on_exit
+
+$:.unshift "#{File.dirname(__FILE__)}/../lib"
+require 'stevedore'
+

data/test/test_R.rb

@@ -0,0 +1,62 @@
+require "#{File.dirname(__FILE__)}/helper.rb"
+
+R = Stevedore::ShellR
+
+
+
+
+describe "ShellR class" do
+  
+  it "tries to find the path to R using `which`" do
+    File.exist?(R.path).should == true
+  end
+  
+  it "allows you to set the path to R" do
+    R.path "/monkey"
+    R.path.should == "/monkey"
+  end
+  
+end
+
+describe "An instance of ShellR" do
+  
+  before do
+    @r = R.new
+  end
+  
+  it "gets the path to R from the class" do
+    R.path "/junkie"
+    @r.path.should == "/junkie"
+  end
+  
+  it "properly constructs the shell command" do
+    @r.command("shines").should == "/junkie --slave -e 'shines'"
+  end
+  
+  it "can shell out to call R" do
+    R.path  `which R`.chomp
+    expression = %Q{power.t.test(power=0.9, delta=0.0001, sd=0.001133, sig.level=0.01)["n"]}
+    res = @r.execute(expression)
+    res.should == "$n\n[1] 3821.76\n\n"
+  end
+  
+  it "can scan the R result when printed as list of attrs" do
+    output = <<-TXT
+$n
+[1] 3821.76
+
+$delta
+[1] 1e-04
+
+    TXT
+    @r.parse(output).should ==  { "n" => 3821.76, "delta" => 1e-04 }
+  end
+  
+  it "has a power_test method" do
+    options = {:power => 0.9, :delta => 0.003, :sig_level => 0.01, :sd => 0.02}
+    res = @r.power_test(options)
+    res.keys.sort.should == ["delta", "n", "power", "sd", "sig.level"]
+  end
+  
+  
+end

data/test/test_sample.rb

@@ -0,0 +1,27 @@
+require "#{File.dirname(__FILE__)}/helper.rb"
+
+Sample = Stevedore::Sample
+
+describe "Stevedore::Sample" do
+  
+  before do
+    @sample = Sample.new((1..40).to_a)
+    @nested_sample = Sample.new [@sample, @sample, @sample ]
+  end
+  
+  it "a flattened Sample is still a Sample" do
+    @nested_sample.flatten.class.should == Sample
+    [@nested_sample].flatten.class.should == Array
+  end
+  
+  it "has a mean" do
+    @sample.mean.should == 20.5
+    @nested_sample.mean.should == 20.5
+  end
+  
+  it "has a standard deviation" do
+    @sample.standard_deviation.to_i.should == 11
+    @nested_sample.standard_deviation.to_i.should == 11
+  end
+  
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification 
+name: automatthew-stevedore
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Matthew King
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-01-01 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: echoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Benchmarking with a whiff of statistical awareness
+email: automatthew@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- lib/stevedore/class.rb
+- lib/stevedore/instance.rb
+- lib/stevedore/rsruby.rb
+- lib/stevedore/shell_r.rb
+- lib/stevedore/stats.rb
+- lib/stevedore.rb
+- README
+files: 
+- examples/data.yml
+- examples/instance_eval.rb
+- examples/stats_libs.rb
+- lib/stevedore/class.rb
+- lib/stevedore/instance.rb
+- lib/stevedore/rsruby.rb
+- lib/stevedore/shell_r.rb
+- lib/stevedore/stats.rb
+- lib/stevedore.rb
+- Manifest
+- Rakefile
+- README
+- stevedore.gemspec
+- test/helper.rb
+- test/test_R.rb
+- test/test_sample.rb
+has_rdoc: true
+homepage: ""
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Stevedore
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "1.2"
+  version: 
+requirements: []
+
+rubyforge_project: stevedore
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Benchmarking with a whiff of statistical awareness
+test_files: 
+- test/helper.rb
+- test/test_R.rb
+- test/test_sample.rb