experiment 0.2.0 → 0.3.0

data/lib/experiment/notify.rb

@@ -1,180 +1,184 @@
-# This class is responsible for UI goodness in letting you know 
-# about the progress of your experiments
 require "drb/drb"
-class Notify
+module Experiment
+  # This class is responsible for UI goodness in letting you know 
+  # about the progress of your experiments
+  # @private
+  class Notify
   
-  class << self
-    include DRb::DRbUndumped
-    # initialize display
-    def init(total, out = STDERR, growl = true, mode = :normal)
-      @curent_experiment = ""
-      @current_cv = 0
-      @cv_prog = {}
-      @total = total
-      @out = out
-      @terminal_width = 80
-      @bar_mark = "o"
-      @current = 0
-      @previous = 0
-      @finished_p = false
-      @start_time = Time.now
-      @previous_time = @start_time
-      @growl = growl
-      @mode = mode
-      show if @mode == :normal
-    end
+    class << self
+      include DRb::DRbUndumped
+
+      # initialize display
+      def init(total, out = STDERR, growl = true, mode = :normal)
+        @curent_experiment = ""
+        @current_cv = 0
+        @cv_prog = {}
+        @total = total
+        @out = out
+        @terminal_width = 80
+        @bar_mark = "o"
+        @current = 0
+        @previous = 0
+        @finished_p = false
+        @start_time = Time.now
+        @previous_time = @start_time
+        @growl = growl
+        @mode = mode
+        show if @mode == :normal
+      end
     
-    # Called when starting work on a particular experiment
-    def started(experiment)
-      @curent_experiment = experiment
-      @current_cv = 1
-      @cv_prog[experiment] = []
-      show_if_needed
-    end
+      # Called when starting work on a particular experiment
+      def started(experiment)
+        @curent_experiment = experiment
+        @current_cv = 1
+        @cv_prog[experiment] = []
+        show_if_needed
+      end
     
-    # Called when experiment completed. 
-    # Shows a Growl notification on OSX.
-    # The message can be expanded by overriding the result_line 
-    # method in the experiment class
-    def completed(experiment, msg = "")
-      if @growl
-        begin
-          `G_TITLE="Experiment Complete" #{File.dirname(__FILE__)}/../../bin/growl.sh -nosticky "Experimental condition #{experiment} complete. #{msg}"`
-        rescue
-          # probably not on OSX
+      # Called when experiment completed. 
+      # Shows a Growl notification on OSX.
+      # The message can be expanded by overriding the result_line 
+      # method in the experiment class
+      def completed(experiment, msg = "")
+        if @growl
+          begin
+            `G_TITLE="Experiment Complete" #{File.dirname(__FILE__)}/../../bin/growl.sh -nosticky "Experimental condition #{experiment} complete. #{msg}"`
+          rescue
+            # probably not on OSX
+          end
         end
+        m = "Condition #{experiment} complete. #{msg}"
+        puts m + " " * @terminal_width
+        @curent_experiment = nil
       end
-      m = "Condition #{experiment} complete. #{msg}"
-      puts m + " " * @terminal_width
-      @curent_experiment = nil
-    end
     
-    # called after a crossvalidation has completed
-    def cv_done(experiment, num)
-      @cv_prog[experiment][num] ||= 0
-      inc(1 - @cv_prog[experiment][num])
-      #@cv_prog = 0
-    end
+      # called after a crossvalidation has completed
+      def cv_done(experiment, num)
+        @cv_prog[experiment][num] ||= 0
+        inc(1 - @cv_prog[experiment][num])
+        #@cv_prog = 0
+      end
     
-    # Wrap up
-    def done
-      @current = @total
-      @finished_p = true
-      #show
-    end
+      # Wrap up
+      def done
+        @current = @total
+        @finished_p = true
+        #show
+      end
     
-    # Use this in experiment after each (potentially time consuming) task
-    # The argument should be a fraction (0 < num < 1) which tells 
-    # how big a portion the task was of the complete run (eg. your 
-    # calls should sum up to 1).
-    def step(experiment, cv, num)
-      if @mode == :normal
-        if num > 1
-          num = num / 100
+      # Use this in experiment after each (potentially time consuming) task
+      # The argument should be a fraction (0 < num < 1) which tells 
+      # how big a portion the task was of the complete run (eg. your 
+      # calls should sum up to 1).
+      def step(experiment, cv, num)
+        if @mode == :normal
+          if num > 1
+            num = num / 100
+          end
+          inc(num)
+          @cv_prog[experiment][cv] ||= 0
+          @cv_prog[experiment][cv] += num
+        else
+          @mode.notify.step(experiment, cv, num)
         end
-        inc(num)
-        @cv_prog[experiment][cv] ||= 0
-        @cv_prog[experiment][cv] += num
-      else
-        @mode.notify.step(experiment, cv, num)
       end
-    end
     
-  end
-  
-  # a big part of this module is copied/inspired by Satoru Takabayashi's <satoru@namazu.org> ProgressBar class at http://0xcc.net/ruby-progressbar/index.html.en
-  module ProgressBar #:nodoc
-    def inc(step = 1)
-      @current += step
-      @current = @total if @current > @total
-      show_if_needed
-      @previous = @current
     end
-    
-    def show_if_needed
-      if @total.zero?
-        cur_percentage = 100
-        prev_percentage = 0
-      else
-        cur_percentage  = (@current  * 100 / @total).to_i
-        prev_percentage = (@previous * 100 / @total).to_i
+  
+    # a big part of this module is copied/inspired by Satoru Takabayashi's <satoru@namazu.org> ProgressBar class at http://0xcc.net/ruby-progressbar/index.html.en
+    module ProgressBar #:nodoc
+      def inc(step = 1)
+        @current += step
+        @current = @total if @current > @total
+        show_if_needed
+        @previous = @current
       end
-      @finished_p = cur_percentage == 100
-      # Use "!=" instead of ">" to support negative changes
-      if cur_percentage != prev_percentage || 
-          Time.now - @previous_time >= 1 || @finished_p
-        show
+    
+      def show_if_needed
+        if @total.zero?
+          cur_percentage = 100
+          prev_percentage = 0
+        else
+          cur_percentage  = (@current  * 100 / @total).to_i
+          prev_percentage = (@previous * 100 / @total).to_i
+        end
+        @finished_p = cur_percentage == 100
+        # Use "!=" instead of ">" to support negative changes
+        if cur_percentage != prev_percentage || 
+            Time.now - @previous_time >= 1 || @finished_p
+          show
+        end
       end
-    end
     
     
     
-    def show
-      percent = @current  * 100 / @total
-      bar_width = percent * @terminal_width / 100
-      line = sprintf "%3d%% |%s%s| %s", percent, "=" * bar_width, "-" * (@terminal_width - bar_width), stat
+      def show
+        percent = @current  * 100 / @total
+        bar_width = percent * @terminal_width / 100
+        line = sprintf "%3d%% |%s%s| %s", percent, "=" * bar_width, "-" * (@terminal_width - bar_width), stat
       
 
-      width = get_width
-      if line.length == width - 1 
-        @out.print(line +  (@finished_p ? "\n" : "\r"))
-        @out.flush
-      elsif line.length >= width
-        @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
-        if @terminal_width == 0 then @out.print(line + eol) else show end
-      else # line.length < width - 1
-        @terminal_width += width - line.length + 1
-        show
+        width = get_width
+        if line.length == width - 1 
+          @out.print(line +  (@finished_p ? "\n" : "\r"))
+          @out.flush
+        elsif line.length >= width
+          @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+          if @terminal_width == 0 then @out.print(line + eol) else show end
+        else # line.length < width - 1
+          @terminal_width += width - line.length + 1
+          show
+        end
+        @previous_time = Time.now
       end
-      @previous_time = Time.now
-    end
     
-    def stat
-      if @finished_p then elapsed else eta end
-    end
+      def stat
+        if @finished_p then elapsed else eta end
+      end
     
-    def eta
-      if @current == 0
-        "ETA:  --:--:--"
-      else
-        elapsed = Time.now - @start_time
-        eta = elapsed * @total / @current - elapsed;
-        sprintf("ETA:  %s", format_time(eta))
+      def eta
+        if @current == 0
+          "ETA:  --:--:--"
+        else
+          elapsed = Time.now - @start_time
+          eta = elapsed * @total / @current - elapsed;
+          sprintf("ETA:  %s", format_time(eta))
+        end
       end
-    end
 
-    def elapsed
-      elapsed = Time.now - @start_time
-      sprintf("Time: %s", format_time(elapsed))
-    end
+      def elapsed
+        elapsed = Time.now - @start_time
+        sprintf("Time: %s", format_time(elapsed))
+      end
     
-    def format_time (t)
-      t = t.to_i
-      sec = t % 60
-      min  = (t / 60) % 60
-      hour = t / 3600
-      sprintf("%02d:%02d:%02d", hour, min, sec);
-    end
+      def format_time (t)
+        t = t.to_i
+        sec = t % 60
+        min  = (t / 60) % 60
+        hour = t / 3600
+        sprintf("%02d:%02d:%02d", hour, min, sec);
+      end
     
     
-    def get_width
-       # FIXME: I don't know how portable it is.
-       default_width = 80
-       begin
-         tiocgwinsz = 0x5413
-         data = [0, 0, 0, 0].pack("SSSS")
-         if @out.ioctl(tiocgwinsz, data) >= 0 then
-           rows, cols, xpixels, ypixels = data.unpack("SSSS")
-           if cols >= 0 then cols else default_width end
-         else
+      def get_width
+         # FIXME: I don't know how portable it is.
+         default_width = 80
+         begin
+           tiocgwinsz = 0x5413
+           data = [0, 0, 0, 0].pack("SSSS")
+           if @out.ioctl(tiocgwinsz, data) >= 0 then
+             rows, cols, xpixels, ypixels = data.unpack("SSSS")
+             if cols >= 0 then cols else default_width end
+           else
+             default_width
+           end
+         rescue Exception
            default_width
          end
-       rescue Exception
-         default_width
        end
-     end
-  end
+    end
   
-  extend ProgressBar
+    extend ProgressBar
   
+  end
 end

data/lib/experiment/params.rb

@@ -0,0 +1,18 @@
+module Experiment
+  class Params
+   
+    # Return if set the value of the current param.
+    #
+    # If it is not defined fallback to {Experiment::Config#[]}.
+    def self.[](h)
+      @@params[h] || Config[h]
+    end
+    
+    
+    # @private
+    def self.set(a) # :nodoc: 
+      @@params = a
+    end
+  end
+  Params.set({})
+end

data/lib/experiment/runner.rb

@@ -1,10 +1,20 @@
+require File.dirname(__FILE__) + "/config"
 module Experiment
   
-  # This is the class behind the command line magic
+  # This is the class behind the command line magic.
+  # It is possible to use it programatically, though.
+  # @see https://github.com/gampleman/Experiment/wiki/Command-Line-Interface
+  # @example For documentation on the CLI run
+  #   experiment -h
   class Runner
     
     attr_reader :options
     
+    # If you are using this programmatically you need to set these params correctly:
+    # @param [Array<String>] arg Typically the name of the experiment the operation
+    #   needs to operate on.
+    # @param [Struct, OpenStruct] opt an options object that should respond according
+    #   to the CLI.
     def initialize(arg, opt)
       @arguments, @options = arg, opt
     end
@@ -51,13 +61,13 @@ module Experiment
       end
       FileUtils::cp File.join(basedir, "generator/readme_template.txt"), File.join(dir, "README")
       FileUtils::cp File.join(basedir, "generator/Rakefile"), File.join(dir, "Rakefile")
-      FileUtils::cp File.join(basedir, "generator/experiment_template.rb"), File.join(dir, "experiments", "experiment.rb")
+      FileUtils::cp File.join(basedir, "generator/experiment_template.rb.txt"), File.join(dir, "experiments", "experiment.rb")
     end
     
     # Lists available experiments
 		def list
 		  puts "Available experiments:"
-		  puts "  " + Dir["./experiments/*"].map{|a| File.basename(a) }.join(", ")
+		  puts "  " + Dir["./experiments/*"].map{|a| File.dirname(a) }.join(", ")
 		end
 		
 		# Generates 2 files in the report directory
@@ -109,14 +119,14 @@ module Experiment
 		end 
 		
 		
-		# runs experiments passed aa arguments
+		# runs experiments passed as arguments
 		# use the -o option to override configuration
 		def run
 		  require File.dirname(__FILE__) + "/base"
 
 		  require "./experiments/experiment"
       Experiment::Config::init @options.env
-		  
+		  @options.cv = Experiment::Config.get :cross_validations, 5 if @options.cv.nil?
 		  if @options.distributed
 		    require "drb/drb"
 		    require File.dirname(__FILE__) + "/work_server"
@@ -130,15 +140,45 @@ module Experiment
 			  @arguments.each do |exp|
   			  require "./experiments/#{exp}/#{exp}"
   			  cla = eval(as_class_name(exp))
-  				experiment = cla.new :normal, exp, @options.opts, @options.env
+  				experiment = cla.new :normal, exp, @options
   				experiment.normal_run! @options.cv
   			end
 			  Notify::done
 		  end
 		end
 		
+		# Creates an IRB console useful for debugging experiments
+		# Loads up the environment for the condition passed
+		def console
+			cla = as_class_name(@arguments.first)	if @arguments.length == 1
+		  File.open("./tmp/irb-setup.rb", 'w') do |f|
+		    f.puts "Experiment::Config::init #{@options.env.inspect}"
+		    f.puts "def reload!"
+		    f.puts "  "
+		    f.puts "end"
+		    if @arguments.length == 1
+		      f.puts "def experiment"
+  		    f.puts "  @experiment ||= #{cla}.new :normal, #{@arguments.first.inspect}, OpenStruct.new(#{@options.marshal_dump})"
+  		    f.puts "end"
+  		    f.puts "experiment #load up the configs"
+  		  else
+  		    f.puts 'Dir["./app/*.rb"].each{|e| require e }'
+  		    f.puts "Experiment::Config::load '', #{options.opts.inspect}"
+		    end
+		    
+		  end
+      irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+      libs =  " -r irb/completion"
+      libs <<  " -r #{File.dirname(__FILE__) + "/base"}"
+      libs << " -r./experiments/experiment"
+      libs << " -r ./experiments/#{@arguments.first}/#{@arguments.first}" if @arguments.length == 1
+      libs << " -r ./tmp/irb-setup.rb"
+      puts "Loading #{@options.env} environment..."
+      exec "#{irb} #{libs} --simple-prompt"
+	  end
+		
 		
-		# This is a Worker implementation. It requires an --address option
+		# Starts a Worker implementation. It requires an --address option
 		# of it's master server and will recieve tasks (experiments and
 		# cross-validations) and compute them.
 		def worker
@@ -155,13 +195,15 @@ module Experiment
           require "./experiments/experiment"
           require "./experiments/#{exp}/#{exp}"
   			  cla = eval(as_class_name(exp))
-  				experiment = cla.new :slave, exp, @options.opts, @options.env
+  				experiment = cla.new :slave, exp, @options
   			  experiment.master = @master.instance item
   			  experiment.slave_run!
         end
       end
 	  end
 		
+		
+		
 		private
 		
 		require 'socket'

data/lib/experiment/stats/descriptive.rb

@@ -0,0 +1,58 @@
+module Experiment
+  module Stats
+    module Descriptive
+
+  		def sum(ar = self, &block)
+  			ar.reduce(0.0) {|asum, a| (block_given? ? yield(a) : a) + asum}
+  	  end
+
+  	  def variance(ar = self)
+  	    v = sum(ar) {|x| (mean(ar) - x)**2.0 }
+        v/(ar.count - 1.0)
+  	  end
+
+  	  def standard_deviation(ar = self)
+  	    Math.sqrt(variance(ar))
+  	  end
+
+  	  def z_scores(ar = self)
+  	    ar.map {|x| z_score(ar, x)}
+  	  end
+
+  	  def z_score(ar = self, x)
+  	    (x - mean(ar)) / standard_deviation(ar)
+  	  end
+
+  		def range(ar = self)
+  			ar.max - ar.min
+  		end
+
+  	  def mean(ar = self)
+  	    sum(ar) / ar.count
+  	  end
+
+  	  def median(ar = self)
+  			a = ar.sort
+  			if ar.count.odd?
+  				a[(ar.count-1)/2]
+  			else
+  				(a[ar.count/2 - 1] + a[ar.count/2]) / 2.0
+  			end
+  	  end
+
+    end
+    
+    class << self
+      	# Monkey pathces the Array class to accept the methods in this class 
+      	# as it's own - so instead of `Stats::variance([1, 2, 3])`
+      	# you can call [1, 2, 3].variance
+    		def monkey_patch!
+    		  Array.send :include, Descriptive
+    		end
+    		
+    		include Descriptive
+    end
+    
+  end
+end
+

data/lib/experiment/work_server.rb

@@ -1,4 +1,8 @@
 module Experiment
+  # This class is responsible for disrtibuting work
+  # and instantiating experimental conditions according
+  # to available resources
+  # @private
   class WorkServer
     def initialize(experiments, options, ip = "localhost")
       uri="druby://#{ip}:8787"
@@ -15,20 +19,29 @@ module Experiment
       DRb.thread.join
     end
     
-    def ready?
+    # @deprecated
+    def ready? # TODO: get rid of this
       true
     end
     
+    # Workers call this method and recieve a new work object incrementally
     def new_item
       @experiments.each_with_index do |e, i|
         if @experiment_instances[i].nil?
           exp = @experiments[i]
           require "./experiments/#{exp}/#{exp}"
   			  cla = eval(as_class_name(exp))
-  				experiment = cla.new :master, exp, @options.opts, @options.env
-  				experiment.master_run! @options.cv
-  				@experiment_instances[i] = experiment
-  				return i
+  				experiment = cla.new :master, exp, @options
+  				if experiment.respond_to? :master_sub_experiments
+  				  subs = experiment.master_sub_experiments @options.cv
+  				  @experiments += subs.map { exp }
+  				  @experiment_instances += subs
+  				  return i + 1
+				  else
+  				  experiment.master_run! @options.cv
+  				  @experiment_instances[i] = experiment
+  				  return i
+  				end
   			elsif !@experiment_instances[i].distribution_done?
           return i
         end
@@ -38,10 +51,12 @@ module Experiment
       false
     end
     
+    # accessor for the remote notification service
     def notify
       Notify
     end
     
+    
     def experiment(num)
       @experiments[num]
     end

data/lib/experiment.rb

@@ -2,5 +2,5 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module Experiment
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/test/test_stats.rb

@@ -1,8 +1,9 @@
-require File.dirname(__FILE__) + "/../lib/experiment/stats"
-#require "wrong"
+require File.dirname(__FILE__) + "/../lib/experiment/stats/descriptive"
+#require "wrong"require "../lib/experiment/stats/descriptive"
+
 class TestStats < Test::Unit::TestCase
   #include Wrong
-  
+  include Experiment
   def setup
     @data = [1, 2, 3, 4]
   end
@@ -31,4 +32,9 @@ class TestStats < Test::Unit::TestCase
   def test_median
     assert_equal 2.5, Stats::median(@data)
   end
+  
+  def test_monkey_patch
+    Stats::monkey_patch!
+    assert_equal 1.6666666666666667, [1, 2, 3, 4].variance
+  end
 end