experiment 0.0.1 → 0.2.0

data/lib/experiment.rb

@@ -2,5 +2,5 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module Experiment
-  VERSION = '0.0.1'
+  VERSION = '0.2.0'
 end

data/lib/experiment/base.rb

@@ -1,25 +1,45 @@
 require File.dirname(__FILE__) + "/notify"
 require File.dirname(__FILE__) + "/stats"
 require File.dirname(__FILE__) + "/config"
+require File.dirname(__FILE__) + "/distributed"
 require 'benchmark'
+require "drb/drb"
 
 module Experiment
   class Base
-    attr_reader :dir, :current_cv, :cvs
     
-  	def initialize(experiment, options, env)
+    include Distributed
+    
+    attr_reader :dir, :current_cv, :cvs
+
+  	def initialize(mode, experiment, options, env)
   		@experiment = experiment
-  		
+  		case mode
+  		  
+  		when :normal
+  		  @abm = [] 
+		  when :master
+		    @abm = []
+		    extend DRb::DRbUndumped
+		    @done = false
+	    when :slave
+		    
+  		end
   		Experiment::Config::load(experiment, options, env)
-  		require "./experiments/#{experiment}/#{experiment}"
-  		@abm = []
+  		@mode = mode
   	end
+  	
+  	def done?
+  	  @done
+	  end
+  	
+  	
     
     # runs the whole experiment
-  	def run!(cv)
+  	def normal_run!(cv)
   		@cvs = cv || 1
       @results = {}
-  		Notify.print "Running #{@experiment} "
+  		Notify.started @experiment
       split_up_data
   		write_dir!
   		specification!
@@ -28,21 +48,39 @@ module Experiment
   			@bm = []
   			@current_cv = cv_num
   			File.open(@dir + "/raw-#{cv_num}.txt", "w") do |output|
+  			  @ouptut_file = output
   			    run_the_experiment(@data[cv_num], output)
   			end
   			array_merge @results, analyze_result!(@dir + "/raw-#{cv_num}.txt", @dir + "/analyzed-#{cv_num}.txt")
   			write_performance!
-  			Notify.print "."
+  			Notify.cv_done @experiment, cv_num
   		end
   		summarize_performance!
   		summarize_results! @results
-  		Notify.print result_line
+  		Notify.completed @experiment
   	end
-
-
+    
+    
+    # use this evry time you want to do a measurement.
+    # It will be put on the record file and benchmarked
+    # automatically
+    # The weight parameter is used for calculating 
+    # Notify::step. It should be an integer denoting how many 
+    # such measurements you wish to do.
+    def measure(label = "", weight = nil, &block)
+      out = ""
+      benchmark label do
+        out = yield
+      end
+      @ouptut_file << out
+      Notify::step(@experiment, @current_cv, 1.0/weight) unless weight.nil?
+    end
+    
+    
     # Registers and performs a benchmark which is then 
     # calculated to the total and everage times
   	def benchmark(label = "", &block)
+  	  @bm ||= []
   	  @bm << Benchmark.measure("CV #{@current_cv} #{label}", &block)
   	end
 
@@ -70,6 +108,7 @@ module Experiment
   			f << @bm.map {|m| m.format("%19n "+Benchmark::FMTSTR)}.join
   			total = @bm.reduce(0) {|t, m| m + t}
   			f << total.format("         Total: "+Benchmark::FMTSTR)
+  			@abm ||= []
   			@abm << total
   		end
   	end
@@ -85,7 +124,7 @@ module Experiment
     # creates a summary of the results and writes to 'all.csv'
   	def summarize_results!(results)
   	  File.open(@dir + '/results.yaml', 'w' ) do |out|
-  			YAML.dump(results, out )
+  			YAML.dump(results, out)
   		end
   		
   		# create an array of arrays
@@ -99,7 +138,6 @@ module Experiment
   		
   		ls = ["Standard Deviation".length] + ls
   		res = [["cv"] + (1..cvs).to_a.map(&:to_s) + ["Mean", "Standard Deviation"]] + res
-  		
   		out = ""
   		res.transpose.each do |col|
   		  col.each_with_index do |cell, i|

data/lib/experiment/config.rb

@@ -9,7 +9,8 @@ module Experiment
 	    # the options string (which should be in this format:
 	    # "key: value, key2:value2,key3: value3")
 	    def load(experiment, options, env = :development)
-	      init env
+	      #init env
+	      @config ||= {}
   			expath = File.expand_path("./experiments/#{experiment}/config.yaml")
   			if File.exists? expath
   				exp = YAML::load_file(expath)
@@ -17,20 +18,48 @@ module Experiment
   			end
   			@config.merge! parse(options)
   		end
-
+      
+      # loads the main config file
   		def init(env = :development)
   		  conf = YAML::load_file("./config/config.yaml")
   		  @config = conf["environments"][env.to_s]
   		end
-
+      
+      
+      # Allows access to any config option by key (either String or Symbol)
   		def [](v)
   			@config[v.to_s]
   		end
   		
+  		# Allows access to any config option by key. Supports Interploations.
+  		# Interpolations are supported as opts argument
+  		# words preceded with a colon (:) are interpolated
+  		# Otionaly second argument may be a default value to use if option
+  		# not present.
+  		def get(v, *opts)
+  		  default = opts.shift if opts.length == 2 || !opts.first.is_a?(Hash)
+  		  out = @config[v.to_s] || default
+  		  if opts = opts.first
+          opts.keys.reduce(out.dup) do |result, inter|
+            result.gsub /:#{inter}/, opts[inter]
+          end
+        else
+  			  out
+			  end
+		  end
+		  
+		  def set(opts)
+		    @config ||= opts
+		    @config.merge opts
+	    end
+		  
+		  # parses a string as passed into the CLI -o option
   		def parse(options)
+  		  return {} if options == ""
   		  Hash[options.split(/\, ?/).map{|a| a.split /\: ?/ }]
 		  end
 		  
+		  # returns current options as a Hash object
 		  def to_h
 		    @config
 	    end

data/lib/experiment/distributed.rb

@@ -0,0 +1,67 @@
+module Experiment
+module Distributed
+  attr_accessor :master
+  def get_work()
+	  if cv = @started.index(false)
+	    @started[cv] = true
+	    {:cv => cv, :input => @data[cv], :dir => @dir, :options => Experiment::Config.to_h }
+	  else
+	    false
+    end
+  end
+  
+  def distribution_done?
+    @started.all?
+  end
+  
+  def submit_result(cv, result, performance)
+    @completed[cv] = true
+    array_merge(@results, result)
+    @abm << performance
+    Notify.cv_done @experiment, cv
+    master_done! if @completed.all?
+  end
+  
+  
+  def slave_run!
+    while work = @master.get_work
+      puts work.inspect
+      Experiment::Config.set work[:options]
+      @current_cv = work[:cv]
+
+      @dir = work[:dir]
+      File.open(@dir + "/raw-#{@current_cv}.txt", "w") do |output|
+			  @ouptut_file = output
+			  run_the_experiment(work[:input], output)
+			end
+			result = analyze_result!(@dir + "/raw-#{@current_cv}.txt", @dir + "/analyzed-#{@current_cv}.txt")
+			write_performance!
+			@master.submit_result @current_cv, result, @abm.first
+    end
+
+  end
+  
+  
+  def master_run!(cv)
+    
+    @cvs = cv || 1
+    @results = {}
+		Notify.started @experiment
+    split_up_data
+		write_dir!
+		specification!
+		@completed = (1..@cvs).map {|a| false }
+		@started = @completed.dup
+  end
+  
+  def master_done!
+    @done = true
+    summarize_performance!
+		summarize_results! @results
+		Notify.completed @experiment
+		
+		#sleep 1
+    #DRb.stop_service
+  end
+end
+end

data/lib/experiment/notify.rb

@@ -1,6 +1,180 @@
+# This class is responsible for UI goodness in letting you know 
+# about the progress of your experiments
+require "drb/drb"
 class Notify
-  def self.method_missing(meth, *args, &blk)
-    $stdout.sync = true
-    $stdout.send meth, *args, &blk
+  
+  class << self
+    include DRb::DRbUndumped
+    # initialize display
+    def init(total, out = STDERR, growl = true, mode = :normal)
+      @curent_experiment = ""
+      @current_cv = 0
+      @cv_prog = {}
+      @total = total
+      @out = out
+      @terminal_width = 80
+      @bar_mark = "o"
+      @current = 0
+      @previous = 0
+      @finished_p = false
+      @start_time = Time.now
+      @previous_time = @start_time
+      @growl = growl
+      @mode = mode
+      show if @mode == :normal
+    end
+    
+    # Called when starting work on a particular experiment
+    def started(experiment)
+      @curent_experiment = experiment
+      @current_cv = 1
+      @cv_prog[experiment] = []
+      show_if_needed
+    end
+    
+    # Called when experiment completed. 
+    # Shows a Growl notification on OSX.
+    # The message can be expanded by overriding the result_line 
+    # method in the experiment class
+    def completed(experiment, msg = "")
+      if @growl
+        begin
+          `G_TITLE="Experiment Complete" #{File.dirname(__FILE__)}/../../bin/growl.sh -nosticky "Experimental condition #{experiment} complete. #{msg}"`
+        rescue
+          # probably not on OSX
+        end
+      end
+      m = "Condition #{experiment} complete. #{msg}"
+      puts m + " " * @terminal_width
+      @curent_experiment = nil
+    end
+    
+    # called after a crossvalidation has completed
+    def cv_done(experiment, num)
+      @cv_prog[experiment][num] ||= 0
+      inc(1 - @cv_prog[experiment][num])
+      #@cv_prog = 0
+    end
+    
+    # Wrap up
+    def done
+      @current = @total
+      @finished_p = true
+      #show
+    end
+    
+    # Use this in experiment after each (potentially time consuming) task
+    # The argument should be a fraction (0 < num < 1) which tells 
+    # how big a portion the task was of the complete run (eg. your 
+    # calls should sum up to 1).
+    def step(experiment, cv, num)
+      if @mode == :normal
+        if num > 1
+          num = num / 100
+        end
+        inc(num)
+        @cv_prog[experiment][cv] ||= 0
+        @cv_prog[experiment][cv] += num
+      else
+        @mode.notify.step(experiment, cv, num)
+      end
+    end
+    
   end
+  
+  # a big part of this module is copied/inspired by Satoru Takabayashi's <satoru@namazu.org> ProgressBar class at http://0xcc.net/ruby-progressbar/index.html.en
+  module ProgressBar #:nodoc
+    def inc(step = 1)
+      @current += step
+      @current = @total if @current > @total
+      show_if_needed
+      @previous = @current
+    end
+    
+    def show_if_needed
+      if @total.zero?
+        cur_percentage = 100
+        prev_percentage = 0
+      else
+        cur_percentage  = (@current  * 100 / @total).to_i
+        prev_percentage = (@previous * 100 / @total).to_i
+      end
+      @finished_p = cur_percentage == 100
+      # Use "!=" instead of ">" to support negative changes
+      if cur_percentage != prev_percentage || 
+          Time.now - @previous_time >= 1 || @finished_p
+        show
+      end
+    end
+    
+    
+    
+    def show
+      percent = @current  * 100 / @total
+      bar_width = percent * @terminal_width / 100
+      line = sprintf "%3d%% |%s%s| %s", percent, "=" * bar_width, "-" * (@terminal_width - bar_width), stat
+      
+
+      width = get_width
+      if line.length == width - 1 
+        @out.print(line +  (@finished_p ? "\n" : "\r"))
+        @out.flush
+      elsif line.length >= width
+        @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+        if @terminal_width == 0 then @out.print(line + eol) else show end
+      else # line.length < width - 1
+        @terminal_width += width - line.length + 1
+        show
+      end
+      @previous_time = Time.now
+    end
+    
+    def stat
+      if @finished_p then elapsed else eta end
+    end
+    
+    def eta
+      if @current == 0
+        "ETA:  --:--:--"
+      else
+        elapsed = Time.now - @start_time
+        eta = elapsed * @total / @current - elapsed;
+        sprintf("ETA:  %s", format_time(eta))
+      end
+    end
+
+    def elapsed
+      elapsed = Time.now - @start_time
+      sprintf("Time: %s", format_time(elapsed))
+    end
+    
+    def format_time (t)
+      t = t.to_i
+      sec = t % 60
+      min  = (t / 60) % 60
+      hour = t / 3600
+      sprintf("%02d:%02d:%02d", hour, min, sec);
+    end
+    
+    
+    def get_width
+       # FIXME: I don't know how portable it is.
+       default_width = 80
+       begin
+         tiocgwinsz = 0x5413
+         data = [0, 0, 0, 0].pack("SSSS")
+         if @out.ioctl(tiocgwinsz, data) >= 0 then
+           rows, cols, xpixels, ypixels = data.unpack("SSSS")
+           if cols >= 0 then cols else default_width end
+         else
+           default_width
+         end
+       rescue Exception
+         default_width
+       end
+     end
+  end
+  
+  extend ProgressBar
+  
 end

data/lib/experiment/runner.rb

@@ -0,0 +1,208 @@
+module Experiment
+  
+  # This is the class behind the command line magic
+  class Runner
+    
+    attr_reader :options
+    
+    def initialize(arg, opt)
+      @arguments, @options = arg, opt
+    end
+    
+    
+    # Generates a new experiment condition
+    # Usage of the -m flag for writing a hypothesis is recommended 
+    def generate
+		  dir = "./experiments/" + @arguments.first
+			Dir.mkdir(dir)
+			File.open(dir + "/" + @arguments.first + ".rb", "w") do |req_file|
+			  req_file.puts "# ## #{as_human_name @arguments.first} ##"
+			  req_file.puts "# "+@options.description.split("\n").join("\n# ")
+			  req_file.puts
+			  req_file.puts
+			  req_file.puts "# The first contious block of comment will be included in your report."
+			  req_file.puts "# This includes the reference implementation."
+			  req_file.puts "# Override any desired files in this directory."
+			  Dir["./app/**/*.rb"].each do |f|
+			    p = f.split("/") - File.expand_path(".").split("/")
+			    req_file.puts "require File.dirname(__FILE__) + \"/../../#{p.join("/")}\""
+			  end
+			  req_file.puts "\nclass #{as_class_name @arguments.first} < MyExperiment\n\t\nend"
+			end
+			File.open(dir + "/config.yaml", "w") do |f|
+			  f << "---\nexperiment:\n  development:\n  compute:\n"
+      end
+    end
+    
+    # generate a new project in the current directory
+    def new_project 
+      require 'fileutils'
+      dir = "./" + @arguments.first
+      Dir.mkdir(dir)
+      %w[app config experiments report results test tmp vendor].each do |d|
+        Dir.mkdir(dir + "/" + d)
+      end
+      basedir = File.dirname(__FILE__)
+      File.open(File.join(dir, "config", "config.yaml"), "w") do |f|
+        f << "---\nenvironments:\n  development:\n  compute:\n"
+      end
+      File.open(File.join(dir, ".gitignore"), "w") do |f|
+        f << "tmp/*"
+      end
+      FileUtils::cp File.join(basedir, "generator/readme_template.txt"), File.join(dir, "README")
+      FileUtils::cp File.join(basedir, "generator/Rakefile"), File.join(dir, "Rakefile")
+      FileUtils::cp File.join(basedir, "generator/experiment_template.rb"), File.join(dir, "experiments", "experiment.rb")
+    end
+    
+    # Lists available experiments
+		def list
+		  puts "Available experiments:"
+		  puts "  " + Dir["./experiments/*"].map{|a| File.basename(a) }.join(", ")
+		end
+		
+		# Generates 2 files in the report directory
+		# method.mmd which sums up comments from experimental conditions
+		# data.csv which sums all results in a table
+		def report
+		  dir = "./report/"
+      File.open(dir + "method.mmd", "w") do |f|
+        f.puts "# Methods #"
+        Dir["./experiments/*/*.rb"].each do |desc|
+          if File.basename(desc) == File.basename(File.dirname(desc)) + ".rb"
+            File.read(desc).split("\n").each do |line|
+              if m = line.match(/^\# (.+)/)
+                f.puts m[1]
+              else
+                break
+              end
+            end
+          f.puts
+          f.puts
+          end
+        end
+      end
+      require 'csv'
+      require "yaml"
+      require File.dirname(__FILE__) + "/stats"
+      CSV.open(dir + "/data.csv", "w") do |csv|
+        data = {}
+        Dir["./results/*/results.yaml"].each do |res|
+          d = YAML::load_file(res)
+          da = {}
+          d.each do |k, vals|
+            da[k.to_s + " mean"], da[k.to_s + " sd"] = Stats::mean(vals), Stats::standard_deviation(vals)
+            vals.each_with_index do |v, i|
+              da[k.to_s + " cv:" + i.to_s] = v
+            end
+          end
+          array_merge(data, da)
+        end
+        data.keys.map do |key| 
+    		  # calculate stats
+    		  a = data[key]
+    		  [key] + a
+  		  end.transpose.each do |row|
+  		    csv << row
+  		  end
+      end
+		
+		end 
+		
+		
+		# runs experiments passed aa arguments
+		# use the -o option to override configuration
+		def run
+		  require File.dirname(__FILE__) + "/base"
+
+		  require "./experiments/experiment"
+      Experiment::Config::init @options.env
+		  
+		  if @options.distributed
+		    require "drb/drb"
+		    require File.dirname(__FILE__) + "/work_server"
+		    puts "Running in distributed mode. Run other machines with:\nexperiment worker --address #{local_ip}\n"
+		    Notify::init @arguments.length * @options.cv, STDOUT, Experiment::Config::get(:growl_notifications, true)
+		    ws = WorkServer.new @arguments, @options, local_ip
+		    Notify::done
+		    return true
+  	  else
+  	    Notify::init @arguments.length * @options.cv, STDOUT, Experiment::Config::get(:growl_notifications, true)
+			  @arguments.each do |exp|
+  			  require "./experiments/#{exp}/#{exp}"
+  			  cla = eval(as_class_name(exp))
+  				experiment = cla.new :normal, exp, @options.opts, @options.env
+  				experiment.normal_run! @options.cv
+  			end
+			  Notify::done
+		  end
+		end
+		
+		
+		# This is a Worker implementation. It requires an --address option
+		# of it's master server and will recieve tasks (experiments and
+		# cross-validations) and compute them.
+		def worker
+		  require "drb/drb"
+		  require File.dirname(__FILE__) + "/base"
+		  Experiment::Config::init @options.env
+		  loop do
+		    @server_uri="druby://#{@options.master}:8787"
+  		  connect
+  		  Notify::init 0, STDOUT, false, @master
+        while item = @master.new_item
+          #puts item
+          exp = @master.experiment item
+          require "./experiments/experiment"
+          require "./experiments/#{exp}/#{exp}"
+  			  cla = eval(as_class_name(exp))
+  				experiment = cla.new :slave, exp, @options.opts, @options.env
+  			  experiment.master = @master.instance item
+  			  experiment.slave_run!
+        end
+      end
+	  end
+		
+		private
+		
+		require 'socket'
+
+    def connect
+      begin
+        puts "Connecting..."
+        DRb.start_service
+        @master = DRbObject.new_with_uri(@server_uri)
+        @master.ready?
+      rescue
+        sleep 10
+        connect
+      end
+    end
+
+    def local_ip
+      orig, Socket.do_not_reverse_lookup = Socket.do_not_reverse_lookup, true  
+      UDPSocket.open do |s|
+        s.connect '64.233.187.99', 1
+        s.addr.last
+      end
+    ensure
+      Socket.do_not_reverse_lookup = orig
+    end
+    
+		
+    def array_merge(h1, h2)
+  	  h2.each do |key, value|
+  	   h1[key] ||= []
+  	   h1[key] << value
+  	  end
+	  end
+    
+    def as_class_name(str)
+      str.split(/[\_\-]+/).map(&:capitalize).join
+    end
+    
+    def as_human_name(str)
+      str.split(/[\_\-]+/).map(&:capitalize).join(" ")
+    end
+  end
+
+end