experiment 0.0.1 → 0.2.0

data/Manifest.txt

@@ -5,11 +5,15 @@ Rakefile
 lib/experiment.rb
 lib/experiment/config.rb
 lib/experiment/stats.rb
+lib/experiment/runner.rb
 lib/experiment/generator/readme_template.txt
 lib/experiment/generator/experiment_template.rb
 lib/experiment/generator/Rakefile
 lib/experiment/base.rb
 lib/experiment/notify.rb
+lib/experiment/work_server.rb
+lib/experiment/distributed.rb
 test/test_experiment.rb
 test/test_helper.rb
-bin/experiment
+bin/experiment
+bin/growl.sh

data/README.rdoc

@@ -27,7 +27,7 @@ Experiments are set up in the experiments directory. The first thing you need to
 
 For a typical experiment you will need to do some setup work (eg. initialize your classes, calculate parametres, etc.), run the experiment and maybe do cleanup (remove temp. files).
 
-You do all this work in the `run_the_experiment` method. Remember to pass the raw output via `<<` to the `output` variable and wrap the experiment in a benchmark block. This will be automatically saved to the results directory for further analysis.
+You do all this work in the `run_the_experiment` method. Use the `measure` method to wrap your measurements. These will be autmatically benchmarked and their ouput will be automatically saved to the results directory for further analysis.
 
 The `test_data` method lets you specify an array of data points that you want split for cross-validation (see below). This will be passed to in `run_the_experiment` in the input variable.
 
@@ -102,7 +102,7 @@ Then your final config will look like this:
       :master_dir => "/Users/kubowo/Desktop/points-vals/s015", 
       :alpha => 0.6 }
 
-Flexible, eh?
+Flexible, eh? **NEW** Check out the *get* method. It has features like interpolation and defaults.
 
 == Cross Validation
 
@@ -116,8 +116,17 @@ Surprise, surprise. This will create two files in your `report` directory (BTW,
 
 The second file created is the `data.csv` file which contains the data from all your experiments. It should be importable to Numbers, Excel even Matlab for further analysis and charting.
 
+
+== Distributed computing support
+
+Newly this library supports a simple distributed model of running experiments. Setup worker computers with the 
+
+    $ experiment worker --address IP_OF_COMPUTER_WHERE_YOU_RUN_EXPERIMENTS
+    
+and then run experiments with --distributed flag.
+
 == Misc
 
-So that's pretty much the gist of experiment. There's a few other features (and a few soon to come to a gem near you ;-) 
+So that's pretty much the gist of experiment. There's a few other features (and a few soon to come to a gem near you ;-) Growl notifications are now supported. Turn them of by setting growl_notifications to false in your config file.
 
 Also check out the RDocs: http://rdoc.info/github/gampleman/Experiment/master/frames

data/Rakefile

@@ -14,7 +14,7 @@ $hoe = Hoe.spec 'experiment' do
   self.developer 'Jakub Hampl', 'honitom@seznam.cz'
   #self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
   self.rubyforge_name       = self.name # TODO this is default value
-  # self.extra_deps         = [['activesupport','>= 2.0.2']]
+  #self.extra_deps         = [['ruby-growl','>= 1.0']]
 
 end
 

data/bin/experiment

@@ -1,47 +1,14 @@
 #!/usr/bin/env ruby
 
-# == Synopsis 
-#   This program will run an experimental batch or generate files
-# 	for a new experiment
-#
-# == Examples
-#   Running an experiment
-#     experiment --env dice experiment1 experiment2 ...
-#
-#   Generating a new experiment with 2 cross validations
-#     experiment new experiment_name --cv 2
-#
-#   List all available experiments
-#     experiment list
-# 	
-#
-# == Usage 
-#   experiment command [options]
-#
-#   For help use: experiment -h
-#
-# == Options
-#   -h, --help          Displays help message
-#   -v, --version       Display the version, then exit
-#   -q, --quiet         Output as little as possible, overrides verbose
-#   -V, --verbose       Verbose output
-#   -e, --env           Sets the environment to run in
-#                       Defaults to development
-#   -c, --cv            Number of Cross validations to run
-#   -m, --description   A description of the current experiment
-#
-# == Author
-#   Jakub Hampl
-#
-
-#require "rubygems"
 require 'optparse'
-#require 'rdoc/usage'
 require 'ostruct'
-#require File.dirname(__FILE__) + "/experiment"
+require "rdoc"
+require "rdoc/rdoc"
+
+require File.dirname(__FILE__) + "/../lib/experiment/runner"
 
 class App
-  VERSION = '1.0'
+  VERSION = '0.2.0'
   
   attr_reader :options
 
@@ -55,11 +22,10 @@ class App
     @options.quiet = false
 		@options.env = :development
 		@options.cv = 5
-		@options.n_classes = 10
-		@options.kind = "d"
 		@options.description = ""
-		@options.opts = {}
-
+		@options.opts = ""
+    @options.distributed = false
+    @options.master = "localhost"
   end
 
   # Parse options, check arguments, then process the command
@@ -69,10 +35,7 @@ class App
       
       puts "Start at #{DateTime.now}\n\n" if @options.verbose
       
-      output_options if @options.verbose # [Optional]
-      require File.dirname(__FILE__) + "/vendor/backports/backports" if @options.env == :dice
-      
-      process_arguments            
+      output_options if @options.verbose # [Optional]         
       process_command
       
       puts "\nFinished at #{DateTime.now}" if @options.verbose
@@ -88,20 +51,20 @@ class App
     def parsed_options?
       
       # Specify options
-      opts = OptionParser.new 
-      opts.on('-v', '--version')    { output_version ; exit 0 }
-      opts.on('-h', '--help')       { output_help }
-      opts.on('-V', '--verbose')    { @options.verbose = true }  
-      opts.on('-q', '--quiet')      { @options.quiet = true }
-			opts.on('-e', '--env [ENV]', [:development, :dice])	{ |v| @options.env = v }
-			opts.on('-c', '--cv CV', Integer)	{ |v| @options.cv = v }
-      opts.on('-n', '--number NUMBER', Integer)	{ |v| @options.n_classes = v }
-      opts.on('-k', '--kind KIND', String)      { |v| @options.kind = v }
-      opts.on('-m', '--description M', String)      { |v| @options.description = v }
-      opts.on('-o', '--options OPTSTRING', String) do |v|
+      @opts = OptionParser.new 
+      @opts.on('-v', '--version')    { output_version ; exit 0 }
+      @opts.on('-h', '--help')       { output_help }
+      @opts.on('-V', '--verbose')    { @options.verbose = true }  
+      @opts.on('-q', '--quiet')      { @options.quiet = true }
+			@opts.on('-e', '--env [ENV]', [:development, :compute], "Sets the environment to run in.")	{ |v| @options.env = v }
+			@opts.on('-c', '--cv CV', Integer, "The number of cross validations to run.")	{ |v| @options.cv = v }
+      @opts.on('-m', '--description M', String, "Description or hypothesis for the condition being generated.")      { |v| @options.description = v }
+      @opts.on('-o', '--options OPTSTRING', String, "Options to override config with. key1:val1,key2:val2") do |v|
         @options.opts = v
       end
-      opts.parse!(@arguments) rescue return false
+      @opts.on('-D', '--distributed', "Run with a distributed computing mode. This will be the master server/work cue.")	{  @options.distributed = true }
+      @opts.on('-a', '--address MODE', String, "Address to the master machine.")	{ |v| @options.master = v }
+      @opts.parse!(@arguments) #rescue return false
       
       process_options
       true      
@@ -122,21 +85,45 @@ class App
 
     # True if required arguments were provided
     def arguments_valid?
-      true if @arguments.length > 0
-    end
-    
-    # Setup the arguments
-    def process_arguments
-      # TO DO - place in local vars, etc
+      true #if @arguments.length > 0
     end
     
     def output_help
       output_version
-      RDoc::usage() #exits app
+      puts "= Synopsis
+This program will run an experimental batch or generate files      
+for a new experiment"
+      puts
+      output_usage
+      puts
+      puts "= Options"
+      puts @opts.help
+      puts
+      puts "= Commands"
+      # Bizzare hax to make RDoc parse the files
+      top_level = RDoc::TopLevel.new File.dirname(__FILE__) + "/../lib/experiment/runner.rb"
+      opts = RDoc::Options.new
+      stats = RDoc::Stats.new 1
+      parser = RDoc::Parser.for top_level, File.dirname(__FILE__) + "/../lib/experiment/runner.rb", File.read(File.dirname(__FILE__) + "/../lib/experiment/runner.rb"), opts, stats
+      d = parser.scan
+      d.modules.first.classes.first.method_list.each do |m|
+        if m.comment != ""
+          puts "== #{m.name == 'new_project' ? 'new' : m.name}"
+          puts m.comment
+          puts
+        end
+      end 
+      
+      puts "----"
+      exit 0
     end
     
     def output_usage
-      RDoc::usage('usage') # gets usage from comments above
+      puts "= Usage 
+experiment command [options]
+      
+For help use: experiment -h"
+      
     end
     
     def output_version
@@ -144,130 +131,19 @@ class App
     end
     
     def process_command
-			if @arguments.first == 'generate'
-			  dir = "./experiments/" + @arguments[1]
-				Dir.mkdir(dir)
-				File.open(dir + "/" + @arguments[1] + ".rb", "w") do |req_file|
-				  req_file.puts "# ## #{as_human_name @arguments[1]} ##"
-				  req_file.puts "# "+@options.description.split("\n").join("\n# ")
-				  req_file.puts
-				  req_file.puts
-				  req_file.puts "# The first contious block of comment will be included in your report."
-				  req_file.puts "# This includes the reference implementation."
-				  req_file.puts "# Override any desired files in this directory."
-				  Dir["./app/**/*.rb"].each do |f|
-				    p = f.split("/") - File.expand_path(".").split("/")
-				    req_file.puts "require File.dirname(__FILE__) + \"/../../#{p.join("/")}\""
-				  end
-				  req_file.puts "\nclass #{as_class_name @arguments[1]} < MyExperiment\n\t\nend"
-				end
-				File.open(dir + "/config.yaml", "w") do |f|
-				  f << "---\nexperiment:\n  development:\n  compute:\n"
-        end
-        
-      elsif @arguments.first == "new" # generate a new project
-        require 'fileutils'
-        dir = "./" + @arguments[1]
-        Dir.mkdir(dir)
-        %w[app config experiments report results test tmp vendor].each do |d|
-          Dir.mkdir(dir + "/" + d)
-        end
-        basedir = File.dirname(__FILE__) + "/.."
-        File.open(File.join(dir, "config", "config.yaml"), "w") do |f|
-          f << "---\nenvironments:\n  development:\n  compute:\n"
-        end
-        File.open(File.join(dir, ".gitignore"), "w") do |f|
-          f << "tmp/*"
-        end
-        FileUtils::cp File.join(basedir, "lib/experiment/generator/readme_template.txt"), File.join(dir, "README")
-        FileUtils::cp File.join(basedir, "lib/experiment/generator/Rakefile"), File.join(dir, "Rakefile")
-        FileUtils::cp File.join(basedir, "lib/experiment/generator/experiment_template.rb"), File.join(dir, "experiments", "experiment.rb")
-			elsif @arguments.first == "list"
-			  puts "Available experiments:"
-			  puts "  " + Dir["./experiments/*"].map{|a| File.basename(a) }.join(", ")
-			elsif @arguments.first == "report"
-			  dir = "./report/"
-        File.open(dir + "method.mmd", "w") do |f|
-          f.puts "# Methods #"
-          Dir["./experiments/*/*.rb"].each do |desc|
-            if File.basename(desc) == File.basename(File.dirname(desc)) + ".rb"
-              File.read(desc).split("\n").each do |line|
-                if m = line.match(/^\# (.+)/)
-                  f.puts m[1]
-                else
-                  break
-                end
-              end
-            f.puts
-            f.puts
-            end
-          end
-        end
-        require 'csv'
-        require "yaml"
-        require File.dirname(__FILE__)+"/../lib/experiment/stats"
-        CSV.open(dir + "/data.csv", "w") do |csv|
-          data = {}
-          Dir["./results/*/results.yaml"].each do |res|
-            d = YAML::load_file(res)
-            da = {}
-            d.each do |k, vals|
-              da[k.to_s + " mean"], da[k.to_s + " sd"] = Stats::mean(vals), Stats::standard_deviation(vals)
-              vals.each_with_index do |v, i|
-                da[k.to_s + " cv:" + i.to_s] = v
-              end
-            end
-            array_merge(data, da)
-          end
-          data.keys.map do |key| 
-      		  # calculate stats
-      		  a = data[key]
-      		  [key] + a
-    		  end.transpose.each do |row|
-    		    csv << row
-    		  end
-        end
-			elsif @arguments.shift == "run"
-			  require File.dirname(__FILE__) + "/../lib/experiment/base"
-			  require "experiments/experiment"
-				@arguments.each do |exp|
-				  require "./experiments/#{exp}/#{exp}"
-				  cla = eval(as_class_name(exp))
-					experiment = cla.new exp, @options.opts, @options.env
-					experiment.run! @options.cv
-				end
-			else
-			  output_usage
-			end
-      
-    end
-    
-    
-    private
-    def array_merge(h1, h2)
-  	  h2.each do |key, value|
-  	   h1[key] ||= []
-  	   h1[key] << value
-  	  end
-	  end
-    
-    def as_class_name(str)
-      str.split(/[\_\-]+/).map(&:capitalize).join
-    end
-    
-    def as_human_name(str)
-      str.split(/[\_\-]+/).map(&:capitalize).join(" ")
+      command = @arguments.shift
+      runner = Experiment::Runner.new @arguments, options
+      command = "new_project" if command == 'new'
+      #begin
+        runner.send command.to_sym
+      #rescue NoMethodError => e
+      #  puts "Wrong input #{e.inspect}"
+      #  output_usage
+      #end
     end
 
     def process_standard_input
       input = @stdin.read      
-      # TO DO - process input
-      
-      # [Optional]
-      # @stdin.each do |line| 
-      #  # TO DO - process each line
-      #end
-
     end
 end
 

data/bin/growl.sh

@@ -0,0 +1,241 @@
+#!/bin/sh
+#-----------------------------------------------------------------------------
+# growl - Send the a message to Mac OS X Growl <http://growl.info> via a
+#         Unix shell script.  
+#
+#         And if this is a Linux or Windows_NT box, use ssh to forward the
+#         growl notice to my Mac OS X workstation (taking advantage of
+#         previously exchanged ssh so no additional authentication is
+#         needed).
+#
+#         This script should be somewhere in your PATH.  I like to use
+#         $HOME/bin for personal scripts, and /usr/local/bin for system
+#         wide scripts, with appropriate entries added to PATH.  The script
+#         needs to be executable:
+#
+#             chmod +x $HOME/bin/growl
+#
+#         The basics for the Growl Applescript came from the Growl
+#         documentation at:
+#             <http://growl.info/documentation/applescript-support.php>
+#-----------------------------------------------------------------------------
+# Bob Harris - 2-Jun-2007
+#-----------------------------------------------------------------------------
+
+
+#--- usage -------------------------------------------------------------------
+usage()
+{
+    echo 1>&2 ""
+    echo 1>&2 "Usage:  growl [options] \"message to display\"" 
+    echo 1>&2 ""
+    echo 1>&2 "        -sticky     - Stays on screen until dismissed [Default]."
+    echo 1>&2 "        -nosticky   - Goes away after several seconds."
+    echo 1>&2 "        -priority n - Priority -2,-1,0,1,2 [Default: 0]"
+    echo 1>&2 "        -verylow    - priority [-2]"
+    echo 1>&2 "        -moderate   - priority [-1]"
+    echo 1>&2 "        -normal     - priority  [0] [Default]"
+    echo 1>&2 "        -high       - priority  [1]"
+    echo 1>&2 "        -emergency  - priority  [2]"
+    echo 1>&2 ""
+    echo 1>&2 "    Interesting environment variables:"
+    echo 1>&2 "        G_TITLE            - Used as the Growl message title."
+    echo 1>&2 "        G_APPLICATION_NAME - Used by Growl to manage a set of"
+    echo 1>&2 "                             G_WITH_NAME message configurations."
+    echo 1>&2 "                             See System Preferences -> Growl"
+    echo 1>&2 "        G_ALL_NAMES        - Used to specify 'ALL' the possible"
+    echo 1>&2 "                             G_WITH_NAME values this"
+    echo 1>&2 "                             G_APPLICATION_NAME will ever use."
+    echo 1>&2 "                             Specify something like:"
+    echo 1>&2 "                             G_ALL_NAMES='\"class1\",\"class2\"'"
+    echo 1>&2 "                             export G_ALL_NAMES"
+    echo 1>&2 "        G_WITH_NAME        - Used by Growl to associate a"
+    echo 1>&2 "                             message with a set of Growl"
+    echo 1>&2 "                             notification settings, such as"
+    echo 1>&2 "                             message style and colors associated"
+    echo 1>&2 "                             with different priorities.  See"
+    echo 1>&2 "                             System preferences -> Growl"
+    echo 1>&2 "                             Specify something like:"
+    echo 1>&2 "                             G_WITH_NAME='class2'"
+    echo 1>&2 "                             export G_WITH_NAME"
+    echo 1>&2 "                             If G_WITH_NAME is not in"
+    echo 1>&2 "                             G_ALL_NAMES, nothing will be"
+    echo 1>&2 "                             displayed"
+    echo 1>&2 "        G_APPLICATION_ICON - Display this application's icon in"
+    echo 1>&2 "                             the Growl message (default is"
+    echo 1>&2 "                             Terminal.app, as this is a shell"
+    echo 1>&2 "                             script generated Growl notice)."
+    echo 1>&2 ""
+    exit 1
+}
+
+
+#--- defaults ----------------------------------------------------------------
+defaults()
+{
+#
+# These 2 variables are used when this script notices that it is on a Linux
+# or Windows_NT system, then this script uses ssh to forward the request to
+# my Mac OS X workstation.  This assumes that ssh keys have been exchanged
+# between the systems so that no passwords are needed.  If you don't know
+# what this means, then do a Google search for something like 
+# "ssh nopassword key" and you should turn up a number of guides for
+# configuring ssh.
+#
+G_REMOTE="${G_REMOTE:-juggle-mac.us.oracle.com}" # assumes ssh exchanged keys
+G_REMOTE_GROWL="${G_REMOTE_GROWL:-bin/growl}"    # growl script remote location
+
+#
+# G_APPLICATION_NAME is the name Growl will use in the Growl System
+# Preferences to provide defaults for Display, and to collect each of your
+# notification classes (See G_ALL_NAMES below).
+#
+G_APPLICATION_NAME="${G_APPLICATION_NAME:-Shell Script Growl Message}"
+#
+# G_ALL_NAMES contains a list of all the notification classes to be
+# associated with the G_APPLICATION_NAME.  Each notification class can have
+# its own default Display, Priority, and Stickness settings in the Growl
+# System Preferences.
+#
+G_ALL_NAMES="${G_ALL_NAMES:-\"Shell Script Growl Message\",\"Growl Message\"}"
+#
+# The default notification class this message should use, must be in the
+# G_ALL_NAMES list above.
+#
+G_WITH_NAME="${G_WITH_NAME:-Shell Script Growl Message}" # default notification
+#
+G_TITLE="${G_TITLE:-Shell Script Growl Message}"         # default title
+G_APPLICATION_ICON="${G_APPLICATION_ICON:-Terminal.app}" # default icon to use
+G_STICKY="${G_STICKY:-yes}"                      # default sticky setting
+G_PRIORITY="${G_PRIORITY:-0}"                    # default priority (normal)
+}
+
+
+#--- growl -------------------------------------------------------------------
+# notify v : Post a notification to be displayed via Growl
+#   notify
+#     with name string          : name of the notification to display
+#     title string              : title of the notification to display
+#     description string        : full text of the notification to display
+#     application name string   : name of the application posting the
+#                                 notification.
+#     [image from location location_reference] 
+#                               : Location of the image file to use for this
+#                                 notification. Accepts aliases, paths and
+#                                 file:/// URLs.
+#     [icon of file location_reference] 
+#                               : Location of the file whose icon should be
+#                                 used as the image for this notification.
+#                                 Accepts aliases, paths and file:/// URLs.
+#                                 e.g. 'file:///Applications'.
+#     [icon of application string] 
+#                               : Name of the application whose icon should
+#                                 be used for this notification.  For
+#                                 example, 'Mail.app'.
+#     [image Image]             : TIFF Image to be used for the
+#                                 notification.
+#     [pictImage Picture]       : PICT Image to be used for the
+#                                 notification.
+#     [sticky boolean]          : whether or not the notification displayed
+#                                 should time out. Defaults to 'no'.
+#     [priority integer]        : The priority of the notification, from -2
+#                                 (low) to 0 (normal) to 2 (emergency).
+# 
+growl()
+{
+ typeset description="$*"
+
+ osascript <<EOD
+  -- From <http://growl.info/documentation/applescript-support.php>
+  --
+  tell application "GrowlHelperApp"
+     -- Make a list of all the notification types that this script will ever send:
+     set the allNotificationsList to {${G_ALL_NAMES}}
+
+     -- Make a list of the notifications that will be enabled by default.      
+     -- Those not enabled by default can be enabled later in the 'Applications'
+     -- tab of the growl prefpane.
+     set the enabledNotificationsList to {"${G_WITH_NAME}"}
+
+     -- Register our script with growl.  You can optionally (as here) set a
+     -- default icon for this script's notifications.
+     register as application "${G_APPLICATION_NAME}" all notifications allNotificationsList default notifications enabledNotificationsList icon of application "${G_APPLICATION_ICON}"
+             
+     -- Send a Notification...
+     notify with name "${G_WITH_NAME}" title "${G_TITLE}" description "${description}" application name "${G_APPLICATION_NAME}" sticky ${G_STICKY} priority ${G_PRIORITY}
+
+  end tell
+EOD
+}
+
+
+#--- main --------------------------------------------------------------------
+#{
+    if [[ $# = 0 ]]; then
+        #
+        # No arguments, so give the usage message.
+        #
+        usage
+        exit 1
+    fi
+    while [[ "X$1" = X-* ]]
+    do
+        if [[ "X$1" = X-nos* ]]; then
+            G_STICKY=no
+        elif [[ "X$1" = X-s* ]]; then
+            G_STICKY=yes
+        elif [[ "X$1" = X-p* ]]; then
+            G_PRIORITY="$2"
+            G_TITLE="${G_TITLE:-Priority $2}"
+            OPTIONS="$OPTIONS $1"
+            shift
+        elif [[ "X$1" = X-v* ]]; then
+            G_PRIORITY="-2"
+            G_TITLE="${G_TITLE:-Very Low Priority}"
+        elif [[ "X$1" = X-m* ]]; then
+            G_PRIORITY="-1"
+            G_TITLE="${G_TITLE:-Moderate Priority}"
+        elif [[ "X$1" = X-n* ]]; then
+            G_PRIORITY="0"
+            G_TITLE="${G_TITLE:-Normal Priority}"
+        elif [[ "X$1" = X-h* ]]; then
+            G_PRIORITY="1"
+            G_TITLE="${G_TITLE:-High Priority}"
+        elif [[ "X$1" = X-e* ]]; then
+            G_PRIORITY="2"
+            G_TITLE="${G_TITLE:-Emergency Priority}"
+        else
+            break;
+        fi
+        OPTIONS="$OPTIONS $1"
+        shift
+    done
+
+    #
+    # If any of the option variables have not been set yet, then apply the
+    # default values now.
+    #
+    defaults
+
+    UNAME=$(uname)
+    if [[ "$UNAME" = Darwin ]]; then
+        #
+        # I'm assuming this is one of my systems where I have Growl
+        # installed.
+        #
+        growl "$*"
+    elif [[ "$UNAME" = *Linux* || "$UNAME" = *Windows_NT* ]]; then
+        #
+        # Must be one of the development systems I work on, so lets ship
+        # this request to my Mac OS X workstation.
+        #
+        ssh ${G_REMOTE} ${G_REMOTE_GROWL} $OPTIONS "$*"
+    else
+        #
+        # I don't know what this is, so I'll just try to Growl anyway.  It
+        # will most likely fail (no osascript would be my guess), but what
+        # do I have to loose at this point!
+        #
+        growl "$*"
+    fi
+#}