experiment 0.0.1

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.0.1 2010-10-29
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,15 @@
+History.txt
+Manifest.txt
+README.rdoc
+Rakefile
+lib/experiment.rb
+lib/experiment/config.rb
+lib/experiment/stats.rb
+lib/experiment/generator/readme_template.txt
+lib/experiment/generator/experiment_template.rb
+lib/experiment/generator/Rakefile
+lib/experiment/base.rb
+lib/experiment/notify.rb
+test/test_experiment.rb
+test/test_helper.rb
+bin/experiment

data/README.rdoc

@@ -0,0 +1,123 @@
+= Experiment
+* http://github.com/gampleman/experiment
+
+== What's it about?
+
+Experiment is a ruby library and environment for running scientific experiments (eg. AI, GA...), especially good for experiments in optimizing results by variations in algorithm or parameters.
+
+== Installation
+
+    $ sudo gem install experiment
+
+== Getting started
+
+Experiment is modeled after rails and the workflow should be recognizable enough.
+
+First start by generating your project:
+
+    $ experiment new my_project
+
+This will create several files and directories. We will shortly introduce you to these.
+
+First off is the `app` directory. This is where a basic implementation of what you mean to do. You can write your code however you want, just make sure the code is well structured - you will be overriding this later in your experiments.
+
+== Setting up an experiment
+
+Experiments are set up in the experiments directory. The first thing you need to do is define what consist an experiment in your case. For this open up the file `experiments/experiment.rb`. You will notice that this file contains a bunch of comments and a stub letting you easily understand what to do.
+
+For a typical experiment you will need to do some setup work (eg. initialize your classes, calculate parametres, etc.), run the experiment and maybe do cleanup (remove temp. files).
+
+You do all this work in the `run_the_experiment` method. Remember to pass the raw output via `<<` to the `output` variable and wrap the experiment in a benchmark block. This will be automatically saved to the results directory for further analysis.
+
+The `test_data` method lets you specify an array of data points that you want split for cross-validation (see below). This will be passed to in `run_the_experiment` in the input variable.
+
+Next you may want to analyze the data you got. For that there is the `analyze_result!` method which has 2 arguments. One is the raw data file that was output by your code and the other is the path to an expected output file (this can be very rich in detail, ideal for confusion matrices and the like). The method should return a hash of summary results (eg. `:total_performance => 16`).
+
+All of this will be also saved to disk and available for later analysis.
+
+== Creating an experimental condition
+
+Now to get to making different conditions and measuring them. First call
+
+    $ experiment generate my_condition -m "This should be a description of what you plan to\
+     do, maybe including a hypothesis. Don't worry, you can edit this later."
+    
+This will create a directory in `experiments` based on the name you provide (in this case `experiments/my_condition`). In this directory you will find a class that inherits from the experiment you defined earlier and that it also explicitly requires all the files you wrote in `app`. This gives you the flexibility to delete any of these includes and create a copy of that file to modify it. It also allows you to override the experiment logic as needed.
+
+Also notice that the description you provided is stored as a comment in that file. You can expand your hypothesis as you work on the file and it will be included in your report automatically.
+
+== Running the experiment
+
+Once you make the desired changes you can run the experiment with:
+
+    $ experiment run my_condition --cv 5
+    
+This will create a directory in `results` named something like `my_condition-cv5-46424`. The naming convention is to give the condition name, a summary of the configuration used, and a shortened timestamp to differentiate reruns of the same experiment.
+
+The experimental results and benchmarks will be written to this directory with a specification yaml file that details all conditions of the experiment.
+
+Please notice that you can provide several different conditions to the run command and it will run them sequentially, all with required options.
+
+== Configuration
+
+So far we have been talking mainly about variations in the source code of the experiments. But what if you just want to tweak a few parameters? There is always the almighty *Config* class to the rescue. 
+
+    Experiment::Config[:my_config_variable] # anywhere in your code
+    
+Where does this come from? You have a config directory containing a `config.yaml` file. This file contains several environments. The idea is that you might want to tweak your options differently when running on your laptop then when running on a university supercomputer ;)
+
+`development` is the default environment, you can set any other with the `--env` option.
+
+Notice that also when generating an experimental condition it also gets its own config.yaml file. This file overrides the main config file so you can introduce in condition specific options.
+
+And finally when running an experiment you can use the -o or --options option to override any config you want.
+
+Let me give you an example. Your main config file looks like this:
+
+    environments:
+      development:
+        ref_dir: /Users/kubowo/Desktop/points-vals
+        master_dir: /Users/kubowo/Desktop/points-vals/s014
+        alpha: 0.4
+      compute:
+        ref_dir: /afs/group/DB/points
+        master_dir: /afs/group/DB/points/s145
+        alpha: 0.4
+
+Your `experiments/my_condition/config.yaml` looks like this:
+
+    experiment:
+      development:
+        alpha: 0.5
+      compute:
+        alpha: 0.6
+
+And you run the experiment with
+
+    $ experiment run my_condition --env compute -o "master_dir: /Users/kubowo/Desktop/points-vals/s015"
+    
+Then your final config will look like this:
+
+    { :ref_dir => "/afs/group/DB/points", 
+      :master_dir => "/Users/kubowo/Desktop/points-vals/s015", 
+      :alpha => 0.6 }
+
+Flexible, eh?
+
+== Cross Validation
+
+Cross validation (CV) is one of the most crucial research methods in CS and AI. For that reason it is built right in. You specify how many CVs you want to run using the --cv flag and your data is automatically split up for you and the experiment is run for each CV with the appropriate data.
+
+== Reporting Results
+
+    $ experiment report
+
+Surprise, surprise. This will create two files in your `report` directory (BTW, this directory is also meant for you to store your report or paper draft). The first is methods.mmd. This takes all the stuff you wrote in the beginnings of your experimental condition files and creates a multi-markdown (http://fletcherpenney.net/multimarkdown/) file out of them (I chose multi-markdown for it's LaTEX support and also it is directly importable into Scrivener, my writing application of choice, available at http://www.literatureandlatte.com/scrivener.html).  
+
+The second file created is the `data.csv` file which contains the data from all your experiments. It should be importable to Numbers, Excel even Matlab for further analysis and charting.
+
+== Misc
+
+So that's pretty much the gist of experiment. There's a few other features (and a few soon to come to a gem near you ;-) 
+
+Also check out the RDocs: http://rdoc.info/github/gampleman/Experiment/master/frames

data/Rakefile

@@ -0,0 +1,26 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/experiment'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'experiment' do
+  self.developer 'Jakub Hampl', 'honitom@seznam.cz'
+  #self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
+  self.rubyforge_name       = self.name # TODO this is default value
+  # self.extra_deps         = [['activesupport','>= 2.0.2']]
+
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features]

data/bin/experiment

@@ -0,0 +1,277 @@
+#!/usr/bin/env ruby
+
+# == Synopsis 
+#   This program will run an experimental batch or generate files
+# 	for a new experiment
+#
+# == Examples
+#   Running an experiment
+#     experiment --env dice experiment1 experiment2 ...
+#
+#   Generating a new experiment with 2 cross validations
+#     experiment new experiment_name --cv 2
+#
+#   List all available experiments
+#     experiment list
+# 	
+#
+# == Usage 
+#   experiment command [options]
+#
+#   For help use: experiment -h
+#
+# == Options
+#   -h, --help          Displays help message
+#   -v, --version       Display the version, then exit
+#   -q, --quiet         Output as little as possible, overrides verbose
+#   -V, --verbose       Verbose output
+#   -e, --env           Sets the environment to run in
+#                       Defaults to development
+#   -c, --cv            Number of Cross validations to run
+#   -m, --description   A description of the current experiment
+#
+# == Author
+#   Jakub Hampl
+#
+
+#require "rubygems"
+require 'optparse'
+#require 'rdoc/usage'
+require 'ostruct'
+#require File.dirname(__FILE__) + "/experiment"
+
+class App
+  VERSION = '1.0'
+  
+  attr_reader :options
+
+  def initialize(arguments, stdin)
+    @arguments = arguments
+    @stdin = stdin
+    
+    # Set defaults
+    @options = OpenStruct.new
+    @options.verbose = false
+    @options.quiet = false
+		@options.env = :development
+		@options.cv = 5
+		@options.n_classes = 10
+		@options.kind = "d"
+		@options.description = ""
+		@options.opts = {}
+
+  end
+
+  # Parse options, check arguments, then process the command
+  def run
+        
+    if parsed_options? && arguments_valid? 
+      
+      puts "Start at #{DateTime.now}\n\n" if @options.verbose
+      
+      output_options if @options.verbose # [Optional]
+      require File.dirname(__FILE__) + "/vendor/backports/backports" if @options.env == :dice
+      
+      process_arguments            
+      process_command
+      
+      puts "\nFinished at #{DateTime.now}" if @options.verbose
+      
+    else
+      output_usage
+    end
+      
+  end
+  
+  protected
+  
+    def parsed_options?
+      
+      # Specify options
+      opts = OptionParser.new 
+      opts.on('-v', '--version')    { output_version ; exit 0 }
+      opts.on('-h', '--help')       { output_help }
+      opts.on('-V', '--verbose')    { @options.verbose = true }  
+      opts.on('-q', '--quiet')      { @options.quiet = true }
+			opts.on('-e', '--env [ENV]', [:development, :dice])	{ |v| @options.env = v }
+			opts.on('-c', '--cv CV', Integer)	{ |v| @options.cv = v }
+      opts.on('-n', '--number NUMBER', Integer)	{ |v| @options.n_classes = v }
+      opts.on('-k', '--kind KIND', String)      { |v| @options.kind = v }
+      opts.on('-m', '--description M', String)      { |v| @options.description = v }
+      opts.on('-o', '--options OPTSTRING', String) do |v|
+        @options.opts = v
+      end
+      opts.parse!(@arguments) rescue return false
+      
+      process_options
+      true      
+    end
+
+    # Performs post-parse processing on options
+    def process_options
+      @options.verbose = false if @options.quiet
+    end
+    
+    def output_options
+      puts "Options:\n"
+      
+      @options.marshal_dump.each do |name, val|        
+        puts "  #{name} = #{val}"
+      end
+    end
+
+    # True if required arguments were provided
+    def arguments_valid?
+      true if @arguments.length > 0
+    end
+    
+    # Setup the arguments
+    def process_arguments
+      # TO DO - place in local vars, etc
+    end
+    
+    def output_help
+      output_version
+      RDoc::usage() #exits app
+    end
+    
+    def output_usage
+      RDoc::usage('usage') # gets usage from comments above
+    end
+    
+    def output_version
+      puts "#{File.basename(__FILE__)} version #{VERSION}"
+    end
+    
+    def process_command
+			if @arguments.first == 'generate'
+			  dir = "./experiments/" + @arguments[1]
+				Dir.mkdir(dir)
+				File.open(dir + "/" + @arguments[1] + ".rb", "w") do |req_file|
+				  req_file.puts "# ## #{as_human_name @arguments[1]} ##"
+				  req_file.puts "# "+@options.description.split("\n").join("\n# ")
+				  req_file.puts
+				  req_file.puts
+				  req_file.puts "# The first contious block of comment will be included in your report."
+				  req_file.puts "# This includes the reference implementation."
+				  req_file.puts "# Override any desired files in this directory."
+				  Dir["./app/**/*.rb"].each do |f|
+				    p = f.split("/") - File.expand_path(".").split("/")
+				    req_file.puts "require File.dirname(__FILE__) + \"/../../#{p.join("/")}\""
+				  end
+				  req_file.puts "\nclass #{as_class_name @arguments[1]} < MyExperiment\n\t\nend"
+				end
+				File.open(dir + "/config.yaml", "w") do |f|
+				  f << "---\nexperiment:\n  development:\n  compute:\n"
+        end
+        
+      elsif @arguments.first == "new" # generate a new project
+        require 'fileutils'
+        dir = "./" + @arguments[1]
+        Dir.mkdir(dir)
+        %w[app config experiments report results test tmp vendor].each do |d|
+          Dir.mkdir(dir + "/" + d)
+        end
+        basedir = File.dirname(__FILE__) + "/.."
+        File.open(File.join(dir, "config", "config.yaml"), "w") do |f|
+          f << "---\nenvironments:\n  development:\n  compute:\n"
+        end
+        File.open(File.join(dir, ".gitignore"), "w") do |f|
+          f << "tmp/*"
+        end
+        FileUtils::cp File.join(basedir, "lib/experiment/generator/readme_template.txt"), File.join(dir, "README")
+        FileUtils::cp File.join(basedir, "lib/experiment/generator/Rakefile"), File.join(dir, "Rakefile")
+        FileUtils::cp File.join(basedir, "lib/experiment/generator/experiment_template.rb"), File.join(dir, "experiments", "experiment.rb")
+			elsif @arguments.first == "list"
+			  puts "Available experiments:"
+			  puts "  " + Dir["./experiments/*"].map{|a| File.basename(a) }.join(", ")
+			elsif @arguments.first == "report"
+			  dir = "./report/"
+        File.open(dir + "method.mmd", "w") do |f|
+          f.puts "# Methods #"
+          Dir["./experiments/*/*.rb"].each do |desc|
+            if File.basename(desc) == File.basename(File.dirname(desc)) + ".rb"
+              File.read(desc).split("\n").each do |line|
+                if m = line.match(/^\# (.+)/)
+                  f.puts m[1]
+                else
+                  break
+                end
+              end
+            f.puts
+            f.puts
+            end
+          end
+        end
+        require 'csv'
+        require "yaml"
+        require File.dirname(__FILE__)+"/../lib/experiment/stats"
+        CSV.open(dir + "/data.csv", "w") do |csv|
+          data = {}
+          Dir["./results/*/results.yaml"].each do |res|
+            d = YAML::load_file(res)
+            da = {}
+            d.each do |k, vals|
+              da[k.to_s + " mean"], da[k.to_s + " sd"] = Stats::mean(vals), Stats::standard_deviation(vals)
+              vals.each_with_index do |v, i|
+                da[k.to_s + " cv:" + i.to_s] = v
+              end
+            end
+            array_merge(data, da)
+          end
+          data.keys.map do |key| 
+      		  # calculate stats
+      		  a = data[key]
+      		  [key] + a
+    		  end.transpose.each do |row|
+    		    csv << row
+    		  end
+        end
+			elsif @arguments.shift == "run"
+			  require File.dirname(__FILE__) + "/../lib/experiment/base"
+			  require "experiments/experiment"
+				@arguments.each do |exp|
+				  require "./experiments/#{exp}/#{exp}"
+				  cla = eval(as_class_name(exp))
+					experiment = cla.new exp, @options.opts, @options.env
+					experiment.run! @options.cv
+				end
+			else
+			  output_usage
+			end
+      
+    end
+    
+    
+    private
+    def array_merge(h1, h2)
+  	  h2.each do |key, value|
+  	   h1[key] ||= []
+  	   h1[key] << value
+  	  end
+	  end
+    
+    def as_class_name(str)
+      str.split(/[\_\-]+/).map(&:capitalize).join
+    end
+    
+    def as_human_name(str)
+      str.split(/[\_\-]+/).map(&:capitalize).join(" ")
+    end
+
+    def process_standard_input
+      input = @stdin.read      
+      # TO DO - process input
+      
+      # [Optional]
+      # @stdin.each do |line| 
+      #  # TO DO - process each line
+      #end
+
+    end
+end
+
+
+# Create and run the application
+app = App.new(ARGV, STDIN)
+app.run

data/lib/experiment.rb

@@ -0,0 +1,6 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+module Experiment
+  VERSION = '0.0.1'
+end

data/lib/experiment/base.rb

@@ -0,0 +1,162 @@
+require File.dirname(__FILE__) + "/notify"
+require File.dirname(__FILE__) + "/stats"
+require File.dirname(__FILE__) + "/config"
+require 'benchmark'
+
+module Experiment
+  class Base
+    attr_reader :dir, :current_cv, :cvs
+    
+  	def initialize(experiment, options, env)
+  		@experiment = experiment
+  		
+  		Experiment::Config::load(experiment, options, env)
+  		require "./experiments/#{experiment}/#{experiment}"
+  		@abm = []
+  	end
+    
+    # runs the whole experiment
+  	def run!(cv)
+  		@cvs = cv || 1
+      @results = {}
+  		Notify.print "Running #{@experiment} "
+      split_up_data
+  		write_dir!
+  		specification!
+
+  		@cvs.times do |cv_num|
+  			@bm = []
+  			@current_cv = cv_num
+  			File.open(@dir + "/raw-#{cv_num}.txt", "w") do |output|
+  			    run_the_experiment(@data[cv_num], output)
+  			end
+  			array_merge @results, analyze_result!(@dir + "/raw-#{cv_num}.txt", @dir + "/analyzed-#{cv_num}.txt")
+  			write_performance!
+  			Notify.print "."
+  		end
+  		summarize_performance!
+  		summarize_results! @results
+  		Notify.print result_line
+  	end
+
+
+    # Registers and performs a benchmark which is then 
+    # calculated to the total and everage times
+  	def benchmark(label = "", &block)
+  	  @bm << Benchmark.measure("CV #{@current_cv} #{label}", &block)
+  	end
+
+  	
+    # Creates the results directory for the current experiment
+  	def write_dir!
+  		@dir = "./results/#{@experiment}-cv#{@cvs}-#{Time.now.to_i.to_s[4..9]}"
+  		Dir.mkdir @dir
+  	end
+    
+    # Writes a yaml specification of all the options used to run the experiment
+  	def specification!
+  		File.open(@dir + '/specification.yaml', 'w' ) do |out|
+  			YAML.dump({:name => @experiment, :date => Time.now, :configuration => Experiment::Config.to_h, :cross_validations => @cvs}, out )
+  		end
+  	end
+    
+    
+    
+    # Writes a file called 'performance_table.txt' which 
+    # details all the benchmarks performed
+  	def write_performance!
+  		performance_f do |f|
+  			f << "Cross Validation #{@current_cv} " + Benchmark::CAPTION 
+  			f << @bm.map {|m| m.format("%19n "+Benchmark::FMTSTR)}.join
+  			total = @bm.reduce(0) {|t, m| m + t}
+  			f << total.format("         Total: "+Benchmark::FMTSTR)
+  			@abm << total
+  		end
+  	end
+    
+    # Calculates the average performance and writes it
+  	def summarize_performance!
+  		performance_f do |f|
+  			total = @abm.reduce(0) {|t, m| m + t} / @abm.count
+  			f << total.format("       Average: "+Benchmark::FMTSTR)
+  		end
+  	end
+    
+    # creates a summary of the results and writes to 'all.csv'
+  	def summarize_results!(results)
+  	  File.open(@dir + '/results.yaml', 'w' ) do |out|
+  			YAML.dump(results, out )
+  		end
+  		
+  		# create an array of arrays
+  		res = results.keys.map do |key| 
+  		  # calculate stats
+  		  a = results[key]
+  		  [key] + a + [Stats::mean(a), Stats::standard_deviation(a)]
+		  end
+		  
+		  ls = results.keys.map{|v| v.to_s.length }
+  		
+  		ls = ["Standard Deviation".length] + ls
+  		res = [["cv"] + (1..cvs).to_a.map(&:to_s) + ["Mean", "Standard Deviation"]] + res
+  		
+  		out = ""
+  		res.transpose.each do |col|
+  		  col.each_with_index do |cell, i|
+  		    l = ls[i]
+  		    out << "| "
+  		    if cell.is_a?(String) || cell.is_a?(Symbol)
+  		      out << sprintf("%#{l}s", cell)
+		      else
+		        out << sprintf("%#{l}.3f", cell)
+		      end
+		      out << " "
+  		  end
+  		  out << "|\n"
+  		end
+  		File.open(@dir + "/summary.mmd", 'w') do |f|
+  		  f << "## Results for #{@experiment} ##\n\n"
+  		  f << out
+		  end
+  	  #results = results.reduce({}) do |tot, res|
+  	  # cv = res.delete :cv
+  	  # tot.merge Hash[res.to_a.map {|a| ["cv_#{cv}_#{a.first}".to_sym, a.last]}]
+  	  #end
+  	  #FasterCSV.open("./results/all.csv", "a") do |csv|
+  	  #  csv << results.to_a.sort_by{|a| a.first.to_s}.map(&:last)
+  	  #end
+  	end
+  	
+  	def result_line
+  	 " Done\n"
+  	end
+  	
+  	# A silly method meant to be overriden.
+  	# should return an array, which will be then split up for cross-validating
+  	def test_data
+  	  (1..cvs).to_a
+  	end
+  	
+  	def split_up_data
+  	  @data = []
+  	  test_data.each_with_index do |item, i|
+  	    @data[i % cvs] ||= []
+  	    @data[i % cvs] << item
+  	  end
+  	  @data
+  	end
+  	
+  	private
+  	# Yields a handle to the performance table
+  	def performance_f(&block) # just a simple wrapper to make code a little DRYer
+  		File.open(@dir+"/performance_table.txt", "a", &block) 
+  	end
+  	
+  	def array_merge(h1, h2)
+  	  h2.each do |key, value|
+  	   h1[key] ||= []
+  	   h1[key] << value
+  	  end
+	  end
+  end
+end

data/lib/experiment/config.rb

@@ -0,0 +1,40 @@
+require "yaml"
+
+module Experiment
+	class Config
+	  class << self
+	    
+	    # the load method takes the basic config file, which is then 
+	    # overriden by the experimental config file and finally by
+	    # the options string (which should be in this format:
+	    # "key: value, key2:value2,key3: value3")
+	    def load(experiment, options, env = :development)
+	      init env
+  			expath = File.expand_path("./experiments/#{experiment}/config.yaml")
+  			if File.exists? expath
+  				exp = YAML::load_file(expath)
+  			  @config.merge! exp["experiment"][env.to_s] if exp["experiment"][env.to_s].is_a? Hash
+  			end
+  			@config.merge! parse(options)
+  		end
+
+  		def init(env = :development)
+  		  conf = YAML::load_file("./config/config.yaml")
+  		  @config = conf["environments"][env.to_s]
+  		end
+
+  		def [](v)
+  			@config[v.to_s]
+  		end
+  		
+  		def parse(options)
+  		  Hash[options.split(/\, ?/).map{|a| a.split /\: ?/ }]
+		  end
+		  
+		  def to_h
+		    @config
+	    end
+	  end
+		
+	end
+end

data/lib/experiment/generator/Rakefile

@@ -0,0 +1,17 @@
+require 'rake'
+require 'rake/testtask'
+require "rake/rdoctask"
+
+task :default => [:test]
+
+Rake::TestTask.new do |t|
+   t.libs += ["app/input", "app/dtw", "app/models", "."]
+   t.test_files = FileList['test/unit/*.rb']
+   t.verbose = true
+end
+
+Rake::RDocTask.new do |rd|
+    rd.main = "README"
+    rd.rdoc_files.include("README", "app/**/*.rb")
+		rd.rdoc_dir = "doc"
+end

data/lib/experiment/generator/experiment_template.rb

@@ -0,0 +1,35 @@
+class MyExperiment < Experiment::Base
+  
+  def test_data
+    # TODO: Specify an array of all the test data. 
+    # It will be split up automatically for you accross Cross-validations
+  end
+  
+  def run_the_experiment(data, output)
+    # TODO: Define how you will run the experiment
+    # Remeber, each seperate experiment inherits from this base class and includes
+    # it's own files, so this should be a rather generic implementation
+    
+    # 1. prepare any nessecary setup like I/O lists, etc...
+    
+    # 2. do the experiment
+    benchmark do
+      output << # run your code here
+    end
+    
+    # 3. clean up
+    
+  end
+  
+  def analyze_result!(input, output)
+    # TODO perform an analysis of what your program did
+    
+    # remember to return a hash of meaningful data, best of all a summary
+  end
+  
+  # you might want to override this method as well:
+  # def summarize_results!(results)
+  #   super(results)
+  # end
+  
+end

data/lib/experiment/generator/readme_template.txt

@@ -0,0 +1,24 @@
+This project has a slightly complex structure which is though beneficial for automating
+many mundane tasks. So a brief description of the files/folders included:
+
+app
+ - Contains the base implementation.
+config
+ - Configuration for running in different environments.
+doc
+ - Documentation for the reference implementation.
+experiments
+ - This directory includes all experiments that were coded. They generally `require` 
+   files from the reference implementation and add modifications of there own. 
+   Each is explained in its `about.md` file.
+report
+ - Source files used to create the report (multi-markdown format, see http://fletcherpenney.net/multimarkdown).
+results
+ - Has all the measurements from individual experiments. Naming convention: 
+   {name}-{classes}-cv{number of cross validations}-{shortened timestamp}.
+test
+ - Unit tests.
+tmp
+ - Various stuff stored here. Ignore.
+vendor
+ - Library code and scripts not written by me.

data/lib/experiment/notify.rb

@@ -0,0 +1,6 @@
+class Notify
+  def self.method_missing(meth, *args, &blk)
+    $stdout.sync = true
+    $stdout.send meth, *args, &blk
+  end
+end

data/lib/experiment/stats.rb

@@ -0,0 +1,43 @@
+class Stats
+	class << self
+		
+		def sum(ar, &block)
+			ar.reduce(0.0) {|asum, a| (block_given? ? yield(a) : a) + asum}
+	  end
+		
+	  def variance(ar)
+	    v = sum(ar) {|x| (mean(ar) - x)**2.0 }
+      v/(ar.count - 1.0)
+	  end
+
+	  def standard_deviation(ar)
+	    Math.sqrt(variance(ar))
+	  end
+
+	  def z_scores(ar)
+	    ar.map {|x| z_score(ar, x)}
+	  end
+
+	  def z_score(ar, x)
+	    (x - mean(ar)) / standard_deviation(ar)
+	  end
+
+		def range(ar)
+			ar.max - ar.min
+		end
+
+	  def mean(ar)
+	    sum(ar) / ar.count
+	  end
+
+	  def median(ar)
+			a = ar.sort
+			if ar.count.odd?
+				a[(ar.count-1)/2]
+			else
+				(a[ar.count/2 - 1] + a[ar.count/2]) / 2.0
+			end
+	  end
+		
+	end
+end

data/test/test_experiment.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TestExperiment < Test::Unit::TestCase
+
+  def setup
+  end
+  
+  def test_truth
+    assert true
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'stringio'
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/experiment'

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification 
+name: experiment
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Jakub Hampl
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-31 01:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rubyforge
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 0
+        - 4
+        version: 2.0.4
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 6
+        - 2
+        version: 2.6.2
+  type: :development
+  version_requirements: *id002
+description: ""
+email: 
+- honitom@seznam.cz
+executables: 
+- experiment
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- lib/experiment/generator/readme_template.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.rdoc
+- Rakefile
+- lib/experiment.rb
+- lib/experiment/config.rb
+- lib/experiment/stats.rb
+- lib/experiment/generator/readme_template.txt
+- lib/experiment/generator/experiment_template.rb
+- lib/experiment/generator/Rakefile
+- lib/experiment/base.rb
+- lib/experiment/notify.rb
+- test/test_experiment.rb
+- test/test_helper.rb
+- bin/experiment
+has_rdoc: true
+homepage: http://github.com/gampleman/experiment
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: experiment
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: ""
+test_files: 
+- test/test_experiment.rb
+- test/test_helper.rb