biopsy 0.1.0.alpha → 0.1.1.alpha

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e4b7eed6bdea005473b5bab0e14b2d05eadb7dbf
+  data.tar.gz: 788a2e6a6bd4f67d5d35c39e5b710634d72c45cb
+SHA512:
+  metadata.gz: ddfe6c7044e51f9f59774b2750670cc5e44f88ea61cacdcdf42f5cb8e8000979da68f9c7326d8c3fcf05aea5125c5d12e27c23a9b9be73de55306fe5b71b1590
+  data.tar.gz: 1e8ad4d20403f7c6abf300663ad2dd670518c67d93626573011cdf9220ebc38ee330952af24e175b16e12d3ab99562f72ab9ecef286a707333af388f0c331be5

data/README.md

@@ -7,6 +7,28 @@ Biopsy is a framework for optimising any program or pipeline which produces a me
 
 A simple example of the power of this approach is *de-novo* transcriptome assembly. Typically, the assembly process takes many GB of data as input, uses many GB of RAM and takes many hours to complete. This prevents researchers from performing full parameter sweeps, and they are therefore forced to use word-of-mouth and very basic optimisation to choose assembler settings. Assemblotron, which uses the Biopsy framework, can fully optimise any *de-novo* assembler to produce the optimal assembly possible given a particular input. This typically takes little more time than running a single assembly.
 
+## Installation
+
+Make sure you have Ruby installed, then:
+
+`gem install biopsy --pre`
+
+## Usage
+
+[]
+
+Detailed usage instructions are on the wiki. Here's a quick overview:
+
+1. Define your optimisation target. This is a program or pipeline you want to optimise, and you define it by filling in a template [YAML file](http://en.wikipedia.org/wiki/YAML). Easy!
+2. Define your objective function. This is a program that analyses the output of your program and gives it a score. You define it by writing a small amount of Ruby code. Don't worry - there's a template and detailed instructions on the wiki.
+3. Run Biopsy, and wait while the experiment runs.
+
+### Command line examples
+
+`biopsy list targets`
+`biopsy list objectives`
+`biposy run --target test_target --objective test_objective --input test_file.txt --time-limit 24h`
+
 ## Development status
 
 [![Gem Version](https://badge.fury.io/rb/biopsy.png)][gem]
@@ -46,4 +68,5 @@ Documentation is in development and will be released with the beta.
 
 ### Citation
 
-This is *pre-release*, *pre-publication* academic software. In lieu of a paper to cite, please cite this Github repo if your use of the software leads to a publication.
+This is *pre-release*, *pre-publication* academic software. In lieu of a paper to cite, please cite this Github repo and/or the [Figshare DOI (http://dx.doi.org/10.6084/m9.figshare.790660
+)](http://dx.doi.org/10.6084/m9.figshare.790660) if your use of the software leads to a publication.

data/lib/biopsy/base_extensions.rb

@@ -32,6 +32,7 @@ class Hash
     target = dup    
     target.inject({}) do |memo, (key, value)|
       value = value.deep_symbolize if value.is_a?(Hash)
+      value = value.map{ |x| x.is_a?(Hash) ? x.deep_symbolize : x } if value.is_a?(Array)
       memo[(key.to_sym rescue key) || key] = value
       memo
     end
@@ -54,11 +55,7 @@ class Array
   # Requires the array to contain only objects of class Fixnum.
   # If any other class is encountered, an error will be raised.
   def mean
-    self.sum / self.size.to_f
-  end
-
-  def sum
-    self.inject(0, :+)
+    self.inject(0, :+) / self.size.to_f
   end
 
 end # Array

data/lib/biopsy/experiment.rb

@@ -19,13 +19,12 @@ module Biopsy
     attr_reader :inputs, :outputs, :retain_intermediates, :target, :start, :algorithm
 
     # Returns a new Experiment
-    def initialize(target_name, domain_name, start=nil, algorithm=nil)
-      @domain = Domain.new domain_name
+    def initialize(target_name, start=nil, algorithm=nil)
       @start = start
       @algorithm = algorithm
 
       self.load_target target_name
-      @objective = ObjectiveHandler.new(@domain, @target)
+      @objective = ObjectiveHandler.new @target
       self.select_algorithm
       self.select_starting_point
       @scores = {}
@@ -44,19 +43,23 @@ module Biopsy
 
     # Return a random set of parameters from the parameter space.
     def random_start_point
-      Hash[@target.parameter_ranges.map { |p, r| [p, r.sample] }] 
+      Hash[@target.parameters.map { |p, r| [p, r.sample] }] 
     end
 
     # select the optimisation algorithm to use
     def select_algorithm
-      @algorithm = ParameterSweeper.new(@target.parameter_ranges)
-      return if @algorithm.combinations.size < Settings.instance.sweep_cutoff
-      @algorithm = TabuSearch.new(@target.parameter_ranges)
+      max = Settings.instance.sweep_cutoff
+      n = @target.count_parameter_permutations
+      if n < max
+        @algorithm = ParameterSweeper.new(@target.parameters)
+      else
+        @algorithm = TabuSearch.new(@target.parameters)
+      end
     end
 
     # load the target named +:target_name+
     def load_target target_name
-      @target = Target.new @domain
+      @target = Target.new
       @target.load_by_name target_name
     end
 
@@ -82,20 +85,54 @@ module Biopsy
     # encompassing the program, objective(s) and optimiser.
     # Returns the output of the optimiser.
     def run_iteration
-      # run the target
-      run_data = @target.run @current_params
-      # evaluate with objectives
-      param_key = @current_params.to_s
-      result = nil
-      if @scores.has_key? param_key
-        result = @scores[param_key]
-      else
-        result = @objective.run_for_output run_data
-        @iteration_count += 1
+      # create temp dir
+        Dir.chdir(self.create_tempdir) do
+        # run the target
+        raw_output = @target.run @current_params
+        # evaluate with objectives
+        param_key = @current_params.to_s
+        result = nil
+        if @scores.has_key? param_key
+          result = @scores[param_key]
+        else
+          result = @objective.run_for_output raw_output
+          @iteration_count += 1
+        end
+        @scores[@current_params.to_s] = result
+        # get next steps from optimiser
+        @current_params = @algorithm.run_one_iteration(@current_params, result)
+      end
+      self.cleanup
+    end
+
+    def cleanup
+      # TODO: make this work
+      # remove all but essential files
+      if Settings.instance.keep_intermediates
+        @objectives.values.each{ |objective| essential_files += objective.essential_files }
+      end
+      Dir["*"].each do |file|
+        next if File.directory? file
+        if essential_files && essential_files.include?(file)
+          `gzip #{file}` if Settings.instance.gzip_intermediates
+          FileUtils.mv("#{file}.gz", '../output')
+        end
+      end
+      FileUtils.rm_rf @last_tempdir
+    end
+
+    # create a guaranteed random temporary directory for storing outputs
+    # return the dirname
+    def create_tempdir
+      token = loop do
+        # generate random dirnames until we find one that
+        # doesn't exist
+        test_token = SecureRandom.hex
+        break test_token unless File.exists? test_token
       end
-      @scores[@current_params.to_s] = result
-      # get next steps from optimiser
-      @current_params = @algorithm.run_one_iteration(@current_params, result)
+      Dir.mkdir(token)
+      @last_tempdir = token
+      return token
     end
 
   end # end of class RunHandler

data/lib/biopsy/objective_handler.rb

@@ -39,10 +39,8 @@ module Biopsy
     attr_reader :last_tempdir
     attr_accessor :objectives
 
-    def initialize domain, target
-      @domain = domain
+    def initialize target
       @target = target
-      base_dir = Settings.instance.base_dir
       @objectives_dir = Settings.instance.objectives_dir.first
       @objectives = {}
       $LOAD_PATH.unshift(@objectives_dir)
@@ -76,12 +74,14 @@ module Biopsy
 
     # Run a specific +:objective+ on the +:output+ of a target
     # with max +:threads+.
-    def run_objective(objective, name, output, threads)
+    def run_objective(objective, name, raw_output, output_files, threads)
       begin
-        # output is a hash containing the file(s) output
-        # by the target in the format expected by the
-        # objective function(s).
-        return objective.run(output, threads)
+        # output is a, array: [raw_output, output_files].
+        # output_files is a hash containing the absolute paths
+        # to file(s) output by the target in the format expected by the
+        # objective function(s), with keys as the keys expected by the
+        # objective function
+        return objective.run(raw_output, output_files, threads)
       rescue NotImplementedError => e
         puts "Error: objective function #{objective.class} does not implement the run() method"
         puts "Please refer to the documentation for instructions on adding objective functions"
@@ -90,7 +90,6 @@ module Biopsy
     end
 
     # Perform a euclidean distance dimension reduction of multiple objectives
-    # using weighting specified in the domain definition.
     def dimension_reduce(results)
       # calculate the weighted Euclidean distance from optimal
       # d(p, q) = \sqrt{(p_1 - q_1)^2 + (p_2 - q_2)^2+...+(p_i - q_i)^2+...+(p_n - q_n)^2}
@@ -107,40 +106,26 @@ module Biopsy
     end
 
     # Run all objectives functions for +:output+. 
-    def run_for_output(output, threads=6, cleanup=true, allresults=false)
+    def run_for_output(raw_output, threads=6, allresults=false)
       # check output files exist
-      @target.output_files.each_pair do |key, name|
-        unless File.exists?(output[key]) && File.size(output[key]) > 0
-          info("file #{output[key]} does not exist or is empty")
+      output_files = {}
+      @target.output.each_pair do |key, glob|
+        files = Dir[glob]
+        zerosize = files.reduce(false) { |empty, f| File.size(f) == 0 }
+        if files.empty? || zerosize
+          puts Dir.pwd
+          raise ObjectiveHandlerError.new "output files for #{key} matching #{glob} do not exist or are empty"
           return nil
         end
+        output_files[key] = files.map { |f| File.expand_path(f) }
       end
+
       # run all objectives for output
       results = {}
-      # create temp dir
-      Dir.chdir(self.create_tempdir) do
-        @objectives.each_pair do |name, objective|
-          results[name] = self.run_objective(objective, name, output, threads)
-        end
-        if cleanup == 1
-          # remove all but essential files
-          essential_files = @domain.keep_intermediates
-          if essential_files
-            @objectives.values.each{ |objective| essential_files += objective.essential_files }
-          end
-          Dir["*"].each do |file|
-            next if File.directory? file
-            if essential_files && essential_files.include?(file)
-              `gzip #{file}` if @domain.gzip_intermediates
-              FileUtils.mv("#{file}.gz", '..')
-            end
-          end
-        end
-      end
-      if cleanup
-        # clean up temp dir
-        FileUtils.rm_rf @last_tempdir
+      @objectives.each_pair do |name, objective|
+        results[name] = self.run_objective(objective, name, raw_output, output_files, threads)
       end
+
       if allresults
         return {:results => results,
                 :reduced => self.dimension_reduce(results)}
@@ -151,20 +136,6 @@ module Biopsy
       end
     end
 
-    # create a guaranteed random temporary directory for storing outputs
-    # return the dirname
-    def create_tempdir
-      token = loop do
-        # generate random dirnames until we find one that
-        # doesn't exist
-        test_token = SecureRandom.hex
-        break test_token unless File.exists? test_token
-      end
-      Dir.mkdir(token)
-      @last_tempdir = token
-      return token
-    end
-
   end
 
 end

data/lib/biopsy/optimisers/parameter_sweeper.rb

@@ -42,8 +42,12 @@ module Biopsy
       end
     end
 
+    def setup *args
+      nil 
+    end
+
     # return the next parameter set to evaluate
-    def run_one_iteration(*args)
+    def run_one_iteration *args
       @combinations.pop
     rescue
       nil
@@ -62,5 +66,14 @@ module Biopsy
         generate_combinations(index + 1, opts)
       end
     end
+
+    def knows_starting_point?
+      true
+    end
+
+    def select_starting_point
+      self.run_one_iteration
+    end
+
   end
 end

data/lib/biopsy/optimisers/tabu_search.rb

@@ -201,8 +201,10 @@ module Biopsy
       @backtracks = 1.0
 
       # convergence
-      @num_threads = 2
+      @num_threads = 5
       @threads = []
+      @convergence_alpha = 0.05
+      @global_best = {:parameters => nil, :score => nil}
 
     end # initialize
 
@@ -249,6 +251,8 @@ module Biopsy
         thread.loaded = false
       end
       @current_thread = @num_threads - 2
+      # adjust the alpha for multiple testing in convergence
+      @adjusted_alpha = @convergence_alpha / @num_threads
     end
 
     def load_next_thread
@@ -276,6 +280,13 @@ module Biopsy
       else
         @iterations_since_best += 1
       end
+      if @global_best[:score].nil? || @best[:score] > @global_best[:score]
+        @global_best = @best.clone
+      end
+    end
+
+    def best
+      @global_best
     end
 
     # use probability distributions to define the
@@ -385,9 +396,26 @@ module Biopsy
     # check termination conditions 
     # and return true if met
     def finished?
-      return false if @threads.first.recent_scores.size < @jump_cutoff
-      scores = @threads.map { |t| t.recent_scores }
-      scores.map { |s| s.mean }.uniq.length == 1
+      return false unless @threads.all? { |t| t.recent_scores.size == @jump_cutoff }
+      probabilities = self.recent_scores_combination_test
+      n_significant = 0
+      probabilities.each do |mann_u, levene| 
+        if mann_u <= @adjusted_alpha && levene <= @convergence_alpha
+          n_significant += 1 
+        end
+      end
+      finish = n_significant >= probabilities.size * 0.5
+    end
+
+    # returns a matrix of correlation probabilities for recent
+    # scores between all threads
+    def recent_scores_combination_test
+      combinations = 
+      @threads.map{ |t| t.recent_scores.to_scale }.combination(2).to_a
+      combinations.map do |a, b|
+        [Statsample::Test.u_mannwhitney(a, b).probability_exact,
+         Statsample::Test::Levene.new([a,b]).probability]
+      end
     end
 
     # True if this algorithm chooses its own starting point

data/lib/biopsy/settings.rb

@@ -25,11 +25,11 @@ module Biopsy
 
     attr_accessor :base_dir
     attr_accessor :target_dir
-    attr_accessor :domain_dir
-    attr_accessor :domain
     attr_accessor :objectives_dir
     attr_accessor :objectives_subset
     attr_accessor :sweep_cutoff
+    attr_accessor :keep_intermediates
+    attr_accessor :gzip_intermediates
 
     def initialize
       self.set_defaults
@@ -40,11 +40,11 @@ module Biopsy
       @config_file = '~/.biopsyrc'
       @base_dir = ['.']
       @target_dir = ['targets']
-      @domain_dir = ['domains']
-      @domain = 'test_domain'
       @objectives_dir = ['objectives']
       @objectives_subset = nil
       @sweep_cutoff = 100
+      @keep_intermediates = false
+      @gzip_intermediates = false
     end
 
     # Loads settings from a YAML config file. If no file is
@@ -86,25 +86,6 @@ module Biopsy
       all_settings.to_yaml
     end
 
-    # Locate the first YAML config file whose name
-    # excluding extension matches +:name+ (case insensitive)
-    # in dirs listed by the +:dir_key+ setting.
-    def locate_config(dir_key, name)
-      dir_key = "@#{dir_key.to_s}".to_sym
-      unless self.instance_variables.include? dir_key
-        raise SettingsError.new "no setting found for compulsory key #{dir_key}"
-      end
-      self.instance_variable_get(dir_key).each do |dir|
-        Dir.chdir ::File.expand_path(dir) do
-          Dir[name + '.yml'].each do |file|
-            return ::File.expand_path(file) if ::File.basename(file, '.yml').downcase == name.downcase
-          end
-        end
-      end
-
-      nil
-    end
-
   end # Settings
 
 end # Biopsy

data/lib/biopsy/target.rb

@@ -5,25 +5,13 @@ module Biopsy
 
   class Target
     require 'yaml'
-    require 'ostruct'
-
-    # array of input files expected by the target constructor
-    attr_accessor :input_files
-    # array of output files to keep for submission to objective
-    # functions during optimisation
-    attr_accessor :output_files
-    # hash mapping parameters to the ranges of values they can take
-    attr_reader :parameter_ranges
-    # path to the constructor code
-    attr_reader :constructor_path
-    attr_reader :domain
+    require 'set'
 
-    # create a new Target instance.
-    # arguments:
-    # +:domain+ the domain to which this target belongs (see Domain documentation)
-    def initialize domain
-      @domain = domain
-    end
+    attr_accessor :parameters
+    attr_accessor :options
+    attr_accessor :output
+    attr_accessor :name
+    attr_reader :constructor_path
 
     # load target with +name+.
     def load_by_name name
@@ -31,17 +19,9 @@ module Biopsy
       raise TargetLoadError.new("Target definition file does not exist for #{name}") if path.nil?
       config = YAML::load_file(path)
       raise TargetLoadError.new("Target definition file #{path} is not valid YAML") if config.nil?
-      missing = self.check_config config.deep_symbolize
-      if missing
-        msg = "Target definition file #{path} is missing required fields: #{missing}"
-        raise TargetLoadError.new(msg)
-      end
-      errors = self.validate_config config
-      unless errors.empty?
-        raise TargetLoadError.new("Target definition file #{path} contains the following errors:\n - #{errors.join("\n - ")}")
-      end
+      config = config.deep_symbolize
       self.store_config config
-      self.check_constructor
+      self.check_constructor name
       self.load_constructor
     end
 
@@ -49,48 +29,52 @@ module Biopsy
     # to the definition YAML file. All +:target_dir+s defined in Settings are
     # searched and the first matching YAML file is loaded. 
     def locate_definition name
-      Settings.instance.locate_config(:target_dir, name)
+      self.locate_file(name + '.yml')
     end
 
-    # verify that +:config+ contains values for all essential target settings
-    # returning false if no keys are missing, or an array of the missing keys
-    # if any cannot be found
-    def check_config config
-      required = %w(input_files output_files parameter_ranges constructor_path)
-      missing = false
-      required.each do |key|
-        unless config.has_key? key.to_sym
-          missing ||= []
-          missing << key
+    # store the values in +:config+, checking they are valid
+    def store_config config
+      required = Set.new([:name, :parameters, :output])
+      missing = required - config.keys
+      raise TargetLoadError.new("Required keys are missing from target definition: #{missing.to_a.join(",")}") unless missing.empty?
+      config.each_pair do |param, data|
+        case param
+        when :name
+          raise TargetLoadError.new('Target name must be a string') unless data.is_a? String
+          @name = data
+        when :shortname
+          raise TargetLoadError.new('Target shortname must be a string') unless data.is_a? String
+          @shortname = data
+        when :parameters
+          self.generate_parameters data
+        when :output
+          raise TargetLoadError.new('Target output must be a hash') unless data.is_a?(Hash)
+          @output = data
         end
       end
-      missing
-    end
-
-    # validate the config against the domain definition. Return an array
-    # whose length will be the number of errors found. Thus an array of
-    # length 0 indicates that the config is valid according to the domain
-    # specification.
-    def validate_config config
-      @domain.target_valid? config
     end
 
-    # Store the values in +:config+
-    def store_config config
-      config.each_pair do |key, value|
-        self.instance_variable_set('@' + key.to_s, value)
+    # Locate a file with name in one of the target_dirs
+    def locate_file name
+      Settings.instance.target_dir.each do |dir|
+        Dir.chdir File.expand_path(dir) do
+          return File.expand_path(name) if File.exists? name
+        end
       end
+      raise TargetLoadError.new("Couldn't find file #{name}")
+      nil
     end
 
     # Validate the constructor. True if valid, false otherwise.
-    def check_constructor
-      raise "constructor path is not defined for this target" if @constructor_path.nil?
+    def check_constructor name
+      @constructor_path = self.locate_file name + '.rb'
+      raise TargetLoadError.new("constructor path is not defined for this target") if @constructor_path.nil?
       self.valid_ruby? @constructor_path
     end
 
     # Load constructor
     def load_constructor
-      require File.join(Settings.instance.target_dir, @constructor_path)
+      require @constructor_path 
       file_name = File.basename(@constructor_path, '.rb')
       constructor_name = file_name.camelize
       @constructor = Module.const_get(constructor_name).new
@@ -104,10 +88,41 @@ module Biopsy
     # true if file is valid ruby
     def valid_ruby? file
       return false unless ::File.exists? file
-      result = `ruby -c #{file} &> /dev/null`
+      result = `ruby -c #{file} > /dev/null 2>&1`
       !result.size.zero?
     end
 
+    # convert parameter specification to a hash of arrays and ranges
+    def generate_parameters params
+      @parameters = {}
+      @options = {}
+      params.each_pair do |param, data|
+        if data[:opt]
+          # optimise this parameter
+          if data[:values] 
+            # definition has provided an array of values
+            raise TargetLoadError.new("'values' for parameter #{param} is not an array") unless data[:values].is_a? Array
+            @parameters[param] = data[:values]
+          else
+            # definition has specified a range
+            min, max, step = data[:min], data[:max], data[:step]
+            raise TargetLoadError.new("min and max must be set for parameter #{param}") unless min && max
+            range = (min..max)
+            range = range.step(step) if step
+            @parameters[param] = range.to_a
+          end
+        else
+          # present option to user
+
+        end
+      end
+    end
+
+    # return the total number of possible permutations of
+    def count_parameter_permutations
+      @parameters.each_pair.map{ |k, v| v }.reduce(1) { |n, arr| n * arr.size }
+    end
+
   end # end of class Domain
 
 end # end of module Biopsy

data/lib/biopsy/version.rb

@@ -4,7 +4,7 @@ module Biopsy
   module VERSION
     MAJOR = 0
     MINOR = 1
-    PATCH = 0
+    PATCH = 1
     BUILD = 'alpha'
 
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')

data/lib/biopsy.rb

@@ -1,7 +1,6 @@
 require "biopsy/version"
 require "biopsy/base_extensions"
 require "biopsy/settings"
-require "biopsy/domain"
 require "biopsy/experiment"
 require "biopsy/target"
 require "biopsy/objective_handler"