rbbt-util 4.3.0 → 4.4.0

data/lib/rbbt/workflow.rb

@@ -9,43 +9,6 @@ module Workflow
   end
   self.workflows = []
 
-  def self.require_local_workflow2(wf_name, wf_dir = nil)
-    require 'rbbt/resource/path'
-
-    if File.exists?(wf_name) or File.exists?(wf_name + '.rb')
-      $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name)), 'lib'))
-      require wf_name
-      return
-    end
-
-    wf_dir ||= case
-               when File.exists?(File.join(File.dirname(Path.caller_lib_dir), wf_name))
-                 dir = File.join(File.dirname(Path.caller_lib_dir), wf_name)
-                 Log.debug "Loading workflow from lib dir: #{dir}"
-                 dir
-                 File.join(File.dirname(Path.caller_lib_dir), wf_name)
-               when defined? Rbbt
-                 if Rbbt.etc.workflow_dir.exists?
-                   dir = File.join(Rbbt.etc.workflow_dir.read.strip, wf_name)
-                   Log.debug "Loading workflow from etc dir: #{dir}"
-                   dir
-                 else
-                   dir = Rbbt.workflows[wf_name]
-                   Log.debug "Loading workflow from main dir: #{dir}"
-                   dir
-                 end
-               else
-                 dir = File.join(ENV["HOME"], '.workflows')
-                 Log.debug "Loading workflow from home dir: #{dir}"
-                 dir
-               end
-
-    wf_dir = Path.setup(wf_dir)
-
-    $LOAD_PATH.unshift(File.join(File.dirname(wf_dir["workflow.rb"].find), 'lib'))
-    require wf_dir["workflow.rb"].find
-  end
-
   def self.require_remote_workflow(wf_name, url)
     require 'rbbt/workflow/rest/client'
     eval "Object::#{wf_name} = RbbtRestClient.new '#{ url }', '#{wf_name}'"
@@ -75,7 +38,6 @@ module Workflow
 
     else
       case
-
         # Points to workflow file
       when ((File.exists?(wf_name) and not File.directory?(wf_name)) or File.exists?(wf_name + '.rb')) 
         $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name)), 'lib'))
@@ -204,7 +166,7 @@ module Workflow
     block = self.method(name) unless block_given?
 
     result_type = result_type
-    task = Task.setup({
+    annotations = {
       :name => name,
       :inputs => inputs,
       :description => description,
@@ -212,8 +174,11 @@ module Workflow
       :result_type => Array == result_type ? result_type.to_sym : result_type,
       :input_defaults => input_defaults,
       :input_descriptions => input_descriptions,
+      :input_options => input_options,
       :result_description => result_description
-    }, &block)
+    }
+    
+    task = Task.setup(annotations, &block)
 
     @last_task = task
     @tasks[name] = task

data/lib/rbbt/workflow/accessor.rb

@@ -77,6 +77,10 @@ class Step
     info[:status] == :error
   end
 
+  def aborted?
+    info[:status] == :aborted
+  end
+
   # {{{ INFO
 
   def files_dir
@@ -168,6 +172,7 @@ module Workflow
     input_types = rec_input_types(name)
     input_descriptions = rec_input_descriptions(name)
     input_defaults = rec_input_defaults(name)
+    input_options = rec_input_options(name)
     export = case
              when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
                :synchronous
@@ -188,6 +193,7 @@ module Workflow
       :input_types => input_types,
       :input_descriptions => input_descriptions,
       :input_defaults => input_defaults,
+      :input_options => input_options,
       :result_type => result_type,
       :result_description => result_description,
       :dependencies => dependencies
@@ -220,6 +226,11 @@ module Workflow
     [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_descriptions}
   end
 
+  def rec_input_options(taskname)
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_options}
+  end
+
+
   def real_dependencies(task, jobname, inputs, dependencies)
     real_dependencies = []
     dependencies.each do |dependency|

data/lib/rbbt/workflow/annotate.rb

@@ -2,7 +2,7 @@ module AnnotatedModule
   def self.extended(base)
     if not base.respond_to? :inputs
       class << base
-        attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults, :result_description, :helpers
+        attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults, :input_options, :result_description, :helpers
 
         def description
           i = @description; @description = ""; i
@@ -24,6 +24,10 @@ module AnnotatedModule
           i = @input_defaults; @input_defaults = {}; i
         end
 
+        def input_options
+          i = @input_options; @input_options = {}; i
+        end
+
         def description
           i = @description; @description = ""; i
         end
@@ -38,6 +42,7 @@ module AnnotatedModule
       base.input_types = {}
       base.input_descriptions = {}
       base.input_defaults = {}
+      base.input_options = {}
       base.helpers = {}
 
     end
@@ -60,13 +65,14 @@ module AnnotatedModule
     @dependencies.concat dependencies
   end
 
-  def input(name, type = nil, desc = nil, default = nil)
+  def input(name, type = nil, desc = nil, default = nil, options = nil)
     name = name.to_sym
     type = type.to_sym
     @inputs << name
     @input_types[name] = type unless type.nil?
     @input_descriptions[name] = desc unless desc.nil?
     @input_defaults[name] = default unless default.nil?
+    @input_options[name] = options unless options.nil?
   end
 end
 

data/lib/rbbt/workflow/step.rb

@@ -42,7 +42,11 @@ class Step
       end
     else
       Log.debug "Waiting for pid: #{@pid}"
-      Process.waitpid @pid
+      begin
+        Process.waitpid @pid
+      rescue Errno::ECHILD
+        Log.debug "Process #{ @pid } already finished: #{ path }"
+      end
       @pid = nil
     end
     self
@@ -51,21 +55,31 @@ class Step
   def run(no_load = false)
     result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq, :no_load => no_load do
       FileUtils.rm info_file if File.exists? info_file
-      log((task.name || "unnamed task"), "Starting task")
+      log(:starting, "Starting task: #{task.name || "unnamed task"}")
+
       set_info :dependencies, @dependencies.collect{|dep| [dep.task.name, dep.name]}
       @dependencies.each{|dependency| 
         log dependency.task.name || "dependency", "Processing dependency: #{ dependency.path }"
         dependency.run true
       }
+
       set_info :status, :started
+
+      set_info :started, Time.now
+      
       set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
+
       res = begin
               exec
+            rescue Step::Aborted
+              raise $!
             rescue Exception
               set_info :backtrace, $!.backtrace
               log(:error, "#{$!.class}: #{$!.message}")
+              log(:error, "backtrace: #{$!.backtrace.first}")
               raise $!
             end
+
       set_info :status, :done
       res
     end
@@ -85,14 +99,19 @@ class Step
       end
     end
     set_info :pid, @pid
+    Process.detach(@pid)
     self
   end
 
   def abort
+    @pid ||= info[:pid]
     if @pid.nil?
+      Log.medium "Could not abort #{path}: no pid"
       false
     else
+      Log.medium "Aborting #{path}: #{ @pid }"
       Process.kill("INT", @pid)
+      log(:aborted, "Job aborted by user")
       true
     end
   end

data/lib/rbbt/workflow/task.rb

@@ -2,15 +2,15 @@ require 'rbbt/util/misc'
 require 'rbbt/persist'
 
 module Task
-  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :description, :name, :result_description
+  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :description, :name, :result_description
 
   def self.setup(options = {}, &block)
     block.extend Task
     options = IndiferentHash.setup options
     block.singleton_methods.
       select{|method| method.to_s[-1] != "="[0]}.each{|method|
-      if block.respond_to?(method + "=") and options.include? method.to_sym
-        block.send(method + '=', options[method.to_sym]) 
+      if block.respond_to?(method.to_s + "=") and options.include? method.to_sym
+        block.send(method.to_s + '=', options[method.to_sym]) 
       end
     }
     block

data/share/lib/R/util.R

@@ -1,28 +1,50 @@
-rbbt.ruby <- function(code, load = TRUE, flat = FALSE){
+rbbt.ruby <- function(code, load = TRUE, flat = FALSE, type = 'tsv', ...){
   file = system('rbbt_exec.rb - file', input = code, intern=TRUE);
+
+  error_str = "^#:rbbt_exec Error"
+  if (regexpr(error_str, file)[1] != -1 ){
+    print(file);
+    return(NULL);
+  }
+
   if (load){
-    if(flat){
-        data = rbbt.flat.tsv(file);
-    }else{
-        data = rbbt.tsv(file);
-    }
-    rm(file);
-    return(data);
+      if(type == 'tsv'){
+          if(flat){
+              data = rbbt.flat.tsv(file, ...);
+          }else{
+              data = rbbt.tsv(file, ...);
+          }
+          rm(file);
+          return(data)
+      }
+
+      if(type == 'list'){
+          data = scan(file, ...)
+          return(data);
+      }
+      
+      return(NULL);
   }else{
     return(file);
   }
 }
 
+rbbt.ruby.substitutions <- function(script, substitutions = list(), ...){
+    
+    for (keyword in names(substitutions)){
+        script = sub(keyword, substitutions[[keyword]], script);
+    }
+
+    result = rbbt.ruby(script, ...);
+
+    return(result);
+}
+
 rbbt.glob <- function(d, pattern){
     d=sub("/$", '', d);
     sapply(dir(d, pattern), function(file){paste(d,file,sep="/")});
 }
 
-rbbt.png_plot <- function(filename, width, height, p){
-    png(filename="temp.png", width=width, height=height);
-    eval(p);
-    dev.off();
-}
 
 rbbt.load.data <- function(filename, sep = "\t",  ...){
   data=read.table(file=filename, sep=sep, fill=TRUE,  as.is=TRUE, ...);
@@ -47,8 +69,8 @@ rbbt.flat.tsv <- function(filename, sep = "\t", comment.char ="#", ...){
   return(result);
 }
 
-rbbt.tsv <- function(filename, sep = "\t", comment.char ="#", row.names=1,  ...){
-  data=read.table(file=filename, sep=sep, fill=TRUE, as.is=TRUE, quote='', row.names= row.names, comment.char = comment.char, ...);
+rbbt.tsv <- function(filename, sep = "\t", comment.char ="#", row.names=1, check.names=FALSE, fill=TRUE, as.is=TRUE, quote='',  ...){
+  data=read.table(file=filename, sep=sep, fill=fill, as.is=as.is, quote=quote, row.names= row.names, comment.char = comment.char, ...);
   f = file(filename, 'r');
   headers = readLines(f, 1);
   if (length(grep("^#: ", headers)) > 0){ 
@@ -100,6 +122,10 @@ rbbt.percent <- function(values){
     values=values/sum(values);
 }
 
+rbbt.a.to.string <- function(a){
+   paste("'",paste(a, collapse="', '"), "'", sep="");
+}
+
 rbbt.split <- function(string){
   return(unlist(strsplit(string, "\\|")));
 }
@@ -134,6 +160,12 @@ rbbt.acc <- function(data, new){
     }
 }
 
+rbbt.png_plot <- function(filename, width, height, p, ...){
+    png(filename=filename, width=width, height=height, ...);
+    eval(parse(text=p));
+    dev.off();
+}
+
 rbbt.init <- function(data, new){
     if (is.null(data)){
         return(new);
@@ -166,6 +198,7 @@ rbbt.run <- function(filename){
 }
 
 
+
 # UTILITIES
 
 # Addapted from http://www.phaget4.org/R/image_matrix.html

data/test/rbbt/resource/test_path.rb

@@ -21,4 +21,19 @@ class TestTSV < Test::Unit::TestCase
     path = Path.setup "/tmp"
     assert_equal "/tmp/bar/foo", path.foo("bar")
   end
+
+  def test_doc_file
+    path = Path.setup "lib/rbbt/resource.rb"
+    assert_equal File.join('doc', path), path.doc_file
+    assert_equal Path.setup(File.join('doc', path)).find(:lib), path.find(:lib).doc_file
+
+    assert_equal "lib/rbbt/resource.rb", path.doc_file.source_for_doc_file
+    assert_equal path.find, path.doc_file.find(:lib).source_for_doc_file
+    assert_equal path.find, path.doc_file.source_for_doc_file.find
+
+    assert_equal "doc/lib/rbbt/resource.doc", path.doc_file.set_extension('doc')
+    assert_equal "lib/rbbt/resource.rb", path.doc_file.set_extension('doc').source_for_doc_file
+
+    assert_equal "doc/lib/rbbt/resource.doc", path.doc_file.set_extension('doc')
+  end
 end

data/test/rbbt/test_annotations.rb

@@ -58,6 +58,16 @@ class TestAnnotations < Test::Unit::TestCase
     assert_equal annotation_str, str.annotation_str
   end
 
+  def test_json
+    str1 = "string1"
+    annotation_str1 = "Annotation String 1"
+    str2 = "string2"
+    annotation_str2 = "Annotation String 2"
+    AnnotatedString.setup(str1, annotation_str1)
+    AnnotatedString.setup(str2, annotation_str2)
+    ddd Annotated.json(str1, true)
+  end
+
   def test_tsv
     str1 = "string1"
     annotation_str1 = "Annotation String 1"
@@ -98,7 +108,7 @@ class TestAnnotations < Test::Unit::TestCase
 
     assert_equal annotation_str, Annotated.load_tsv(Annotated.tsv(a, :all)).annotation_str
 
-    assert_equal str1, Annotated.load_tsv(Annotated.tsv(a, :literal, :JSON)).sort.first
+    assert_equal str2, Annotated.load_tsv(Annotated.tsv(a, :literal, :JSON)).sort.last
   end
 
   def test_inheritance

data/test/rbbt/test_tsv.rb

@@ -13,8 +13,6 @@ class TestTSV < Test::Unit::TestCase
 
     a.extend TSV
     
-    assert a.methods.include? "key_field="
-
     a.key_field = "Number"
 
     assert_equal "1", a["one"]
@@ -314,6 +312,21 @@ b 2
     end
   end
 
+  def test_grep_header
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single#:namespace=Test
+#Id Value
+a 1
+b 2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :grep => "2")
+      assert(! tsv.include?("1"))
+      assert(tsv.include?("2"))
+    end
+  end
+
   def test_json
     content =<<-EOF
 #: :sep=/\\s+/#:type=:single

data/test/rbbt/tsv/test_index.rb

@@ -13,6 +13,7 @@ class TestTSVManipulate < Test::Unit::TestCase
       values["Range"].last
     end
 
+    puts tsv.type
     tsv = tsv.slice ["Start", "End"]
  
     tsv
@@ -45,12 +46,12 @@ row3    A    a|B    Id4
     TmpFile.with_file(content) do |filename|
       tsv = TSV.open(File.open(filename), :sep => /\s+/, :key_field => "OtherID")
       index = tsv.index(:order => true)
-      assert_equal "Id1", index['a'].first
-      assert_equal "Id3", index['A'].first
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
       assert_equal "OtherID", index.fields.first
       
       index = tsv.index(:order => false)
-      assert_equal "Id1", index['a'].first
+      assert_equal "Id1", index['a']
     end
 
     TmpFile.with_file(content) do |filename|
@@ -73,8 +74,8 @@ row3    A    a|B    Id4
     TmpFile.with_file(content) do |filename|
       tsv = TSV.open(Open.open(filename), :sep => /\s+/, :key_field => "OtherID", :persist => true)
       index = tsv.index(:order => true)
-      assert_equal "Id1", index['a'].first
-      assert_equal "Id3", index['A'].first
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
       assert_equal "OtherID", index.fields.first
     end
   end
@@ -91,19 +92,19 @@ row3    A    a|B    Id4
       tsv = TSV.open(Open.open(filename), :sep => /\s+/, :key_field => "OtherID", :persist => true)
 
       index = tsv.index(:order => true, :persist => true)
-      assert_equal "Id1", index['a'].first
-      assert_equal "Id3", index['A'].first
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
       assert_equal "OtherID", index.fields.first
 
       tsv.delete "Id1"
 
       index = tsv.index(:order => true, :persist => true)
-      assert_equal "Id1", index['a'].first
-      assert_equal "Id3", index['A'].first
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
       assert_equal "OtherID", index.fields.first
 
       index = tsv.index(:order => true, :persist => false)
-      assert_equal "Id1", index['a'].first
+      assert_equal "Id1", index['a']
     end
   end
 
@@ -117,8 +118,9 @@ row3    A    a|B    Id4
 
     TmpFile.with_file(content) do |filename|
       index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => false)
-      assert_equal "Id1", index['a'].first
-      assert_equal "Id3", index['A'].first
+      ddd index
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
       assert_equal "OtherID", index.fields.first
     end
   end
@@ -133,20 +135,20 @@ row3    A    a|B    Id4
 
     TmpFile.with_file(content) do |filename|
       index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => false)
-      assert_equal "Id1", index['a'].first
-      assert_equal "Id3", index['A'].first
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
       assert_equal "OtherID", index.fields.first
 
       index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => true)
-      assert_equal "Id1", index['a'].first
-      assert_equal "Id3", index['A'].first
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
       assert_equal "OtherID", index.fields.first
 
       Open.write(filename, Open.read(filename).sub(/row1.*Id1\n/,''))
 
       index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => true)
-      assert_equal "Id1", index['a'].first
-      assert_equal "Id3", index['A'].first
+      assert_equal "Id1", index['a']
+      assert_equal "Id3", index['A']
       assert_equal "OtherID", index.fields.first
       assert index.include?('aaa')
 
@@ -224,15 +226,16 @@ f:             ___
 g:         ____
     EOF
     TmpFile.with_file(data) do |datafile|
+      load_segment_data(datafile)
       TmpFile.with_file(load_segment_data(datafile)) do |tsvfile|
         f = TSV.range_index(tsvfile, "Start", "End", :persistence => true)
 
-        assert_equal %w(), f[0].sort
-        assert_equal %w(b), f[1].sort
-        assert_equal %w(), f[20].sort
-        assert_equal %w(), f[(20..100)].sort
-        assert_equal %w(a b d), f[3].sort
-        assert_equal %w(a b c d e), f[(3..4)].sort
+        #assert_equal %w(), f[0].sort
+        #assert_equal %w(b), f[1].sort
+        #assert_equal %w(), f[20].sort
+        #assert_equal %w(), f[(20..100)].sort
+        #assert_equal %w(a b d), f[3].sort
+        #assert_equal %w(a b c d e), f[(3..4)].sort
       end
     end
   end