rbbt-util 4.0.2 → 4.1.0

data/lib/rbbt/util/open.rb

@@ -78,9 +78,9 @@ module Open
   end
 
   # Cache
-
+  #
   def self.in_cache(url, options = {})
-    digest = Digest::MD5.hexdigest([url, options["--post-data"], (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
+    digest = Digest::MD5.hexdigest([url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
 
     filename = File.join(REMOTE_CACHEDIR, digest)
     if File.exists? filename
@@ -89,9 +89,20 @@ module Open
       nil
     end
   end
+ 
+  def self.remove_from_cache(url, options = {})
+    digest = Digest::MD5.hexdigest([url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
+
+    filename = File.join(REMOTE_CACHEDIR, digest)
+    if File.exists? filename
+      FileUtils.rm filename 
+    else
+      nil
+    end
+  end
   
   def self.add_cache(url, data, options = {})
-    digest = Digest::MD5.hexdigest([url, options["--post-data"], (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
+    digest = Digest::MD5.hexdigest([url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)
     Misc.sensiblewrite(File.join(REMOTE_CACHEDIR, digest), data)
   end
 
@@ -156,7 +167,7 @@ module Open
     wget_options[:nice] = options.delete(:nice)
     wget_options[:nice_key] = options.delete(:nice_key)
     wget_options[:quiet] = options.delete(:quiet)
-    wget_options[:post] = options.delete(:post)
+    wget_options["--post-data="] = options.delete(:post) if options.include? :post
     wget_options[:cookies] = options.delete(:cookies)
 
     io = case

data/lib/rbbt/workflow.rb

@@ -148,6 +148,16 @@ module Workflow
       base.extend AnnotatedModule
       class << base
         attr_accessor :libdir, :workdir, :tasks, :task_dependencies, :task_description, :dependencies, :asynchronous_exports, :synchronous_exports, :exec_exports, :last_task
+ 
+        alias prev_workflow_extended extended if methods.include? "extended"
+
+        def extended(object)
+          self.send(:prev_workflow_extended, object) if methods.include? "prev_workflow_extended"
+          object.extend Workflow unless Workflow === object
+
+          object.tasks.merge! self.tasks
+          object.task_dependencies.merge! self.task_dependencies
+        end
 
         def dependencies
           i = @dependencies; @dependencies = []; i
@@ -242,12 +252,10 @@ module Workflow
     task = tasks[taskname]
     raise "Task not found: #{ taskname }" if task.nil?
 
-
     IndiferentHash.setup(inputs)
 
     resolve_locals(inputs)
 
-
     dependencies = real_dependencies(task, jobname, inputs, task_dependencies[taskname] || [])
 
     input_values = task.take_input_values(inputs)
@@ -256,10 +264,17 @@ module Workflow
 
     step = Step.new step_path, task, input_values, dependencies
 
+    helpers = @helpers
+    (class << step; self; end).class_eval do
+      helpers.each do |name, block|
+        define_method name, &block
+      end
+    end
+
     step
   end
 
-  def load(path)
+  def load_step(path)
     task = task_for path
     Step.new path, tasks[task]
   end

data/lib/rbbt/workflow/annotate.rb

@@ -2,7 +2,7 @@ module AnnotatedModule
   def self.extended(base)
     if not base.respond_to? :inputs
       class << base
-        attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults, :result_description
+        attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults, :result_description, :helpers
 
         def description
           i = @description; @description = ""; i
@@ -38,10 +38,15 @@ module AnnotatedModule
       base.input_types = {}
       base.input_descriptions = {}
       base.input_defaults = {}
+      base.helpers = {}
 
     end
   end
 
+  def helper(name, &block)
+    @helpers[name] = block
+  end
+
   def returns(text)
     @result_description = text
   end

data/lib/rbbt/workflow/soap.rb

@@ -11,7 +11,7 @@ class WorkflowSOAP < SimpleWS
       workdir_find = workdir
     end
  
-    @workflow.load(File.join(workdir_find, jobid))
+    @workflow.load_step(File.join(workdir_find, jobid))
   end
 
   def initialize(workflow, *args)

data/lib/rbbt/workflow/step.rb

@@ -23,8 +23,15 @@ class Step
     @inputs = inputs || []
   end
 
+  def prepare_entity(value, description = nil, info = {})
+    return value if description.nil?
+    Entity.formats[description].setup(value, info.merge(:format => description)) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? description
+    value
+  end
+
   def exec
-    @task.exec_in self, *@inputs
+    result = @task.exec_in self, *@inputs
+    prepare_entity result, @task.result_description
   end
 
   def join
@@ -41,7 +48,7 @@ class Step
   end
 
   def run
-    Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq do
+    result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq do
       log task.name, "Starting task: #{ name }"
       set_info :dependencies, @dependencies.collect{|dep| [dep.task.name, dep.name]}
       @dependencies.each{|dependency| dependency.run}
@@ -51,6 +58,8 @@ class Step
       set_info :status, :done
       res
     end
+
+    prepare_entity result, @task.result_description, info
   end
 
   def fork
@@ -78,9 +87,10 @@ class Step
 
   def load
     raise "Can not load: Step is waiting for proces #{@pid} to finish" if not done?
-    Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path} do
+    result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path} do
       exec
     end
+    prepare_entity result, @task.result_description, info
   end
 
   def clean

data/lib/rbbt/workflow/task.rb

@@ -36,8 +36,8 @@ module Task
     return [] if @inputs.nil?
     values = []
     @inputs.each do |input|
-      value = input_values[input] ||
-        IndiferentHash.setup(@input_defaults || {})[input]
+      value = input_values[input]
+      value = IndiferentHash.setup(@input_defaults || {})[input] if value.nil?
       values << value
     end
     values

data/share/install/software/lib/install_helpers

@@ -15,6 +15,12 @@ function expand_path(){
   echo "$(cd $dir && pwd)/$name"
 }
 
+function add2file(){
+    file=$1
+    line=$2
+
+    grep "$2" "$1" || echo "$2" >> "$1"
+}
 
 #{{{ GET PKG OPT_BUILD_DIR
 

data/share/lib/R/util.R

@@ -48,11 +48,16 @@ rbbt.tsv2matrix <- function(data){
   return(new);
 }
 
-rbbt.tsv.write <- function(filename, data, key.field = NULL){
+rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL){
   if (is.null(key.field)){ key.field = "ID";}
 
   f = file(filename, 'w');
 
+  if (!is.null(extra_headers)){
+      extra_headers = paste("#: ", extra_headers, "\n", sep="");
+      cat(extra_headers, file=f);
+  }
+
   header = paste("#", key.field, sep="");
   for (name in colnames(data)){ header = paste(header, name, sep="\t");}
   header = paste(header, "\n", sep="");

data/test/rbbt/test_annotations.rb

@@ -103,4 +103,11 @@ class TestAnnotations < Test::Unit::TestCase
     AnnotatedString.setup(str, annotation_str)
     assert_equal str + annotation_str, str.add_annot
   end
+
+ def test_annotation_positional2hash
+    str = "string"
+    annotation_str = "Annotation String"
+    AnnotatedString.setup(str, :annotation_str => annotation_str)
+    assert_equal str + annotation_str, str.add_annot
+  end
 end

data/test/rbbt/test_persist.rb

@@ -0,0 +1,32 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+require 'rbbt/persist'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+class TestPersist < Test::Unit::TestCase
+  def test_array_persist
+    TmpFile.with_file do |tmp|
+      10.times do
+        assert_equal ["1", "2"],(Persist.persist("Test", :array, :file => tmp) do
+          ["1", "2"]
+        end)
+      end
+    end
+  
+    TmpFile.with_file do |tmp|
+      10.times do
+        assert_equal [],(Persist.persist("Test", :array, :file => tmp) do
+          []
+        end)
+      end
+    end
+
+    TmpFile.with_file do |tmp|
+      10.times do
+        assert_equal ["1"],(Persist.persist("Test", :array, :file => tmp) do
+          ["1"]
+        end)
+      end
+    end
+  end
+end

data/test/rbbt/test_tsv.rb

@@ -19,7 +19,7 @@ class TestTSV < Test::Unit::TestCase
 
     assert_equal "1", a["one"]
   end
-   
+  
   def test_tsv
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
@@ -78,7 +78,7 @@ row2    A    B    Id3
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.open(filename, :sep => /\s+/, :fields => 1)
+      tsv = TSV.open(filename, :sep => /\s+/, :fields => [1])
       assert_equal ["a", "aa", "aaa"], tsv["row1"][0]
       assert_equal :double, tsv.type
       assert_equal [%w(a aa aaa)], tsv["row1"]
@@ -359,5 +359,104 @@ row2    A    B    Id3
     end
   end
 
+  def test_flat
+    content =<<-EOF
+#Id    ValueA 
+row1   a   aa   aaa
+row2   b  bbb bbbb bb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :flat, :fields => ["ValueA"])
+      assert_equal ["a", "aa", "aaa"], tsv["row1"]
+    end
+  end
+
+  def test_zipped
+    content =<<-EOF
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b|bb|bbb
+row2    a|aa|aaa    c|cc|ccc
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :double, :key_field => "ValueA", :zipped => true)
+      assert_equal [["row1", "row2"], ["b", "c"]], tsv["a"]
+    end
+  end
+
+  def test_named_array_key
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+      assert_equal "row1", tsv["row1"].key
+
+    end
+ 
+  end
+
+  def test_unnamed_key
+    content =<<-EOF
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :key_field => 1)
+      assert tsv.keys.include? "a"
+    end
+ 
+  end
+
+  def test_float_array
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1   0.2   0.3 0
+row2    0.1  4.5 0
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :list, :cast => :to_f)
+      assert_equal [0.2, 0.3, 0], tsv["row1"]
+      assert_equal :float_array, tsv.serializer
+    end
+ 
+  end
+
+  def test_flat_field_select
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1   a  aa  aaa
+row2    b  bb bbb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      assert TSV.open(filename, :sep => /\s+/, :key_field => "Value").include? "aa"
+    end
+  end
+
+  def test_flat2
+    content =<<-EOF
+#: :type=:flat
+#Id    Value
+row1    a|aa|aaa
+row2    A|AA|AAA
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat).include? "row1"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat)["row1"].include? "a"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id")["row1"].include? "a"
+      assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id", :fields => ["Value"])["row1"].include? "a"
+    end
+ 
+  end
+
 
 end

data/test/rbbt/test_workflow.rb

@@ -7,6 +7,14 @@ require 'test/unit'
 module TestWF
   extend Workflow
 
+  helper :user do
+    "User"
+  end 
+
+  task :user => :string do
+    user
+  end
+
   str = "TEST"
   task :str => :string do
     str
@@ -56,6 +64,9 @@ TestWF.workdir = Rbbt.tmp.test.workflow
 
 class TestWorkflow < Test::Unit::TestCase
 
+  def test_helper
+    assert_equal "User", TestWF.job(:user, "Default", :number => 3).run
+  end
   def test_job
     str = "TEST"
     job = TestWF.job(:repeat2, "Default", :number => 3).fork

data/test/rbbt/tsv/test_accessor.rb

@@ -5,6 +5,21 @@ require 'test/unit'
 
 class TestTSV < Test::Unit::TestCase
 
+  def test_to_hash
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+      hash = tsv.to_hash
+      assert hash.methods.select{|m| m =~ /#{ TSV::KEY_PREFIX }/}.empty?
+    end
+  end
+
+
   def test_tsv
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID

data/test/rbbt/tsv/test_attach.rb

@@ -170,7 +170,7 @@ B    Id3
       tsv3 = TSV.open(File.open(filename), :double, :sep => /\s+/)
     end
 
-    tsv1.attach tsv2, "OtherID"
+    tsv1.attach tsv2, :fields => "OtherID"
 
     assert_equal %w(ValueA ValueB OtherID), tsv1.fields
     assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]

data/test/rbbt/tsv/test_manipulate.rb

@@ -4,6 +4,20 @@ require 'rbbt/tsv/manipulate'
 
 class TestTSVManipulate < Test::Unit::TestCase
 
+  def test_zipped
+    content =<<-EOF
+#Id    ValueA    ValueB ValueC
+rowA    A|AA    B|BB  C|CC
+rowa    a|aa    b|BB  C|CC
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :double)
+      assert_equal ["A", "AA", "a", "aa"].sort, tsv.reorder("ValueA", nil, :zipped => true).keys.sort
+    end
+  end
+
+
   def test_through
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
@@ -104,14 +118,17 @@ row3    a    C    Id4
       new = tsv.select "ValueB" => /b|Id4/
       assert_equal %w(row1).sort, new.keys
 
-      tsv = TSV.open(filename, :sep => /\s+/, :type => :flat)
-      assert tsv.type != :double
       
       new = tsv.select %w(b Id4)
       assert_equal %w(row1 row3).sort, new.keys.sort
 
-      new = tsv.select do |k,v| v["ValueA"].include? "A" end
+      new = tsv.select do |k,v| 
+        v["ValueA"].include? "A" 
+      end
       assert_equal %w(row2).sort, new.keys.sort
+
+      tsv = TSV.open(filename, :sep => /\s+/, :type => :flat)
+      assert tsv.type != :double
     end
   end
 
@@ -191,4 +208,21 @@ row3    a    C    Id4
  
   end
 
+  def test_reorder_flat
+    content =<<-EOF
+#Id    ValueA    
+row1    a aa aaa
+row2    A
+row3    a
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :flat)
+      
+      assert_equal ["row1", "row3"].sort, tsv.reorder("ValueA")["a"]
+
+    end
+  end
+
+
 end