rbbt-util 5.27.1 → 5.27.6

data/share/install/software/lib/install_helpers

@@ -71,7 +71,7 @@ get_pkg(){
     local url="$2"
 
     if [ ! -f "$OPT_SRC_DIR/$name.pkg" ]; then
-        wget "$url" -O "$OPT_SRC_DIR/$name.pkg"  || wget "$url" -O "$OPT_SRC_DIR/$name.pkg" --no-check-certificate
+      wget "$url" -O "$OPT_SRC_DIR/$name.pkg"  || wget "$url" -O "$OPT_SRC_DIR/$name.pkg" --no-check-certificate || (rm -f "$OPT_SRC_DIR/$name.pkg"; echo "Error downloading"; exit -1)
     fi
 }
 
@@ -85,7 +85,7 @@ uncompress_pkg(){
     mkdir -p "$OPT_BUILD_DIR"
     cd "$OPT_BUILD_DIR"
 
-    (tar xvfz $pkg || tar xvfJ $pkg || tar xvfj $pkg || unzip $pkg || echo "Error decompressing") 2> /dev/null
+    (tar xvfz $pkg || tar xvfJ $pkg || tar xvfj $pkg || unzip $pkg || (echo "Error decompressing" & cd & rmdir "$OPT_BUILD_DIR" & exit -1 ) ) 2> /dev/null
 
     cd "$old_pwd"
 }
@@ -203,8 +203,12 @@ prepare_make(){
     local old_pwd="`expand_path $(pwd)`"
     cd "`build_dir`"
 
+    [ -f bootstrap ] && (./bootstrap || exit -1)
+
     [ -d src -a ! -e CMakeLists.txt -a ! -e Makefile -a ! -e configure ] && cd src
 
+    [ -f bootstrap ] && (./bootstrap || exit -1)
+
     if [ -f config/m4 ]; then
         libtoolize --force
         aclocal
@@ -273,6 +277,8 @@ build_make(){
         setup "$name"
         clean_build
     fi
+
+    cd "$old_pwd"
 }
 
 buid_cmake(){
@@ -487,7 +493,7 @@ install_jar(){
     local url="$2"
 
     [ -d  "$OPT_DIR/$name/" ] || mkdir -p  "$OPT_DIR/$name/"
-    wget "$url" -O "$OPT_DIR/$name/$name.jar" --no-check-certificate
+    wget "$url" -O "$OPT_DIR/jars/$name.jar" || wget "$url" -O "$OPT_DIR/jars/$name.jar" --no-check-certificate || (rm "$OPT_DIR/jars/$name.jar"; exit -1)
     link "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar" 
 }
 

data/share/rbbt_commands/migrate_job

@@ -3,7 +3,7 @@
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
-require 'rbbt/workflow/remote/ssh/get'
+require 'rbbt/workflow/remote_workflow'
 
 $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -34,4 +34,6 @@ end
 
 path, search_path, _sep, *other = ARGV
 
+search_path = 'user' if search_path.nil?
+
 Step.migrate(path, search_path, options)

data/share/rbbt_commands/system/status

@@ -187,7 +187,7 @@ workflows.sort.each do |workflow,tasks|
         if info_fields and info_fields.any?
           info = begin
                    Open.open(i[:info_file]) do |f|
-                     Step::INFO_SERIALIAZER.load(f)
+                     Step::INFO_SERIALIZER.load(f)
                    end
                  rescue
                    {:status => :noinfo}
@@ -198,9 +198,10 @@ workflows.sort.each do |workflow,tasks|
       else
         info = begin
                  Open.open(i[:info_file]) do |f|
-                   Step::INFO_SERIALIAZER.load(f)
+                   Step::INFO_SERIALIZER.load(f)
                  end
                rescue
+                 Log.exception $!
                  {:status => :noinfo}
                end
         IndiferentHash.setup(info)

data/share/rbbt_commands/tsv/get

@@ -47,6 +47,28 @@ fields = options[:fields]
 key_field = options[:key_field]
 fields = fields.split(/[,|]/, -1) unless fields.nil?
 
+if TSV === tsv
+  v = tsv[key]
+  fields ||= tsv.fields
+  puts Log.color(:blue, "Key: #{ key }")
+  if fields
+    if fields.length == 1
+      if options[:lines]
+        puts (Array === v ? v.flatten*"\n" : v.to_s ) 
+      else
+        puts (Array === v ? v.flatten*"\t" : v.to_s ) 
+      end
+    else
+      fields.zip(v).each do |field,v|
+        puts "#{Log.color :magenta, field+":"} #{v}"
+      end
+    end
+  else
+    puts "#{Log.color :magenta, "value:"} #{v}"
+  end
+  exit
+end
+
 parser = TSV::Parser.new tsv, :key_field => key_field,  :fields => fields, :type => options[:type], :header_hash => options[:header_hash], :sep => options[:sep]
 fields ||= parser.fields
 
@@ -54,15 +76,19 @@ TSV.traverse(parser) do |k,v|
   next unless k.include? key
   k = k.first if Array === k
   puts Log.color(:blue, "Key: #{ k }")
-  if fields.length == 1
-    if options[:lines]
-      puts (Array === v ? v.flatten*"\n" : v.to_s ) 
+  if fields
+    if fields.length == 1
+      if options[:lines]
+        puts (Array === v ? v.flatten*"\n" : v.to_s ) 
+      else
+        puts (Array === v ? v.flatten*"\t" : v.to_s ) 
+      end
     else
-      puts (Array === v ? v.flatten*"\t" : v.to_s ) 
+      fields.zip(v).each do |field,v|
+        puts "#{Log.color :magenta, field+":"} #{v}"
+      end
     end
   else
-    fields.zip(v).each do |field,v|
-      puts "#{Log.color :magenta, field+":"} #{v}"
-    end
+    puts "#{Log.color :magenta, "value:"} #{Misc.fingerprint v}"
   end
 end

data/share/rbbt_commands/workflow/monitor

@@ -41,7 +41,7 @@ def list_jobs(options)
     clean_file = file.sub('.info','')
     begin
       next if File.exist? clean_file and $quick
-      info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
+      info = Step::INFO_SERIALIZER.load(Open.read(file, :mode => 'rb'))
       next if (File.exist?(clean_file) or info[:status] == :done) and (info[:children_pids].nil? or info[:children_done] or info[:children_pids].select{|pid| Misc.pid_exists? pid}.empty?)
     rescue Exception
       puts "Error parsing info file: #{ file }"
@@ -86,7 +86,7 @@ def clean_jobs(options)
     info = nil
     next if File.exist? clean_file
     begin
-      info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
+      info = Step::INFO_SERIALIZER.load(Open.read(file, :mode => 'rb'))
     rescue Exception
       Log.debug "Error process #{ file }"
       remove_job file if options[:errors]
@@ -117,7 +117,7 @@ begin
     end
   else
     if options[:file]
-      info = Step::INFO_SERIALIAZER.load(Open.read(options[:file], :mode => 'rb'))
+      info = Step::INFO_SERIALIZER.load(Open.read(options[:file], :mode => 'rb'))
       print_job options[:file], info
   else
     list_jobs options

data/share/rbbt_commands/workflow/task

@@ -416,7 +416,7 @@ begin
 
     job.fork
   else
-    job.produce
+    job.run(:stream)
     res = job
   end
 

data/test/rbbt/knowledge_base/test_query.rb

@@ -24,7 +24,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   EFFECT_TSV = TSV.open EFFECT, EFFECT_OPTIONS.dup 
 
-  KNOWLEDGE_BASE = KnowledgeBase.new '/tmp/kb.foo2', "Hsa"
+  KNOWLEDGE_BASE = KnowledgeBase.new Rbbt.tmp.test.kb_foo2, "Hsa"
   KNOWLEDGE_BASE.format = {"Gene" => "Ensembl Gene ID"}
 
   KNOWLEDGE_BASE.register :effects, EFFECT_TSV, EFFECT_OPTIONS.dup

data/test/rbbt/test_entity.rb

@@ -71,7 +71,9 @@ module ReversableString
       }
     end
   end
+
   persist :multiple_annotation_list, :annotations, :dir => TmpFile.tmp_file
+  #persist :multiple_annotation_list, :annotations, :dir => Rbbt.tmp.test.annots
 end
 
 class TestEntity < Test::Unit::TestCase
@@ -195,33 +197,40 @@ class TestEntity < Test::Unit::TestCase
     string3 = 'AAABBBCCC_3'
     string4 = 'AAABBBCCC_4'
 
+
+    $processed_multiple = []
+    Log.severity = 0
+
     array = ReversableString.setup([string1, string2])
     assert_equal [string1, string2].collect{|s| s.chars}, array.multiple_annotation_list
-
     assert_equal string1.length, array[0].multiple_annotation_list.length
     assert_equal $processed_multiple, [string1, string2]
 
     array = ReversableString.setup([string2, string3])
-    assert_equal string2.length, array[0].multiple_annotation_list.length
+    assert_equal [string2, string3].collect{|s| s.chars}, array.multiple_annotation_list
+    assert_equal string3, array.multiple_annotation_list.last * ""
     assert_equal $processed_multiple, [string1, string2, string3]
 
+    $processed_multiple = []
     array = ReversableString.setup([string2, string3])
+    assert_equal [string2, string3].collect{|s| s.chars}, array.multiple_annotation_list
     assert_equal string2.length, array[0].multiple_annotation_list.length
-    assert_equal $processed_multiple, [string1, string2, string3]
+    assert_equal $processed_multiple, []
 
     $processed_multiple = []
     array = ReversableString.setup([string2, string3, string4])
-    assert_equal string2.length, array[0].multiple_annotation_list.length
+    assert_equal string2.length, array.multiple_annotation_list[0].length
     assert_equal $processed_multiple, [string4]
 
     string1 = 'aaabbbccc'
     string2 = 'AAABBBCCC'
     string3 = 'AAABBBCCC_3'
     string4 = 'AAABBBCCC_4'
+
     $processed_multiple = []
     array = ReversableString.setup([string2, string3, string4])
     assert_equal string2.length, array[0].multiple_annotation_list.length
-    assert_equal $processed_multiple, [string4]
+    assert_equal $processed_multiple, []
 
   end
 

data/test/rbbt/test_knowledge_base.rb

@@ -92,9 +92,9 @@ class TestKnowledgeBase < Test::Unit::TestCase
       assert downstream.length < downstream.follow(kb, :tfacts,false).flatten.length
 
       Misc.benchmark(50) do
-        downstream.follow(kb, :tfacts,false)
-        downstream.backtrack(kb, :tfacts,false)
-        downstream.expand(kb, :tfacts,false)
+        downstream.follow(kb, :tfacts, false)
+        downstream.backtrack(kb, :tfacts, false)
+        downstream.expand(kb, :tfacts, false)
       end
 
       Misc.benchmark(50) do

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -40,8 +40,8 @@ module DepWorkflow
     end
   end
 
-  dep :task2
   dep :task5
+  dep :task2
   task :task6 => :array do
     s1 = TSV.get_stream step(:task2)
     s2 = TSV.get_stream step(:task5)
@@ -81,6 +81,26 @@ module ComputeWorkflow
 
 end
 
+module ResumeWorkflow
+  extend Workflow
+
+  resumable
+  task :resume => :string do
+    if file('foo').exists?
+      'done'
+    else
+      Open.mkdir files_dir
+      Open.touch(file('foo'))
+      raise 
+    end
+  end
+
+  dep :resume
+  task :reverse => :string do
+    step(:resume).load.reverse
+  end
+end
+
 class TestWorkflowDependency < Test::Unit::TestCase
   def test_task1
     size = 10000
@@ -152,13 +172,14 @@ class TestWorkflowDependency < Test::Unit::TestCase
     Log.severity = 0
     TmpFile.with_file(content) do |input_file|
       begin
-      job = DepWorkflow.job(:task6, "TEST", :input_file => input_file)
-      job.run(:stream)
-      io = TSV.get_stream job
-      while line = io.gets
-        last_line = line.strip
-      end
-      io.join
+        job = DepWorkflow.job(:task6, "TEST", :input_file => input_file)
+        job.recursive_clean
+        job.run(:stream)
+        io = TSV.get_stream job
+        while line = io.gets
+          last_line = line.strip
+        end
+        io.join
       rescue Exception
         job.abort
         raise $!
@@ -204,5 +225,16 @@ class TestWorkflowDependency < Test::Unit::TestCase
       assert_equal "Line #{size}\tTask1\tTask2", last_line
     end
   end
+
+  def test_resume
+    Log.severity = 0
+    job = ResumeWorkflow.job(:reverse)
+    job.recursive_clean
+    assert_raise do
+      job.run
+    end
+    assert job.dependencies.first.file('foo').exists?
+    assert_equal 'done'.reverse, job.run
+  end
 end
 

data/test/rbbt/workflow/test_remote_workflow.rb

@@ -61,6 +61,18 @@ class TestRemoteWorkflow < Test::Unit::TestCase
   def test_rest
     Log.with_severity 0 do
 
+      remote_workflow_server(TestWFRest) do |client|
+        job = client.job(:hi, nil, {})
+        job.clean
+        job = client.job(:hi, nil, {})
+        assert ! job.done?
+        job.run
+        job.produce
+        job = client.job(:hi, nil, {})
+        assert job.done?
+        sleep 1
+      end
+
       remote_workflow_server(TestWFRest) do |client|
         assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
         assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
@@ -78,7 +90,7 @@ class TestRemoteWorkflow < Test::Unit::TestCase
 
   def _test_ssh
     Log.severity = 0
-    client = RemoteWorkflow.new "ssh://turbo:Translation", "Translation"
+    client = RemoteWorkflow.new "ssh://#{ENV["HOSTNAME"]}:Translation", "Translation"
     job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])
     assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.27.1
+  version: 5.27.6
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-19 00:00:00.000000000 Z
+date: 2020-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake