rbbt-util 5.13.4 → 5.13.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 43ee6ce8a12a65ea0a42e62a3eef0a917c084051
-  data.tar.gz: 91fc982769f590d4f33738c87c459afeaa5dd7be
+  metadata.gz: 419096bee8e448dd3c606c3c9770d76f347ede9b
+  data.tar.gz: 85c4317ea460338198ee814424988f0a12026dd3
 SHA512:
-  metadata.gz: 18a0478c57dd34188715d0eede18d661739f862dcc40a303383254638a93c73ae4a2bf294bcfdf2d5480f895321a8c94eafdcecf0b1a2cbcddd510103ed8311c
-  data.tar.gz: 3a3e30b11a53ea72d7b76065a646a05faec9607c53046f69db78842be35072208e23b235121fe9e909646031b45ddcff4d6f4874e1158cb0b4f30367173868f1
+  metadata.gz: 3b9cb44704148225709dc3ba2e8aaac7b586829849700347d7152e78ecc3bc2d55f73eb44429eb24038495109ea49a65154050a33f427245352de6feee9513c6
+  data.tar.gz: 9baaffb5b6eacc7781b503969098ff53bbfa4052deb846f2fbc4f592fc69e3932cc6290f39f754fd15a95f083883aaf80c5f8280c70cdcbd172bc4abbadb9101

data/lib/rbbt/persist.rb

@@ -202,10 +202,12 @@ module Persist
       rescue Aborted
         Log.warn "Persist stream thread aborted: #{ Log.color :blue, path }"
         file.abort if file.respond_to? :abort
+        raise $!
       rescue Exception
         Log.warn "Persist stream thread exception: #{ Log.color :blue, path }"
         file.abort if file.respond_to? :abort
         parent.raise $!
+        raise $!
       end
     end
     ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
@@ -233,7 +235,8 @@ module Persist
             Log.warn "Persist stream pipe exception: #{ Log.color :blue, path }"
             Log.exception $!
             stream.abort if stream.respond_to? :abort
-            parent.raise $!
+            stream.join if stream.respond_to? :join
+            raise $!
           end
         end
 
@@ -250,7 +253,7 @@ module Persist
             sin.abort if sin.respond_to? :abort
             sout.abort if sout.respond_to? :abort
             Log.exception $!
-            parent.raise $!
+            raise $!
           ensure
             sin.close unless sin.closed?
           end

data/lib/rbbt/tsv/accessor.rb

@@ -513,9 +513,9 @@ module TSV
             end
           end
         end
-      rescue
+      rescue Exception
         Log.exception $!
-        parent.raise $!
+        raise $!
       end
     end
 

data/lib/rbbt/tsv/dumper.rb

@@ -7,9 +7,8 @@ module TSV
         begin
           yield dumper
           dumper.close
-        rescue Aborted
         rescue Exception
-          parent.raise $!
+          raise $!
         end
       end
       dumper.stream

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -1,7 +1,7 @@
 module TSV
   def self.obj_stream(obj)
     case obj
-    when Step
+    when (defined? Step and Step)
       obj.result
     when IO, File
       obj
@@ -430,7 +430,7 @@ module TSV
         stream.abort if stream and stream.respond_to? :abort
         stream = obj_stream(into)
         stream.abort if stream and stream.respond_to? :abort
-        parent.raise $!
+        raise $!
       end
     end
     ConcurrentStream.setup(obj_stream(into), :threads => thread)

data/lib/rbbt/util/concurrency/threads.rb

@@ -10,9 +10,8 @@ class RbbtThreadQueue
               p = queue.pop
               block.call *p
             end
-          rescue Aborted
           rescue Exception
-            parent.raise $! 
+            raise $!
           end
         end
       else
@@ -23,9 +22,8 @@ class RbbtThreadQueue
               p = Array === p ? p << mutex : [p,mutex]
               block.call *p
             end
-          rescue Aborted
           rescue Exception
-            parent.raise $! 
+            raise $!
           end
         end
       end

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -53,10 +53,11 @@ module ConcurrentStream
   def join_threads
     if @threads and @threads.any?
       @threads.each do |t| 
-        begin
-        ensure
-          t.join unless t == Thread.current
-        end
+        t.join unless t == Thread.current
+        #begin
+        #ensure
+        #  t.join unless t == Thread.current
+        #end
       end
       @threads = []
     end

data/lib/rbbt/util/misc/lock.rb

@@ -33,10 +33,10 @@ module Misc
           end
         end
       end
+      lockfile.lock 
     end
 
     begin
-      lockfile.lock 
       res = yield lockfile
     rescue Lockfile::StolenLockError
       unlock = false

data/lib/rbbt/util/misc/pipes.rb

@@ -64,8 +64,9 @@ module Misc
         begin
           yield sin
           sin.close if close and not sin.closed?
-        rescue
-          parent.raise $!
+        rescue Exception
+          Log.warn "Exception in open_pipe: #{$!.message}"
+          raise $!
         end
       end
       ConcurrentStream.setup sout, :threads => [thread]
@@ -104,10 +105,14 @@ module Misc
         stream.abort if stream.respond_to? :abort
         stream_out1.abort if stream_out1.respond_to? :abort
         stream_out2.abort if stream_out2.respond_to? :abort
-        parent.raise $!
+        Log.warn "tee_stream_thread aborted: #{$!.message}"
       rescue Exception
         stream.abort if stream.respond_to? :abort
-        parent.raise $!
+        stream_out1.abort if stream_out1.respond_to? :abort
+        stream_out2.abort if stream_out2.respond_to? :abort
+        stream.join
+        Log.warn "Exception in tee_stream_thread: #{$!.message}"
+        raise $!
       end
     end
 
@@ -156,8 +161,6 @@ module Misc
     else
       begin
         while block = io.read(2048)
-          #return if io.eof?
-          #Thread.pass 
         end
         io.join if io.respond_to? :join
       rescue Aborted
@@ -166,6 +169,7 @@ module Misc
       rescue Exception
         Log.warn "Exception consuming stream: #{Misc.fingerprint io}: #{$!.message}"
         io.abort if io.respond_to? :abort
+        io.join
         raise $!
       end
     end
@@ -392,9 +396,12 @@ module Misc
     Thread.new(Thread.current) do |parent|
       begin
         Misc.sensiblewrite(file, save)
-      rescue
+      rescue Exception
         save.abort if save.respond_to? :abort
-        parent.raise $!
+        stream.abort if stream.respond_to? :abort
+        stream.join
+        Log.warn "Exception in save_stream: #{$!.message}"
+        raise $!
       end
     end
 

data/lib/rbbt/workflow.rb

@@ -236,7 +236,7 @@ module Workflow
   def get_job_step(step_path, task = nil, input_values = nil, dependencies = nil)
     step_path = step_path.call if Proc === step_path
     persist = input_values.nil? ? false : true
-    #persist = false
+    persist = false
     key = Path === step_path ? step_path.find : step_path
     step = Persist.memory("Step", :key => key, :repo => step_cache, :persist => persist) do
       step = Step.new step_path, task, input_values, dependencies

data/share/rbbt_commands/workflow/example

@@ -35,6 +35,7 @@ def run_task(workflow, task, name)
         success = true
       end
     end
+    sleep 0.5
     path = Open.read(res).strip if File.exists? res
   end
   path = "NO RESULT" if path.nil? or path.empty?
@@ -82,11 +83,11 @@ task_result.each do |code,res|
     if orig_name
       puts Open.read(path)
     else
-      puts "#{Log.color :green, "SUCCESS"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+      STDERR.puts "#{Log.color :green, "SUCCESS"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
       puts path
     end
   else
-    puts "#{Log.color :red, "ERROR"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+    STDERR.puts "#{Log.color :red, "ERROR"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
     puts path
   end
 end

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -6,7 +6,7 @@ class StopException < StandardError; end
 
 class TestTSVParallelThrough < Test::Unit::TestCase
 
-  def _test_traverse_tsv
+  def test_traverse_tsv
     require 'rbbt/sources/organism'
 
     head = 100
@@ -25,7 +25,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.keys.compact.sort.length
   end
 
-  def _test_traverse_tsv_cpus
+  def test_traverse_tsv_cpus
     require 'rbbt/sources/organism'
 
     head = 100
@@ -47,7 +47,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert res.values.compact.flatten.uniq.length > 0
   end
 
-  def _test_traverse_stream
+  def test_traverse_stream
     require 'rbbt/sources/organism'
 
     head = 1000
@@ -61,7 +61,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.keys.compact.sort.length
   end
 
-  def _test_traverse_stream_cpus
+  def test_traverse_stream_cpus
     require 'rbbt/sources/organism'
 
     head = 1000
@@ -75,7 +75,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.keys.compact.sort.length
   end
 
-  def _test_traverse_stream_keys
+  def test_traverse_stream_keys
     require 'rbbt/sources/organism'
 
     head = 1000
@@ -99,7 +99,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal res.sort, Organism.identifiers("Hsa").tsv(:head => head).keys.sort
   end
   
-  def _test_traverse_array
+  def test_traverse_array
     require 'rbbt/sources/organism'
 
     array = []
@@ -121,7 +121,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal array, res
   end
 
-  def _test_traverse_array_threads
+  def test_traverse_array_threads
     require 'rbbt/sources/organism'
 
     array = []
@@ -142,7 +142,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal array.sort, res.sort
   end
 
-  def _test_traverse_array_cpus
+  def test_traverse_array_cpus
     require 'rbbt/sources/organism'
 
     array = []
@@ -157,7 +157,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal array.sort, res.sort
   end
 
-  def _test_traverse_benchmark
+  def test_traverse_benchmark
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -179,7 +179,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     end
   end
 
-  def _test_traverse_into_dumper
+  def test_traverse_into_dumper
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -197,7 +197,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def _test_traverse_into_dumper_threads
+  def test_traverse_into_dumper_threads
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -217,7 +217,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def _test_traverse_into_dumper_cpus
+  def test_traverse_into_dumper_cpus
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -238,7 +238,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
 
   #{{{ TRAVERSE DUMPER
 
-  def _test_traverse_dumper
+  def test_traverse_dumper
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -259,7 +259,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def _test_traverse_dumper_threads
+  def test_traverse_dumper_threads
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -281,7 +281,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def _test_traverse_dumper_cpus
+  def test_traverse_dumper_cpus
     require 'rbbt/sources/organism'
 
     head = 10_000
@@ -309,26 +309,22 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     head = 2_000
     cpus = 2
 
-    20.times do
-      Log.info Log.color :red, "TRAVERSE EXCEPTION"
-      stream = Organism.identifiers("Hsa/jun2011").open 
-      dumper = TSV::Dumper.new Organism.identifiers("Hsa/jun2011").tsv_options
-      dumper.init
-
-      assert_raise StopException do
-        TSV.traverse stream, :head => head,  :cpus => cpus, :into => dumper do |k,v|
-          k = k.first
-          raise StopException if rand(100) < 1
-          [k,v]
-        end
-        stream = dumper.stream
-        stream.read
-        stream.join
+    Log.info Log.color :red, "TRAVERSE EXCEPTION"
+    stream = Organism.identifiers("Hsa/jun2011").open 
+    dumper = TSV::Dumper.new Organism.identifiers("Hsa/jun2011").tsv_options
+    dumper.init
+
+    assert_raise StopException do
+      TSV.traverse stream, :head => head, :_cpus => cpus, :into => dumper do |k,v|
+        k = k.first
+        raise StopException if rand(100) < 1
+        [k,v]
       end
+      Misc.consume_stream dumper.stream 
     end
   end
 
-  def _test_traverse_into_stream
+  def test_traverse_into_stream
     size = 100
     array = (1..size).to_a.collect{|n| n.to_s}
     stream = TSV.traverse array, :into => :stream do |e|
@@ -337,7 +333,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal size, stream.read.split("\n").length
   end
 
-  def _test_traverse_progress
+  def test_traverse_progress
     size = 1000
     array = (1..size).to_a.collect{|n| n.to_s}
     stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|

data/test/rbbt/tsv/test_filter.rb

@@ -3,7 +3,7 @@ require 'rbbt/tsv'
 require 'rbbt/tsv/filter'
 
 class TestTSVFilters < Test::Unit::TestCase
-  def _test_collect
+  def test_collect
     content1 =<<-EOF
 #: :sep=/\\s+/#:case_insensitive=false
 #Id    ValueA    ValueB
@@ -19,7 +19,7 @@ row2    A    B
     end
   end
 
-  def _test_through
+  def test_through
     content1 =<<-EOF
 #: :sep=/\\s+/#:case_insensitive=false
 #Id    ValueA    ValueB
@@ -37,7 +37,7 @@ row2    A    B
     end
   end
 
-  def _test_2_filters
+  def test_2_filters
     content1 =<<-EOF
 #: :sep=/\\s+/#:case_insensitive=false
 #Id    ValueA    ValueB
@@ -64,7 +64,7 @@ row3    A    C
     end
   end
 
-  def _test_filter_persistence
+  def test_filter_persistence
     content1 =<<-EOF
 #: :sep=/\\s+/#:case_insensitive=false
 #Id    ValueA    ValueB
@@ -165,7 +165,7 @@ row2 A B C
     end
   end
 
-  def _test_filename
+  def test_filename
     content1 =<<-EOF
 #: :sep=/\\s+/#:case_insensitive=false
 #Id    ValueA    ValueB

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.4
+  version: 5.13.5
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-01 00:00:00.000000000 Z
+date: 2014-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake