rbbt-util 5.20.22 → 5.20.23

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f2f08ffb6c3f5394b0349460d1c6bd9232bcaff2
+  data.tar.gz: 32688697499d69a24888527b903ebedebed94245
+SHA512:
+  metadata.gz: 7edf42547f74103bf6df9abbb50f7371488adb0d4e4103e70983f791d0278b8cb4a8cfa8f46fcb12436705ac8ceffa5dd3a4454508fa5f06593f3a49f326ab7d
+  data.tar.gz: 31318b7facb960e7a0c6cd149afb3534674d424f9103afbf7b00452ec86a1c3e891a2bbf4c7da3303e20b14b229d4c07f85405757a3dc813beb024b9933888b2

data/lib/rbbt/persist.rb

@@ -217,9 +217,14 @@ module Persist
       end
     end
     ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
+
     out.callback = callback
     out.abort_callback = abort_callback
     out.lockfile = stream.lockfile if stream.respond_to? :lockfile and stream.lockfile
+
+    stream.callback = callback
+    stream.abort_callback = abort_callback
+
     out
   end
 
@@ -239,8 +244,8 @@ module Persist
 
         # This is to avoid calling the callbacks twice, since they have been
         # moved to the new 'res' stream
-        stream.callback = nil
-        stream.abort_callback = nil
+        #stream.callback = nil
+        #stream.abort_callback = nil
 
         res = tee_stream(stream, path, type, callback, abort_callback, lockfile)
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -76,6 +76,9 @@ module TSV
       tsv.through options[:key_field], options[:fields] do |k,v|
         begin
           callback.call yield(k,v)
+        rescue Exception
+          Log.exception $!
+          raise $!
         ensure
           bar.tick if bar
         end
@@ -202,12 +205,19 @@ module TSV
 
     if callback
       bar.init if bar
+      exception = nil
+      begin
       TSV::Parser.traverse(io, options) do |k,v|
         begin
           callback.call yield k, v
-        ensure
-          bar.tick if bar
+        rescue Exception
+          exception = $!
+          raise $!
         end
+        bar.tick if bar
+      end
+      ensure
+        raise exception if exception
       end
     else
       options[:monitor] = bar
@@ -291,36 +301,24 @@ module TSV
       end
     rescue IOError
       Log.low{"IOError traversing #{stream_name(obj)}: #{$!.message}"}
-      stream = obj_stream(obj)
-      stream.abort if stream and stream.respond_to? :abort
-      stream = obj_stream(options[:into])
-      stream.abort if stream.respond_to? :abort
+      abort_stream obj
+      abort_stream options[:into], $!
       raise $!
     rescue Errno::EPIPE
       Log.low{"Pipe closed while traversing #{stream_name(obj)}: #{$!.message}"}
-      stream = obj_stream(obj)
-      stream.abort if stream and stream.respond_to? :abort
-      stream = obj_stream(options[:into])
-      stream.abort if stream.respond_to? :abort
+      abort_stream obj
+      abort_stream options[:into], $!
       raise $!
     rescue Aborted
       Log.low{"Aborted traversing #{stream_name(obj)}"}
-      stream = obj_stream(obj)
-      stream.abort if stream and stream.respond_to? :abort
-      stream = obj_stream(options[:into])
-      stream.abort if stream.respond_to? :abort
-      Log.low{"Aborted traversing 2 #{stream_name(obj)}"}
+      abort_stream obj
+      abort_stream options[:into], $!
+      raise $!
     rescue Exception
       Log.low{"Exception traversing #{stream_name(obj)}"}
-      begin
-        stream = obj_stream(obj)
-        stream.abort if stream and stream.respond_to? :abort
-        stream = obj_stream(options[:into])
-        stream.abort if stream.respond_to? :abort
-      rescue Exception
-      ensure
-        raise $!
-      end
+      abort_stream obj
+      abort_stream options[:into], $!
+      raise $!
     end
   end
 
@@ -437,12 +435,11 @@ module TSV
       true
     rescue Aborted, Interrupt
       Log.low "Aborted storing into #{Misc.fingerprint store}"
-      stream = obj_stream(store)
-      stream.abort if stream.respond_to? :abort
+      abort_stream(store, $!)
+      raise $!
     rescue Exception
       Log.low "Exception storing into #{Misc.fingerprint store}: #{$!.message}"
-      stream = obj_stream(store)
-      stream.abort if stream.respond_to? :abort
+      abort_stream(store, $!)
       raise $!
     end
   end
@@ -503,20 +500,19 @@ module TSV
   end
 
   def self.traverse_stream(obj, threads = nil, cpus = nil, options = {}, &block)
-    into = options[:into]
+    into = options[:into]        
+
     thread = Thread.new(Thread.current) do |parent|
       begin
         traverse_run(obj, threads, cpus, options, &block)
-        into.close if into.respond_to?(:close) and not (into.respond_to? :closed? and into.closed?)
+        into.close if into.respond_to?(:close) and not (into.respond_to? :closed? and into.closed?) 
       rescue Exception
-        stream = obj_stream(obj)
-        stream.abort if stream and stream.respond_to? :abort
-        stream = obj_stream(into)
-        stream.abort if stream and stream.respond_to? :abort
+        abort_stream obj
         parent.raise $!
         raise $!
       end
     end
+
     ConcurrentStream.setup(obj_stream(into), :threads => thread)
   end
 
@@ -529,8 +525,13 @@ module TSV
         begin
           traverse(obj, options.merge(:into => sin), &block)                                                                                                                                  
         rescue Exception
-          sout.abort if sout.respond_to? :abort
-          sout.join if sout.respond_to? :join
+          Log.exception $!
+          begin
+            sout.abort if sout.respond_to? :abort
+            sout.join if sout.respond_to? :join
+          ensure
+            raise $!
+          end
         end
       end                                                                                                                                                                                   
       return sout
@@ -598,15 +599,11 @@ module TSV
           store_into into, e
         rescue Aborted
           Log.low "Aborted callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
-          stream = nil
-          stream = get_stream obj
-          stream.abort if stream.respond_to? :abort
+          abort_stream(into, $!)
           raise $!
         rescue Exception
           Log.low "Exception callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
-          stream = nil
-          stream = get_stream obj
-          stream.abort if stream.respond_to? :abort
+          abort_stream(into, $!)
           raise $!
         ensure
           bar.tick if bar
@@ -614,12 +611,17 @@ module TSV
       end
 
       bar.init if bar
-      case into
-      when TSV::Dumper, IO
-        traverse_stream(obj, threads, cpus, options, &block)
-      else
-        traverse_run(obj, threads, cpus, options, &block)
-        into.close if into.respond_to?(:close) and not (into.respond_to? :closed and into.closed?)
+      begin
+        case into
+        when TSV::Dumper, IO
+          traverse_stream(obj, threads, cpus, options, &block)
+        else
+          traverse_run(obj, threads, cpus, options, &block)
+        end
+      rescue Exception
+        Log.exception $!
+        abort_stream(into, $!)
+        raise $!
       end
 
       into

data/lib/rbbt/tsv/util.rb

@@ -58,6 +58,23 @@ module TSV
     end
   end
 
+  def self.abort_stream(file, exception = nil)
+    return if file.nil?
+    if Step === file
+      if exception
+        file.exception exception 
+      else
+        if not (file.aborted? or file.done?)
+          file.abort 
+        end
+      end
+    else
+      stream = get_stream(file)
+      stream.abort(exception) if stream.respond_to? :abort
+      stream.extend AbortedStream
+    end
+  end
+
   def self.get_stream(file, open_options = {})
     case file
     when Zlib::GzipReader

data/lib/rbbt/util/cmd.rb

@@ -3,80 +3,6 @@ require 'stringio'
 
 module CMD
 
-  #module SmartIO 
-  #  attr_accessor :pid, :cmd, :post, :in, :out, :err, :log
-  #  def self.tie(io, pid = nil, cmd = "",  post = nil, sin = nil, out = nil, err = nil, log = true)
-  #    io.extend SmartIO
-  #    io.pid = pid
-  #    io.cmd = cmd
-  #    io.in  = sin 
-  #    io.out  = out 
-  #    io.err  = err 
-  #    io.post = post
-  #    io.log = log
-
-  #    io.class.send(:alias_method, :original_close, :close)
-  #    io.class.send(:alias_method, :original_read, :read)
-  #    io
-  #  end
-
-  #  def wait_and_status
-  #    if @pid
-  #      begin
-  #        Process.waitpid(@pid)
-  #      rescue
-  #      end
-
-  #      Log.debug{"Process #{ cmd } succeded" if $? and $?.success? and log}
-
-  #      if $? and not $?.success?
-  #        Log.debug{"Raising exception" if log}
-  #        exception = ProcessFailed.new "Command [#{@pid}] '#{@cmd}' failed with error status #{$?.exitstatus}"
-  #        begin
-  #          original_close
-  #        ensure
-  #          raise exception
-  #        end
-  #      end
-  #    end
-  #  end
-
-  #  def close
-  #    self.original_read unless self.closed? or self.eof?
-
-  #    wait_and_status
-
-  #    @post.call if @post
-
-  #    original_close unless self.closed?
-  #  end
-
-  #  def force_close
-  #    if @pid
-  #      Log.debug{"Forcing close by killing '#{@pid}'" if log}
-  #      begin
-  #        Process.kill("KILL", @pid)
-  #        Process.waitpid(@pid)
-  #      rescue
-  #        Log.low{"Exception in forcing close of command [#{ @pid }, #{cmd}]: #{$!.message}"}
-  #      end
-  #    end
-
-  #    @post.call if @post
-
-  #    original_close unless self.closed?
-  #  end
-
-  #  def read(*args)
-  #    data = original_read(*args) unless self.closed? #or self.eof?
-
-  #    self.close if not self.closed? and self.eof?
-
-  #    data #|| ""
-  #  end
-  #end
-
-
   def self.process_cmd_options(options = {})
     string = ""
     options.each do |option, value|
@@ -201,7 +127,7 @@ module CMD
       end
 
       #SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
-      ConcurrentStream.setup sout, :pids => [pid], :autojoin => true, :no_fail => no_fail unless no_wait
+      ConcurrentStream.setup sout, :pids => [pid], :autojoin => no_wait, :no_fail => no_fail 
 
       sout
     else
@@ -213,7 +139,7 @@ module CMD
         serr.close
       end
 
-      ConcurrentStream.setup sout, :pids => [pid], :autojoin => true, :no_fail => no_fail
+      ConcurrentStream.setup sout, :pids => [pid], :autojoin => no_wait, :no_fail => no_fail
       out = StringIO.new sout.read
       sout.close unless sout.closed?
 

data/lib/rbbt/util/log/progress.rb

@@ -18,6 +18,7 @@ module Log
       @depth = depth
       @desc = desc.nil? ? "Progress" : desc.gsub(/\n/,' ')
       @file = file
+      @bytes = bytes
     end
 
     def percent

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -1,8 +1,17 @@
+module AbortedStream
+  attr_accessor :exception
+  def self.setup(obj, exception = nil)
+    obj.extend AbortedStream
+    obj.exception = exception
+  end
+end
+
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread
 
   def self.setup(stream, options = {}, &block)
-    threads, pids, callback, filename, autojoin, lockfile, no_fail = Misc.process_options options, :threads, :pids, :callback, :filename, :autojoin, :lockfile, :no_fail
+    
+    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = Misc.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
@@ -12,21 +21,39 @@ module ConcurrentStream
     stream.autojoin = autojoin
     stream.no_fail = no_fail
 
+    stream.pair = pair if pair
+
     callback = block if block_given?
-    if stream.callback and callback
-      old_callback = stream.callback
-      stream.callback = Proc.new do
-        old_callback.call
-        callback.call
+    if callback
+      if stream.callback
+        old_callback = stream.callback
+        stream.callback = Proc.new do
+          old_callback.call
+          callback.call
+        end
+      else
+        stream.callback = callback 
+      end
+    end
+
+    if abort_callback
+      if stream.abort_callback
+        old_abort_callback = stream.abort_callback
+        stream.abort_callback = Proc.new do
+          old_abort_callback.call
+          abort_callback.call
+        end
+      else
+        stream.abort_callback = abort_callback 
       end
-    else
-      stream.callback = callback
     end
 
     stream.filename = filename unless filename.nil?
 
     stream.lockfile = lockfile if lockfile
 
+    stream.aborted = false
+
     stream
   end
 
@@ -52,7 +79,6 @@ module ConcurrentStream
       @threads.each do |t| 
         next if t == Thread.current
         begin
-          #t.join if t.status == 'run'
           t.join unless FalseClass === t.status 
         rescue Exception
           Log.warn "Exception joining thread in ConcurrenStream: #{filename}"
@@ -97,12 +123,11 @@ module ConcurrentStream
   end
 
   def abort_threads(exception)
-    Log.medium "Aborting threads (#{Thread.current.inspect}) #{@threads.collect{|t| t.inspect } * ", "}"
+    Log.low "Aborting threads (#{Thread.current.inspect}) #{@threads.collect{|t| t.inspect } * ", "}"
 
     @threads.each do |t| 
-      @aborted = false if t == Thread.current
       next if t == Thread.current
-      Log.medium "Aborting thread #{t.inspect}"
+      Log.low "Aborting thread #{t.inspect}"
       t.raise exception ? exception : Aborted.new 
     end if @threads
 
@@ -112,18 +137,17 @@ module ConcurrentStream
       if t.alive? 
         sleep 1 unless sleeped
         sleeped = true
-        Log.medium "Kill thread #{t.inspect}"
+        Log.low "Kill thread #{t.inspect}"
         t.kill
       end
       begin
-        Log.medium "Join thread #{t.inspect}"
+        Log.low "Join thread #{t.inspect}"
         t.join unless t == Thread.current
       rescue Aborted
       rescue Exception
-        Log.exception $!
+        Log.warn "Thread exception: #{$!.message}"
       end
     end
-    Log.medium "Aborted threads (#{Thread.current.inspect}) #{@threads.collect{|t| t.inspect } * ", "}"
   end
 
   def abort_pids
@@ -137,21 +161,31 @@ module ConcurrentStream
   end
 
   def abort(exception = nil)
-    return if @aborted
-    Log.medium "Aborting stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"
+    if @aborted
+      Log.medium "YET aborted stream #{Misc.fingerprint self} [#{@aborted}]"
+      return
+    else
+      Log.medium "Aborting stream #{Misc.fingerprint self} [#{@aborted}]"
+    end
+    AbortedStream.setup(self, exception)
     @aborted = true 
     begin
-      @callback = nil
-      @abort_callback.call if @abort_callback
-      @abort_callback = nil
-      close unless closed?
+      @abort_callback.call exception if @abort_callback
 
       abort_threads(exception)
       abort_pids
+
+      @callback = nil
+      @abort_callback = nil
+
+      @pair.abort exception if @pair
+
+      close unless closed?
     ensure
-      lockfile.unlock if lockfile and lockfile.locked?
+      if lockfile and lockfile.locked?
+        lockfile.unlock 
+      end
     end
-    Log.medium "Aborted stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"
   end
 
   def super(*args)

data/lib/rbbt/util/misc/pipes.rb

@@ -74,22 +74,30 @@ module Misc
 
       ConcurrentStream.setup sout, :pids => [pid]
     else
+
+
+      ConcurrentStream.setup sin, :pair => sout
+
       thread = Thread.new(Thread.current) do |parent|
         begin
           
           yield sin
-          sin.close if close and not sin.closed?
+
+          sin.close if close and not sin.closed? and not sin.aborted?
 
         rescue Aborted
           Log.medium "Aborted open_pipe: #{$!.message}"
         rescue Exception
           Log.medium "Exception in open_pipe: #{$!.message}"
+          Log.exception $!
           parent.raise $!
           raise $!
         end
       end
 
-      ConcurrentStream.setup sout, :threads => [thread]
+
+      sin.threads = [thread]
+      ConcurrentStream.setup sout, :threads => [thread], :pair => sin
     end
 
     sout
@@ -171,7 +179,7 @@ module Misc
                 skip[i] = true
               end unless skip[i] 
             end
-        end
+          end
         rescue IOError
         end
 
@@ -201,6 +209,12 @@ module Misc
       ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename
     end
 
+    abort_callback = Proc.new do
+      out_pipes.each do |s|
+        s.abort if s.respond_to? :abort
+      end
+    end
+
     out_pipes
   end
 
@@ -261,7 +275,7 @@ module Misc
 
       begin
         into = into.find if Path === into
-        into = Open.open(into, :mode => 'w') if String === into 
+        into_path, into = into, Open.open(into, :mode => 'w') if String === into 
         into.sync = true if IO === into
         into_close = false unless into.respond_to? :close
         io.sync = true
@@ -281,11 +295,13 @@ module Misc
       rescue Aborted
         Log.medium "Consume stream aborted #{Misc.fingerprint io}"
         io.abort if io.respond_to? :abort
-        io.close unless io.closed?
+        #io.close unless io.closed?
+        FileUtils.rm into_path if into_path and File.exists? into_path
       rescue Exception
         Log.medium "Exception consuming stream: #{Misc.fingerprint io}: #{$!.message}"
-        io.abort if io.respond_to? :abort
-        io.close unless io.closed?
+        io.abort $! if io.respond_to? :abort
+        #io.close unless io.closed?
+        FileUtils.rm into_path if into_path and File.exists? into_path
         raise $!
       end
     end
@@ -372,7 +388,6 @@ module Misc
           Open.rm path if File.exist? path
         rescue Exception
           Log.medium "Exception in sensiblewrite: [#{Process.pid}] #{$!.message} -- #{ Log.color :blue, path }"
-          Log.exception $!
           content.abort if content.respond_to? :abort
           Open.rm path if File.exist? path
           raise $!

data/lib/rbbt/util/open.rb

@@ -126,13 +126,14 @@ module Open
   # Grep
   
   def self.grep(stream, grep, invert = false)
+    grep_cmd = 'egrep'
     case 
     when Array === grep
       TmpFile.with_file(grep * "\n", false) do |f|
-        CMD.cmd("grep #{invert ? '-v' : ''}", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        CMD.cmd("#{grep_cmd} #{invert ? '-v' : ''}", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
       end
     else
-      CMD.cmd("grep #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})
+      CMD.cmd("#{grep_cmd} #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})
     end
   end
 
@@ -293,7 +294,6 @@ module Open
   end
    
   def self.gunzip(stream)
-    #Zlib::GzipReader.new(stream)
     CMD.cmd('zcat', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
   end
 

data/lib/rbbt/workflow/accessor.rb

@@ -304,7 +304,6 @@ class Step
   end
 
   def exception(ex, msg = nil)
-    self._abort
     ex_class = ex.class.to_s
     set_info :backtrace, ex.backtrace
     set_info :exception, {:class => ex_class, :message => ex.message, :backtrace => ex.backtrace}
@@ -313,6 +312,7 @@ class Step
     else
       log :error, "#{msg} -- #{ex.message}"
     end
+    self._abort
   end
 
   def started?
@@ -678,7 +678,8 @@ module Workflow
                 if rec_dependency.streaming? and rec_dependency.running?
                   _inputs[i] = rec_dependency.join.load
                 else
-                  rec_dependency.run(true).join
+                  rec_dependency.run(true)
+                  rec_dependency.join
                   _inputs[i] = rec_dependency.load
                 end
               end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -202,10 +202,13 @@ class Step
       Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :respawn => :always) do |dep|
         Misc.insist do
           begin
-            dep.produce
-          rescue
+            dep.produce 
+          rescue Aborted
             dep.abort
             raise $!
+          rescue Exception
+            dep.exception $!
+            raise $!
           end
         end
         nil
@@ -315,7 +318,7 @@ class Step
 
   def stop_dependencies
     dependencies.each do |dep|
-      dep.abort
+      dep.abort unless dep.done? or dep.aborted?
     end
     kill_children
   end

data/lib/rbbt/workflow/step/run.rb

@@ -159,7 +159,15 @@ class Step
 
           if stream
             log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
-            ConcurrentStream.setup stream do
+
+            callback = Proc.new do
+              if AbortedStream === stream
+                if stream.exception
+                  raise stream.exception 
+                else
+                  raise Aborted
+                end
+              end
               begin
                 if status != :done
                   Misc.insist do
@@ -177,15 +185,30 @@ class Step
                 FileUtils.rm pid_file if File.exist?(pid_file)
               end
             end
-            stream.abort_callback = Proc.new do
+
+            abort_callback = Proc.new do |exception|
               begin
-                log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
+                if exception
+                  self.exception exception
+                else
+                  log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
+                end
+                _clean_finished
               rescue
                 Log.exception $!
                 stop_dependencies
                 FileUtils.rm pid_file if File.exist?(pid_file)
               end
             end
+
+            ConcurrentStream.setup stream, :callback => callback, :abort_callback => abort_callback
+
+            if AbortedStream === stream 
+              exception = stream.exception || Aborted
+              self.exception exception
+              _clean_finished
+              raise exception
+            end
           else
             set_info :done, (done_time = Time.now)
             set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
@@ -314,8 +337,7 @@ class Step
   end
 
   def abort_stream
-    stream = get_stream if @result
-    stream ||= @saved_stream 
+    stream = @result if IO === @result
     @saved_stream = nil
     if stream and stream.respond_to? :abort and not stream.aborted?
       begin
@@ -330,6 +352,17 @@ class Step
     end
   end
 
+  def _clean_finished
+    if Open.exists? path
+      Log.warn "Aborted job had finished. Removing result -- #{ path }"
+      begin
+        Open.rm path
+      rescue Exception
+        Log.warn "Exception removing result of aborted job: #{$!.message}"
+      end
+    end
+  end
+
   def _abort
     return if @aborted
     @aborted = true
@@ -346,14 +379,7 @@ class Step
       Log.exception $!
       retry
     ensure
-      if Open.exists? path
-        Log.warn "Aborted job had finished. Removing result -- #{ path }"
-        begin
-          Open.rm path
-        rescue Exception
-          Log.warn "Exception removing result of aborted job: #{$!.message}"
-        end
-      end
+      _clean_finished
     end
   end
 
@@ -371,9 +397,8 @@ class Step
         Misc.consume_stream stream 
         stream.join if stream.respond_to? :join
       rescue Exception
-        stream.abort
+        stream.abort $!
         self._abort
-        raise $!
       end
     end
   end
@@ -403,6 +428,7 @@ class Step
     return self if not Open.exists? info_file
 
     return self if info[:joined]
+
     pid = @pid 
 
     Misc.insist [0.1, 0.2, 0.5, 1] do
@@ -410,6 +436,7 @@ class Step
     end
 
     begin
+
       if pid.nil? or Process.pid == pid
         dependencies.each{|dep| dep.join }
       else
@@ -422,7 +449,9 @@ class Step
         pid = nil
         dependencies.each{|dep| dep.join }
       end
+
       sleep 1 until path.exists? or error? or aborted?
+
       self
     ensure
       set_info :joined, true

metadata

@@ -1,174 +1,153 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.20.22
-  prerelease: 
+  version: 5.20.23
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-12 00:00:00.000000000 Z
+date: 2016-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: progress-monitor
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: lockfile
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: spreadsheet
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: highline
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bio-bgzf
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: to_regexp
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nakayoshi_fork
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Utilities for handling tsv files, caches, etc
@@ -185,6 +164,12 @@ extra_rdoc_files:
 - README.rdoc
 files:
 - LICENSE
+- README.rdoc
+- bin/rbbt
+- bin/rbbt_Rutil.rb
+- bin/rbbt_dangling_locks.rb
+- bin/rbbt_exec.rb
+- bin/rbbt_query.rb
 - etc/app.d/base.rb
 - etc/app.d/entities.rb
 - etc/app.d/finder.rb
@@ -392,193 +377,186 @@ files:
 - share/rbbt_commands/workflow/task
 - share/unicorn.rb
 - share/workflow_config.ru
-- README.rdoc
-- test/test_helper.rb
-- test/rbbt/resource/test_path.rb
-- test/rbbt/association/test_item.rb
+- test/rbbt/annotations/test_util.rb
 - test/rbbt/association/test_database.rb
-- test/rbbt/association/test_open.rb
 - test/rbbt/association/test_index.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_open.rb
 - test/rbbt/association/test_util.rb
-- test/rbbt/util/test_concurrency.rb
-- test/rbbt/util/test_log.rb
-- test/rbbt/util/test_chain_methods.rb
-- test/rbbt/util/test_simpleopt.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/test_semaphore.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/test_open.rb
-- test/rbbt/util/test_filecache.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/resource/test_path.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/test_association.rb
+- test/rbbt/test_entity.rb
+- test/rbbt/test_fix_width_table.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/test_persist.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/test_tsv.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_field_index.rb
+- test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_matrix.rb
+- test/rbbt/tsv/test_parallel.rb
+- test/rbbt/tsv/test_parser.rb
+- test/rbbt/tsv/test_stream.rb
+- test/rbbt/tsv/test_util.rb
 - test/rbbt/util/R/test_eval.rb
 - test/rbbt/util/R/test_model.rb
-- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/concurrency/test_threads.rb
 - test/rbbt/util/log/test_progress.rb
-- test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_R.rb
-- test/rbbt/util/misc/test_lock.rb
-- test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_bgzf.rb
-- test/rbbt/util/misc/test_omics.rb
+- test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
+- test/rbbt/util/misc/test_omics.rb
+- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/test_R.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/test_colorize.rb
+- test/rbbt/util/test_concurrency.rb
 - test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/test_log.rb
 - test/rbbt/util/test_misc.rb
-- test/rbbt/test_entity.rb
+- test/rbbt/util/test_open.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/test_tmpfile.rb
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_task.rb
-- test/rbbt/test_association.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
-- test/rbbt/tsv/parallel/test_through.rb
-- test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_accessor.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_stream.rb
-- test/rbbt/tsv/test_filter.rb
-- test/rbbt/tsv/test_matrix.rb
-- test/rbbt/tsv/test_attach.rb
-- test/rbbt/tsv/test_manipulate.rb
-- test/rbbt/tsv/test_field_index.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_parser.rb
-- test/rbbt/test_packed_index.rb
-- test/rbbt/test_persist.rb
-- test/rbbt/test_fix_width_table.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/test_workflow.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/test_tsv.rb
-- bin/rbbt_query.rb
-- bin/rbbt_exec.rb
-- bin/rbbt_Rutil.rb
-- bin/rbbt
-- bin/rbbt_dangling_locks.rb
+- test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.4.6
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/test_helper.rb
+- test/rbbt/test_workflow.rb
 - test/rbbt/resource/test_path.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/util/test_concurrency.rb
-- test/rbbt/util/test_log.rb
-- test/rbbt/util/test_chain_methods.rb
-- test/rbbt/util/test_simpleopt.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_setup.rb
 - test/rbbt/util/test_cmd.rb
-- test/rbbt/util/test_semaphore.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/test_open.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/R/test_eval.rb
-- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/test_chain_methods.rb
 - test/rbbt/util/test_simpleDSL.rb
-- test/rbbt/util/log/test_progress.rb
-- test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_R.rb
+- test/rbbt/util/test_log.rb
+- test/rbbt/util/test_open.rb
 - test/rbbt/util/misc/test_lock.rb
-- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_omics.rb
-- test/rbbt/util/misc/test_multipart_payload.rb
+- test/rbbt/util/test_concurrency.rb
+- test/rbbt/util/test_R.rb
+- test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/test_colorize.rb
+- test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_misc.rb
+- test/rbbt/util/test_tmpfile.rb
+- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/test_association.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/test_resource.rb
 - test/rbbt/test_entity.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/test_tsv.rb
 - test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_doc.rb
-- test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_task.rb
-- test/rbbt/test_association.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
-- test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/workflow/test_step.rb
+- test/rbbt/workflow/test_doc.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_persist.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_stream.rb
+- test/rbbt/tsv/test_util.rb
+- test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_filter.rb
-- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_manipulate.rb
-- test/rbbt/tsv/test_field_index.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_parser.rb
-- test/rbbt/test_packed_index.rb
-- test/rbbt/test_persist.rb
+- test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
 - test/rbbt/test_fix_width_table.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/test_workflow.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/test_tsv.rb
+- test/test_helper.rb