rbbt-util 5.23.7 → 5.23.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d76b8cd3df1f148aba3f5d8dd7106ec89e78110
-  data.tar.gz: 27e02877f0fcc8d75d5a1822edfb4fbd13b9f272
+  metadata.gz: f8aaf566bf17aad351fdc54c77e2cff349e23daf
+  data.tar.gz: 393fc23341ab6ecb7b7a01ae408e8ca2dea2ef9e
 SHA512:
-  metadata.gz: 5115c493080b64334eca39ae4f880df24e8ac4e84b6cb635f25bfb02c7159a93d2a9b79175cb08e5e86e1d56f84e0c55976e619b7538ec5012f87e55f275cf7d
-  data.tar.gz: 7a92cd8e8660ee5793dd90a699570cd4c97b060d795f316e34ab6eb9dd5f92fc9240f82e015924c4c0fed0a8e459c76b0f1398336093abdcd165faa82a639c7e
+  metadata.gz: 729e28c3ed2a19f84c0b702b3eb62f94b141e64b1b305caa0ae0388409fbdf34e53110922ad0ee6499b4364f5bdba8e1735b1fc491fe7c2dc137e559c66c1189
+  data.tar.gz: 943598ede0e34302e6784b1dc7201165f44ba8dae6e8bed071a828d568bc7d241f1dab49aa3c46c24c4aa77c989d06bb37a483ba4c379d73b789cb8982b2fffe

data/lib/rbbt/persist.rb

@@ -197,7 +197,7 @@ module Persist
   def self.tee_stream_thread(stream, path, type, callback = nil, abort_callback = nil, lockfile = nil)
     file, out = Misc.tee_stream(stream)
 
-    saver_thread = Thread.new(Thread.current) do |parent|
+    saver_thread = Thread.new do
       begin
         file.threads = []
         Thread.current["name"] = "file saver: " + path
@@ -209,7 +209,6 @@ module Persist
       rescue Exception
         Log.medium "Persist stream thread exception: #{ Log.color :blue, path }"
         file.abort if file.respond_to? :abort
-        #parent.raise $!
         raise $!
       rescue Exception
         Log.exception $!

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -71,162 +71,200 @@ module TSV
   def self.traverse_tsv(tsv, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
-    if callback
-      bar.init if bar
-      tsv.through options[:key_field], options[:fields] do |k,v|
-        begin
-          callback.call yield(k,v)
-        rescue Exception
-          Log.exception $!
-          raise $!
-        ensure
-          bar.tick if bar
+    begin
+      error = false
+      if callback
+        bar.init if bar
+        tsv.through options[:key_field], options[:fields] do |k,v|
+          begin
+            callback.call yield(k,v)
+          rescue Exception
+            Log.exception $!
+            raise $!
+          ensure
+            bar.tick if bar
+          end
         end
-      end
-    else
-      bar.init if bar
-      tsv.through options[:key_field], options[:fields] do |k,v|
-        begin
-          yield k,v 
-        ensure
-          bar.tick if bar
+      else
+        bar.init if bar
+        tsv.through options[:key_field], options[:fields] do |k,v|
+          begin
+            yield k,v 
+          ensure
+            bar.tick if bar
+          end
         end
       end
-    end
-    Log::ProgressBar.remove_bar(bar) if bar
-    join.call if join
+      rescue
+        Log.exception $!
+        error = true
+        raise $!
+      ensure
+        Log::ProgressBar.remove_bar(bar) if bar
+        join.call(error) if join
+      end
   end
 
   def self.traverse_hash(hash, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
-    if callback
-      bar.init if bar
-      hash.each do |k,v|
-        begin
-          callback.call yield(k,v)
-        ensure
-          bar.tick if bar
+    begin
+      error = false
+      if callback
+        bar.init if bar
+        hash.each do |k,v|
+          begin
+            callback.call yield(k,v)
+          ensure
+            bar.tick if bar
+          end
         end
-      end
-    else
-      bar.init if bar
-      hash.each do |k,v|
-        begin
-          yield k,v 
-        ensure
-          bar.tick if bar
+      else
+        bar.init if bar
+        hash.each do |k,v|
+          begin
+            yield k,v 
+          ensure
+            bar.tick if bar
+          end
         end
       end
+    rescue
+      error = true
+      raise $!
+    ensure
+      Log::ProgressBar.remove_bar(bar) if bar
+      join.call(error) if join
     end
-    Log::ProgressBar.remove_bar(bar) if bar
-    join.call if join
   end
 
   def self.traverse_array(array, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
-    if callback
-      bar.init if bar
-      array.each do |e|
-        begin
-          callback.call yield(e)
-        ensure
-          bar.tick if bar
+    begin
+      error = false
+      if callback
+        bar.init if bar
+        array.each do |e|
+          begin
+            callback.call yield(e)
+          ensure
+            bar.tick if bar
+          end
         end
-      end
-    else
-      bar.init if bar
-      array.each do |e|
-        begin
-          yield e
-        rescue Exception
-          Log.exception $!
-          raise $!
-        ensure
-          bar.tick if bar
+      else
+        bar.init if bar
+        array.each do |e|
+          begin
+            yield e
+          rescue Exception
+            Log.exception $!
+            raise $!
+          ensure
+            bar.tick if bar
+          end
         end
       end
+
+    rescue
+      error = true
+      raise $!
+    ensure
+      Log::ProgressBar.remove_bar(bar) if bar
+      join.call(error) if join
     end
-    Log::ProgressBar.remove_bar(bar) if bar
-    join.call if join
   end
 
   def self.traverse_io_array(io, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
-    if File === io and io.closed? 
-      begin
-        Log.low{"Rewinding stream #{stream_name(io)}"}
-        io.reopen io.filename, "r"
-      rescue
-        Log.exception $!
-        raise "File closed and could not reopen #{stream_name(io)}"
+    begin
+      error = false
+      if File === io and io.closed? 
+        begin
+          Log.low{"Rewinding stream #{stream_name(io)}"}
+          io.reopen io.filename, "r"
+        rescue
+          Log.exception $!
+          raise "File closed and could not reopen #{stream_name(io)}"
+        end
       end
-    end
 
-    if callback
-      bar.init if bar
-      while line = io.gets
-        if line[-1] != "\n"
-          while c = io.getc
-            line << c
-            break if c=="\n"
+      if callback
+        bar.init if bar
+        while line = io.gets
+          if line[-1] != "\n"
+            while c = io.getc
+              line << c
+              break if c=="\n"
+            end
+          end
+          begin
+            callback.call yield line.chomp
+          ensure
+            bar.tick if bar
           end
         end
-        begin
-          callback.call yield line.chomp
-        ensure
-          bar.tick if bar
-        end
-      end
-    else
-      bar.init if bar
-      while line = io.gets
-        begin
-          yield line.chomp
-        ensure
-          bar.tick if bar
+      else
+        bar.init if bar
+        while line = io.gets
+          begin
+            yield line.chomp
+          ensure
+            bar.tick if bar
+          end
         end
       end
+    rescue
+      error = true
+      raise $!
+    ensure
+      Log::ProgressBar.remove_bar(bar) if bar
+      join.call(error) if join
     end
-    Log::ProgressBar.remove_bar(bar) if bar
-    join.call if join
   end
 
   def self.traverse_io(io, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
-    if File === io and io.closed? 
-      begin
-        Log.low{"Rewinding stream #{stream_name(io)}"}
-        io.reopen io.filename, "r"
-      rescue
-        Log.exception $!
-        raise "File closed and could not reopen #{stream_name(io)}"
-      end
-    end
 
-    options[:monitor] = bar
-    if callback
-      bar.init if bar
-      exception = nil
-      begin
-      TSV::Parser.traverse(io, options) do |k,v|
+    begin
+      error = false
+      if File === io and io.closed? 
         begin
-          callback.call yield k, v
-        rescue Exception
-          exception = $!
-          raise $!
+          Log.low{"Rewinding stream #{stream_name(io)}"}
+          io.reopen io.filename, "r"
+        rescue
+          Log.exception $!
+          raise "File closed and could not reopen #{stream_name(io)}"
         end
-        bar.tick if bar
       end
-      ensure
-        raise exception if exception
+
+      options[:monitor] = bar
+      if callback
+        bar.init if bar
+        exception = nil
+        begin
+          TSV::Parser.traverse(io, options) do |k,v|
+            begin
+              callback.call yield k, v
+            rescue Exception
+              exception = $!
+              raise $!
+            end
+            bar.tick if bar
+          end
+        ensure
+          raise exception if exception
+        end
+      else
+        TSV::Parser.traverse(io, options.merge(:monitor => bar), &block)
       end
-    else
-      TSV::Parser.traverse(io, options.merge(:monitor => bar), &block)
+    rescue
+      error = true
+      raise $!
+    ensure
+      Log::ProgressBar.remove_bar(bar) if bar
+      join.call(error) if join
     end
-    Log::ProgressBar.remove_bar(bar) if bar
-    join.call if join
   end
 
   def self.traverse_obj(obj, options = {}, &block)
@@ -261,6 +299,7 @@ module TSV
           end
         rescue Aborted
           obj.abort if obj.respond_to? :abort
+          raise $!
         rescue Exception
           obj.abort if obj.respond_to? :abort
           raise $!
@@ -380,6 +419,7 @@ module TSV
       stream.abort if stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
+      q.join
       raise "Traversal aborted"
     rescue Exception
       error = true
@@ -389,6 +429,7 @@ module TSV
       stream.abort if stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
+      q.join
       raise $!
     ensure
       if bar

data/lib/rbbt/util/cmd.rb

@@ -116,10 +116,16 @@ module CMD
             in_content.join if in_content.respond_to? :join 
           end
         rescue
-          parent.raise $!
-          Process.kill "INT", pid
-        ensure
-          sin.close  unless sin.closed?
+          Log.exception $!
+          raise $!
+        #  begin
+        #    Process.kill "INT", pid
+        #    #parent.raise $!
+        #  ensure
+        #    raise $!
+        #  end
+        #ensure
+        #  #sin.close  unless sin.closed?
         end
       end
     else

data/lib/rbbt/util/concurrency/processes.rb

@@ -135,7 +135,7 @@ class RbbtProcessQueue
             retry
           rescue Aborted
             Log.low "Aborting manager thread #{Process.pid}"
-            raise Aborted
+            raise $!
           rescue Exception
             raise Exception
           end
@@ -254,16 +254,19 @@ class RbbtProcessQueue
       @callback_thread.join 
       error = false
     rescue Aborted, Interrupt
+      exception = $!
       Log.exception $!
-      self.abort
       error = true
+      self.abort
       Log.high "Process queue #{@master_pid} aborted"
       retry
     rescue Errno::ESRCH, Errno::ECHILD
       retry if Misc.pid_exists? @master_pid
       error = ! @status.success?
     rescue ProcessFailed
+      exception = $!
     rescue Exception
+      exception = $!
       Log.exception $!
       retry
     ensure
@@ -292,6 +295,12 @@ class RbbtProcessQueue
       ensure
         self.clean
       end
+
+      if exception
+        raise exception 
+      else
+        raise "Process queue #{@master_pid} failed" 
+      end if error
     end
   end
 
@@ -321,8 +330,12 @@ class RbbtProcessQueue
 
   def abort
     _abort
-    (@callback_thread.raise(Aborted.new); @callback_thread.join) if @callback_thread and @callback_thread.alive?
+    @callback_thread.raise(Aborted.new) if @callback_thread and @callback_thread.alive?
     @aborted = true
+    begin
+      _join
+    rescue
+    end
   end
 
   def clean

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -117,24 +117,26 @@ module ConcurrentStream
   end
 
   def join
-    join_threads
-    join_pids
-
-    join_callback
-
-    @joined = true
-
-    lockfile.unlock if lockfile and lockfile.locked?
-    close unless closed?
+    begin
+      join_threads
+      join_pids
+      join_callback
+      close unless closed?
+    ensure
+      @joined = true
+      lockfile.unlock if lockfile and lockfile.locked?
+    end
   end
 
   def abort_threads(exception = nil)
     return unless @threads and @threads.any?
-    Log.low "Aborting threads (#{Thread.current.inspect}) #{@threads.collect{|t| t.inspect } * ", "}"
+    name = Thread.current.inspect
+    name = filename if filename
+    Log.low "Aborting threads (#{name}) #{@threads.collect{|t| t.inspect } * ", "}"
 
     @threads.each do |t| 
       next if t == Thread.current
-      Log.debug "Aborting thread #{t.inspect} with exception: #{exception}"
+      Log.debug "Aborting thread (#{name}) #{t.inspect} with exception: #{exception}"
       t.raise((exception.nil? ? Aborted.new : exception))
     end 
 
@@ -142,14 +144,14 @@ module ConcurrentStream
       next if t == Thread.current
       if t.alive? 
         sleep 1
-        Log.low "Kill thread #{t.inspect}"
+        Log.low "Kill thread (#{name}) #{t.inspect}"
         t.kill
       end
       begin
         t.join unless t == Thread.current
       rescue Aborted
       rescue Exception
-        Log.debug "Thread exception: #{$!.message}"
+        Log.debug "Thread (#{name}) exception: #{$!.message}"
       end
     end
   end
@@ -219,11 +221,15 @@ module ConcurrentStream
   end
 
   def raise(exception)
-    threads.each do |thread|
-      thread.raise exception
-    end
+    begin
+      threads.each do |thread|
+        thread.raise exception
+      end
 
-    self.abort
+      self.abort
+    ensure
+      Kernel.raise $!
+    end
   end
 
 end

data/lib/rbbt/util/misc/development.rb

@@ -155,8 +155,6 @@ def self.add_libdir(dir=nil)
     rescue StopInsist
       raise $!.exception
     rescue Aborted, Interrupt
-      Log.exception $!
-      Log.stack caller
       if msg
         Log.warn("Not Insisting after Aborted: #{$!.message} -- #{msg}")
       else
@@ -172,10 +170,6 @@ def self.add_libdir(dir=nil)
         Log.warn("Insisting after exception:  #{$!.class} #{$!.message}")
       end
 
-      Log.stack caller
-      Log.exception $!
-
-
       if sleep and try > 0
         sleep sleep
         sleep = sleep_array.shift || sleep if sleep_array

data/lib/rbbt/util/misc/pipes.rb

@@ -68,7 +68,7 @@ module Misc
 
     if do_fork
 
-      parent_pid = Process.pid
+      #parent_pid = Process.pid
       pid = Process.fork {
         purge_pipes(sin)
         sout.close
@@ -79,7 +79,7 @@ module Misc
 
         rescue Exception
           Log.exception $!
-          Process.kill :INT, parent_pid
+          #Process.kill :INT, parent_pid
           Kernel.exit! -1
         end
         Kernel.exit! 0
@@ -92,7 +92,7 @@ module Misc
 
       ConcurrentStream.setup sin, :pair => sout
 
-      thread = Thread.new(Thread.current) do |parent|
+      thread = Thread.new do 
         begin
           
           yield sin
@@ -104,7 +104,7 @@ module Misc
         rescue Exception
           Log.medium "Exception in open_pipe: #{$!.message}"
           Log.exception $!
-          parent.raise $!
+          sin.close
           raise $!
         end
       end
@@ -442,26 +442,34 @@ module Misc
 
   def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
     Misc.open_pipe do |sin|
-      begin
-        stream = TSV.get_stream stream
+      stream = TSV.get_stream stream
 
+      line = stream.gets
+      while line =~ /^#{header_hash}/ do
+        sin.puts line
         line = stream.gets
-        while line =~ /^#{header_hash}/ do
-          sin.puts line
-          line = stream.gets
-        end
+      end
 
-        line_stream = Misc.open_pipe do |line_stream_in|
-          line_stream_in.puts line
+      line_stream = Misc.open_pipe do |line_stream_in|
+        line_stream_in.puts line
+        begin
           Misc.consume_stream(stream, false, line_stream_in)
+        rescue
+          raise $!
         end
+      end
 
-        sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
+      sorted = CMD.cmd("env LC_ALL=C sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
 
+      begin
         Misc.consume_stream(sorted, false, sin)
       rescue
-        if defined? step and step
-          step.abort
+        begin
+          Log.exception $!
+          sorted.abort
+          stream.abort
+        ensure
+          raise $!
         end
       end
     end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -236,29 +236,41 @@ class Step
       respawn = :always if respawn.nil?
 
       Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path} [#{cpus}]", :respawn => respawn) do |dep|
-        Misc.insist do
-          begin
-            dep.produce 
-            Log.warn "Error in bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
-
-          rescue Aborted
+        begin
+          Signal.trap(:INT) do
             dep.abort
-            Log.warn "Aborted bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
-            raise $!
+            raise Aborted
+          end
 
-          rescue RbbtException
-            if canfail || dep.canfail?
-              Log.warn "Allowing failing of #{dep.path}: #{dep.messages.last}"
-            else
-              Log.warn "NOT Allowing failing of #{dep.path}: #{dep.messages.last}"
-              dep.exception $!
-              if dep.recoverable_error?
-                raise $!
+          Misc.insist do
+            begin
+              dep.produce 
+              Log.warn "Error in bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
+
+            rescue Aborted
+              dep.abort
+              Log.warn "Aborted bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
+              raise $!
+
+            rescue RbbtException
+              if canfail || dep.canfail?
+                Log.warn "Allowing failing of #{dep.path}: #{dep.messages.last}"
               else
-                raise StopInsist.new($!)
+                Log.warn "NOT Allowing failing of #{dep.path}: #{dep.messages.last}"
+                dep.exception $!
+                if dep.recoverable_error?
+                  dep.abort
+                  raise $!
+                else
+                  raise StopInsist.new($!)
+                end
               end
             end
           end
+        rescue
+          iif [:ABORTIN, dep]
+          dep.abort
+          raise $!
         end
         nil
       end
@@ -416,15 +428,9 @@ class Step
         next
       end
 
-      begin
-        next if dep.done? or dep.aborted?
-      rescue
-      end
+      next if dep.done? or dep.aborted?
 
-      begin
-        dep.abort if dep.running?
-      rescue
-      end
+      dep.abort if dep.running?
     end
     kill_children
   end

data/lib/rbbt/workflow/step/run.rb

@@ -344,7 +344,6 @@ class Step
                 end
                 _clean_finished
               rescue
-                Log.exception $!
                 stop_dependencies
                 set_info :pid, nil
                 FileUtils.rm pid_file if File.exist?(pid_file)
@@ -563,15 +562,16 @@ class Step
     doretry = true
     begin
       return if done?
-      stop_dependencies
-      abort_stream
       abort_pid if running?
+      abort_stream
+      stop_dependencies
     rescue Aborted, Interrupt
       Log.medium{"#{Log.color :red, "Aborting ABORTED RETRY"} #{Log.color :blue, path}"}
       if doretry
         doretry = false
         retry
       end
+      raise $!
     rescue Exception
       if doretry
         doretry = false

data/test/rbbt/util/misc/test_pipes.rb

@@ -269,4 +269,24 @@ line4
       assert_equal text, lines.to_a * ""
     end
   end
+
+  def test_sort
+    assert_raise RbbtException do
+      io = Misc.open_pipe do |sin|
+        sin.puts "#START"
+        20.times do
+          sin.puts rand(1000).to_s
+          sleep 0.1
+        end
+        raise RbbtException
+      end
+
+      sio = Misc.sort_stream(io)
+      begin
+        Misc.consume_stream(sio, false, STDOUT)
+      rescue
+        Log.exception $!
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.23.7
+  version: 5.23.8
 platform: ruby
 authors:
 - Miguel Vazquez