rbbt-util 5.23.5 → 5.23.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8fb980bbf88e61f1087f0fd28c27cb3057bd891c
-  data.tar.gz: ac0c6c3e802ebdf902301b8289b866252eaa1f35
+  metadata.gz: 6c71bcff349d90d7b342c381f84bc4958cc5f733
+  data.tar.gz: a78929b585bb9a6187e6b45bd31e825ba2000907
 SHA512:
-  metadata.gz: 6742777bd0aefb41ede80c70d76b1dab223fb443e12a7e492e99a02472e8dcafd9e37c012bda15c260ccd77b6b7c19990b28d2afd9edb4acf23d68bcf9b9197c
-  data.tar.gz: 7463782b74d10ec37222a8f9771a8cdf8a094987c96b803a8e8e122365da7bcd17f5cd3d0cca5bccc545b448eb83fa234f4172a5bf3e6fe63d560ec11bdfc0be
+  metadata.gz: 236bd1122fd5ac49d4bba5fb86eb5cf48123292703509bb7deaf323930f8550a19a30a3dd7db2e4cfa2f72b93191bab211548d5569722b3d0299a05703221538
+  data.tar.gz: 21623006af00e24c2912b3ed5d7f2dd41144bc59cd0ae5b0cd5941e805ecdc05584cec9399b294768c5e81aaf628e35676c464097d9884bb608207dba4dcc611

data/lib/rbbt/persist.rb

@@ -209,14 +209,17 @@ module Persist
       rescue Exception
         Log.medium "Persist stream thread exception: #{ Log.color :blue, path }"
         file.abort if file.respond_to? :abort
-        parent.raise $!
+        #parent.raise $!
         raise $!
       rescue Exception
         Log.exception $!
         raise $!
       end
     end
-    ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
+
+    threads = [saver_thread]
+    threads += stream.threads if stream.respond_to?(:threads) && stream.threads
+    ConcurrentStream.setup(out, :threads => threads, :filename => path)
 
     out.callback = callback
     out.abort_callback = abort_callback
@@ -225,6 +228,7 @@ module Persist
     stream.callback = callback
     stream.abort_callback = abort_callback
 
+
     out
   end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -349,7 +349,6 @@ module TSV
     end
 
     q.join
-    q.clean
     nil
   end
 
@@ -375,8 +374,8 @@ module TSV
 
     rescue Interrupt, Aborted
       error = true
-      q.abort
       Log.low{"Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.backtrace*","}"}
+      q.abort
       stream = obj_stream(obj)
       stream.abort if stream.respond_to? :abort
       stream = obj_stream(options[:into])
@@ -384,15 +383,14 @@ module TSV
       raise "Traversal aborted"
     rescue Exception
       error = true
+      Log.low{"Exception during traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.message}"}
       q.abort
-      Log.low "Exception during traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.message}"
       stream = obj_stream(obj)
       stream.abort if stream.respond_to? :abort
       stream = obj_stream(options[:into])
       stream.abort if stream.respond_to? :abort
       raise $!
     ensure
-      q.clean
       if bar
         Log::ProgressBar.remove_bar(bar, error)
       end
@@ -509,13 +507,13 @@ module TSV
   def self.traverse_stream(obj, threads = nil, cpus = nil, options = {}, &block)
     into = options[:into]        
 
-    thread = Thread.new(Thread.current) do |parent|
+    thread = Thread.new do 
       begin
         traverse_run(obj, threads, cpus, options, &block)
         into.close if into.respond_to?(:close) and not (into.respond_to? :closed? and into.closed?) 
       rescue Exception
         abort_stream obj
-        parent.raise $!
+        abort_stream into
         raise $!
       end
     end

data/lib/rbbt/util/concurrency/processes.rb

@@ -30,49 +30,17 @@ class RbbtProcessQueue
 
       @callback_queue = RbbtProcessSocket.new
 
-      @callback_thread = Thread.new(Thread.current) do |parent|
-        begin
-          loop do
-            p = @callback_queue.pop unless @callback_queue.cleaned
-
-            if Exception === p or (Array === p and Exception === p.first)
-              e = Array === p ? p.first : p
-              Log.warn "Callback recieved exception from worker: #{e.message}" unless Aborted === e or ClosedStream === e
-              raise e 
-            end
-
-            if @callback.arity == 0
-              @callback.call
-            else
-              @callback.call p
-            end
-          end
-        rescue ClosedStream
-        rescue Aborted
-          Log.warn "Callback thread aborted"
-          self._abort
-          raise $!
-        rescue Exception
-          Log.warn "Exception captured in callback: #{$!.message}"
-          self._abort
-          raise $!
-        ensure
-
-          @callback_queue.sread.close unless @callback_queue.sread.closed?
-        end
-      end
     else
       @callback, @callback_queue, @callback_thread = nil, nil, nil
     end
   end
 
-  def init(&block)
-    @init_block = block
-
+  def init_master
     RbbtSemaphore.wait_semaphore(@sem)
     @master_pid = Process.fork do
       @close_up = false
       processes_initiated = false
+      processes = []
       process_mutex = Mutex.new
 
       Signal.trap(CLOSE_SIGNAL) do
@@ -103,24 +71,22 @@ class RbbtProcessQueue
       end
 
       Signal.trap(ABORT_SIGNAL) do
-        Misc.insist([0,0.01,0.1,0.2,0.5]) do
-          @monitor_thread.raise Aborted if @monitor_thread && @monitor_thread.alive?
-        end
+        @abort_monitor = true
+        @monitor_thread.raise Aborted if @monitor_thread && @monitor_thread.alive?
       end
 
       if @callback_queue
-        Misc.purge_pipes(@queue.swrite, @queue.sread, @callback_queue.swrite, @callback_queue.sread) 
+        Misc.purge_pipes(@queue.swrite, @queue.sread, @callback_queue.swrite) 
       else
         Misc.purge_pipes(@queue.swrite, @queue.sread) 
       end
 
       @total = 0
       @count = 0
-      processes = []
 
       @manager_thread = Thread.new do
-        begin
         while true 
+          break if processes_initiated && processes.empty? && (@monitor_thread && ! @monitor_thread.alive?)
           begin
             Thread.current["working"] = true
             if @close_up
@@ -171,14 +137,10 @@ class RbbtProcessQueue
             Log.low "Aborting manager thread #{Process.pid}"
             raise Aborted
           rescue Exception
-            Log.exception $!
             raise Exception
           end
         end
-        rescue Exception
-          Log.exception $!
-          raise $!
-        end
+        Log.low "Manager thred stopped #{Process.pid}"
       end
 
       @callback_queue.close_read if @callback_queue
@@ -195,43 +157,84 @@ class RbbtProcessQueue
       @monitor_thread = Thread.new do
         begin
           while processes.any?
-            processes[0].join 
+            raise Aborted if @abort_monitor
+            processes[0].join
+            Log.debug "Joined process #{processes[0].pid} of queue #{Process.pid}"
             processes.shift
           end
+          Log.low "All processes completed #{Process.pid}"
         rescue Aborted
-          Log.warn "Aborting process monitor"
-          processes.each{|p| p.abort_and_join}
+          Log.low "Aborting process monitor #{Process.pid}"
+          processes.each{|p|  p.abort_and_join}
+          Log.low "Processes aborted #{Process.pid}"
           processes.clear
 
-          @callback_thread.kill if @callback_thread && @callback_thread.alive?
-          @manager_thread.kill if @manager_thread.alive?
+          @manager_thread.raise Aborted if @manager_thread.alive?
+          raise Aborted, "Aborted monitor thread"
         rescue Exception
-          Log.warn "Process monitor exception: #{$!.message}"
+          Log.low "Process monitor exception [#{Process.pid}]: #{$!.message}"
           processes.each{|p| p.abort_and_join}
+          Log.low "Processes aborted #{Process.pid}"
           processes.clear
 
-          @callback_thread.kill if @callback_thread && @callback_thread.alive?
-          @manager_thread.kill if @manager_thread.alive?
+          @manager_thread.raise $! if @manager_thread.alive?
+          raise Aborted, "Aborted monitor thread with exception"
         end
       end
 
       RbbtSemaphore.post_semaphore(@sem)
 
       begin
-
-        while @monitor_thread.alive?
-          @monitor_thread.join(1)
-        end
+        @monitor_thread.join
+        @manager_thread.raise TryAgain if @manager_thread.alive?
+        @manager_thread.join 
+        @callback_queue.push ClosedStream.new if @callback_queue
       rescue Exception
-        Log.exception $!
+        Kernel.exit -1
       end
 
-      Kernel.exit! 0
+      Kernel.exit 0
     end
 
-    Log.info "Cpu process (#{num_processes}) started with master: #{@master_pid}"
-    
     @queue.close_read
+    Log.info "Cpu process (#{num_processes}) started with master: #{@master_pid}"
+  end
+
+  def init(&block)
+    @init_block = block
+
+    init_master
+
+    RbbtSemaphore.synchronize(@sem) do
+    @callback_thread = Thread.new do
+      begin
+        loop do
+          p = @callback_queue.pop unless @callback_queue.cleaned
+
+          if Exception === p or (Array === p and Exception === p.first)
+            e = Array === p ? p.first : p
+            Log.low "Callback recieved exception from worker: #{e.message}" unless Aborted === e or ClosedStream === e
+            raise e 
+          end
+
+          if @callback.arity == 0
+            @callback.call
+          else
+            @callback.call p
+          end
+        end
+      rescue ClosedStream
+        Log.low "Callback thread closing"
+      rescue Aborted
+        Log.low "Callback thread aborted"
+        raise $!
+      rescue Exception
+        Log.low "Exception captured in callback: #{$!.message}"
+        raise $!
+      end
+    end if @callback_queue
+    end
+
   end
 
   def add_process
@@ -242,78 +245,84 @@ class RbbtProcessQueue
     Process.kill :USR2, @master_pid
   end
 
-  def close_callback
-    return unless @callback_thread.alive?
-    begin
-      t = Thread.new do
-        @callback_queue.push ClosedStream.new
-      end
-    rescue Exception
-      Log.warn "Error closing callback: #{$!.message}"
-    end
-    @callback_thread.join  if @callback_thread && @callback_thread.alive?
-    t.join if t && t.alive?
-  end
-
   def _join
-    error = true
+    error = :redo 
     begin
-      pid, status = Process.waitpid2 @master_pid
-      error = false if status.success?
-      raise ProcessFailed if error
-    rescue Errno::ECHILD
-    rescue Aborted
-      Log.error "Aborted joining queue"
-      raise $!
+      pid, @status = Process.waitpid2 @master_pid unless @status
+      error = true unless @status.success?
+      raise ProcessFailed.new @master_pid unless @status.success?
+      @callback_thread.join 
+      error = false
+    rescue Aborted, Interrupt
+      Log.exception $!
+      self.abort
+      error = true
+      Log.high "Process queue #{@master_pid} aborted"
+      retry
+    rescue Errno::ESRCH, Errno::ECHILD
+      retry if Misc.pid_exists? @master_pid
+      error = ! @status.success?
+    rescue ProcessFailed
     rescue Exception
-      Log.error "Exception joining queue: #{$!.message}"
-      raise $!
+      Log.exception $!
+      retry
     ensure
-      close_callback if @callback
-      @callback_thread.join if @callback_thread
-      if @join
-        if @join.arity == 1
-          @join.call(error) 
-        else
-          @join.call
+      begin
+        begin
+          self.abort
+        ensure
+          _join 
+        end if error == :redo
+
+        begin
+          @callback_thread.join 
+        rescue Exception
+        end
+
+        Log.high "Joining #{"(error) " if error}#{@master_pid} #{@join}" 
+        begin
+          if @join
+            if @join.arity == 1
+              @join.call(error) 
+            else
+              @join.call
+            end
+          end
         end
+      ensure
+        self.clean
       end
     end
-
   end
 
-  def join
+  def close_up_queue
     begin
       RbbtSemaphore.synchronize(@sem) do
         Process.kill CLOSE_SIGNAL, @master_pid
       end
     rescue Errno::ECHILD, Errno::ESRCH
-      Log.debug "Cannot kill #{@master_pid}: #{$!.message}"
-    end 
+      Log.debug "Cannot kill (CLOSE) #{@master_pid}: #{$!.message}"
+    end if Misc.pid_exists? @master_pid 
+  end
 
-    begin
-      _join
-    ensure
-      close_callback if @callback
-      @queue.swrite.close unless @queue.swrite.closed?
-    end
-    #@callback_thread.join if @callback_thread
-    self.clean
+  def join
+    close_up_queue
+
+    _join
   end
 
   def _abort
     begin
       Process.kill ABORT_SIGNAL, @master_pid
-      pid, status = Process.waitpid2 @master_pid
     rescue Errno::ECHILD, Errno::ESRCH
-      Log.warn "Cannot kill #{@master_pid}: #{$!.message}"
+      Log.debug "Cannot abort #{@master_pid}: #{$!.message}"
     end
   end
 
   def abort
     _abort
     (@callback_thread.raise(Aborted.new); @callback_thread.join) if @callback_thread and @callback_thread.alive?
-    raise Aborted.new
+    @aborted = true
   end
 
   def clean
@@ -323,7 +332,8 @@ class RbbtProcessQueue
 
     ensure
       @queue.clean if @queue
-      @callback_queue.clean if @callback_queue
+      #@callback_thread.push ClosedStream if @callback_thread && @callback_thread.alive?
+      @callback_queue.clean if @queue
     end
   end
 

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -21,7 +21,7 @@ class RbbtProcessQueue
       @cleaned = true
       @sread.close unless @sread.closed?
       @swrite.close unless @swrite.closed?
-      Log.debug "Destroying socket semaphores: #{[@key] * ", "}"
+      Log.low "Destroying socket semaphores: #{[@key] * ", "}"
       RbbtSemaphore.delete_semaphore(@write_sem)
       RbbtSemaphore.delete_semaphore(@read_sem)
     end

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -30,7 +30,6 @@ class RbbtProcessQueue
           @abort = false
           Signal.trap(20){ 
             @abort = true
-            raise Aborted
           }
 
           loop do
@@ -64,8 +63,10 @@ class RbbtProcessQueue
           retry unless @stop 
           Log.high "Worker #{Process.pid} leaving"
         rescue Exception
-          Log.exception $!
-          @callback_queue.push($!) if @callback_queue
+          begin
+            @callback_queue.push($!) if @callback_queue
+          rescue
+          end
           Kernel.exit! -1
         ensure
           @callback_queue.close_write if @callback_queue 
@@ -119,6 +120,35 @@ class RbbtProcessQueue
           sleep 0.1
         end
 
+        @current = nil
+        Signal.trap(:INT){ 
+          begin
+            Process.kill :INT, @current if @current
+          rescue Errno::ESRCH, Errno::ECHILD
+          end
+        }
+
+        Signal.trap(:USR1){ 
+          begin
+            Process.kill :USR1, @current if @current
+          rescue Errno::ESRCH, Errno::ECHILD
+          end
+        }
+
+        Signal.trap(:USR2){ 
+          begin
+            Process.kill :USR2, @current if @current
+          rescue Errno::ESRCH, Errno::ECHILD
+          end
+        }
+
+        Signal.trap(20){ 
+          begin
+            Process.kill 20, @current if @current
+          rescue Errno::ESRCH, Errno::ECHILD
+          end
+        }
+
         @current = Process.fork do
           run
         end
@@ -135,12 +165,11 @@ class RbbtProcessQueue
           Log.high "Worker #{Process.pid} respawning from #{@prev} to #{@current}"
         end
       rescue Aborted, Interrupt
-        Log.warn "Worker #{Process.pid} aborted. Current #{@current} #{Misc.pid_exists?(@current) ? "exists" : "does not exist"}"
+        Log.high "Worker #{Process.pid} aborted. Current #{@current} #{Misc.pid_exists?(@current) ? "exists" : "does not exist"}"
         Process.kill "INT", @current if Misc.pid_exists? @current
         @callback_queue.close_write if @callback_queue 
         Kernel.exit! 0
       rescue Exception
-        Log.exception $!
         raise $!
       ensure
         @monitor_thread.kill if @monitor_thread
@@ -185,10 +214,10 @@ class RbbtProcessQueue
       return unless Misc.pid_exists? @pid
       begin
         pid, status = Process.waitpid2 @pid
-        raise ProcessFailed if not status.success?
+        raise ProcessFailed.new @pid if not status.success?
       rescue Aborted
         self.abort
-        raise $!
+        raise Aborted
       rescue Errno::ESRCH, Errno::ECHILD
         Log.exception $!
       rescue ProcessFailed
@@ -204,37 +233,39 @@ class RbbtProcessQueue
       begin
         Process.kill 20, @pid
       rescue Errno::ESRCH, Errno::ECHILD
-      rescue Exception
-        Log.exception $!
       end
     end
 
     def abort_and_join
-      begin
-        Process.kill 20, @pid
-      rescue Errno::ESRCH, Errno::ECHILD
-        Log.low "Already joined worker #{@pid}"
-        return
-      end
+      self.abort
 
+      begin
       Misc.insist([0,0.05,0.5,1,2]) do
         begin
           pid, status = Process.waitpid2 @pid, Process::WNOHANG
-          raise if status.nil?
           Log.low "Abort and join of #{@pid}"
           return
+        rescue Aborted
+          abort
+          raise 
+        rescue ProcessFailed
+          Log.low "Abort and join of #{@pid} (ProcessFailed)"
+          return
         rescue Errno::ESRCH, Errno::ECHILD
           Log.low "Already joined worker #{@pid}"
           return
         end
       end
-
+      rescue Aborted
+        retry
+      end
 
       begin
         Log.low "Forcing abort of #{@pid}"
         Process.kill 9, @pid
         pid, status = Process.waitpid2 @pid
       rescue Errno::ESRCH, Errno::ECHILD
+        Log.low "Force killed worker #{@pid}"
       end
     end
 

data/lib/rbbt/util/log.rb

@@ -238,7 +238,7 @@ module Log
   def self.exception(e)
     stack = caller
     error([e.class.to_s, e.message].compact * ": " )
-    error("BACKTRACE: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
+    error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
   end
 
   def self.deprecated(m)
@@ -265,7 +265,7 @@ module Log
   def self.stack(stack)
     LOG_MUTEX.synchronize do
 
-      STDERR.puts Log.color :magenta, "Stack trace: " << Log.last_caller(caller)
+      STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
       color_stack(stack).each do |line|
         STDERR.puts line
       end

data/lib/rbbt/util/log/progress.rb

@@ -37,29 +37,24 @@ module Log
       return if ENV["RBBT_NO_PROGRESS"] == "true"
       @ticks += step
 
-      begin
-        time = Time.now
-        if @last_time.nil?
-          @last_time = time
-          @last_count = @ticks
-          @start = time
-          return
-        end
+      time = Time.now
+      if @last_time.nil?
+        @last_time = time
+        @last_count = @ticks
+        @start = time
+        return
+      end
 
-        diff = time - @last_time
-        report and return if diff > @frequency
-        return unless max and max > 0
+      diff = time - @last_time
+      report and return if diff > @frequency
+      return unless max and max > 0
 
-        percent = self.percent
-        if @last_percent.nil?
-          @last_percent = percent
-          return
-        end
-        report and return if percent > @last_percent and diff > 0.3
-      rescue Exception
-        Log.warn "Exception during report: " << $!.message
-        Log.exception $!
+      percent = self.percent
+      if @last_percent.nil?
+        @last_percent = percent
+        return
       end
+      report and return if percent > @last_percent and diff > 0.3
     end
 
     def pos(pos)

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -110,10 +110,9 @@ module ConcurrentStream
     if @callback and not joined?
       begin
         @callback.call
-      rescue Exception
-        Log.exception $!
+      ensure
+        @callback = nil
       end
-      @callback = nil
     end
   end
 
@@ -135,7 +134,7 @@ module ConcurrentStream
 
     @threads.each do |t| 
       next if t == Thread.current
-      Log.low "Aborting thread #{t.inspect} with exception: #{exception}"
+      Log.debug "Aborting thread #{t.inspect} with exception: #{exception}"
       t.raise((exception.nil? ? Aborted.new : exception))
     end 
 
@@ -150,7 +149,7 @@ module ConcurrentStream
         t.join unless t == Thread.current
       rescue Aborted
       rescue Exception
-        Log.warn "Thread exception: #{$!.message}"
+        Log.debug "Thread exception: #{$!.message}"
       end
     end
   end

data/lib/rbbt/util/misc/development.rb

@@ -155,6 +155,8 @@ def self.add_libdir(dir=nil)
     rescue StopInsist
       raise $!.exception
     rescue Aborted, Interrupt
+      Log.exception $!
+      Log.stack caller
       if msg
         Log.warn("Not Insisting after Aborted: #{$!.message} -- #{msg}")
       else
@@ -170,6 +172,10 @@ def self.add_libdir(dir=nil)
         Log.warn("Insisting after exception:  #{$!.class} #{$!.message}")
       end
 
+      Log.stack caller
+      Log.exception $!
+
+
       if sleep and try > 0
         sleep sleep
         sleep = sleep_array.shift || sleep if sleep_array

data/lib/rbbt/util/misc/exceptions.rb

@@ -3,7 +3,14 @@ class ParameterException < RbbtException; end
 class FieldNotFoundError < RbbtException;end
 class ClosedStream < RbbtException; end
 
-class ProcessFailed < StandardError; end
+class ProcessFailed < StandardError; 
+  def initialize(pid = Process.pid)
+    @pid = pid
+    @msg = "Process #{@pid} failed"
+    super(@msg)
+  end
+
+end
 
 class Aborted < StandardError; end
 

data/lib/rbbt/util/misc/pipes.rb

@@ -209,7 +209,7 @@ module Misc
         Log.medium "Tee aborting #{Misc.fingerprint stream}"
         raise $!
       rescue Exception
-        stream.abort if stream.respond_to? :abort
+        stream.abort($!) if stream.respond_to? :abort
         out_pipes.each do |sout|
           sout.abort if sout.respond_to? :abort
         end

data/share/rbbt_commands/workflow/task

@@ -394,7 +394,7 @@ begin
 
   if options.delete(:printpath)
     job.join
-    raise job.messages.last if job.error?
+    raise job.messages.last if job.error? or job.aborted?
     if Open.remote? job.path
       puts job.url + Log.color(:blue, "?_format=raw")
     else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.23.5
+  version: 5.23.6
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-06 00:00:00.000000000 Z
+date: 2018-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake