rbbt-util 5.22.1 → 5.22.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5b9813fbac7c55028b0e3ad16c838bc28c11ba00
-  data.tar.gz: 2f2aca52f1ee3cf761e02abdda0d0b8909030749
+  metadata.gz: a706bb947413cbe8267aac3c21f7d23668313123
+  data.tar.gz: 835cc701d39015fbdb0379ff483c38513e77a134
 SHA512:
-  metadata.gz: bfab013cd31214571c69b04043e6847f2fb654a2b3b018f4bb316c8a01161b94ae4bb512a226b500db6486f1f1f6d81a0f1aa67ba78531c6d76f8dfc255354ef
-  data.tar.gz: 49b27fb08c8a80bbddb2939cff3321f8908ecc6e64d909be74756f95cd25413f8cea2d86a5f3202022a5d51db265ca610570cc4d298e6ecd3d6e04277a28021d
+  metadata.gz: c7db876d745f2dfc5dbefb8798145a29f3ef32962a31f1940bc611589fd61a9feb918616cfa1f45c769b2d67767f9eb5a9b487b2f1d35fa845c0bb645d3915f6
+  data.tar.gz: 68fe15aa3feaab425c9d90424f4bdba20206f5f10a541ace9c7a8aef6d956b4c980566feb1253501999d53a58d58bd1d3d91c0b5ec624bae5e333889cdfe24e4

data/lib/rbbt/monitor.rb

@@ -146,7 +146,8 @@ module Rbbt
           task = File.basename(taskdir)
           next if tasks and not tasks.include? task
 
-          cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/*.files\" -prune \\) -not -path '*/*.pid' -not -type d 2>/dev/null"
+          cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/*.files\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' -not -type d 2>/dev/null"
+
           files = CMD.cmd(cmd, :pipe => true)
           TSV.traverse files, :type => :array, :into => jobs, :_bar => "Finding jobs in #{ taskdir }" do |file|
             _files << file

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -393,7 +393,9 @@ module TSV
       raise $!
     ensure
       q.clean
-      Log::ProgressBar.remove_bar(bar, error) if bar
+      if bar
+        Log::ProgressBar.remove_bar(bar, error)
+      end
     end
   end
 
@@ -595,8 +597,9 @@ module TSV
     if into
       bar = Misc.process_options options, :bar
 
-      options[:join] = Proc.new do
-        Log::ProgressBar.remove_bar(bar)
+      options[:join] = Proc.new do |error|
+        error = false if error.nil?
+        Log::ProgressBar.remove_bar(bar, error)
       end if bar
 
       options[:callback] = Proc.new do |e|

data/lib/rbbt/util/concurrency/processes.rb

@@ -12,6 +12,7 @@ class RbbtProcessQueue
     @respawn = reswpan
     @offset = offset
     @queue = RbbtProcessSocket.new
+    @process_mutex = Mutex.new
   end
 
   attr_accessor :callback, :callback_queue, :callback_thread
@@ -25,7 +26,7 @@ class RbbtProcessQueue
       @callback_thread = Thread.new(Thread.current) do |parent|
         begin
           loop do
-            p = @callback_queue.pop
+            p = @callback_queue.pop unless @callback_queue.cleaned
 
             if Exception === p or (Array === p and Exception === p.first)
               e = Array === p ? p.first : p
@@ -39,14 +40,14 @@ class RbbtProcessQueue
               @callback.call p
             end
           end
+        rescue ClosedStream
         rescue Aborted
           Log.warn "Callback thread aborted"
-          @process_monitor.raise $!
+          self._abort
           raise $!
-        rescue ClosedStream
         rescue Exception
           Log.warn "Exception captured in callback: #{$!.message}"
-          @process_monitor.raise $!
+          self._abort
           raise $!
         ensure
 
@@ -59,61 +60,160 @@ class RbbtProcessQueue
   end
 
   def init(&block)
-    num_processes.times do |i|
-      @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, @offset, &block)
-    end
-    @queue.close_read
+    @init_block = block
 
-    @process_monitor = Thread.new(Thread.current) do |parent|
-      begin
-        while @processes.any?
-          @processes[0].join 
-          @processes.shift
-        end
-      rescue Aborted
-        Log.warn "Aborting process monitor"
-        @processes.each{|p| p.abort }
-        @processes.each{|p| 
+    @master_pid = Process.fork do
+      if @callback_queue
+        Misc.purge_pipes(@queue.swrite,@queue.sread,@callback_queue.swrite, @callback_queue.sread) 
+      else
+        Misc.purge_pipes(@queue.swrite,@queue.sread) 
+      end
+
+      @total = num_processes
+      @count = 0
+      @processes = []
+      @close_up = false
+
+
+      Signal.trap(:INT) do
+        @close_up = true
+        @manager_thread.raise TryAgain
+      end
+
+      @manager_thread = Thread.new do
+        while true 
           begin
-            p.join 
-          rescue ProcessFailed
+            begin
+              sleep 10
+            rescue TryAgain
+            end
+
+            @process_mutex.synchronize do
+              if @close_up
+                @total.times do
+                  @queue.push ClosedStream.new unless @queue.cleaned 
+                end unless @processes.empty?
+                @count = 0
+              end
+              while @count > 0
+                @count -= 1
+                @total += 1
+                @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, @offset, &@init_block)
+                Log.low "Added process #{@processes.last.pid} to #{Process.pid} (#{@processes.length})"
+              end
+
+              while @count < 0
+                @count += 1
+                next unless @processes.length > 1
+                first = @processes.shift
+                first.stop
+                Log.low "Removed process #{first.pid} from #{Process.pid} (#{@processes.length})"
+              end
+            end
+          rescue TryAgain
+            retry
+          rescue Aborted
+            Log.low "Closing manager thread"
+            raise Aborted
+          rescue Exception
+            Log.exception $!
+            raise Exception
           end
-        }
-      rescue Exception
-        Log.warn "Process monitor exception: #{$!.message}"
-        @processes.each{|p| p.abort }
-        @processes.each{|p| 
-          begin
-            p.join 
-          rescue ProcessFailed
+        end
+      end
+
+      Signal.trap(:USR1) do
+        @count += 1
+        @manager_thread.raise TryAgain
+      end
+
+      Signal.trap(:USR2) do
+        @count -= 1
+        @manager_thread.raise TryAgain
+      end
+
+
+      @callback_queue.close_read if @callback_queue
+
+      num_processes.times do |i|
+        @process_mutex.synchronize do
+          @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, @offset, &@init_block)
+        end
+      end
+
+      @monitor_thread = Thread.new do
+        begin
+          while @processes.any?
+            @processes[0].join 
+            @processes.shift
           end
-        }
-        @callback_thread.raise $! if @callback_thread and @callback_thread.alive?
-        raise $!
+        rescue Aborted
+          Log.warn "Aborting process monitor"
+          @processes.each{|p| p.abort_and_join}
+          @processes.clear
+
+          @callback_thread.kill if @callback_thread && @callback_thread.alive?
+          @manager_thread.kill if @manager_thread.alive?
+        rescue Exception
+          Log.warn "Process monitor exception: #{$!.message}"
+          @processes.each{|p| p.abort_and_join}
+          @processes.clear
+
+          @callback_thread.kill if @callback_thread && @callback_thread.alive?
+          @manager_thread.kill if @manager_thread.alive?
+        end
+      end
+
+      Signal.trap(20) do
+        begin
+          @monitor_thread.raise Aborted.new
+        rescue Exception
+          Log.exception $!
+        end
+      end
+
+      begin
+        @monitor_thread.join
+      rescue Exception
+        Log.exception $!
       end
+
+      Kernel.exit! 0
     end
+
+    Log.info "Cpu process (#{num_processes}) started with master: #{@master_pid}"
+    
+    @queue.close_read
+  end
+
+  def add_process
+    Process.kill :USR1, @master_pid
+  end
+
+  def remove_process
+    Process.kill :USR2, @master_pid
   end
 
   def close_callback
+    return unless @callback_thread.alive?
     begin
-      @callback_queue.push ClosedStream.new if @callback_thread.alive?
+      t = Thread.new do
+        @callback_queue.push ClosedStream.new
+      end
     rescue Exception
       Log.warn "Error closing callback: #{$!.message}"
     end
     @callback_thread.join  #if @callback_thread.alive?
+    t.join
   end
 
-  def join
-    begin
-      @processes.length.times do 
-        @queue.push ClosedStream.new
-      end if @process_monitor.alive?
-    rescue Exception
-    end
-
+  def _join
+    error = true
     begin
-      @process_monitor.join
-      close_callback if @callback
+      pid, status = Process.waitpid2 @master_pid
+      error = false if status.success?
+      raise ProcessFailed if error
+    rescue Errno::ECHILD
     rescue Aborted
       Log.error "Aborted joining queue"
       raise $!
@@ -121,34 +221,64 @@ class RbbtProcessQueue
       Log.error "Exception joining queue: #{$!.message}"
       raise $!
     ensure
-      @queue.swrite.close unless @queue.swrite.closed?
+      if @join
+        if @join.arity == 1
+          @join.call(error) 
+        else
+          @join.call
+        end
+      end
     end
 
-    @join.call if @join
   end
 
-  def clean
-    if (@process_monitor and @process_monitor.alive?) or (@callback_thread and @callback_thread.alive?)
-      self.abort 
-      self.join
+  def join
+    begin
+      Process.kill :INT, @master_pid
+    rescue Errno::ECHILD, Errno::ESRCH
+      Log.debug "Cannot kill #{@master_pid}: #{$!.message}"
+    end 
+
+    begin
+      _join
+    ensure
+      close_callback if @callback
+      @queue.swrite.close unless @queue.swrite.closed?
     end
+    @callback_thread.join if @callback_thread
+    self.clean
+  end
 
-    @queue.clean if @queue
-    @callback_queue.clean if @callback_queue
+  def _abort
+    begin
+      Process.kill 20, @master_pid
+    rescue Errno::ECHILD, Errno::ESRCH
+      Log.debug "Cannot kill #{@master_pid}: #{$!.message}"
+    end
+
+    begin
+      _join
+    rescue ProcessFailed
+    end
   end
 
   def abort
+    _abort
+    (@callback_thread.raise(Aborted.new); @callback_thread.join) if @callback_thread and @callback_thread.alive?
+    raise Aborted.new
+  end
+
+  def clean
     begin
-      (@process_monitor.raise(Aborted.new); @process_monitor.join) if @process_monitor and @process_monitor.alive?
-      (@callback_thread.raise(Aborted.new); @callback_thread.join) if @callback_thread and @callback_thread.alive?
+      self.abort if Misc.pid_exists?(@master_pid)
+
     ensure
-      begin
-        join
-      rescue ProcessFailed
-      end
+      @queue.clean if @queue
+      @callback_queue.clean if @callback_queue
     end
   end
 
+
   def process(*e)
     begin
       @queue.push e

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -3,12 +3,11 @@ require 'rbbt/util/semaphore'
 class RbbtProcessQueue
   class RbbtProcessSocket
 
-    attr_accessor :sread, :swrite, :write_sem, :read_sem
+    attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned
     def initialize(serializer = nil)
       @sread, @swrite = Misc.pipe
 
       @serializer = serializer || Marshal
-                   
 
       @key = "/" << rand(1000000000).to_s << '.' << Process.pid.to_s;
       @write_sem = @key + '.in'
@@ -19,6 +18,7 @@ class RbbtProcessQueue
     end
 
     def clean
+      @cleaned = true
       @sread.close unless @sread.closed?
       @swrite.close unless @swrite.closed?
       Log.debug "Destroying socket semaphores: #{[@key] * ", "}"

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -12,48 +12,63 @@ class RbbtProcessQueue
 
     def run
       begin
-        Signal.trap(:INT){ 
-          Kernel.exit! -1
-        }
+        begin
+          Signal.trap(:INT){ 
+            Kernel.exit! -1
+          }
 
-        @stop = false
-        Signal.trap(:USR1){ 
-          @stop = true
-        }
+          @respawn = false
+          Signal.trap(:USR1){ 
+            @respawn = true
+          }
 
-        @abort = false
-        Signal.trap(:USR2){ 
-          @abort = true
-        }
+          @stop = false
+          Signal.trap(:USR2){ 
+            @stop = true
+          }
 
+          @abort = false
+          Signal.trap(20){ 
+            @abort = true
+            raise Aborted
+          }
 
-        loop do
-          p = @queue.pop
-          next if p.nil?
-          raise p if Exception === p
-          raise p.first if Array === p and Exception === p.first
-          begin
-            res = @block.call *p
-            @callback_queue.push res if @callback_queue
-          rescue Respawn
-            @callback_queue.push $!.payload 
-            raise $!
+          loop do
+            p = @queue.pop
+            next if p.nil?
+            raise p if Exception === p
+            raise p.first if Array === p and Exception === p.first
+            begin
+              res = @block.call *p
+              @callback_queue.push res if @callback_queue
+            rescue Respawn
+              @callback_queue.push $!.payload 
+              raise $!
+            end
+            raise Respawn if @respawn
+            if @stop
+              Log.high "Worker #{Process.pid} leaving"
+              break
+            end
           end
-          raise Respawn if @stop
-          raise Aborted if @abort
+          Kernel.exit! 0
+        rescue Respawn
+          Kernel.exit! 28
+        rescue ClosedStream
+        rescue Interrupt,Aborted
+          Log.high "Worker #{Process.pid} aborted"
+        rescue SemaphoreInterrupted
+          retry unless @stop 
+          Log.high "Worker #{Process.pid} leaving"
+        rescue Exception
+          Log.exception $!
+          @callback_queue.push($!) if @callback_queue
+          Kernel.exit! -1
+        ensure
+          @callback_queue.close_write if @callback_queue 
         end
-        Kernel.exit! 0
-      rescue Respawn
-        Kernel.exit! 28
-      rescue ClosedStream
-      rescue Aborted, Interrupt
-        Log.info "Worker #{Process.pid} aborted"
-      rescue Exception
-        Log.exception $!
-        @callback_queue.push($!) if @callback_queue
-        Kernel.exit! -1
-      ensure
-        @callback_queue.close_write if @callback_queue 
+      rescue Aborted
+        Log.high "Worker #{Process.pid} aborted"
       end
       Kernel.exit! 0
     end
@@ -81,15 +96,16 @@ class RbbtProcessQueue
           begin
             while true
               @monitored = true
-              current = @current ? 0 : Misc.memory_use(@current) 
-              if current > memory_cap and not @asked
-                Log.medium "Worker #{@current} for #{Process.pid} asked to respawn -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap} - current: #{current}"
+
+              current_mem = @current ? Misc.memory_use(@current) : 0
+              if current_mem > memory_cap and not @asked
+                Log.medium "Worker #{@current} for #{Process.pid} asked to respawn -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap} - current: #{current_mem}"
                 RbbtSemaphore.synchronize(@callback_queue.write_sem) do
                   Process.kill "USR1", @current if @current
                 end
                 @asked = true
               end
-              sleep 3 + rand(5)
+              sleep 2
             end
           rescue
             Log.exception $!
@@ -99,6 +115,7 @@ class RbbtProcessQueue
         while ! @monitored
           sleep 0.1
         end
+
         @current = Process.fork do
           run
         end
@@ -161,14 +178,65 @@ class RbbtProcessQueue
     end
 
     def join
-      Process.waitpid @pid
-      raise ProcessFailed if not $?.success?
+      return unless Misc.pid_exists? @pid
+      begin
+        pid, status = Process.waitpid2 @pid
+        raise ProcessFailed if not status.success?
+      rescue Aborted
+        self.abort
+        raise $!
+      rescue Errno::ESRCH, Errno::ECHILD
+        Log.exception $!
+      rescue ProcessFailed
+        raise $!
+      rescue Exception
+        Log.exception $!
+        raise $!
+      end
     end
 
+
     def abort
+      begin
+        Process.kill 20, @pid
+      rescue Errno::ESRCH, Errno::ECHILD
+      rescue Exception
+        Log.exception $!
+      end
+    end
+
+    def abort_and_join
+      begin
+        Process.kill 20, @pid
+      rescue Errno::ESRCH, Errno::ECHILD
+        Log.low "Already joined worker #{@pid}"
+        return
+      end
+
+      Misc.insist([0,0.05,0.5,1,2]) do
+        begin
+          pid, status = Process.waitpid2 @pid, Process::WNOHANG
+          raise if status.nil?
+          Log.low "Abort and join of #{@pid}"
+          return
+        rescue Errno::ESRCH, Errno::ECHILD
+          Log.low "Already joined worker #{@pid}"
+          return
+        end
+      end
+
+
+      begin
+        Log.low "Forcing abort of #{@pid}"
+        Process.kill 9, @pid
+        pid, status = Process.waitpid2 @pid
+      rescue Errno::ESRCH, Errno::ECHILD
+      end
+    end
+
+    def stop
       begin
         Process.kill :USR2, @pid
-        Process.kill :INT, @pid
       rescue Errno::ESRCH 
       rescue Exception
         Log.exception $!

data/lib/rbbt/util/log/progress/util.rb

@@ -58,6 +58,9 @@ module Log
     end
 
     def self.remove_bar(bar, error = false)
+      BAR_MUTEX.synchronize do
+        return if REMOVE.include? bar
+      end
       if error
         bar.error if bar.respond_to? :error
       else

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -84,7 +84,7 @@ module ConcurrentStream
           if no_fail
             Log.low "Not failing on exception joining thread in ConcurrenStream: #{filename}"
           else
-            Log.warn "Exception joining thread in ConcurrenStream: #{filename}"
+            Log.low "Exception joining thread in ConcurrenStream: #{filename}"
             raise $! 
           end
         end

data/lib/rbbt/util/misc/exceptions.rb

@@ -1,13 +1,15 @@
 class RbbtException < StandardError; end
 class ParameterException < RbbtException; end
 class FieldNotFoundError < RbbtException;end
-class TryAgain < RbbtException; end
 class ClosedStream < RbbtException; end
 
 class ProcessFailed < RbbtException; end
 
 class Aborted < StandardError; end
 
+class TryAgain < StandardError; end
+class SemaphoreInterrupted < TryAgain; end
+
 class RemoteServerError < RbbtException; end
 
 class DependencyError < Aborted

data/lib/rbbt/util/misc/inspect.rb

@@ -95,6 +95,14 @@ module Misc
         end
         new
       end
+    when Float
+      if obj.abs > 10
+        "%.1f" % obj
+      elsif obj.abs > 1
+        "%.3f" % obj
+      else
+        "%.6f" % obj
+      end
     else
       obj.to_s
     end

data/lib/rbbt/util/open.rb

@@ -472,8 +472,11 @@ module Open
     case
     when block_given?
       begin
-        File.open(file, mode) do |f| 
+        f = File.open(file, mode)
+        begin
           yield f
+        ensure
+          f.close unless f.closed?
         end
       rescue Exception
         FileUtils.rm file if File.exist? file

data/lib/rbbt/util/semaphore.rb

@@ -33,11 +33,13 @@ if continue
       EOF
 
       builder.c_singleton <<-EOF
-  void wait_semaphore(char* name){
+  int wait_semaphore(char* name){
+    int ret;
     sem_t* sem;
     sem = sem_open(name, 0);
-    sem_wait(sem);
+    ret = sem_wait(sem);
     sem_close(sem);
+    return(ret);
   }
       EOF
 
@@ -53,7 +55,8 @@ if continue
 
     SEM_MUTEX = Mutex.new
     def self.synchronize(sem)
-      RbbtSemaphore.wait_semaphore(sem)
+      ret = RbbtSemaphore.wait_semaphore(sem)
+      raise SemaphoreInterrupted if ret == -1
       begin
         yield
       ensure

data/lib/rbbt/workflow/accessor.rb

@@ -50,6 +50,14 @@ class Step
     path.nil? ? nil : path + '.info'
   end
 
+  def self.tmp_path(path)
+    path = path.find if Path === path
+    path = File.expand_path(path)
+    dir = File.dirname(path)
+    filename = File.basename(path)
+    File.join(dir, '.' << filename)
+  end
+
   def self.pid_file(path)
     path.nil? ? nil : path + '.pid'
   end
@@ -447,6 +455,10 @@ class Step
     @files_dir ||= Step.files_dir path
   end
 
+  def tmp_path
+    @tmp_path ||= Step.tmp_path path
+  end
+
   def files
     files = Dir.glob(File.join(files_dir, '**', '*')).reject{|path| File.directory? path}.collect do |path| 
       Misc.path_relative_to(files_dir, path) 
@@ -898,7 +910,7 @@ module Workflow
     if inputs.length > 0 or dependencies.any?
       tagged_jobname = case TAG
                        when :hash
-                         hash_str = Misc.obj2digest({:inputs => inputs, :dependencies => dependencies})
+                         hash_str = Misc.obj2digest({:inputs => Annotated.purge(inputs), :dependencies => dependencies})
                          jobname + '_' << hash_str
                        when :inputs
                          all_inputs = {}

data/lib/rbbt/workflow/step.rb

@@ -266,9 +266,7 @@ class Step
 
       Misc.insist do
         Open.rm info_file if Open.exists? info_file
-        #Open.rm info_file + '.lock' if Open.exists? info_file + '.lock'
         Open.rm path if Open.exists? path
-        #Open.rm path + '.lock' if Open.exists? path + '.lock'
         Open.rm_rf files_dir if Open.exists? files_dir
         Open.rm pid_file if Open.exists? pid_file
       end

data/lib/rbbt/workflow/step/run.rb

@@ -364,6 +364,9 @@ class Step
 
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
+          if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
+            FileUtils.mv self.tmp_path, self.path
+          end
           result
         end
 
@@ -526,7 +529,6 @@ class Step
         stream.abort 
       rescue Aborted, Interrupt
         Log.medium "Aborting job stream #{stream.inspect} ABORTED RETRY -- #{Log.color :blue, path}"
-        Log.exception $!
         if doretry
           doretry = false
           retry
@@ -563,7 +565,6 @@ class Step
         retry
       end
     rescue Exception
-      Log.exception $!
       if doretry
         doretry = false
         retry

data/share/rbbt_commands/system/status

@@ -124,7 +124,7 @@ jobs = Rbbt.job_info workflow, task
 workflows = {}
 
 TSV.traverse jobs, :_bar => "Checking job status" do |file,info|
-  next unless all or not info[:done] or not File.exist? file
+  next unless all || ! info[:done] || ! File.exist?(file)
   workflow = info[:workflow]
   task = info[:task]
   workflows[workflow] ||= {}
@@ -182,7 +182,7 @@ workflows.sort.each do |workflow,tasks|
             value = info[field]
             next if  value.nil?
             value_str = Misc.fingerprint(value)
-            str << " #{field}=#{value_str}"
+            str << "\t#{Log.color :magenta, field}=#{value_str}"
           end
         end
       end

data/share/rbbt_commands/workflow/prov

@@ -80,7 +80,7 @@ def report_msg(status, name, path, info = nil)
       value = job_inputs[input]
       next if  value.nil?
       value_str = Misc.fingerprint(value)
-      str << " #{Log.color :magenta, input}=#{value_str}"
+      str << "\t#{Log.color :magenta, input}=#{value_str}"
     end
   end
 
@@ -88,9 +88,9 @@ def report_msg(status, name, path, info = nil)
     $info_fields.each do |field|
       IndiferentHash.setup(info)
       value = info[field]
-      next if  value.nil?
+      next if value.nil?
       value_str = Misc.fingerprint(value)
-      str << " #{Log.color :magenta, field}=#{value_str}"
+      str << "\t#{Log.color :magenta, field}=#{value_str}"
     end
   end
 

data/test/rbbt/util/concurrency/test_processes.rb

@@ -6,6 +6,68 @@ require 'rbbt/util/concurrency/processes'
 
 class TestConcurrencyProcess < Test::Unit::TestCase
 
+  def setup
+    Log.severity = 0
+  end
+
+  def test_process_throttle
+    q = RbbtProcessQueue.new 10
+
+    times = 500
+
+    res = []
+    q.callback do |v|
+      res << v
+    end
+
+    q.init do |i|
+      sleep 0.001
+      Process.pid
+    end
+
+    times.times do |i|
+      q.process i
+    end
+
+    sleep 1 while res.length < times
+    assert_equal 10, res.uniq.length
+
+    q.add_process
+    sleep 0.1
+    q.add_process
+    sleep 0.1
+    q.add_process
+    sleep 0.1
+
+    times.times do |i|
+      q.process i
+    end
+
+    sleep 1 while res.length < times * 2
+    assert_equal 13, res[(times + 20)..-1].uniq.length
+
+    q.remove_process
+    sleep 0.1
+    q.remove_process
+    sleep 0.1
+    q.remove_process
+    sleep 0.1
+    q.remove_process
+    sleep 0.1
+    q.remove_process
+    sleep 0.1
+
+    times.times do |i|
+      q.process i
+    end
+
+    sleep 1 while res.length < times * 3
+    assert_equal 8, res[(2*times + 30)..-1].uniq.length
+
+    q.join
+    assert_equal times * 3, res.length
+  end
+
   def test_process
     q = RbbtProcessQueue.new 10
 
@@ -20,7 +82,6 @@ class TestConcurrencyProcess < Test::Unit::TestCase
     end
 
     times = 500
-    t = TSV.setup({"a" => 1}, :type => :single)
 
     times.times do |i|
       q.process i
@@ -45,6 +106,34 @@ class TestConcurrencyProcess < Test::Unit::TestCase
     end
   end
 
+  def test_error
+    assert_raise RbbtException do
+      q = RbbtProcessQueue.new 3
+
+      res = []
+
+      q.callback do |v|
+        res << v
+      end
+
+      q.init do |i|
+        raise RbbtException.new "MY ERROR" if i == 300
+        i * 2
+      end
+
+      times = 500
+
+      times.times do |i|
+        q.process i
+      end
+
+      q.join
+
+      assert_equal times, res.length
+      assert_equal [0, 2, 4], res.sort[0..2]
+    end
+  end
+
   def test_process_abort
     assert_raise Aborted do
       q = RbbtProcessQueue.new 10
@@ -60,15 +149,14 @@ class TestConcurrencyProcess < Test::Unit::TestCase
       end
 
       times = 500
-      t = TSV.setup({"a" => 1}, :type => :single)
 
       times.times do |i|
         q.process i
       end
 
-      sleep 2
+      sleep 1
       q.clean
-
+      sleep 1
 
       q.join
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.22.1
+  version: 5.22.3
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-26 00:00:00.000000000 Z
+date: 2018-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -271,7 +271,6 @@ files:
 - lib/rbbt/util/concurrency.rb
 - lib/rbbt/util/concurrency/processes.rb
 - lib/rbbt/util/concurrency/processes/socket.rb
-- lib/rbbt/util/concurrency/processes/socket_old.rb
 - lib/rbbt/util/concurrency/processes/worker.rb
 - lib/rbbt/util/concurrency/threads.rb
 - lib/rbbt/util/config.rb

data/lib/rbbt/util/concurrency/processes/socket_old.rb

@@ -1,144 +0,0 @@
-class RbbtProcessQueue
-  class RbbtProcessSocket
-
-    class ClosedSocket < Exception; end
-
-    attr_accessor :sin, :sout, :in_lockfile, :out_lockfile
-    def initialize(lockfile = nil)
-      @sout, @sin = File.pipe
-
-      lockfile ||= TmpFile.tmp_file
-
-      @lockfile = lockfile
-      @in_lockfile = lockfile + '.in'
-      @out_lockfile = lockfile + '.out'
-      raise "in_lockfile exists?" if File.exists? @in_lockfile
-      raise "out_lockfile exists?" if File.exists? @in_lockfile
-      FileUtils.touch @in_lockfile
-      FileUtils.touch @out_lockfile
-    end
-
-    def self.serialize(obj)
-      dump = nil
-      begin
-        case obj
-        when TSV
-          type = "T"
-          info = obj.info
-          info.delete_if{|k,v| v.nil?}
-          dump = Marshal.dump([info, {}.merge(obj)])
-        else
-          type = "M"
-          dump = Marshal.dump(obj)
-        end
-        payload = [type, dump].pack('A1a*')
-        length = payload.bytesize
-        #Log.info "Writing #{ length }"
-        [length].pack('L') << payload
-      rescue Exception
-        Log.error "Serialize error for: #{Misc.fingerprint obj} - #{Misc.fingerprint dump}"
-        raise $!
-      end
-    end
-
-    def self.unserialize(str)
-      begin
-        c, dump = str.unpack("A1a*")
-        case c
-        when "M"
-          return Marshal.load(dump)
-        when "T"
-          info, hash = Marshal.load(dump)
-          return TSV.setup(hash, info)
-        end
-      rescue Exception
-        Log.error "Unserialize error for: #{Misc.fingerprint str}"
-        raise $!
-      end
-    end
-
-    def read_sout(length)
-      str = ""
-      str << sout.readpartial(length-str.length) while str.length < length
-      str
-    end
-
-    def write_sin(str)
-      str_length = str.length
-      wrote = 0
-      wrote += sin.write_nonblock(str[wrote..-1]) while wrote < str_length
-    end
-
-    def push(obj)
-      Filelock in_lockfile do
-        payload = RbbtProcessSocket.serialize(obj)
-        sin << payload
-      end
-    end
-
-
-    def pop
-      r = []
-
-      payload = begin
-                  Filelock out_lockfile do
-                    raise ClosedQueue if sout.eof?
-                    r,w,e = IO.select([sout], [], [], 1)
-                    raise TryAgain if r.empty?
-
-                    first_char = read_sout(4)
-                    length = first_char.unpack('L').first
-                    #Log.info "Reading #{ length }"
-                    read_sout(length)
-                  end
-                rescue TryAgain
-                  sleep 1
-                end
-
-      RbbtProcessSocket.unserialize(payload)
-    end
-
-    def pop
-      loop do
-        r,w,e = IO.select([sout], [], [], 1)
-        next if r.empty?
-        break
-      end
-
-      first_char = read_sout(4)
-      length = first_char.unpack('L').first
-      #Log.info "Reading #{ length }"
-      read_sout(length)
-    end
-  rescue TryAgain
-    sleep 1
-  end
-
-      RbbtProcessSocket.unserialize(payload)
-    end
-
-    def rest
-      sin.close
-      str = sout.read
-      res = []
-
-      while not str.empty?
-        first_char = str[0]
-        next if first_char.nil?
-        length = first_char.unpack("C").first
-        dump = str[1..length]
-        res << Marshal.load(dump)
-        str = str[length+1..-1]
-      end
-
-      res
-    end
-
-    def clean
-      FileUtils.rm @in_lockfile if File.exists? @in_lockfile
-      FileUtils.rm @out_lockfile if File.exists? @out_lockfile
-      sin.close unless sin.closed?
-      sout.close unless sout.closed?
-    end
-  end
-end