rbbt-util 5.17.35 → 5.17.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4dd1246667e1abdadb7f87df051266fe75214707
-  data.tar.gz: 8ccd292c4188f094a27d55e239c05f0ffdd65e68
+  metadata.gz: 018ff5d75579a0f9063037ef20da554b396daa92
+  data.tar.gz: fbaa875c7332df4f9a5da2511e85df9e2a3fc938
 SHA512:
-  metadata.gz: 8617fc351a070254930005d39835dc750ddf14aa00b261fc23013333b5e8b17349f5ab607cd2d9e36e84dff50639bfd14b507bef00f1b5c92bb5ba0ca727d1c4
-  data.tar.gz: 3cee824334c00459ef780ea69cf75df14034c1c97d3e40aa6bf435938d5050806e3f83ffa332f8941d63eea7e039b95a94fb6c667c23ad6a6e22f5990124f49a
+  metadata.gz: e0b6ca146e9d06f87d73136afdbcc7f816e2ad333453310a633932f5f798ac810cbe97a1634949c8a3307c9acf35cab928f99830766c6c8cd9bcf67f8cf259e7
+  data.tar.gz: 8a25f98e0b3d4ed21571fc438433a2374f8903d0363663ef6f71bb4afce88af19909419122b98507af90ba86330f6125ec88930c285fa7d2013f3ceb1bd73541

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -66,6 +66,7 @@ module TSV
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
     if callback
+      bar.init if bar
       tsv.through options[:key_field], options[:fields] do |k,v|
         begin
           callback.call yield(k,v)
@@ -74,6 +75,7 @@ module TSV
         end
       end
     else
+      bar.init if bar
       tsv.through options[:key_field], options[:fields] do |k,v|
         begin
           yield k,v 
@@ -114,6 +116,7 @@ module TSV
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
 
     if callback
+      bar.init if bar
       array.each do |e|
         begin
           callback.call yield(e)
@@ -122,9 +125,13 @@ module TSV
         end
       end
     else
+      bar.init if bar
       array.each do |e|
         begin
           yield e
+        rescue Exception
+          Log.exception $!
+          raise $!
         ensure
           bar.tick if bar
         end
@@ -147,6 +154,7 @@ module TSV
     end
 
     if callback
+      bar.init if bar
       while line = io.gets
         if line[-1] != "\n"
           while c = io.getc
@@ -161,6 +169,7 @@ module TSV
         end
       end
     else
+      bar.init if bar
       while line = io.gets
         begin
           yield line.strip
@@ -186,6 +195,7 @@ module TSV
     end
 
     if callback
+      bar.init if bar
       TSV::Parser.traverse(io, options) do |k,v|
         begin
           callback.call yield k, v
@@ -333,19 +343,22 @@ module TSV
 
   def self.traverse_cpus(num, obj, options, &block)
     begin
-      callback, cleanup, join, respawn = Misc.process_options options, :callback, :cleanup, :join, :respawn
+      callback, cleanup, join, respawn, bar = Misc.process_options options, :callback, :cleanup, :join, :respawn, :bar
       respawn = true if ENV["RBBT_RESPAWN"] and ENV["RBBT_RESPAWN"] == "true"
 
       Log.low "Traversing in #{ num } cpus: #{respawn ? "respawn" : "no respawn"}"
       q = RbbtProcessQueue.new num, cleanup, join, respawn
+      callback = Proc.new{ bar.tick } if callback.nil? and bar
       q.callback &callback
       q.init &block
 
+      bar.init if bar
       traverse_obj(obj, options) do |*p|
         q.process *p
       end
 
       q.join
+
     rescue Interrupt, Aborted
       q.abort
       Log.medium{"Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.backtrace*","}"}
@@ -364,6 +377,7 @@ module TSV
       raise $!
     ensure
       q.clean
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -586,6 +600,7 @@ module TSV
         end
       end
 
+      bar.init if bar
       case into
       when TSV::Dumper, IO
         traverse_stream(obj, threads, cpus, options, &block)

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -3,7 +3,12 @@ class RbbtProcessQueue
   class RbbtProcessQueueWorker
     attr_reader :pid, :queue, :callback_queue, :cleanup, :block
 
-    class Respawn < Exception; end
+    class Respawn < Exception
+      attr_accessor :payload 
+      def initialize(payload)
+        @payload = payload
+      end
+    end
 
     def run
       begin
@@ -20,9 +25,14 @@ class RbbtProcessQueue
           p = @queue.pop
           next if p.nil?
           raise p if Exception === p
-          raise p.first if Exception === p.first
-          res = @block.call *p
-          @callback_queue.push res if @callback_queue
+          raise p.first if Array === p and Exception === p.first
+          begin
+            res = @block.call *p
+            @callback_queue.push res if @callback_queue
+          rescue Respawn
+            @callback_queue.push $!.payload 
+            raise $!
+          end
           raise Respawn if @stop
         end
         Kernel.exit! 0

data/lib/rbbt/util/concurrency/processes.rb

@@ -16,6 +16,7 @@ class RbbtProcessQueue
   attr_accessor :callback, :callback_queue, :callback_thread
   def callback(&block)
     if block_given?
+    
       @callback = block
 
       @callback_queue = RbbtProcessSocket.new
@@ -31,7 +32,11 @@ class RbbtProcessQueue
               raise e 
             end
 
-            @callback.call p
+            if @callback.arity == 0
+              @callback.call
+            else
+              @callback.call p
+            end
           end
         rescue Aborted
           Log.warn "Callback thread aborted"
@@ -43,6 +48,7 @@ class RbbtProcessQueue
           @process_monitor.raise $!
           raise $!
         ensure
+
           @callback_queue.sread.close unless @callback_queue.sread.closed?
         end
       end

data/lib/rbbt/util/log/progress/report.rb

@@ -80,7 +80,13 @@ module Log
 
     def report_msg
       str = Log.color :magenta, desc
-      return str << " " << Log.color(:yellow, "waiting") if @ticks == 0
+      if @ticks == 0
+        if @max
+          return str << " " << Log.color(:yellow, "waiting on #{@max} - #{Process.pid}") 
+        else
+          return str << " " << Log.color(:yellow, "waiting - #{Process.pid}") 
+        end
+      end
       str << " " << thr_msg
       if max
         str << Log.color(:blue, " -- ") << eta_msg
@@ -101,7 +107,11 @@ module Log
         bars = BARS
         print(io, Log.color(:yellow, "...Progress\n"))
         bars.sort_by{|b| b.depth }.reverse.each do |bar|
-          print(io, Log.color(:yellow ,bar.report_msg) << "\n")
+          if SILENCED.include? bar
+            print(io, Log.color(:yellow ,bar.report_msg) << "\n") 
+          else
+            print(io, "\n") 
+          end
         end
       else
         bars = BARS

data/lib/rbbt/util/log/progress/util.rb

@@ -3,6 +3,7 @@ module Log
     BAR_MUTEX = Mutex.new
     BARS = []
     REMOVE = []
+    SILENCED = []
 
     def self.new_bar(max, options = {})
       cleanup_bars
@@ -24,6 +25,13 @@ module Log
               bar.depth = i
             end
           end
+          index = SILENCED.index bar
+          if index
+            SILENCED.delete_at index
+            SILENCED.each_with_index do |bar,i|
+              bar.depth = i
+            end
+          end
         end
         REMOVE.clear
         BARS.length

data/lib/rbbt/util/log/progress.rb

@@ -24,6 +24,12 @@ module Log
       (@ticks * 100) / @max
     end
 
+    def init
+      @start = @last_time = Time.now
+      @last_count = 0
+      report
+    end
+
     def tick(step = 1)
       return if ENV["RBBT_NO_PROGRESS"] == "true"
       @ticks += step

data/lib/rbbt/util/misc/development.rb

@@ -9,6 +9,7 @@ module Misc
     Persist::CONNECTIONS.values.each do |db| 
       db.close if db.write? 
     end
+    Log::ProgressBar::BARS.clear
     ObjectSpace.each_object(Mutex) do |m| 
       begin 
         m.unlock 
@@ -297,7 +298,8 @@ module Misc
            end
 
 
-    options = Misc.add_defaults options, :respawn => true, :cpus => cpus, :into => Set.new 
+    #options = Misc.add_defaults options, :respawn => true, :cpus => cpus, :into => Set.new 
+    options = Misc.add_defaults options, :respawn => true, :cpus => cpus
     options = Misc.add_defaults options, :bar => "Bootstrap in #{ options[:cpus] } cpus: #{ Misc.fingerprint Annotated.purge(elems) }"
     respawn = options[:respawn] and options[:cpus] and options[:cpus].to_i > 1
 
@@ -306,12 +308,13 @@ module Misc
       elem = elems[pos.to_i]
       elems.annotate elem if elems.respond_to? :annotate
       begin
-        yield elem
+        res = yield elem
       rescue Interrupt
         Log.warn "Process #{Process.pid} was aborted"
       end
-      raise RbbtProcessQueue::RbbtProcessQueueWorker::Respawn if respawn == :always and cpus > 1
-      nil
+      res = nil unless options[:into]
+      raise RbbtProcessQueue::RbbtProcessQueueWorker::Respawn, res if respawn == :always and cpus > 1
+      res
     end
   end
 

data/lib/rbbt/workflow/step/run.rb

@@ -279,6 +279,14 @@ class Step
     end
   end
 
+  def produce
+    return if done? and not dirty?
+    clean if dirty?
+    run(true) unless started?
+    join unless done?
+    self
+  end
+
   def fork(semaphore = nil)
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid and Misc.pid_exists?(@pid) and not done? and info[:forked]
     @pid = Process.fork do

data/test/rbbt/util/test_misc.rb

@@ -458,4 +458,12 @@ eum fugiat quo voluptas nulla pariatur?"
     ppp Misc.snake_case("KinaseSARfari")
   end
 
+  def test_bootstrap
+    Log.severity = 0
+    res = Misc.bootstrap((1..10).to_a, 2, :bar => "Test bootstrap ticks", :respawn => :always, :into => []) do |num|
+      sleep 1 + rand(2)
+      num
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.17.35
+  version: 5.17.37
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-27 00:00:00.000000000 Z
+date: 2015-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake