rbbt-util 5.22.3 → 5.22.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a706bb947413cbe8267aac3c21f7d23668313123
-  data.tar.gz: 835cc701d39015fbdb0379ff483c38513e77a134
+  metadata.gz: 694be20134e0bd77c1d240c04fe52f547cd4b431
+  data.tar.gz: f96f17c3f62ec9f5d781036966c022f275e8b879
 SHA512:
-  metadata.gz: c7db876d745f2dfc5dbefb8798145a29f3ef32962a31f1940bc611589fd61a9feb918616cfa1f45c769b2d67767f9eb5a9b487b2f1d35fa845c0bb645d3915f6
-  data.tar.gz: 68fe15aa3feaab425c9d90424f4bdba20206f5f10a541ace9c7a8aef6d956b4c980566feb1253501999d53a58d58bd1d3d91c0b5ec624bae5e333889cdfe24e4
+  metadata.gz: 6d725866e6cbf97a5a627ca5e575d202e348e2688bf37aff4032d8c752b7fd6e747e1df85c00cf69de6258b7112812da43356db552f65940f22ef86101107afb
+  data.tar.gz: 98dfdd3a29d277cb2c2c52f76e6ffbae1c7f1d033f823a36637235b7b47bb61e0ae43652c14f626a9341df41f370fa0fa7ed55b2a625a63896ffce82c4d9d37a

data/lib/rbbt/util/concurrency/processes.rb

@@ -63,6 +63,18 @@ class RbbtProcessQueue
     @init_block = block
 
     @master_pid = Process.fork do
+      @close_up = false
+      Signal.trap(:INT) do
+        @close_up = true
+        Misc.insist([0,0.01,0.1,0.2,0.5]) do
+          raise TryAgain unless @manager_thread
+          if @manager_thread.alive?
+            raise "Manager thread for #{Process.pid} Not working yet" unless @manager_thread["working"]
+            @manager_thread.raise TryAgain
+          end
+        end
+      end
+
       if @callback_queue
         Misc.purge_pipes(@queue.swrite,@queue.sread,@callback_queue.swrite, @callback_queue.sread) 
       else
@@ -72,34 +84,36 @@ class RbbtProcessQueue
       @total = num_processes
       @count = 0
       @processes = []
-      @close_up = false
-
-
-      Signal.trap(:INT) do
-        @close_up = true
-        @manager_thread.raise TryAgain
-      end
 
       @manager_thread = Thread.new do
+        begin
         while true 
           begin
+            Thread.current["working"] = true
+            if @close_up
+              Log.debug "Closing up process queue #{Process.pid}"
+              @count = 0
+              Thread.new do
+                while true
+                  @queue.push ClosedStream.new unless @queue.cleaned 
+                end unless @processes.empty?
+              end
+              @close_up = false
+            end
+
             begin
-              sleep 10
+              sleep 3
             rescue TryAgain
             end
 
+            raise TryAgain if @close_up
+
             @process_mutex.synchronize do
-              if @close_up
-                @total.times do
-                  @queue.push ClosedStream.new unless @queue.cleaned 
-                end unless @processes.empty?
-                @count = 0
-              end
               while @count > 0
                 @count -= 1
                 @total += 1
                 @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, @offset, &@init_block)
-                Log.low "Added process #{@processes.last.pid} to #{Process.pid} (#{@processes.length})"
+                Log.warn "Added process #{@processes.last.pid} to #{Process.pid} (#{@processes.length})"
               end
 
               while @count < 0
@@ -107,7 +121,7 @@ class RbbtProcessQueue
                 next unless @processes.length > 1
                 first = @processes.shift
                 first.stop
-                Log.low "Removed process #{first.pid} from #{Process.pid} (#{@processes.length})"
+                Log.warn "Removed process #{first.pid} from #{Process.pid} (#{@processes.length})"
               end
             end
           rescue TryAgain
@@ -120,6 +134,10 @@ class RbbtProcessQueue
             raise Exception
           end
         end
+        rescue Exception
+          Log.exception $!
+          raise $!
+        end
       end
 
       Signal.trap(:USR1) do

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -35,16 +35,19 @@ class RbbtProcessQueue
 
           loop do
             p = @queue.pop
+
             next if p.nil?
             raise p if Exception === p
             raise p.first if Array === p and Exception === p.first
+
             begin
               res = @block.call *p
               @callback_queue.push res if @callback_queue
             rescue Respawn
-              @callback_queue.push $!.payload 
+              @callback_queue.push $!.payload if @callback_queue
               raise $!
             end
+
             raise Respawn if @respawn
             if @stop
               Log.high "Worker #{Process.pid} leaving"
@@ -121,6 +124,7 @@ class RbbtProcessQueue
         end
 
         while true
+          @prev = @current
           pid, status = Process.waitpid2 @current
           code = status.to_i >> 8 
           break unless code == 28
@@ -128,7 +132,7 @@ class RbbtProcessQueue
             run
           end
           @asked = false
-          Log.high "Worker #{Process.pid} respawning to #{@current}"
+          Log.high "Worker #{Process.pid} respawning from #{@prev} to #{@current}"
         end
       rescue Aborted, Interrupt
         Log.warn "Worker #{Process.pid} aborted. Current #{@current} #{Misc.pid_exists?(@current) ? "exists" : "does not exist"}"

data/lib/rbbt/util/misc/development.rb

@@ -143,7 +143,7 @@ def self.add_libdir(dir=nil)
     try = 0
 
     if sleep.nil?
-      sleep_array = ([0] + [0.001, 0.01, 0.1] * (times / 3)).sort[0..times-1]
+      sleep_array = ([0] + [0.001, 0.01, 0.1, 0.5] * (times / 3)).sort[0..times-1]
       sleep = sleep_array.shift
     end
 

data/lib/rbbt/util/misc/exceptions.rb

@@ -3,7 +3,7 @@ class ParameterException < RbbtException; end
 class FieldNotFoundError < RbbtException;end
 class ClosedStream < RbbtException; end
 
-class ProcessFailed < RbbtException; end
+class ProcessFailed < StandardError; end
 
 class Aborted < StandardError; end
 

data/lib/rbbt/util/misc/inspect.rb

@@ -270,12 +270,15 @@ module Misc
               obj + " (file missing)"
             end
           when String
-
-            obj = obj.chomp if String === obj
-            if obj.length > HASH2MD5_MAX_STRING_LENGTH
-              sample_large_obj(obj, HASH2MD5_MAX_STRING_LENGTH) << "--" << txt_digest_str(obj)
+            if Misc.is_filename?(obj)
+              obj2str Path.setup(obj.dup)
             else
-              obj
+              obj = obj.chomp if String === obj
+              if obj.length > HASH2MD5_MAX_STRING_LENGTH
+                sample_large_obj(obj, HASH2MD5_MAX_STRING_LENGTH) << "--" << txt_digest_str(obj)
+              else
+                obj
+              end
             end
           when Array
             if obj.length > HASH2MD5_MAX_ARRAY_LENGTH

data/lib/rbbt/util/simpleopt/parse.rb

@@ -11,10 +11,16 @@ module SOPT
     current = [chars.shift]
     short = current * ""
 
-    if (shortcuts.include?(short) and not shortcuts[short] == long) and long.index "-" or long.index "_"
-      parts = long.split(/[_-]/)
-      acc = parts.collect{|s| s[0] } * ""
-      return acc unless shortcuts.include? acc
+    if (shortcuts.include?(short) and not shortcuts[short] == long) 
+      if long.index "-" or long.index "_"
+        parts = long.split(/[_-]/)
+        acc = parts.collect{|s| s[0] } * ""
+        return acc unless shortcuts.include? acc
+      elsif m = long.match(/(\d+)/)
+        n = m[0]
+        acc = long[0] + n
+        return acc unless shortcuts.include? acc
+      end
     end
 
     while shortcuts.include?(short) and not shortcuts[short] == long

data/lib/rbbt/workflow/accessor.rb

@@ -906,11 +906,16 @@ module Workflow
 
   TAG = ENV["RBBT_INPUT_JOBNAME"] == "true" ? :inputs : :hash
   def step_path(taskname, jobname, inputs, dependencies, extension = nil)
-    raise "Jobname makes an invalid path: #{ jobname }" if jobname =~ /\.\./
+    raise "Jobname makes an invalid path: #{ jobname }" if jobname.include? '..'
     if inputs.length > 0 or dependencies.any?
       tagged_jobname = case TAG
                        when :hash
-                         hash_str = Misc.obj2digest({:inputs => Annotated.purge(inputs), :dependencies => dependencies})
+                         clean_inputs = Annotated.purge(inputs)
+                         clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
+                         key_obj = {:inputs => clean_inputs, :dependencies => dependencies}
+                         key_str = Misc.obj2str(key_obj)
+                         hash_str = Misc.digest(key_str)
+                         Log.debug "Hash for '#{[taskname, jobname] * "/"}' #{hash_str} for #{key_str}"
                          jobname + '_' << hash_str
                        when :inputs
                          all_inputs = {}

data/lib/rbbt/workflow/step/run.rb

@@ -21,7 +21,8 @@ class Step
   def resolve_input_steps
     step = false
     pos = 0
-    input_options = workflow.task_info(task_name)[:input_options]
+
+    input_options = Workflow === workflow ? workflow.task_info(task_name)[:input_options] : {}
     new_inputs = inputs.collect do |i| 
       begin
         if Step === i

data/share/rbbt_commands/system/status

@@ -45,13 +45,14 @@ def pid_msg(pid)
   end
 end
 
+
 def status_msg(status)
   color = case status.to_sym
           when :error, :aborted, :missing, :dead
             :red
           when :streaming, :started
             :cyan
-          when :done
+          when :done, :noinfo
             :green
           when :dependencies, :waiting, :setyp
             :yellow
@@ -64,7 +65,6 @@ def status_msg(status)
           end
   Log.color(color, status.to_s)
 end
-
 puts Log.color(:magenta, "# System report")
 puts
 sort_files = Proc.new do |a,b|
@@ -166,14 +166,14 @@ workflows.sort.each do |workflow,tasks|
         str << " (dirty)" if status == 'done' && Workflow.load_step(file).dirty?
 
         if inputs and inputs.any? and info[:inputs]
-          job_inputs = info[:inputs]
+          job_inputs = Workflow.load_step(file).recursive_inputs.to_hash
           IndiferentHash.setup(job_inputs)
 
           inputs.each do |input|
             value = job_inputs[input]
             next if  value.nil?
             value_str = Misc.fingerprint(value)
-            str << " #{input}=#{value_str}"
+            str << "\t#{Log.color :magenta, input}=#{value_str}"
           end
         end
 

data/share/rbbt_commands/workflow/info

@@ -37,7 +37,7 @@ def status_msg(status)
             :red
           when :streaming, :started
             :cyan
-          when :done
+          when :done, :noinfo
             :green
           when :dependencies, :waiting, :setyp
             :yellow

data/share/rbbt_commands/workflow/prov

@@ -45,7 +45,7 @@ def status_msg(status)
             :red
           when :streaming, :started
             :cyan
-          when :done
+          when :done, :noinfo
             :green
           when :dependencies, :waiting, :setyp
             :yellow
@@ -73,7 +73,7 @@ def report_msg(status, name, path, info = nil)
   end
 
   if $inputs and $inputs.any? 
-    job_inputs = info[:inputs]
+    job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
     IndiferentHash.setup(job_inputs)
 
     $inputs.each do |input|
@@ -134,6 +134,7 @@ def adjacency(step)
   end
   name = name.split(":").last
   status = :unsync if status == :done and not File.exist? path
+  shapes = Hash.new "circle"
   edge_info = {:status =>  status, :workflow => workflow, :task => task, :name => name, :label => task, :shape => shapes[workflow], :color => status == 'remote' ? 'blue' : 'green'}
   id = Misc.digest(path)
   edges = []
@@ -143,7 +144,7 @@ def adjacency(step)
     info[:dependencies].each do |task,name,path|
       dep = get_step path
       _id, _edges, _node_info = adjacency(dep)
-      edges << [id, _id] 
+      edges << [_id, id] 
       edges.concat _edges
       node_info.merge!(_node_info)
     end 
@@ -173,9 +174,6 @@ if options[:plot]
 
       require 'rbbt/util/R'
 
-      ppp Open.read edge_file
-      ppp Open.read node_info_file
-
       R.run <<-EOF
       nodes <- read.csv("#{node_info_file}", header=T, as.is=T)
       links <- read.csv("#{edge_file}", header=T, as.is=T)
@@ -185,7 +183,7 @@ if options[:plot]
       net <- graph.data.frame(links, nodes, directed=T)
       net <- simplify(net, remove.multiple = F, remove.loops = T)
 
-      png("#{options[:plot]}")
+      png("#{options[:plot]}", width=1000, height=1000)
       plot(net, edge.arrow.size=0.4, vertex.label=net$label, vertex.color=net$color)
       dev.off()
       EOF

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.22.3
+  version: 5.22.4
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-29 00:00:00.000000000 Z
+date: 2018-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake