rbbt-util 5.23.16 → 5.23.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fff5a075ef539b7ffea6006507ff9bbe068ef96f
-  data.tar.gz: 01ebfdec1ed9b262ac799e750ba6de59f5ac28f0
+  metadata.gz: b741d19f2b817da6421a86a354264aabedae4397
+  data.tar.gz: 5003c159ae149e9a1660acfa939080c3895b08d7
 SHA512:
-  metadata.gz: 63bf9331dc91cb5643abbd960c1707a3f66da6c139a946cb7fc3b270186ab5d7fd0b893e6bce570a3ee937031ba68882e76e77822b1d9a2c3d5a57ccfb75cba1
-  data.tar.gz: eaf404d75cd0455bf87f665de3fbb1b934f1194667647b35351c4ebe99b585e0d2b707331babafa4d57ca145e9ad30ba61ab4694b52e5cd137c539c0610f61d0
+  metadata.gz: 7f14b692b30f79dfd99d859c66d2a51313d83f3f1e704bb08a7a89baa0c69aacdadaf331a998c48958f066c73f8eecc050216c23de774c23461070ca3d3c5e9a
+  data.tar.gz: 6755c1770e04c17b779f63d22649230018d0480fad203fe56033a295d75f0989d63c97fffeee0a1c70b031ce27186d5cbaa05c22be1eeff42500cab170813e0c

data/lib/rbbt/util/concurrency/processes.rb

@@ -330,6 +330,7 @@ class RbbtProcessQueue
 
   def _abort
     begin
+      Log.warn "Aborting process queue #{@master_pid}"
       Process.kill ABORT_SIGNAL, @master_pid
     rescue Errno::ECHILD, Errno::ESRCH
       Log.debug "Cannot abort #{@master_pid}: #{$!.message}"

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -27,9 +27,9 @@ class RbbtProcessQueue
             @stop = true
           }
 
-          @abort = false
           Signal.trap(20){ 
-            @abort = true
+            Log.high "Worker #{Process.pid} signaled to abort"
+            Kernel.exit! -1
           }
 
           loop do
@@ -63,6 +63,7 @@ class RbbtProcessQueue
           Log.high "Worker #{Process.pid} leaving"
         rescue Exception
           Log.high "Worker #{Process.pid} had exception: #{$!.message}"
+          Log.exception $!
           begin
             @callback_queue.push($!) if @callback_queue
           rescue
@@ -144,6 +145,7 @@ class RbbtProcessQueue
         }
 
         Signal.trap(20){ 
+          Log.high "Killing respawned process #{@current}"
           begin
             Process.kill 20, @current if @current
           rescue Errno::ESRCH, Errno::ECHILD
@@ -181,7 +183,7 @@ class RbbtProcessQueue
       end
 
       if status
-        Log.high "Worker respawner with #{Process.pid} (#{@current}) completed with status #{status}"
+        Log.high "Worker #{@current} (respawner #{Process.pid} ) completed with status #{status}"
         Kernel.exit! status.to_i >> 8
       else
         Kernel.exit! -1

data/lib/rbbt/util/log/progress/report.rb

@@ -113,7 +113,6 @@ module Log
       else
         thr = @ticks / (Time.now - @start) 
       end
-      iii [short_mean, thr]
       
       if mean.nil? or mean.to_i > 1
         str = "#{ Log.color :blue, thr.to_i.to_s } per sec."

data/lib/rbbt/util/misc/pipes.rb

@@ -411,9 +411,6 @@ module Misc
           FileUtils.touch path if File.exist? path
           content.join if content.respond_to? :join and not (content.respond_to?(:joined?) and content.joined?)
 
-          if Lockfile === lock_options[:lock] and lock_options[:lock].locked?
-            lock_options[:lock].unlock
-          end
           Open.notify_write(path) 
         rescue Aborted
           Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
@@ -430,6 +427,9 @@ module Misc
           raise $!
         ensure
           FileUtils.rm_f tmp_path if File.exist? tmp_path
+          if Lockfile === lock_options[:lock] and lock_options[:lock].locked?
+            lock_options[:lock].unlock
+          end
         end
       end
     end

data/lib/rbbt/workflow/accessor.rb

@@ -112,6 +112,7 @@ class Step
   end
 
   def status_lock
+    return @mutex
     @status_lock = begin
                    path = Persist.persistence_path(info_file + '.status.lock', {:dir => Step.lock_dir})
                    Lockfile.new path, :refresh => false, :dont_use_lock_id => true
@@ -411,6 +412,7 @@ class Step
     end
 
     if dirty_files.any?
+      Log.low "Some dirty files found for #{self.path}: #{Misc.fingerprint dirty_files}"
       true
     else
       ! self.updated?
@@ -429,21 +431,22 @@ class Step
     status == :noinfo
   end
 
-  def running?
+  def running? 
+    return false if ! (started? || status == :ending)
     pid = info[:pid]
     return nil if pid.nil?
 
-    return false if done? or error? or aborted? or not started?
+    return false if done? or error? or aborted? 
 
     if Misc.pid_exists?(pid) 
       pid
     else
-      false
+      done? or error? or aborted? 
     end
   end
 
   def stalled?
-    started? && ! (done? || error? || aborted? || running? || waiting?)
+    started? && ! (done? || running? || done? || error? || aborted?)
   end
 
   def missing?

data/lib/rbbt/workflow/step/run.rb

@@ -182,204 +182,203 @@ class Step
       res = @mutex.synchronize do
         no_load = :stream if no_load
 
-        self.status_lock.synchronize do
-          Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-          result = Persist.persist "Job", @task.result_type, :file => path, :check => checks.collect{|dep| dep.path}, :no_load => no_load do 
-            if Step === Step.log_relay_step and not self == Step.log_relay_step
-              relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
-            end
+        Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
+        result = Persist.persist "Job", @task.result_type, :file => path, :check => checks.collect{|dep| dep.path}, :no_load => no_load do 
+          if Step === Step.log_relay_step and not self == Step.log_relay_step
+            relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
+          end
 
-            Open.write(pid_file, Process.pid.to_s) unless Open.exists? pid_file
+          Open.write(pid_file, Process.pid.to_s) unless Open.exists? pid_file
 
-            @exec = false
-            init_info
+          @exec = false
+          init_info
 
-            log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
+          log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
 
-            merge_info({
-              :issued => (issue_time = Time.now),
-              :name => name,
-              :clean_name => clean_name,
-              :workflow => (@workflow || @task.workflow).to_s,
-              :task_name => @task.name,
-              :result_type => @task.result_type,
-              :result_description => @task.result_description,
-              :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
-            })
+          merge_info({
+            :issued => (issue_time = Time.now),
+            :name => name,
+            :clean_name => clean_name,
+            :workflow => (@workflow || @task.workflow).to_s,
+            :task_name => @task.name,
+            :result_type => @task.result_type,
+            :result_description => @task.result_description,
+            :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
+          })
 
 
-            begin
-              run_dependencies
-            rescue Exception
-              FileUtils.rm pid_file if File.exist?(pid_file)
-              stop_dependencies
-              raise $!
-            end
+          begin
+            run_dependencies
+          rescue Exception
+            FileUtils.rm pid_file if File.exist?(pid_file)
+            stop_dependencies
+            raise $!
+          end
+
+          new_inputs = []
+          @inputs.each_with_index do |input,i|
+            name = @task.inputs[i]
+            type = @task.input_types[name]
+
+            if type == :directory
+              directory_inputs = file('directory_inputs')
+              input_source = directory_inputs['.source'][name].find
+              input_dir = directory_inputs[name].find
 
-            new_inputs = []
-            @inputs.each_with_index do |input,i|
-              name = @task.inputs[i]
-              type = @task.input_types[name]
-
-              if type == :directory
-                directory_inputs = file('directory_inputs')
-                input_source = directory_inputs['.source'][name].find
-                input_dir = directory_inputs[name].find
-
-                case input
-                when Path
-                  if input.directory?
-                    new_inputs << input
-                  else
-                    input.open do |io|
-                      begin
-                        Misc.untar(io, input_source)
-                      rescue
-                        raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
-                      end
+              case input
+              when Path
+                if input.directory?
+                  new_inputs << input
+                else
+                  input.open do |io|
+                    begin
+                      Misc.untar(io, input_source)
+                    rescue
+                      raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
                     end
-                    tar_1 = input_source.glob("*")
-                    raise ParameterException, "When using tar.gz files for directories, the directory must be the single first level entry" if tar_1.length != 1
-                    FileUtils.ln_s Misc.path_relative_to(directory_inputs, tar_1.first), input_dir
-                    new_inputs << input_dir
-                  end
-                when File, IO, Tempfile
-                  begin
-                    Misc.untar(Open.gunzip(input), input_source)
-                  rescue
-                    raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
                   end
                   tar_1 = input_source.glob("*")
                   raise ParameterException, "When using tar.gz files for directories, the directory must be the single first level entry" if tar_1.length != 1
                   FileUtils.ln_s Misc.path_relative_to(directory_inputs, tar_1.first), input_dir
                   new_inputs << input_dir
-                else
-                  raise ParameterException, "Format of directory input '#{name}' not understood: #{Misc.fingerprint input}"
                 end
+              when File, IO, Tempfile
+                begin
+                  Misc.untar(Open.gunzip(input), input_source)
+                rescue
+                  raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
+                end
+                tar_1 = input_source.glob("*")
+                raise ParameterException, "When using tar.gz files for directories, the directory must be the single first level entry" if tar_1.length != 1
+                FileUtils.ln_s Misc.path_relative_to(directory_inputs, tar_1.first), input_dir
+                new_inputs << input_dir
               else
-                new_inputs << input
+                raise ParameterException, "Format of directory input '#{name}' not understood: #{Misc.fingerprint input}"
               end
-            end if @inputs
+            else
+              new_inputs << input
+            end
+          end if @inputs
 
-            @inputs = new_inputs if @inputs
+          @inputs = new_inputs if @inputs
 
-            if not task.inputs.nil?
-              info_inputs = @inputs.collect do |i| 
-                if Path === i 
-                  i.to_s
-                else 
-                  i 
-                end
+          if not task.inputs.nil?
+            info_inputs = @inputs.collect do |i| 
+              if Path === i 
+                i.to_s
+              else 
+                i 
               end
-              set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, info_inputs)) 
             end
+            set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, info_inputs)) 
+          end
 
-            set_info :started, (start_time = Time.now)
-            log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"
+          set_info :started, (start_time = Time.now)
+          log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"
 
-            begin
-              result = _exec
-            rescue Aborted, Interrupt
-              log(:aborted, "Aborted")
-              raise $!
-            rescue Exception
-              backtrace = $!.backtrace
-
-              # HACK: This fixes an strange behaviour in 1.9.3 where some
-              # backtrace strings are coded in ASCII-8BIT
-              backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
-              set_info :backtrace, backtrace 
-              log(:error, "#{$!.class}: #{$!.message}")
-              stop_dependencies
-              raise $!
-            end
+          begin
+            result = _exec
+          rescue Aborted, Interrupt
+            log(:aborted, "Aborted")
+            raise $!
+          rescue Exception
+            backtrace = $!.backtrace
+
+            # HACK: This fixes an strange behaviour in 1.9.3 where some
+            # backtrace strings are coded in ASCII-8BIT
+            backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
+            set_info :backtrace, backtrace 
+            log(:error, "#{$!.class}: #{$!.message}")
+            stop_dependencies
+            raise $!
+          end
 
-            if not no_load or ENV["RBBT_NO_STREAM"] == "true" 
-              result = prepare_result result, @task.description, info if IO === result 
-              result = prepare_result result.stream, @task.description, info if TSV::Dumper === result 
-            end
+          if not no_load or ENV["RBBT_NO_STREAM"] == "true" 
+            result = prepare_result result, @task.description, info if IO === result 
+            result = prepare_result result.stream, @task.description, info if TSV::Dumper === result 
+          end
 
-            stream = case result
-                     when IO
-                       result
-                     when TSV::Dumper
-                       result.stream
-                     end
-
-            if stream
-              log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
-
-              callback = Proc.new do
-                if AbortedStream === stream
-                  if stream.exception
-                    raise stream.exception 
-                  else
-                    raise Aborted
-                  end
-                end
-                begin
-                  status = self.status
-                  if status != :done and status != :error and status != :aborted
-                    Misc.insist do
-                      merge_info({
-                        :done => (done_time = Time.now),
-                        :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
-                        :time_elapsed => (time_elapsed = done_time - start_time)
-                      })
-                      log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
-                    end
-                  end
-                rescue
-                  Log.exception $!
-                ensure
-                  Step.purge_stream_cache
-                  set_info :pid, nil
-                  FileUtils.rm pid_file if File.exist?(pid_file)
+          stream = case result
+                   when IO
+                     result
+                   when TSV::Dumper
+                     result.stream
+                   end
+
+          if stream
+            log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
+
+            callback = Proc.new do
+              if AbortedStream === stream
+                if stream.exception
+                  raise stream.exception 
+                else
+                  raise Aborted
                 end
               end
-
-              abort_callback = Proc.new do |exception|
-                begin
-                  if exception
-                    self.exception exception
-                  else
-                    log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
+              begin
+                status = self.status
+                if status != :done and status != :error and status != :aborted
+                  Misc.insist do
+                    merge_info({
+                      :done => (done_time = Time.now),
+                      :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
+                      :time_elapsed => (time_elapsed = done_time - start_time)
+                    })
+                    log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
                   end
-                  _clean_finished
-                rescue
-                  stop_dependencies
-                  set_info :pid, nil
-                  FileUtils.rm pid_file if File.exist?(pid_file)
                 end
+              rescue
+                Log.exception $!
+              ensure
+                Step.purge_stream_cache
+                set_info :pid, nil
+                FileUtils.rm pid_file if File.exist?(pid_file)
               end
+            end
 
-              ConcurrentStream.setup stream, :callback => callback, :abort_callback => abort_callback
-
-              if AbortedStream === stream 
-                exception = stream.exception || Aborted.new("Aborted stream: #{Misc.fingerprint stream}")
-                self.exception exception
+            abort_callback = Proc.new do |exception|
+              begin
+                if exception
+                  self.exception exception
+                else
+                  log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
+                end
                 _clean_finished
-                raise exception
+              rescue
+                stop_dependencies
+                set_info :pid, nil
+                FileUtils.rm pid_file if File.exist?(pid_file)
               end
-            else
-              merge_info({
-                :done => (done_time = Time.now),
-                :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
-                :time_elapsed => (time_elapsed = done_time - start_time)
-              })
-              log :ending
-              Step.purge_stream_cache
-              FileUtils.rm pid_file if File.exist?(pid_file)
             end
 
-            set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
+            ConcurrentStream.setup stream, :callback => callback, :abort_callback => abort_callback
 
-            if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
-              FileUtils.mv self.tmp_path, self.path
+            if AbortedStream === stream 
+              exception = stream.exception || Aborted.new("Aborted stream: #{Misc.fingerprint stream}")
+              self.exception exception
+              _clean_finished
+              raise exception
             end
-            result
+          else
+            merge_info({
+              :done => (done_time = Time.now),
+              :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
+              :time_elapsed => (time_elapsed = done_time - start_time)
+            })
+            log :ending
+            Step.purge_stream_cache
+            FileUtils.rm pid_file if File.exist?(pid_file)
           end
-        end
+
+          set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
+
+          if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
+            FileUtils.mv self.tmp_path, self.path
+          end
+          result
+        end # END PERSIST
+        log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
 
         if no_load
           @result ||= result
@@ -388,8 +387,7 @@ class Step
           Step.purge_stream_cache
           @result = prepare_result result, @task.result_description
         end
-      end
-      log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
+      end # END SYNC
       res
     rescue DependencyError
       exception $!
@@ -409,9 +407,8 @@ class Step
   def produce(force=false, dofork=false)
     return self if done? and not dirty?
 
-
     self.status_lock.synchronize do
-      if error? or aborted? or stalled?
+      if error? || aborted? || stalled?
         if stalled?
           Log.warn "Aborting stalled job #{self.path}"
           abort

data/share/rbbt_commands/workflow/prov

@@ -164,7 +164,7 @@ if options[:plot]
   TmpFile.with_file do |edge_file|
     Open.write(edge_file) do |f|
       f.puts "from,to"
-      edges.each do |from,to|
+      edges.uniq.each do |from,to|
         f.puts [from,to]*","
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.23.16
+  version: 5.23.17
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-14 00:00:00.000000000 Z
+date: 2018-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake