rbbt-util 5.23.14 → 5.23.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 337b9a1997f89619b3fa00e0c79ff97104021dbc
-  data.tar.gz: 0dfa5fda330f54920786ea1285bb1353f9a6b954
+  metadata.gz: c7019b7a1b634937fbf75f958245dca674834f76
+  data.tar.gz: 7541763d2c390591b153b3d95c37277d63bdf6ea
 SHA512:
-  metadata.gz: 1ff6227b8954f15ea388a9bf39536146d5df3fa3ad38fe1d33285397baee097ece05409b2f35ab665066ad2097401341e7885cfc670ab63c2b4e6a03fc05519d
-  data.tar.gz: beeec704c2efd2fbf361b1219a3545f41eb5a5d3ea41b7a4c2ec14006b11bab71603dab03c8a549d40bf3feedacb53cdfa3435f88828e9573dfe51ce6527baf0
+  metadata.gz: 86ac8dd6da1e81e8c9a801cda18728c736b91c4540c9a8d9b60b7c0e83885f3d501d290e19afea81392c0ebf57c96d50afb89f21ffc239ca47ba8cee3817d10f
+  data.tar.gz: 81afe72fbfd01c65c67722d71d58b0444a9e78ba178149a2ebc9a3f708fdc6e46937e751d6995ef499b248adb4a9355ad6fac4b1d4dfccd7fa6ab14fd89fd56a

data/lib/rbbt/persist.rb

@@ -197,6 +197,9 @@ module Persist
   def self.tee_stream_thread(stream, path, type, callback = nil, abort_callback = nil, lockfile = nil)
     file, out = Misc.tee_stream(stream)
 
+    out.pair = file
+    file.pair = out
+
     saver_thread = Thread.new do
       begin
         file.threads = []
@@ -220,13 +223,12 @@ module Persist
     threads += stream.threads if stream.respond_to?(:threads) && stream.threads
     ConcurrentStream.setup(out, :threads => threads, :filename => path)
 
-    out.callback = callback
+    #out.callback = callback
     out.abort_callback = abort_callback
     out.lockfile = stream.lockfile if stream.respond_to? :lockfile and stream.lockfile
 
-    stream.callback = callback
-    stream.abort_callback = abort_callback
-
+    #stream.callback = callback
+    #stream.abort_callback = abort_callback
 
     out
   end

data/lib/rbbt/util/cmd.rb

@@ -138,7 +138,7 @@ module CMD
     if pipe
       err_thread = Thread.new do
         while line = serr.gets
-          Log.log "STDERR [#{pid}]: " +  line, stderr if Integer === stderr and log
+          Log.log "STDERR [#{pid}]: " +  line, stderr if Integer === stderr and log
         end
         serr.close
       end
@@ -183,7 +183,7 @@ module CMD
     pid = io.pids.first
     while line = io.gets
       if pid
-        Log.debug "STDOUT [#{pid}]: " + line
+        Log.debug "STDOUT [#{pid}]: " + line
       else
         Log.debug "STDOUT: " + line
       end

data/lib/rbbt/util/concurrency/processes.rb

@@ -158,12 +158,17 @@ class RbbtProcessQueue
         begin
           while processes.any?
             raise Aborted if @abort_monitor
-            processes[0].join
-            Log.debug "Joined process #{processes[0].pid} of queue #{Process.pid}"
-            processes.shift
+            #processes[0].join
+            #Log.debug "Joined process #{processes[0].pid} of queue #{Process.pid}"
+            #processes.shift
+            pid, status = Process.wait2
+            Log.debug "Joined process #{pid} of queue #{Process.pid} (status: #{status})"
+            processes.reject!{|p| p.pid == pid}
+            raise ProcessFailed.new pid if not status.success?
           end
           Log.low "All processes completed #{Process.pid}"
         rescue Aborted
+          Log.exception $!
           Log.low "Aborting process monitor #{Process.pid}"
           processes.each{|p|  p.abort_and_join}
           Log.low "Processes aborted #{Process.pid}"
@@ -250,25 +255,28 @@ class RbbtProcessQueue
     begin
       pid, @status = Process.waitpid2 @master_pid unless @status
       error = true unless @status.success?
-      raise ProcessFailed.new @master_pid unless @status.success?
-      @callback_thread.join 
+      begin
+        @callback_thread.join if @callback_thread
+        raise ProcessFailed.new @master_pid unless @status.success?
+      rescue
+        exception = $!
+        raise $!
+      end
       error = false
     rescue Aborted, Interrupt
       exception = $!
       Log.exception $!
       error = true
-      self.abort
-      Log.high "Process queue #{@master_pid} aborted"
-      retry
+      if @aborted
+        raise $!
+      else
+        self.abort
+        Log.high "Process queue #{@master_pid} aborted"
+        retry
+      end
     rescue Errno::ESRCH, Errno::ECHILD
       retry if Misc.pid_exists? @master_pid
       error = ! @status.success?
-    rescue ProcessFailed
-      exception = $!
-    rescue Exception
-      exception = $!
-      Log.exception $!
-      retry
     ensure
       begin
         begin
@@ -346,7 +354,7 @@ class RbbtProcessQueue
     ensure
       @queue.clean if @queue
       #@callback_thread.push ClosedStream if @callback_thread && @callback_thread.alive?
-      @callback_queue.clean if @queue
+      @callback_queue.clean if @callback_queue
     end
   end
 

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -78,11 +78,11 @@ class RbbtProcessQueue
     end
 
     def close_write
-      @swrite.close
+      @swrite.close unless closed_write?
     end
 
     def close_read
-      @sread.close 
+      @sread.close unless closed_read?
     end
     #{{{ ACCESSOR
   

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -53,7 +53,6 @@ class RbbtProcessQueue
               break
             end
           end
-          Kernel.exit! 0
         rescue Respawn
           Kernel.exit! 28
         rescue ClosedStream
@@ -63,6 +62,7 @@ class RbbtProcessQueue
           retry unless @stop 
           Log.high "Worker #{Process.pid} leaving"
         rescue Exception
+          Log.high "Worker #{Process.pid} had exception: #{$!.message}"
           begin
             @callback_queue.push($!) if @callback_queue
           rescue
@@ -74,6 +74,8 @@ class RbbtProcessQueue
       rescue Aborted
         Log.high "Worker #{Process.pid} aborted"
       end
+
+      Log.high "Worker #{Process.pid} completed"
       Kernel.exit! 0
     end
 
@@ -92,7 +94,6 @@ class RbbtProcessQueue
 
         initial = Misc.memory_use(Process.pid)
         memory_cap = multiplier * initial
-        Log.debug "Worker for #{Process.pid} started with pid #{@current} -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap}"
 
         @asked = false
         @monitored = false
@@ -153,6 +154,8 @@ class RbbtProcessQueue
           run
         end
 
+        Log.debug "Worker for #{Process.pid} started with pid #{@current} -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap}"
+
         while true
           @prev = @current
           pid, status = Process.waitpid2 @current
@@ -178,6 +181,7 @@ class RbbtProcessQueue
       end
 
       if status
+        Log.high "Worker respawner with #{Process.pid} (#{@current}) completed with status #{status}"
         Kernel.exit! status.to_i >> 8
       else
         Kernel.exit! -1

data/lib/rbbt/util/log.rb

@@ -133,12 +133,12 @@ module Log
     end
   end
 
-  def self.up_lines(num = 0)
+  def self.up_lines(num = 1)
     "\033[#{num+1}F\033[2K"
   end
 
-  def self.down_lines(num = 0)
-    "\n\033[#{num+2}E"
+  def self.down_lines(num = 1)
+    "\033[#{num+1}E"
   end
 
   def self.return_line

data/lib/rbbt/util/log/progress.rb

@@ -6,12 +6,12 @@ module Log
 
     attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes
     def initialize(max = nil, options = {})
-      options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :desc => "Progress", :io => STDERR, :severity => Log.severity
-      depth, num_reports, desc, io, severity, file, bytes = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes
+      options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :desc => "Progress", :io => STDERR, :severity => Log.severity, :frequency => 2
+      depth, num_reports, desc, io, severity, file, bytes, frequency = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency
 
       @max = max
       @ticks = 0
-      @frequency = 2
+      @frequency = frequency
       @last_time = nil
       @last_count = nil
       @last_percent = nil
@@ -46,7 +46,7 @@ module Log
       end
 
       diff = time - @last_time
-      report and return if diff > @frequency
+      report and return if diff >= @frequency
       return unless max and max > 0
 
       percent = self.percent

data/lib/rbbt/util/log/progress/report.rb

@@ -10,22 +10,73 @@ module Log
     end
 
     attr_accessor :history, :mean_max, :max_history
-    def thr
-      count = @ticks - @last_count
-      if @last_time.nil?
-        seconds = 0.001
-      else
-        seconds = Time.now - @last_time
-      end
-      thr = count / seconds
-    end
+    #def thr
+    #  count = (@ticks - @last_count).to_f
+    #  if @last_time.nil?
+    #    seconds = 0.001
+    #  else
+    #    seconds = Time.now - @last_time
+    #  end
+    #  thr = count / seconds
+    #end
+
+
+    #def thr_msg
+    #  thr = self.thr
+    #  if @history.nil?
+    #    @history ||= [thr]
+    #  else
+    #    @history << thr
+    #    max_history ||= case 
+    #                  when @ticks > 20
+    #                    count = @ticks - @last_count
+    #                    if @max
+    #                      times = @max / count
+    #                      num = times / 20
+    #                      num = 2 if num < 2
+    #                    else
+    #                      num = 10
+    #                    end
+    #                    count * num
+    #                  else
+    #                    20
+    #                  end
+    #    max_history = 30 if max_history > 30
+    #    max_history = 100
+    #    @history.shift if @history.length > max_history
+    #  end
+
+    #  @mean_max ||= 0
+    #  if @history.length > 3
+
+    #    w_mean = 0
+    #    total_w = 0
+    #    @history.each_with_index do |v,i|
+    #      c = i ** 10
+    #      w_mean += v * c
+    #      total_w += c
+    #    end
+    #    mean = @mean = w_mean.to_f / total_w
+
+    #    @mean_max = mean if mean > @mean_max
+    #  end
+
+    #  if mean.nil? or mean.to_i > 1
+    #    str = "#{ Log.color :blue, thr.to_i.to_s } per sec."
+    #    str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
+    #  else
+    #    str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
+    #    str << " #{ Log.color :yellow, (1/mean).ceil.to_s } avg. #{Log.color :yellow, (1/@mean_max).ceil.to_s} min." if @mean_max > 0
+    #  end
+
+    #  str
+    #end
 
     def thr_msg
-      thr = self.thr
       if @history.nil?
-        @history ||= [thr]
+        @history ||= [[@ticks, Time.now] ]
       else
-        @history << thr
+        @history << [@ticks, Time.now]
         max_history ||= case 
                       when @ticks > 20
                         count = @ticks - @last_count
@@ -47,18 +98,23 @@ module Log
       @mean_max ||= 0
       if @history.length > 3
 
-        w_mean = 0
-        total_w = 0
-        @history.each_with_index do |v,i|
-          c = i ** 10
-          w_mean += v * c
-          total_w += c
-        end
-        mean = @mean = w_mean.to_f / total_w
+        sticks, stime = @history.first
+        ssticks, sstime = @history[3]
+        lticks, ltime = @history.last
+
+        mean = @mean = (lticks - sticks).to_f / (ltime - stime)
+        short_mean = (ssticks - sticks).to_f / (sstime - stime)
 
         @mean_max = mean if mean > @mean_max
       end
 
+      if short_mean
+        thr = short_mean
+      else
+        thr = @ticks / (Time.now - @start) 
+      end
+      iii [short_mean, thr]
+      
       if mean.nil? or mean.to_i > 1
         str = "#{ Log.color :blue, thr.to_i.to_s } per sec."
         str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
@@ -70,6 +126,7 @@ module Log
       str
     end
 
+
     def eta_msg
       percent = self.percent
       time = Time.now
@@ -140,8 +197,9 @@ module Log
       else
         bars = BARS
       end
+      bars << self unless BARS.include? self
 
-      print(io, Log.up_lines(bars.length) << Log.color(:yellow, "...Progress\n") << Log.down_lines(bars.length)) 
+      print(io, Log.up_lines(bars.length+1) << Log.color(:yellow, "...Progress\n") << Log.down_lines(bars.length+1)) 
       print(io, Log.up_lines(@depth) << report_msg << Log.down_lines(@depth)) 
       @last_time = Time.now
       @last_count = ticks

data/lib/rbbt/util/misc/pipes.rb

@@ -203,7 +203,7 @@ module Misc
         end
 
         stream.close unless stream.closed?
-        stream.join if stream.respond_to? :join
+        #stream.join if stream.respond_to? :join
         in_pipes.first.close
         #Log.medium "Tee done #{Misc.fingerprint stream}"
       rescue Aborted, Interrupt
@@ -608,7 +608,7 @@ module Misc
   end
 
   def self.save_stream(file, stream)
-    out, save = Misc.tee_stream stream
+    save, out = Misc.tee_stream stream
     out.filename = file
     save.filename = file
 

data/lib/rbbt/util/open.rb

@@ -192,7 +192,7 @@ module Open
     if (dir_sub_path = find_repo_dir(file))
       remove_from_repo(*dir_sub_path)
     else
-      FileUtils.rm(file)
+      FileUtils.rm(file) if File.exists?(file)
     end
   end
 
@@ -563,4 +563,12 @@ module Open
     end
     notify_write(file) 
   end
+
+  def self.writable?(path)
+    if File.exists?(path)
+      File.writable?(path)
+    else
+      File.writable?(File.dirname(File.expand_path(path)))
+    end
+  end
 end

data/lib/rbbt/workflow/accessor.rb

@@ -111,6 +111,14 @@ class Step
     @info_lock
   end
 
+  def status_lock
+    @status_lock = begin
+                   path = Persist.persistence_path(info_file + '.status.lock', {:dir => Step.lock_dir})
+                   Lockfile.new path, :refresh => false, :dont_use_lock_id => true
+                 end if @status_lock.nil?
+    @status_lock
+  end
+
   def info(check_lock = true)
     return {:status => :noinfo} if info_file.nil? or not Open.exists? info_file
     begin
@@ -163,7 +171,7 @@ class Step
 
   def set_info(key, value)
     return nil if @exec or info_file.nil?
-    return nil if ! File.writable? info_file
+    return nil if ! writable?
     value = Annotated.purge value if defined? Annotated
     Open.lock(info_file, :lock => info_lock) do
       i = info(false).dup
@@ -435,7 +443,7 @@ class Step
   end
 
   def stalled?
-    started? && ! (done? || error? || aborted?) && ! running?
+    started? && ! (done? || error? || aborted? || running? || waiting?)
   end
 
   def missing?
@@ -626,6 +634,11 @@ class Step
       out
     end
   end
+
+  def relocated?
+    done? && info[:path] && info[:path] != path
+  end
+
 end
 
 module Workflow

data/lib/rbbt/workflow/step.rb

@@ -273,6 +273,11 @@ class Step
     pid_file = Step.pid_file path
     files_dir = Step.files_dir path
 
+    if ! (Open.writable?(path) && Open.writable?(info_file))
+      Log.warn "Could not clean #{path}: not writable"
+      return 
+    end
+
     if Open.exists?(path) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir)
 
       @result = nil
@@ -324,6 +329,10 @@ class Step
     new_dependencies.uniq
   end
 
+  def writable?
+    Open.writable?(self.path) && Open.writable?(self.info_file)
+  end
+
   def recursive_clean
     dependencies.each do |step| 
       step.recursive_clean 

data/lib/rbbt/workflow/step/dependencies.rb

@@ -80,18 +80,23 @@ class Step
       return if status == 'streaming' and job.running?
     end
 
-    if (status == 'error' && (job.recoverable_error? || job.dirty?)) ||
-      job.aborted? ||
-      (job.done? && ! job.updated?)  || (job.error? && ! job.updated?) ||
-      (job.done? && job.dirty?)  || (job.error? && job.dirty?) ||
-      (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
+    job.status_lock.synchronize do
+      status = job.status.to_s
 
-      job.clean 
-    end
+      if (status == 'error' && (job.recoverable_error? || job.dirty?)) ||
+          job.aborted? ||
+          (job.done? && ! job.updated?)  || (job.error? && ! job.updated?) ||
+          (job.done? && job.dirty?)  || (job.error? && job.dirty?) ||
+          (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
+
+        job.clean 
+      end
+
+      (job.init_info and job.dup_inputs) unless status == 'done' or job.started?
 
-    (job.init_info and job.dup_inputs) unless status == 'done' or job.started?
+      canfail = ComputeDependency === job && job.canfail?
+    end
 
-    canfail = ComputeDependency === job && job.canfail?
     raise DependencyError, job if job.error? and not canfail
   end
 
@@ -125,16 +130,18 @@ class Step
         return
       end
 
-      if dependency.aborted? or (dependency.error? and dependency.recoverable_error?) or (!Open.remote?(dependency.path) && dependency.missing?)
-        Log.warn "Cleaning dep. #{Log.color :red, dependency.task_name.to_s}"
-        dependency.clean
-        raise TryAgain
+      dependency.status_lock.synchronize do
+        if dependency.aborted? or (dependency.error? and dependency.recoverable_error?) or (!Open.remote?(dependency.path) && dependency.missing?)
+          Log.warn "Cleaning dep. on exec #{Log.color :blue, dependency.path} (missing: #{dependency.missing?}; error #{dependency.error?})"
+          dependency.clean
+          raise TryAgain
+        end
       end
 
       if ! dependency.started? && ! dependency.error?
         log_dependency_exec(dependency, :starting)
         dependency.run(true)
-        raise TryAgain
+          raise TryAgain
       end
 
       dependency.grace
@@ -158,7 +165,7 @@ class Step
       end
 
     rescue TryAgain
-      Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
+      Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
       retry
     rescue Aborted, Interrupt
       Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
@@ -417,8 +424,8 @@ class Step
     dangling_deps = all_deps.reject{|dep| dep.done? || canfail_paths.include?(dep.path) }.
       select{|dep| dep.waiting? }
 
-    Log.medium "Aborting waiting dangling dependencies #{Misc.fingerprint dangling_deps}" if dangling_deps.any?
-    dangling_deps.each{|dep| dep.abort }
+    Log.medium "Aborting (actually not) waiting dangling dependencies #{Misc.fingerprint dangling_deps}" if dangling_deps.any?
+    #dangling_deps.each{|dep| dep.abort }
 
   end
 

data/lib/rbbt/workflow/step/run.rb

@@ -108,7 +108,7 @@ class Step
     rec_dependencies.
       select{|dependency| ! (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) }.
       select{|dependency| ! Open.remote?(dependency.path) }.
-      select{|dependency| Open.exists?(dependency.info_file) }.
+      select{|dependency| Open.exists?(dependency.info_file) && ! dependency.relocated? }.
       select{|dependency| ! dependency.error? }
   end
 
@@ -120,7 +120,6 @@ class Step
     (dependency_checks + input_checks).uniq
   end
 
-
   def out_of_date
 
     checks = self.checks
@@ -130,6 +129,7 @@ class Step
     canfail_paths = self.canfail_paths
     checks.each do |dep| 
       next unless Open.exists?(dep.info_file)
+      next if dep.relocated?
 
       begin
         if dep.done? && self.done? && Open.exists?(dep.path) && Open.exists?(self.path) && (File.mtime(dep.path) > File.mtime(self.path))
@@ -150,7 +150,7 @@ class Step
   end
 
   def updated?
-    return true unless (done? || error?)
+    return true unless (done? || error? || ! writable?)
 
     @updated ||= out_of_date.empty?
   end
@@ -182,201 +182,203 @@ class Step
       res = @mutex.synchronize do
         no_load = :stream if no_load
 
-        Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-        result = Persist.persist "Job", @task.result_type, :file => path, :check => checks.collect{|dep| dep.path}, :no_load => no_load do 
-          if Step === Step.log_relay_step and not self == Step.log_relay_step
-            relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
-          end
-
-          Open.write(pid_file, Process.pid.to_s) unless Open.exists? pid_file
-
-          @exec = false
-          init_info
+        self.status_lock.synchronize do
+          Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
+          result = Persist.persist "Job", @task.result_type, :file => path, :check => checks.collect{|dep| dep.path}, :no_load => no_load do 
+            if Step === Step.log_relay_step and not self == Step.log_relay_step
+              relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
+            end
 
-          log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
+            Open.write(pid_file, Process.pid.to_s) unless Open.exists? pid_file
 
-          merge_info({
-            :issued => (issue_time = Time.now),
-            :name => name,
-            :clean_name => clean_name,
-            :workflow => (@workflow || @task.workflow).to_s,
-            :task_name => @task.name,
-            :result_type => @task.result_type,
-            :result_description => @task.result_description,
-            :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
-          })
+            @exec = false
+            init_info
 
+            log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
 
-          begin
-            run_dependencies
-          rescue Exception
-            FileUtils.rm pid_file if File.exist?(pid_file)
-            stop_dependencies
-            raise $!
-          end
+            merge_info({
+              :issued => (issue_time = Time.now),
+              :name => name,
+              :clean_name => clean_name,
+              :workflow => (@workflow || @task.workflow).to_s,
+              :task_name => @task.name,
+              :result_type => @task.result_type,
+              :result_description => @task.result_description,
+              :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
+            })
 
-          new_inputs = []
-          @inputs.each_with_index do |input,i|
-            name = @task.inputs[i]
-            type = @task.input_types[name]
 
-            if type == :directory
-              directory_inputs = file('directory_inputs')
-              input_source = directory_inputs['.source'][name].find
-              input_dir = directory_inputs[name].find
+            begin
+              run_dependencies
+            rescue Exception
+              FileUtils.rm pid_file if File.exist?(pid_file)
+              stop_dependencies
+              raise $!
+            end
 
-              case input
-              when Path
-                if input.directory?
-                  new_inputs << input
-                else
-                  input.open do |io|
-                    begin
-                      Misc.untar(io, input_source)
-                    rescue
-                      raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
+            new_inputs = []
+            @inputs.each_with_index do |input,i|
+              name = @task.inputs[i]
+              type = @task.input_types[name]
+
+              if type == :directory
+                directory_inputs = file('directory_inputs')
+                input_source = directory_inputs['.source'][name].find
+                input_dir = directory_inputs[name].find
+
+                case input
+                when Path
+                  if input.directory?
+                    new_inputs << input
+                  else
+                    input.open do |io|
+                      begin
+                        Misc.untar(io, input_source)
+                      rescue
+                        raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
+                      end
                     end
+                    tar_1 = input_source.glob("*")
+                    raise ParameterException, "When using tar.gz files for directories, the directory must be the single first level entry" if tar_1.length != 1
+                    FileUtils.ln_s Misc.path_relative_to(directory_inputs, tar_1.first), input_dir
+                    new_inputs << input_dir
+                  end
+                when File, IO, Tempfile
+                  begin
+                    Misc.untar(Open.gunzip(input), input_source)
+                  rescue
+                    raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
                   end
                   tar_1 = input_source.glob("*")
                   raise ParameterException, "When using tar.gz files for directories, the directory must be the single first level entry" if tar_1.length != 1
-                FileUtils.ln_s Misc.path_relative_to(directory_inputs, tar_1.first), input_dir
+                  FileUtils.ln_s Misc.path_relative_to(directory_inputs, tar_1.first), input_dir
                   new_inputs << input_dir
+                else
+                  raise ParameterException, "Format of directory input '#{name}' not understood: #{Misc.fingerprint input}"
                 end
-              when File, IO, Tempfile
-                begin
-                  Misc.untar(Open.gunzip(input), input_source)
-                rescue
-                  raise ParameterException, "Error unpackaging tar directory input '#{name}':\n\n#{$!.message}"
-                end
-                tar_1 = input_source.glob("*")
-                raise ParameterException, "When using tar.gz files for directories, the directory must be the single first level entry" if tar_1.length != 1
-                FileUtils.ln_s Misc.path_relative_to(directory_inputs, tar_1.first), input_dir
-                new_inputs << input_dir
               else
-                raise ParameterException, "Format of directory input '#{name}' not understood: #{Misc.fingerprint input}"
-              end
-            else
-              new_inputs << input
-            end
-          end if @inputs
-          
-          @inputs = new_inputs if @inputs
-
-          if not task.inputs.nil?
-            info_inputs = @inputs.collect do |i| 
-              if Path === i 
-                i.to_s
-              else 
-                i 
+                new_inputs << input
               end
-            end
-            set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, info_inputs)) 
-          end
+            end if @inputs
 
-          set_info :started, (start_time = Time.now)
-          log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"
+            @inputs = new_inputs if @inputs
 
-          begin
-            result = _exec
-          rescue Aborted, Interrupt
-            log(:aborted, "Aborted")
-            raise $!
-          rescue Exception
-            backtrace = $!.backtrace
-
-            # HACK: This fixes an strange behaviour in 1.9.3 where some
-            # backtrace strings are coded in ASCII-8BIT
-            backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
-            set_info :backtrace, backtrace 
-            log(:error, "#{$!.class}: #{$!.message}")
-            stop_dependencies
-            raise $!
-          end
+            if not task.inputs.nil?
+              info_inputs = @inputs.collect do |i| 
+                if Path === i 
+                  i.to_s
+                else 
+                  i 
+                end
+              end
+              set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, info_inputs)) 
+            end
 
-          if not no_load or ENV["RBBT_NO_STREAM"] == "true" 
-            result = prepare_result result, @task.description, info if IO === result 
-            result = prepare_result result.stream, @task.description, info if TSV::Dumper === result 
-          end
+            set_info :started, (start_time = Time.now)
+            log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"
 
-          stream = case result
-                   when IO
-                     result
-                   when TSV::Dumper
-                     result.stream
-                   end
+            begin
+              result = _exec
+            rescue Aborted, Interrupt
+              log(:aborted, "Aborted")
+              raise $!
+            rescue Exception
+              backtrace = $!.backtrace
+
+              # HACK: This fixes an strange behaviour in 1.9.3 where some
+              # backtrace strings are coded in ASCII-8BIT
+              backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
+              set_info :backtrace, backtrace 
+              log(:error, "#{$!.class}: #{$!.message}")
+              stop_dependencies
+              raise $!
+            end
 
-          if stream
-            log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
+            if not no_load or ENV["RBBT_NO_STREAM"] == "true" 
+              result = prepare_result result, @task.description, info if IO === result 
+              result = prepare_result result.stream, @task.description, info if TSV::Dumper === result 
+            end
 
-            callback = Proc.new do
-              if AbortedStream === stream
-                if stream.exception
-                  raise stream.exception 
-                else
-                  raise Aborted
+            stream = case result
+                     when IO
+                       result
+                     when TSV::Dumper
+                       result.stream
+                     end
+
+            if stream
+              log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
+
+              callback = Proc.new do
+                if AbortedStream === stream
+                  if stream.exception
+                    raise stream.exception 
+                  else
+                    raise Aborted
+                  end
                 end
-              end
-              begin
-                status = self.status
-                if status != :done and status != :error and status != :aborted
-                  Misc.insist do
-                    merge_info({
-                     :done => (done_time = Time.now),
-                     :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
-                     :time_elapsed => (time_elapsed = done_time - start_time)
-                    })
-                    log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
+                begin
+                  status = self.status
+                  if status != :done and status != :error and status != :aborted
+                    Misc.insist do
+                      merge_info({
+                        :done => (done_time = Time.now),
+                        :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
+                        :time_elapsed => (time_elapsed = done_time - start_time)
+                      })
+                      log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
+                    end
                   end
+                rescue
+                  Log.exception $!
+                ensure
+                  Step.purge_stream_cache
+                  set_info :pid, nil
+                  FileUtils.rm pid_file if File.exist?(pid_file)
                 end
-              rescue
-                Log.exception $!
-              ensure
-                Step.purge_stream_cache
-                set_info :pid, nil
-                FileUtils.rm pid_file if File.exist?(pid_file)
               end
-            end
 
-            abort_callback = Proc.new do |exception|
-              begin
-                if exception
-                  self.exception exception
-                else
-                  log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
+              abort_callback = Proc.new do |exception|
+                begin
+                  if exception
+                    self.exception exception
+                  else
+                    log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
+                  end
+                  _clean_finished
+                rescue
+                  stop_dependencies
+                  set_info :pid, nil
+                  FileUtils.rm pid_file if File.exist?(pid_file)
                 end
-                _clean_finished
-              rescue
-                stop_dependencies
-                set_info :pid, nil
-                FileUtils.rm pid_file if File.exist?(pid_file)
               end
-            end
 
-            ConcurrentStream.setup stream, :callback => callback, :abort_callback => abort_callback
+              ConcurrentStream.setup stream, :callback => callback, :abort_callback => abort_callback
 
-            if AbortedStream === stream 
-              exception = stream.exception || Aborted.new("Aborted stream: #{Misc.fingerprint stream}")
-              self.exception exception
-              _clean_finished
-              raise exception
+              if AbortedStream === stream 
+                exception = stream.exception || Aborted.new("Aborted stream: #{Misc.fingerprint stream}")
+                self.exception exception
+                _clean_finished
+                raise exception
+              end
+            else
+              merge_info({
+                :done => (done_time = Time.now),
+                :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
+                :time_elapsed => (time_elapsed = done_time - start_time)
+              })
+              log :ending
+              Step.purge_stream_cache
+              FileUtils.rm pid_file if File.exist?(pid_file)
             end
-          else
-            merge_info({
-              :done => (done_time = Time.now),
-              :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
-              :time_elapsed => (time_elapsed = done_time - start_time)
-            })
-            log :ending
-            Step.purge_stream_cache
-            FileUtils.rm pid_file if File.exist?(pid_file)
-          end
 
-          set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
+            set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
-          if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
-            FileUtils.mv self.tmp_path, self.path
+            if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
+              FileUtils.mv self.tmp_path, self.path
+            end
+            result
           end
-          result
         end
 
         if no_load
@@ -407,16 +409,22 @@ class Step
   def produce(force=false, dofork=false)
     return self if done? and not dirty?
 
-    if error? or aborted? or stalled?
-      abort if stalled?
-      if force or aborted? or recoverable_error?
-        clean
-      else
-        e = get_exception
-        if e
-          raise e
+
+    self.status_lock.synchronize do
+      if error? or aborted? or stalled?
+        if stalled?
+          Log.warn "Aborting stalled job #{self.path}"
+          abort
+        end
+        if force or aborted? or recoverable_error?
+          clean
         else
-          raise "Error in job: #{self.path}"
+          e = get_exception
+          if e
+            raise e
+          else
+            raise "Error in job: #{self.path}"
+          end
         end
       end
     end
@@ -662,7 +670,7 @@ class Step
       begin
         set_info :joined, true 
       rescue
-      end if File.exists?(info_file) && File.writable?(info_file)
+      end if File.exists?(info_file) && writable?
       @result = nil
     end
   end

data/test/rbbt/test_workflow.rb

@@ -146,18 +146,22 @@ class TestWorkflow < Test::Unit::TestCase
   end
 
   def test_update_on_input_dependency_update
+    Log.severity = 0
     send_input_dep_to_reverse_job = TestWF.job(:send_input_dep_to_reverse, nil, :name => "Miguel")
     send_input_dep_to_reverse_job.clean
     send_input_dep_to_reverse_job.run
+
     input_dep_job = send_input_dep_to_reverse_job.step(:input_dep)
+    mtime_orig = File.mtime send_input_dep_to_reverse_job.step(:reverse_input_text).path
+
+    sleep 1
     input_dep_job.clean
     input_dep_job.run
     send_input_dep_to_reverse_job = TestWF.job(:send_input_dep_to_reverse, nil, :name => "Miguel")
-    mtime_orig = File.mtime send_input_dep_to_reverse_job.step(:reverse_input_text).path
+
     send_input_dep_to_reverse_job.run
     mtime_new = File.mtime send_input_dep_to_reverse_job.step(:reverse_input_text).path
     assert mtime_orig < mtime_new
-
   end
 
   def test_helper

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.23.14
+  version: 5.23.15
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-11 00:00:00.000000000 Z
+date: 2018-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake