rbbt-util 5.19.37 → 5.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 137e4838f414d232714269ef64c04ba299fa3f62
-  data.tar.gz: 7cc30b69428f3f33461d639c6657d101fb09547f
+  metadata.gz: 333a3110a30ab60b25c3e7a5acb5364c2b75cf87
+  data.tar.gz: 73610983c2f10ee9ec6cfcc3693e6569174a433d
 SHA512:
-  metadata.gz: 6f901c235ed7ef31b1693b1d99282ad86f7f8f37c1b18e9e62a4c67adafa29e10d5fc3b53e12adba5803ce4571958401bd78a65723e1a9e17736e52a7acf755a
-  data.tar.gz: 2aa36eb9d0a80795a230187894e1f05dd51b1125cf890cda3f740e8e66d1c0320f3f2637ac7d10e38b16698935cb313b3a5498283e7ccf26e1848df3c8620523
+  metadata.gz: 433fe181900362ba716a2fdaf9924a08458f2d75d15e41d0b86a9fcab8315c7dfbfa23998f114bfe08cdce03f057fd9af94a3721afdd9f732fd640f2f1789311
+  data.tar.gz: f3ff2dc3ff968917dce9851548c8cafd5c2a7e26666f25fc7be70da1de45a92c5e65c8ea95ac353eb841663f88c76fdf2b6500b5d7e9abab68975876c7bf0226

data/lib/rbbt/rest/client/step.rb

@@ -6,9 +6,8 @@ class WorkflowRESTClient
     def self.get_streams(inputs)
       new_inputs = {}
       inputs.each do |k,v|
-        if Step === v or RemoteStep ===  v
-          v.run
-          new_inputs[k] = v.load
+        if Step === v 
+          new_inputs[k] = TSV.get_stream v
         else
           new_inputs[k] = v
         end
@@ -16,12 +15,27 @@ class WorkflowRESTClient
       new_inputs
     end
 
+    def get_streams
+      @inputs = WorkflowRESTClient::RemoteStep.get_streams @inputs
+    end
+
 
     def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false, stream_input = nil)
       @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec = base_url, task, base_name, inputs, result_type, result_description, is_exec
       @mutex = Mutex.new
       @stream_input = stream_input
-      @inputs = RemoteStep.get_streams @inputs
+      #@inputs = RemoteStep.get_streams @inputs
+    end
+
+    def dup_inputs
+      return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
+      Log.low "Dupping inputs for remote #{path}"
+      new_inputs = {}
+      @inputs.each do |name,input|
+        new_inputs[name] = Step.dup_stream input
+      end
+      @inputs = RemoteStep.get_streams new_inputs
+      @dupped = true
     end
 
     def name
@@ -31,7 +45,7 @@ class WorkflowRESTClient
     end
 
     def task_name
-      return task unless @url
+      return task if task
       init_job
       (Array === @url ? @url.first : @url).split("/")[-2]
     end
@@ -39,16 +53,16 @@ class WorkflowRESTClient
     def info(check_lock=false)
       @done = @info and @info[:status] and @info[:status].to_sym == :done
       @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => !!@done) do
-        init_job unless url
-        info = WorkflowRESTClient.get_json(File.join(url, 'info'))
+        init_job unless @url
+        info = WorkflowRESTClient.get_json(File.join(@url, 'info'))
         info = WorkflowRESTClient.fix_hash(info)
         info[:status] = info[:status].to_sym if String === info[:status]
         info
       end
     end
-    
+
     def status
-      return nil if @url.nil?
+      return nil unless url or started?
       begin
         info[:status]
       ensure
@@ -56,6 +70,10 @@ class WorkflowRESTClient
       end
     end
 
+    def started?
+      @result != nil or @started
+    end
+
     def done?
       @done || status.to_s == 'done'
     end
@@ -68,9 +86,24 @@ class WorkflowRESTClient
       WorkflowRESTClient.get_raw(File.join(url, 'file', file))
     end
 
+    def get_stream
+      case @result
+      when IO 
+        @result
+      when String
+        StringIO.new @result
+      else
+        nil
+      end
+    end
+
+    def grace
+    end
+
     #{{{ MANAGEMENT
-    
+
     def init_job(cache_type = nil, other_params = {})
+      Log.stack caller
       cache_type = :asynchronous if cache_type.nil? and not @is_exec
       cache_type = :exec if cache_type.nil?
       @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
@@ -108,6 +141,8 @@ class WorkflowRESTClient
                         init_job 
                         self.load
                       end
+                    ensure
+                      @started = true
                     end
       end
 
@@ -116,11 +151,15 @@ class WorkflowRESTClient
     end
 
     def exec(noload = false)
-      if noload == :stream
-        _run_job(:exec)
-      else
-        exec_job 
-      end
+      @result ||= begin
+                     if noload == :stream
+                       _run_job(:exec)
+                     else
+                       exec_job 
+                     end
+                    ensure
+                      @started = true
+                   end
     end
 
     def join
@@ -180,20 +219,29 @@ class WorkflowRESTClient
     def _stream_job(stream_input, cache_type = :exec)
       require 'rbbt/util/misc/multipart_payload'
       WorkflowRESTClient.capture_exception do
-        url = URI.encode(File.join(base_url, task.to_s))
-        Log.debug{ "RestClient stream: #{ url } #{stream_input} #{cache_type} - #{Misc.fingerprint inputs}" }
+        task_url = URI.encode(File.join(base_url, task.to_s))
+        Log.debug{ "RestClient stream: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint inputs}" }
         task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
-        res = RbbtMutiplartPayload.issue url, task_params, :mutations, nil, nil, true
+        res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
         type = res.gets
         case type.strip
         when "LOCATION"
-          url = res.gets
-          url.sub!(/\?.*/,'')
-          WorkflowRESTClient.get_raw(url)
-        when "STREAM"
+          @url = res.gets
+          @url.sub!(/\?.*/,'')
+          WorkflowRESTClient.get_raw(@url)
+        when /STREAM: (.*)/
+          @url = $1.strip
+          ConcurrentStream.setup(res)
+          res.callback = Proc.new do
+            @done = true
+          end
           res
         when "BULK"
-          res.read
+          begin
+            res.read
+          ensure
+            @done = true
+          end
         else
           raise "What? " + type
         end
@@ -206,7 +254,7 @@ class WorkflowRESTClient
         return _stream_job(stream_input, cache_type) 
       end
       WorkflowRESTClient.capture_exception do
-        url = URI.encode(File.join(base_url, task.to_s))
+        @url = URI.encode(File.join(base_url, task.to_s))
         task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
 
         sout, sin = Misc.pipe
@@ -244,6 +292,7 @@ class WorkflowRESTClient
     def _restart
       @done = nil
       @name = nil
+      @started = nil
       new_inputs = {}
       inputs.each do |k,i| 
         if File === i 

data/lib/rbbt/tsv/stream.rb

@@ -69,7 +69,7 @@ module TSV
         if fix_flat and parser.type == :flat and parser.first_line
           parts = lines[-1].split("\t")
           lines[-1] = [parts[0], (parts[1..-1] || [])*"|"] * "\t"
-          TSV.stream_flat2double(parser.stream).stream
+          TSV.stream_flat2double(parser.stream, :noheader => true).stream
         else
           parser.stream
         end
@@ -220,10 +220,11 @@ module TSV
   end
 
   def self.stream_flat2double(stream, options = {})
+    noheader = Misc.process_options options, :noheader
     parser = TSV::Parser.new TSV.get_stream(stream), :type => :flat
     dumper_options = parser.options.merge(options).merge(:type => :double)
     dumper = TSV::Dumper.new dumper_options
-    dumper.init
+    dumper.init unless noheader
     TSV.traverse parser, :into => dumper do |key,values|
       key = key.first if Array === key
       values = [values] unless Array === values

data/lib/rbbt/util/misc/exceptions.rb

@@ -8,7 +8,19 @@ class Aborted < Exception; end
 
 class RemoteServerError < Exception; end
 
-class DependencyError < Exception; end
+class DependencyError < Aborted
+  def initialize(msg)
+    if defined? Step and Step === msg
+      step = msg
+      workflow = step.path.split("/")[-3]
+      new_msg = [workflow, step.short_path, step.messages.last] * " - "
+      new_msg = [step.path, step.messages.last] * ": "
+      super(new_msg)
+    else
+      super(msg)
+    end
+  end
+end
 
 class KeepLocked < Exception
   attr_accessor :payload

data/lib/rbbt/util/misc/inspect.rb

@@ -28,7 +28,7 @@ module Misc
     when nil
       "nil"
     when (defined? Step and Step)
-      obj.path || Misc.fingerprint([obj.task.name, obj.inputs])
+      "<Step:"  << (obj.path || Misc.fingerprint([obj.task.name, obj.inputs])) << ">"
     when TrueClass
       "true"
     when FalseClass
@@ -53,7 +53,7 @@ module Misc
       if (length = obj.length) > 10
         "[#{length}--" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
       else
-        "[" << (obj.collect{|e| fingerprint(e) } * ",") << "]"
+        "[" << (obj.collect{|e| fingerprint(e) } * ", ") << "]"
       end
     when (defined? TSV and TSV)
       obj.with_unnamed do

data/lib/rbbt/util/misc/multipart_payload.rb

@@ -1,24 +1,30 @@
 require 'net/http'
 require 'rbbt-util'
 
-class Net::HTTPGenericRequest
-  alias old_send_request_with_body_stream send_request_with_body_stream
-
-  def send_request_with_body_stream(*args)
-    if chunked?
-      Thread.new do
-        old_send_request_with_body_stream(*args)
-      end
-    else
-      old_send_request_with_body_stream(*args)
-    end
-  end
-end
+#class Net::HTTPGenericRequest
+#  alias old_send_request_with_body_stream send_request_with_body_stream
+#
+#  def __send_request_with_body_stream(*args)
+#    begin
+#      pid = Process.fork do
+#        iii [:pre_chunk, Thread.current]
+#        return old_send_request_with_body_stream(*args)
+#      end
+#      Process.wait pid
+#    ensure
+#      iii [:pre_chunk_done, Thread.current]
+#    end
+#  end
+#end
 
 module RbbtMutiplartPayload
   BOUNDARY = "Rbbt_Param_Stream"
   EOL = "\r\n"
 
+  def self.mutex
+    @mutex ||= Mutex.new
+  end
+
   def self.input_header(name, filename = nil)
 
     if filename
@@ -43,9 +49,6 @@ module RbbtMutiplartPayload
     header = input_header(name, filename)
     io.write "--" + BOUNDARY + EOL + header + EOL  + EOL
 
-    #while c = content.read(1024)
-    #  io.write c
-    #end
     while line = content.gets
       io.puts line
     end
@@ -59,9 +62,6 @@ module RbbtMutiplartPayload
   end
 
   def self.post_data_stream(inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil)
-    #sout, sin = Misc.pipe
-
-    #Thread.new do
     Misc.open_pipe do |sin|
       inputs.each do |input,content|
         input = input.to_s
@@ -86,6 +86,7 @@ module RbbtMutiplartPayload
 
       RbbtMutiplartPayload.add_stream(sin, stream_input.to_s, stream_io, stream_filename) if stream_input
       RbbtMutiplartPayload.close_stream(sin)
+
       sin.close unless sin.closed?
     end
   end
@@ -105,7 +106,7 @@ module RbbtMutiplartPayload
                           when File
                             inputs[stream_input].path
                           else
-                            'file'
+                            'file-rand-' + rand(10000000).to_s
                           end
     end
 
@@ -120,14 +121,17 @@ module RbbtMutiplartPayload
       req.body = sout.read
     end
 
-    Misc.open_pipe do |sin|
+    Misc.open_pipe(true) do |sin|
+      sleep rand(10).to_f / 5
       Net::HTTP.start(uri.hostname, uri.port) do |http|
         http.request(req) do |res|
+          url_path = res["RBBT-STREAMING-JOB-URL"]
           if Net::HTTPRedirection === res
             sin.puts "LOCATION" if report_type
             sin.write res["location"]
-          elsif stream_input
-            sin.puts "STREAM" if report_type
+          elsif stream_input and url_path
+            url = URI::HTTP.build(:host => uri.hostname, :post => uri.port, :path => url_path)
+            sin.puts "STREAM: #{url.to_s}" if report_type
             res.read_body do |c|
               sin.write c
             end

data/lib/rbbt/util/misc/pipes.rb

@@ -77,6 +77,7 @@ module Misc
           Log.medium "Aborted open_pipe: #{$!.message}"
         rescue Exception
           Log.medium "Exception in open_pipe: #{$!.message}"
+          Log.exception $!
           parent.raise $!
           raise $!
         end
@@ -90,14 +91,12 @@ module Misc
     stream_out1, stream_in1 = Misc.pipe
     stream_out2, stream_in2 = Misc.pipe
 
-    #if ConcurrentStream === stream 
-    #  stream.annotate stream_out1
-    #end
-
     splitter_thread = Thread.new(Thread.current) do |parent|
       begin
+
         skip1 = skip2 = false
-        while block = stream.read(2048)
+        while block = stream.read(1024)
+
           begin 
             stream_in1.write block; 
           rescue IOError
@@ -111,7 +110,9 @@ module Misc
             Log.medium("Tee stream 2 #{Misc.fingerprint stream} IOError: #{$!.message}");
             skip2 = true
           end unless skip2 
+
         end
+
         stream_in1.close unless stream_in1.closed?
         stream.join if stream.respond_to? :join
         stream_in2.close unless stream_in2.closed?

data/lib/rbbt/workflow/accessor.rb

@@ -585,7 +585,7 @@ module Workflow
     dependencies.each do |dependency|
       real_dependencies << case dependency
       when Array
-        workflow, task, options = dependency
+        workflow, dep_task, options = dependency
 
         _inputs = IndiferentHash.setup(inputs.dup)
         options.each{|i,v|
@@ -595,9 +595,9 @@ module Workflow
             rec_dependency = all_d.select{|d| d.task_name.to_sym == v }.first
 
             if rec_dependency.nil?
-              _inputs[i] = v
+              _inputs[i] = v unless _inputs.include? i
             else
-              input_options = workflow.task_info(task)[:input_options][i] || {}
+              input_options = workflow.task_info(dep_task)[:input_options][i] || {}
               if input_options[:stream]
                 #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
                 _inputs[i] = rec_dependency
@@ -617,7 +617,7 @@ module Workflow
           end
         } if options
 
-        res = workflow.job(task, jobname, _inputs)
+        res = workflow.job(dep_task, jobname, _inputs)
         res
       when Step
         dependency

data/lib/rbbt/workflow/step.rb

@@ -203,11 +203,8 @@ class Step
     info_file = Step.info_file path
     pid_file = Step.pid_file path
     files_dir = Step.files_dir path
+
     if Open.exists?(path) or Open.exists?(pid_file)
-      begin
-        self.abort if self.running?
-      rescue Exception
-      end
 
       @result = nil
       @pid = nil
@@ -224,6 +221,8 @@ class Step
   end
 
   def clean
+    Log.medium "Cleaning step: #{path}"
+    abort if not done? and running?
     Step.clean(path)
     self
   end
@@ -250,7 +249,16 @@ class Step
     rec_dependencies.each do |step| 
       if Open.exists?(step.info_file) 
         step.clean 
-      else
+      end
+    end
+    self
+  end
+
+  def recursive_clean
+    clean
+    dependencies.each do |step| 
+      if Open.exists?(step.info_file) 
+        step.recursive_clean 
       end
     end
     self

data/lib/rbbt/workflow/step/run.rb

@@ -1,31 +1,55 @@
 class Step
 
-  attr_reader :stream, :dupped, :saved_stream
+  attr_reader :stream, :dupped, :saved_stream, :inputs
 
   STREAM_CACHE = {}
   STREAM_CACHE_MUTEX = Mutex.new
+  def self.purge_stream_cache
+    Log.medium "Purging dup. stream cache"
+    STREAM_CACHE_MUTEX.synchronize do
+      #STREAM_CACHE.collect{|k,s| 
+      #  next
+      #  Thread.new do
+      #    Misc.consume_stream s
+      #  end
+      #}
+      STREAM_CACHE.clear
+    end
+  end
+
   def self.dup_stream(stream)
     case stream
-    when IO, File
-      return stream if stream.closed?
+    when IO, File, Step
+      return stream if stream.respond_to?(:closed?) and stream.closed?
+      return stream if stream.respond_to?(:done?) and stream.done?
 
       STREAM_CACHE_MUTEX.synchronize do
-        case current = STREAM_CACHE[stream]
+        stream_key = Misc.fingerprint(stream)
+        current = STREAM_CACHE[stream_key]
+        case current
         when nil
-          Log.medium "Not duplicating stream #{ Misc.fingerprint(stream) }"
-          STREAM_CACHE[stream] = stream
+          Log.medium "Not duplicating stream #{stream_key}"
+          STREAM_CACHE[stream_key] = stream
         when File
           if Open.exists? current.path 
-            Log.medium "Reopening file #{ Misc.fingerprint(current) }"
+            Log.medium "Reopening file #{stream_key}"
             Open.open(current.path)
           else
-            Log.medium "Duplicating file #{ Misc.fingerprint(current) } #{current.inspect}"
-            Misc.dup_stream(current)
+            new = Misc.dup_stream(current)
+            Log.medium "Duplicating file #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
+            new
           end
-
+        when Step
+          job = current
+          current = job.result 
+          new = Misc.dup_stream(current)
+          job.result = current
+          Log.medium "Duplicating step #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
+          new
         else
-          Log.medium "Duplicating stream #{ Misc.fingerprint(stream) }"
-          Misc.dup_stream(current)
+          new = Misc.dup_stream(current)
+          Log.medium "Duplicating stream #{stream_key} #{ Misc.fingerprint(stream) } => #{Misc.fingerprint(new)}"
+          new
         end
       end
     when TSV::Dumper#, TSV::Parser
@@ -47,20 +71,86 @@ class Step
     end
   end
 
-  def self.purge_stream_cache
-    return
-    STREAM_CACHE_MUTEX.synchronize do
-      STREAM_CACHE.collect{|k,s| 
-        Thread.new do
-          Misc.consume_stream s
-        end
-      }
-      STREAM_CACHE.clear
+  def self.prepare_for_execution(job)
+    return if (job.done? and not job.dirty?) or 
+    (job.streaming? and job.running?) or 
+    (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === job and not (job.error? or job.aborted?))
+
+    job.clean if job.aborted? or (job.started? and not job.running? and not job.error?)
+
+    raise DependencyError, job if job.error?
+  end
+
+  def execute_dependency(dependency)
+    task_name = self.task_name
+    begin
+
+      if dependency.done?
+        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} done -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+        return
+      end
+
+      if not dependency.started?
+        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} starting -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+        dependency.run(:stream)
+        raise TryAgain
+      end
+
+      dependency.grace
+
+      if dependency.aborted?
+        Log.warn "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} aborted (clean and retry) -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+        dependency.clean
+        raise TryAgain
+      end
+
+      if dependency.error?
+        Log.error "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} error -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+        raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
+      end
+
+      if dependency.streaming?
+        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} streaming -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+        return
+      end
+
+      begin
+        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+        dependency.join
+        raise TryAgain unless dependency.done?
+        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+      rescue Aborted
+        raise TryAgain
+      end
+
+    rescue TryAgain
+      retry
+    rescue Aborted
+      Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
+      raise $!
+    rescue Interrupt
+      Log.error "Interrupted while in dep. #{Log.color :red, dependency.task_name.to_s}"
+      raise $!
+    rescue Exception
+      Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s }"
+      Log.exception $!
+      raise $!
+    end
+  end
+
+  def dup_inputs
+    return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
+    Log.low "Dupping inputs for #{path}"
+    dupped_inputs = @inputs.collect do |input|
+      Step.dup_stream input
     end
+    @inputs.replace dupped_inputs
+    @dupped = true
   end
 
   def get_stream
     @mutex.synchronize do
+      Log.low "Getting stream from #{path} #{!@saved_stream}"
       begin
         return nil if @saved_stream
         if IO === @result 
@@ -72,13 +162,6 @@ class Step
     end
   end
 
-  def dup_inputs
-    return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
-    @inputs = @inputs.collect do |input|
-      Step.dup_stream input
-    end
-    @dupped = true
-  end
 
   def _exec
     @exec = true if @exec.nil?
@@ -98,7 +181,7 @@ class Step
   def checks
     rec_dependencies.collect{|dependency| (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) ? nil : dependency.path }.compact.uniq
   end
-  
+
 
   def kill_children
     begin
@@ -138,77 +221,39 @@ class Step
     return if @seen.empty?
 
     log :dependencies, "#{Log.color :magenta, "Dependencies"} for step #{Log.color :yellow, task.name.to_s || ""}"
-    dupping = []
-    @seen.each do |dependency|
-      next if (dependency.done? and not dependency.dirty?) or 
-              (dependency.streaming? and dependency.running?) or 
-              (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency and not (dependency.error? or dependency.aborted?))
-
-      dependency.clean if dependency.aborted? or (dependency.started? and not dependency.running? and not dependency.error?)
-
-      raise DependencyError, [dependency.short_path, dependency.messages.last] * ": " if dependency.error?
 
-      dupping << dependency unless dependencies.include? dependency
+    @seen.each do |dependency| 
+      Step.prepare_for_execution(dependency)
     end
 
-    dupping.each{|dep| dep.dup_inputs}
-
+    pre_deps = []
+    last_deps = []
     @seen.each do |dependency| 
-      next unless dependencies.include? dependency
-
-      begin
-
-        if dependency.done?
-          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} done -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
-          next
-        end
-
-        if not dependency.started?
-          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} starting -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
-          dependency.run(:stream)
-          raise TryAgain
-        end
-
-        dependency.grace
-
-        if dependency.aborted?
-          Log.warn "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} aborted (clean and retry) -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
-          dependency.clean
-          raise TryAgain
+      if dependencies.include? dependency
+        if dependency.inputs.flatten.select{|i| Step === i}.any?
+          last_deps << dependency
+        else
+          pre_deps << dependency
         end
 
-        if dependency.error?
-          Log.error "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} error -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
-          raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
-        end
+      else
+        pre_deps << dependency
+      end
+    end
 
-        if dependency.streaming?
-          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} streaming -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
-          next
-        end
+    pre_deps.each do |dependency|
+      dependency.dup_inputs
+      execute_dependency(dependency)
+    end
 
-        begin
-          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
-          dependency.join
-          raise TryAgain unless dependency.done?
-          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
-        rescue Aborted
-          raise TryAgain
-        end
+    last_deps.each do |dependency|
+      dependency.dup_inputs
+    end
 
-      rescue TryAgain
-        retry
-      rescue Aborted
-        Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
-        raise $!
-      rescue Interrupt
-        Log.error "Interrupted while in dep. #{Log.color :red, dependency.task_name.to_s}"
-        raise $!
-      rescue Exception
-        Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s }"
-        raise $!
-      end
+    last_deps.each do |dependency|
+      execute_dependency(dependency)
     end
+
   end
 
   def run(no_load = false)
@@ -243,6 +288,7 @@ class Step
             stop_dependencies
             raise $!
           end
+
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
           set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
@@ -295,6 +341,7 @@ class Step
                 Log.exception $!
               ensure
                 join
+                Step.purge_stream_cache
                 FileUtils.rm pid_file if File.exists?(pid_file)
               end
             end
@@ -312,6 +359,7 @@ class Step
             set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
             set_info :time_elapsed, (time_elapsed = done_time - start_time)
             log :done, "#{Log.color :magenta, "Completed"} step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
+            Step.purge_stream_cache
             FileUtils.rm pid_file if File.exists?(pid_file)
           end
 
@@ -322,6 +370,7 @@ class Step
           @result ||= result
           self
         else
+          Step.purge_stream_cache
           @result = prepare_result result, @task.result_description
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.19.37
+  version: 5.20.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-13 00:00:00.000000000 Z
+date: 2016-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake