rbbt-util 5.20.5 → 5.20.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 732f48d509a3d1292ceecf5001206984b5681829
-  data.tar.gz: 54bad659a94eefc173f4c6d306f4bd78aca37ea9
+  metadata.gz: 5413d3d8418a3823e584e819c00d387d4f1cf756
+  data.tar.gz: 239e68cae4bb62da2e5db9a50e11e9e5fb916e2a
 SHA512:
-  metadata.gz: b8ec9e618e757d8d308e2218c8dab1399bbe2dff8574d48063f07ce3391882b1fa95d78bbf84734b4e29122bda7a1868c650506e2ffe6c02e2e75e89de8254f2
-  data.tar.gz: 06c02c802611a555db90e5bf6402aaec68c3a49b85aa8de3032b30b6e1cb9cea8936b0ba033a52c55e40784b6c2aabfcb487a14908f628519f86c69019458e02
+  metadata.gz: d646cab93661b4328ba21da388a7726064dc6ef1505e1bb8536f4d15ea53a9c9a307eb38454bca061ee3680e7b1b2b2c418e07d83d1c72fb813075cab9944176
+  data.tar.gz: f63f8e7912aa7dd295c44cdc344f780198baf553103cc7855beed580fba2cde5a0e0755138b59aac661fb1e1dd2212042bb949dc73d92bed5989922b4093a7c4

data/lib/rbbt/rest/client/get.rb

@@ -118,4 +118,34 @@ class WorkflowRESTClient
     end
   end
 
+  def self.stream_job(task_url, task_params, stream_input, cache_type = :exec)
+    require 'rbbt/util/misc/multipart_payload'
+    WorkflowRESTClient.capture_exception do
+      Log.debug{ "RestClient stream #{Process.pid}: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint task_params}" }
+      res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
+      type = res.gets
+      case type.strip
+      when "LOCATION"
+        @url = res.gets
+        @url.sub!(/\?.*/,'')
+        WorkflowRESTClient.get_raw(@url)
+      when /STREAM: (.*)/
+        @url = $1.strip
+        res.callback = Proc.new do
+          Log.medium "Done streaming result from #{@url}"
+          @done = true
+        end
+        res
+      when "BULK"
+        begin
+          res.read
+        ensure
+          @done = true
+        end
+      else
+        raise "What? " + type
+      end
+    end
+  end
+
 end

data/lib/rbbt/rest/client/step.rb

@@ -3,11 +3,39 @@ class WorkflowRESTClient
 
     attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec, :stream_input
 
-    def self.get_streams(inputs)
+    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false, stream_input = nil)
+      @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec, @stream_input = base_url, task, base_name, inputs, result_type, result_description, is_exec, stream_input
+      @mutex = Mutex.new
+    end
+
+    def run(noload = false)
+      @mutex.synchronize do
+        @result ||= begin
+                      if @is_exec
+                        exec(noload)
+                      elsif noload == :stream
+                        _run_job(:stream)
+                      else
+                        init_job 
+                        self.load
+                      end
+                    ensure
+                      @started = true
+                    end
+      end
+
+      return @result if noload == :stream
+      noload ? path + '?_format=raw' : @result
+    end
+
+
+    def self.get_streams(inputs, stream_input = nil)
       new_inputs = {}
       inputs.each do |k,v|
+        stream = stream_input.to_s == k.to_s
         if Step === v 
-          new_inputs[k] = TSV.get_stream v
+          v.run(true) and v.grace unless v.done? or v.streaming?
+          new_inputs[k] = (stream and (v.done? or v.streaming?)) ? TSV.get_stream(v) : v.path
         else
           new_inputs[k] = v
         end
@@ -16,26 +44,14 @@ class WorkflowRESTClient
     end
 
     def get_streams
-      @inputs = WorkflowRESTClient::RemoteStep.get_streams @inputs
-    end
-
-
-    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false, stream_input = nil)
-      @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec = base_url, task, base_name, inputs, result_type, result_description, is_exec
-      @mutex = Mutex.new
-      @stream_input = stream_input
-      #@inputs = RemoteStep.get_streams @inputs
+      return if @inputs_done
+      @inputs = WorkflowRESTClient::RemoteStep.get_streams @inputs, @stream_input
+      @inputs_done = true
+      @inputs
     end
 
-    def dup_inputs
-      return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
-      Log.low "Dupping inputs for remote #{path}"
-      new_inputs = {}
-      @inputs.each do |name,input|
-        new_inputs[name] = Step.dup_stream input
-      end
-      @inputs = RemoteStep.get_streams new_inputs
-      @dupped = true
+    def abort
+      #WorkflowRESTClient.get_json(File.join(@url, '?_update=abort')) if @url
     end
 
     def name
@@ -63,8 +79,12 @@ class WorkflowRESTClient
 
     def status
       return nil unless url or started?
+      return :done if @done
+      return :streaming if @streaming
       begin
-        info[:status]
+        status = info[:status]
+        @done = true if status and status.to_sym == :done
+        status
       ensure
         @info = nil
       end
@@ -103,9 +123,9 @@ class WorkflowRESTClient
     #{{{ MANAGEMENT
 
     def init_job(cache_type = nil, other_params = {})
-      Log.stack caller
       cache_type = :asynchronous if cache_type.nil? and not @is_exec
       cache_type = :exec if cache_type.nil?
+      get_streams
       @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
         WorkflowRESTClient.post_jobname(File.join(base_url, task.to_s), inputs.merge(other_params).merge(:jobname => @name||@base_name, :_cache_type => cache_type))
       end
@@ -130,26 +150,6 @@ class WorkflowRESTClient
       end
     end
 
-    def run(noload = false)
-      @mutex.synchronize do
-        @result ||= begin
-                      if @is_exec
-                        exec(noload)
-                      elsif noload == :stream
-                        _run_job(:stream)
-                      else
-                        init_job 
-                        self.load
-                      end
-                    ensure
-                      @started = true
-                    end
-      end
-
-      return @result if noload == :stream
-      noload ? path + '?_format=raw' : @result
-    end
-
     def exec(noload = false)
       @result ||= begin
                      if noload == :stream
@@ -163,6 +163,11 @@ class WorkflowRESTClient
     end
 
     def join
+      if IO === @result
+        res = @result
+        @result = nil
+        Misc.consume_stream(res, true) 
+      end
       sleep 0.2 unless self.done?
       sleep 1 unless self.done?
       sleep 3 while not self.done?
@@ -216,43 +221,26 @@ class WorkflowRESTClient
       load_res get
     end
     
-    def _stream_job(stream_input, cache_type = :exec)
-      require 'rbbt/util/misc/multipart_payload'
-      WorkflowRESTClient.capture_exception do
+    def _run_job(cache_type = :async)
+      get_streams
+      if cache_type == :stream or cache_type == :exec and stream_input
         task_url = URI.encode(File.join(base_url, task.to_s))
-        Log.debug{ "RestClient stream: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint inputs}" }
         task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
-        res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
-        type = res.gets
-        case type.strip
-        when "LOCATION"
-          @url = res.gets
-          @url.sub!(/\?.*/,'')
-          WorkflowRESTClient.get_raw(@url)
-        when /STREAM: (.*)/
-          @url = $1.strip
-          ConcurrentStream.setup(res)
-          res.callback = Proc.new do
-            @done = true
-          end
-          res
-        when "BULK"
-          begin
-            res.read
-          ensure
+        @streaming = true
+        io =  WorkflowRESTClient.stream_job(task_url, task_params, stream_input, cache_type) 
+        if IO === io
+          ConcurrentStream.setup(io)
+          io.add_callback do
             @done = true
+            @streaming = false
           end
         else
-          raise "What? " + type
+          @done = true
+          @streaming = false
         end
+        return io
       end
-    end
 
-    def _run_job(cache_type = :async)
-      #if cache_type == :stream and stream_input
-      if cache_type == :stream or cache_type == :exec and stream_input
-        return _stream_job(stream_input, cache_type) 
-      end
       WorkflowRESTClient.capture_exception do
         @url = URI.encode(File.join(base_url, task.to_s))
         task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
@@ -262,7 +250,7 @@ class WorkflowRESTClient
           sin.write c
         end
 
-        Thread.new do
+        post_thread = Thread.new do
           bl = lambda do |rok|
             rok.read_body do |c,_a, _b|
               sin.write c
@@ -270,17 +258,20 @@ class WorkflowRESTClient
             sin.close
           end
 
+          Log.debug{ "RestClient execute: #{ url } - #{Misc.fingerprint task_params}" }
           RestClient::Request.execute(:method => :post, :url => url, :payload => task_params, :block_response => bl)
         end
 
         reader = Zlib::GzipReader.new(sout)
-        Misc.open_pipe do |sin|
-          while c = reader.read(1015)
+        res_io = Misc.open_pipe do |sin|
+          while c = reader.read(Misc::BLOCK_SIZE)
             sin.write c
           end
           sin.close
           @done = true
         end
+
+        ConcurrentStream.setup(res_io, :threads => [post_thread])
       end
     end
 
@@ -305,6 +296,7 @@ class WorkflowRESTClient
     end
 
     def recursive_clean
+      return
       begin
         params = {:_update => :recursive_clean}
         init_job(nil, params)
@@ -317,6 +309,7 @@ class WorkflowRESTClient
     end
 
     def clean
+      return
       begin
         params = {:_update => :clean}
         init_job(nil, params)

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -54,8 +54,8 @@ module TSV
 
   def self.stream_name(obj)
     return "nil" if obj.nil?
-    filename_obj   = obj.respond_to?(:filename) ? obj.filename : nil
-    filename_obj ||= obj.respond_to?(:path) ? obj.path : nil
+    #filename_obj   = obj.respond_to?(:filename) ? obj.filename : nil
+    #filename_obj ||= obj.respond_to?(:path) ? obj.path : nil
     stream_obj = obj_stream(obj) || obj
     obj.class.to_s << "-" << Misc.fingerprint(stream_obj)
   end
@@ -223,7 +223,8 @@ module TSV
       options[:type] = :single
     end
 
-    Log.medium{"Traversing #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"}
+    name = stream_name(obj)
+    Log.medium{"Traversing #{name} #{Log.color :green, "->"} #{stream_name(options[:into])}"}
     begin
       case obj
       when TSV

data/lib/rbbt/util/log.rb

@@ -156,9 +156,7 @@ module Log
 
     sev_str = severity.to_s
 
-    #prefix = time << "[" << color(severity) << sev_str << color(0)<<"]"
     prefix = time << color(severity) << "["  << sev_str << "]" << color(0)
-    #prefix = color(severity) << time << color(0) << "[" << sev_str  << "]"
     message = "" << highlight << message << color(0) if severity >= INFO
     str = prefix << " " << message
 

data/lib/rbbt/util/misc/development.rb

@@ -318,6 +318,42 @@ module Misc
     end
   end
 
+  def self.bootstrap_in_threads(elems, num = :current, options = {}, &block)
+    IndiferentHash.setup options
+    num = :current if num.nil?
+    threads = case num
+           when :current
+            10
+           when String
+             num.to_i
+           when Integer
+             if num < 100
+               num
+             else
+               32000 / num
+             end
+           else
+             raise "Parameter 'num' not understood: #{Misc.fingerprint num}"
+           end
+
+
+    options = Misc.add_defaults options, :respawn => true, :threads => threads
+    options = Misc.add_defaults options, :bar => "Bootstrap in #{ options[:threads] } threads: #{ Misc.fingerprint Annotated.purge(elems) }"
+
+    index = (0..elems.length-1).to_a.collect{|v| v.to_s }
+    TSV.traverse index, options do |pos|
+      elem = elems[pos.to_i]
+      elems.annotate elem if elems.respond_to? :annotate
+      begin
+        res = yield elem
+      rescue Interrupt
+        Log.warn "Process #{Process.pid} was aborted"
+        raise $!
+      end
+      res = nil unless options[:into]
+      res
+    end
+  end
   def self.memory_use(pid=nil)
     `ps -o rss -p #{pid || $$}`.strip.split.last.to_i
   end

data/lib/rbbt/util/misc/inspect.rb

@@ -229,7 +229,7 @@ module Misc
           when (defined? Step and Step)
             "<Step #{obj.path}>"
           else
-            if obj.respond_to? :filename
+            if obj.respond_to? :filename and obj.filename
               "<IO:" << obj.filename << ">"
             else
               obj_ins = obj.inspect

data/lib/rbbt/util/misc/multipart_payload.rb

@@ -54,13 +54,12 @@ module RbbtMutiplartPayload
         io.write c
       end
     rescue EOFError
+      io.write "\r\n"
     end
-    content.close
   end
 
   def self.close_stream(io)
     io.write "--" + BOUNDARY + "--" + EOL + EOL
-    io.close
   end
 
   def self.post_data_stream(inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil)
@@ -88,16 +87,12 @@ module RbbtMutiplartPayload
 
       RbbtMutiplartPayload.add_stream(sin, stream_input.to_s, stream_io, stream_filename) if stream_input
       RbbtMutiplartPayload.close_stream(sin)
-
-      sin.close unless sin.closed?
     end
   end
 
   def self.issue(url, inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil, report_type = false)
 
     uri = URI(url)
-    req = Net::HTTP::Post.new(uri.path)
-
     IndiferentHash.setup(inputs)
 
     if stream_input
@@ -112,37 +107,47 @@ module RbbtMutiplartPayload
                           end
     end
 
-    sout = RbbtMutiplartPayload.post_data_stream inputs, stream_input, stream_io, stream_filename
+    post_data_stream = RbbtMutiplartPayload.post_data_stream inputs, stream_input, stream_io, stream_filename
 
+    jobname = inputs["jobname"] 
+
+    req = Net::HTTP::Post.new(uri.path)
     if stream_input
       req.content_type = "multipart/form-data; boundary=" + RbbtMutiplartPayload::BOUNDARY + '; stream=' + stream_input.to_s
-      req.body_stream = sout
-      req.add_field "Transfer-Encoding", "chunked"
+      req.body_stream = post_data_stream
     else
       req.content_type = "multipart/form-data; boundary=" + RbbtMutiplartPayload::BOUNDARY
-      req.body = sout.read
+      req.body = post_data_stream.read
     end
 
+    req.add_field "Transfer-Encoding", 'chunked'
+    req.add_field "RBBT_ID", (jobname || "No name")
     Misc.open_pipe do |sin|
       Net::HTTP.start(uri.hostname, uri.port) do |http|
         http.request(req) do |res|
-          url_path = res["RBBT-STREAMING-JOB-URL"]
-          if Net::HTTPRedirection === res
-            sin.puts "LOCATION" if report_type
-            sin.write res["location"]
-          elsif stream_input and url_path
-            url = URI::HTTP.build(:host => uri.hostname, :post => uri.port, :path => url_path)
-            sin.puts "STREAM: #{url.to_s}" if report_type
-            res.read_body do |c|
-              sin.write c
+          if Net::HTTPSuccess === res
+            url_path = res["RBBT-STREAMING-JOB-URL"]
+            if Net::HTTPRedirection === res
+              Log.medium "Response recieved REDIRECT: #{ url_path }"
+              sin.puts "LOCATION" if report_type
+              sin.write res["location"]
+            elsif stream_input and url_path
+              Log.medium "Response recieved STREAM: #{ url_path }"
+              url = URI::HTTP.build(:host => uri.hostname, :post => uri.port, :path => url_path)
+              sin.puts "STREAM: #{url.to_s}" if report_type
+              Log.medium "Read body: #{ url_path }"
+              res.read_body(sin)
+              Log.medium "Read body DONE: #{ url_path }"
+            else
+              Log.medium "Response recieved BULK: #{ url_path }"
+              sin.puts "BULK" if report_type
+              sin.write res.body
             end
           else
-            sin.puts "BULK" if report_type
-            sin.write res.body
+            raise "Error: #{res.code}"
           end
         end
       end
     end
   end
-
 end

data/lib/rbbt/util/misc/pipes.rb

@@ -54,27 +54,32 @@ module Misc
     sout, sin = Misc.pipe
 
     if do_fork
+
       parent_pid = Process.pid
       pid = Process.fork {
         purge_pipes(sin)
         sout.close
         begin
+
           yield sin
           sin.close if close and not sin.closed? 
-        rescue
+
+        rescue Exception
           Log.exception $!
           Process.kill :INT, parent_pid
           Kernel.exit! -1
         end
         Kernel.exit! 0
       }
-      sin.close 
+
       ConcurrentStream.setup sout, :pids => [pid]
     else
       thread = Thread.new(Thread.current) do |parent|
         begin
+          
           yield sin
           sin.close if close and not sin.closed?
+
         rescue Aborted
           Log.medium "Aborted open_pipe: #{$!.message}"
         rescue Exception
@@ -83,8 +88,10 @@ module Misc
           raise $!
         end
       end
+
       ConcurrentStream.setup sout, :threads => [thread]
     end
+
     sout
   end
 
@@ -147,6 +154,8 @@ module Misc
       out_pipes << sout
     end
 
+    filename = stream.filename if stream.respond_to? :filename
+
     splitter_thread = Thread.new(Thread.current) do |parent|
       begin
 
@@ -156,9 +165,9 @@ module Misc
 
             in_pipes.each_with_index do |sin,i|
               begin 
-                sin.write block; 
+                sin.write block
               rescue IOError
-                Log.medium("Tee stream #{i} #{Misc.fingerprint stream} IOError: #{$!.message}");
+                Log.error("Tee stream #{i} #{Misc.fingerprint stream} IOError: #{$!.message}");
                 skip[i] = true
               end unless skip[i] 
             end
@@ -169,6 +178,7 @@ module Misc
         in_pipes.each do |sin|
           sin.close unless sin.closed?
         end
+
         stream.join if stream.respond_to? :join
       rescue Aborted, Interrupt
         stream.abort if stream.respond_to? :abort
@@ -188,7 +198,7 @@ module Misc
     end
 
     out_pipes.each do |sout|
-      ConcurrentStream.setup sout, :threads => splitter_thread
+      ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename
     end
 
     out_pipes
@@ -205,13 +215,15 @@ module Misc
       stream.clear
     end
     tee1, *rest = Misc.tee_stream stream_dup, num + 1
-    stream.reopen(tee1)
     rest
   end
 
+  def Misc.dup_stream (stream)
+    dup_stream_multiple(stream, 1).first
+  end
+
   class << self
     alias tee_stream tee_stream_thread_multiple
-    alias dup_stream dup_stream_multiple
   end
 
   def self.read_full_stream(io)
@@ -254,8 +266,8 @@ module Misc
         io.sync = true
 
         begin
-          while block = io.readpartial(BLOCK_SIZE)
-            into << block if into
+          while c = io.readpartial(BLOCK_SIZE)
+            into << c if into
           end
         rescue EOFError
         end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -79,7 +79,7 @@ class Step
 
     job.clean if job.error? or job.aborted? or (job.started? and not job.running? and not job.error?)
 
-    job.dup_inputs
+    job.dup_inputs unless job.done? or job.started?
 
     raise DependencyError, job if job.error?
   end
@@ -157,64 +157,18 @@ class Step
     end
   end
 
-  #def consolidate_dependencies(path_deps = {})
-  #  return false if @consolidated  or dependencies.nil? or dependencies.empty?
-  #  consolidated_deps = dependencies.collect do |dep|
-  #    dep.consolidate_dependencies(path_deps)
-  #    path = dep.path
-  #    path_deps[path] ||= dep
-  #  end
-  #  dependencies.replace consolidated_deps
-  #  @consolidated = true
-  #end
-
-  #def prepare_dependencies
-  #  dep_step = {}
-
-  #  all_deps = rec_dependencies + [self]
-
-  #  seen_paths = Set.new
-  #  all_deps.uniq.each do |step|
-  #    next if seen_paths.include? step.path
-  #    Step.prepare_for_execution(step)
-  #    seen_paths << step.path
-  #    step.dependencies.each do |step_dep|
-  #      dep_step[step_dep.path] ||= []
-  #      dep_step[step_dep.path] << step_dep
-  #    end
-  #  end
-
-  #  seen_paths = Set.new
-  #  rec_dependencies.uniq.each do |step|
-  #    next if seen_paths.include? step.path
-  # = {}    seen_paths << step.path
-  #    execute_dependency(step)
-  #    if step.streaming? and step.result
-  #      if dep_step[step.path] and dep_step[step.path].length > 1
-  #        stream = step.result
-  #        other_steps = dep_step[step.path] - [step]
-  #        copies = Misc.dup_stream_multiple(stream, other_steps.length)
-  #        other_steps.zip(copies).each do |other,dupped_stream|
-  #          other.instance_variable_set(:@result, dupped_stream)
-  #        end
-  #      end
-  #    end
-  #  end
-
-  #  Step.purge_stream_cache
-  #end
-
   def execute_and_dup(step, dep_step, log = true)
     dup = step.result.nil?
     execute_dependency(step, log)
     if dup and step.streaming? and not step.result.nil?
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
-        other_steps = dep_step[step.path] - [step]
-        copies = Misc.dup_stream_multiple(stream, other_steps.length)
+        other_steps = dep_step[step.path]
+        return unless other_steps.length > 1
+        copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
         log_dependency_exec(step, "duplicating #{copies.length}") 
         other_steps.zip(copies).each do |other,dupped_stream|
-          other.instance_variable_set(:@result, dupped_stream)
+          other.instance_variable_set("@result", dupped_stream)
         end
       end
     end
@@ -233,10 +187,10 @@ class Step
       end
     when :bootstrap
       cpus = rest.nil? ? nil : rest.first 
-      cpus = 30 if cpus.nil?
+      cpus = 5 if cpus.nil?
       cpus = list.length / 2 if cpus > list.length / 2
 
-      Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :_respawn => :always) do |dep|
+      Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :respawn => :always) do |dep|
         dep.produce
         nil
       end
@@ -280,7 +234,7 @@ class Step
         dep_step[step_dep.path] << step_dep
       end
     end
-
+    
     self.dup_inputs
 
     required_dep_paths = []
@@ -325,18 +279,18 @@ class Step
       execute_and_dup(step, dep_step, false)
     end
 
-    Log.medium "Computing pre dependencies: #{Misc.fingerprint(compute_pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
+    Log.medium "Computing pre dependencies: #{Misc.fingerprint(compute_pre_deps)} - #{Log.color :blue, self.path}" if compute_pre_deps.any?
     compute_pre_deps.each do |type,list|
       run_compute_dependencies(type, list, dep_step)
     end
 
-    Log.medium "Processing last dependencies: #{Misc.fingerprint(last_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
+    Log.medium "Processing last dependencies: #{Misc.fingerprint(last_deps)} - #{Log.color :blue, self.path}" if last_deps.any?
     last_deps.each do |step|
       next if compute_deps.include? step
       execute_and_dup(step, dep_step)
     end
 
-    Log.medium "Computing last dependencies: #{Misc.fingerprint(compute_last_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
+    Log.medium "Computing last dependencies: #{Misc.fingerprint(compute_last_deps)} - #{Log.color :blue, self.path}" if compute_last_deps.any?
     compute_last_deps.each do |type,list|
       run_compute_dependencies(type, list, dep_step)
     end
@@ -350,108 +304,4 @@ class Step
     kill_children
   end
 
-  #def run_dependencies
-  #  @seen ||= []
-  #  seen_paths ||= Set.new
-  #  
-  #  consolidate_dependencies
-  #  dependencies.uniq.each do |dependency| 
-  #    dependency_path = dependency.path
-  #    next if seen_paths.include? dependency_path
-  #    @seen.concat dependency.rec_dependencies
-  #    seen_paths.union(dependency.rec_dependencies.collect{|d| d.path})
-  #    @seen << dependency
-  #    seen_paths << dependency_path
-  #  end
-
-  #  @seen.uniq!
-  #  @seen.delete self
-
-  #  return if @seen.empty?
-
-  #  log :dependencies, "#{Log.color :magenta, "Dependencies"} for step #{Log.color :yellow, task.name.to_s || ""}"
-
-  #  @seen.each do |dependency| 
-  #    Step.prepare_for_execution(dependency)
-  #  end
-
-  #  pre_deps = []
-  #  compute_pre_deps = {}
-  #  last_deps = []
-  #  compute_last_deps = {}
-  #  @seen.each do |dependency| 
-  #    if dependencies.include?(dependency) and dependency.inputs.flatten.select{|i| Step === i}.any?
-  #      if ComputeDependency === dependency
-  #        compute_last_deps[dependency.compute] ||= []
-  #        compute_last_deps[dependency.compute] << dependency
-  #      else
-  #        last_deps << dependency
-  #      end
-  #    else
-  #      if ComputeDependency === dependency
-  #        compute_pre_deps[dependency.compute] ||= []
-  #        compute_pre_deps[dependency.compute] << dependency
-  #      else
-  #        pre_deps << dependency if dependencies.include?(dependency)
-  #      end
-  #    end
-  #  end
-
-  #  pre_deps.each do |dependency|
-  #    dependency.dup_inputs
-  #    execute_dependency(dependency)
-  #  end
-
-  #  compute_pre_deps.each do |type,list|
-  #    if Array === type
-  #      type, *rest = type
-  #    end
-
-  #    case type
-  #    when :bootstrap
-  #      cpus = rest.nil? ? nil : rest.first 
-  #      cpus = 10 if cpus.nil?
-
-  #      list.each do |dependency|
-  #        dependency.dup_inputs
-  #      end
-
-  #      Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :_respawn => :always) do |dep|
-  #        dep.produce
-  #        nil
-  #      end
-  #    else
-  #      list.each do |dependency|
-  #        dependency.dup_inputs
-  #        execute_dependency(dependency)
-  #      end
-  #    end
-  #  end
-
-  #  last_deps.each do |dependency|
-  #    dependency.dup_inputs
-  #  end
-
-  #  last_deps.each do |dependency|
-  #    execute_dependency(dependency)
-  #  end
-
-  #  compute_last_deps.each do |type,list|
-  #    case type
-  #    when :_bootstrap
-  #      list.each do |dependency|
-  #        dependency.dup_inputs
-  #      end
-  #      Misc.bootstrap(list, 3, :bar => "Boostrapping dependencies for #{path}", :respawn => :always) do |dependency|
-  #        dependency.produce
-  #        nil
-  #      end
-  #    else
-  #      list.each do |dependency|
-  #        dependency.dup_inputs
-  #        execute_dependency(dependency)
-  #      end
-  #    end
-  #  end
-  #end
 end

data/lib/rbbt/workflow/step/run.rb

@@ -6,7 +6,7 @@ class Step
 
   def get_stream
     @mutex.synchronize do
-      Log.low "Getting stream from #{path} #{!@saved_stream} [#{object_id}]"
+      Log.low "Getting stream from #{path} #{!@saved_stream} [#{object_id}-#{Misc.fingerprint(@result)}]"
       begin
         return nil if @saved_stream
         if IO === @result 
@@ -113,7 +113,6 @@ class Step
             :clean_name => clean_name,
           })
 
-          dup_inputs
           begin
             run_dependencies
           rescue Exception

data/share/rbbt_commands/workflow/info

@@ -125,8 +125,8 @@ if recursive
 
   while deps.any? do
     dep = deps.shift
-    inputs = inputs.merge(dep.info[:inputs])
-    deps.concat dep.info[:dependencies].collect{|v| get_step v.last }
+    inputs = inputs.merge(dep.info[:inputs] || {})
+    deps.concat (dep.info[:dependencies] || []).collect{|v| get_step v.last }
   end
 
   inputs.each do |input,value|

data/share/rbbt_commands/workflow/prov

@@ -40,8 +40,6 @@ def status_msg(status)
             :yellow
           when :done
             :green
-          else
-            nil
           end
   Log.color(color, status)
 end

data/share/rbbt_commands/workflow/server

@@ -64,7 +64,7 @@ TmpFile.with_file do |app_dir|
     ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
 
     if options[:stream]
-      raise "No streaming available for any server other than puma" unless options[:server] == 'puma'
+      raise "No streaming available for any server other than puma" unless true or options[:server].include? 'puma'
       ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
     end
 
@@ -81,9 +81,10 @@ TmpFile.with_file do |app_dir|
 
     case server
     when 'unicorn'
-      `unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:port] || "2887"}`
+      `env RBBT_LOG=#{Log.severity.to_s} unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:port] || "2887"}`
     when 'puma_alt'
-      `puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload`
+      #`puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload`
+      `env RBBT_LOG=#{Log.severity.to_s} puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 20 -t 10:160 --preload`
     else
       options[:config] = config_ru_file
       Rack::Server.start(options)

data/test/rbbt/util/misc/test_multipart_payload.rb

@@ -0,0 +1,202 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/misc/multipart_payload'
+
+class TestMultipartPayload < Test::Unit::TestCase
+
+  URL='http://localhost:2887/Echo'
+  def _test_post_data_stream
+    content =<<-EOF
+Line 1
+Line 2
+Line 3
+END
+    EOF
+    mutipart =<<-EOF
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="input1"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Input1<>
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="input2"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Input2<>
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="stream_input"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Line 1
+Line 2
+Line 3
+END
+<>
+--Rbbt_Param_Stream--<>
+<>
+    EOF
+    TmpFile.with_file(content) do |tmpfile|
+      inputs = {}
+      stream_input = :stream_input
+      inputs[:input1] = "Input1"
+      inputs[:input2] = "Input2"
+      inputs[:stream_input] = Open.open(tmpfile)
+
+      post_data_stream = RbbtMutiplartPayload.post_data_stream(inputs, stream_input, inputs[stream_input])
+      assert_equal mutipart, post_data_stream.read.gsub(/\r\n/,"<>\n")
+    end
+  end
+
+  def _test_issue
+    content =<<-EOF
+Line 1
+Line 2
+Line 3
+END
+    EOF
+    mutipart =<<-EOF
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="input1"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Input1<>
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="input2"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Input2<>
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="stream_input"; filename="FILENAME"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Line 1
+Line 2
+Line 3
+END
+<>
+--Rbbt_Param_Stream--<>
+<>
+DONE_PARAM_STREAM
+    EOF
+    inputs = {}
+    #mutipart.gsub!('<>',"\r")
+    stream_input = :stream_input
+    inputs[:input1] = "Input1"
+    inputs[:input2] = "Input2"
+    num = 50
+    cpus = 1
+    TmpFile.with_file(content) do |tmpfile|
+      inputs  = inputs.dup
+      inputs[:stream_input] = File.open(tmpfile)
+
+     io = RbbtMutiplartPayload.issue(URL, inputs, stream_input)
+      assert_equal mutipart.sub("FILENAME", tmpfile).force_encoding("ASCII"), io.read.gsub(/\r\n/,"<>\n") 
+    end
+  end
+
+  def _test_issue_multiple
+    content =<<-EOF
+Line 1
+Line 2
+Line 3
+END
+    EOF
+    mutipart =<<-EOF
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="input1"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Input1<>
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="input2"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Input2<>
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="stream_input"; filename="FILENAME"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Line 1
+Line 2
+Line 3
+END
+<>
+--Rbbt_Param_Stream--<>
+<>
+DONE_PARAM_STREAM
+    EOF
+    inputs = {}
+    stream_input = :stream_input
+    inputs[:input1] = "Input1"
+    inputs[:input2] = "Input2"
+    num = 100
+    cpus = 10
+    Misc.bootstrap((0..num-1).to_a, cpus) do  |n|
+      TmpFile.with_file(content) do |tmpfile|
+        inputs  = inputs.dup
+        inputs[:stream_input] = File.open(tmpfile)
+
+        io = RbbtMutiplartPayload.issue(URL, inputs, stream_input)
+        str = io.read
+        assert_equal mutipart.sub("FILENAME", tmpfile).force_encoding("ASCII"), str.gsub(/\r\n/,"<>\n") 
+      end
+    end
+  end
+
+  def _test_raw
+    mutipart =<<-EOF
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="input1"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Input1<>
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="input2"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Input2<>
+--Rbbt_Param_Stream<>
+Content-Disposition: form-data; name="stream_input"; filename="FILENAME"<>
+Content-Transfer-Encoding: binary<>
+Content-Type: text/plain<>
+<>
+Line 1
+Line 2
+Line 3
+END
+<>
+--Rbbt_Param_Stream--<>
+<>
+    EOF
+    mutipart.gsub!('<>',"\r")
+    inputs = {}
+    stream_input = :stream_input
+    inputs[:input1] = "Input1"
+    inputs[:input2] = "Input2"
+    num = 50
+    cpus = 1
+    Log.severity = 0
+    Misc.bootstrap((0..num-1).to_a, cpus) do  |n|
+      puts mutipart
+      TmpFile.with_file(mutipart, false) do |tmpfile|
+        inputs  = inputs.dup
+        inputs[:stream_input] = File.open(tmpfile)
+
+        puts "wget '#{URL}' --post-file #{ tmpfile } -O -"
+        io = CMD.cmd("wget '#{URL}' --post-file #{ tmpfile } -O -", :pipe => true)
+        assert_equal mutipart, io.read
+      end
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.20.5
+  version: 5.20.6
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-01 00:00:00.000000000 Z
+date: 2016-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -432,6 +432,7 @@ files:
 - test/rbbt/util/log/test_progress.rb
 - test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_lock.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/simpleopt/test_parse.rb
@@ -491,6 +492,7 @@ test_files:
 - test/rbbt/util/test_log.rb
 - test/rbbt/util/test_open.rb
 - test/rbbt/util/misc/test_lock.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/test_concurrency.rb