rbbt-util 5.19.36 → 5.19.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72a73b569032a1fce45e26a98b2e5e57c1abd83c
-  data.tar.gz: 1bb8e14d02dbc52654120389c6c1ff303ec25d05
+  metadata.gz: 137e4838f414d232714269ef64c04ba299fa3f62
+  data.tar.gz: 7cc30b69428f3f33461d639c6657d101fb09547f
 SHA512:
-  metadata.gz: 6126957c9cb71705cfa47105941752e32033b714e7972dac3b211179c0e23c3c7e7a7d693922f0c7bc141e9c4955b15936ebd1418daa80150fdab5ef950c07cb
-  data.tar.gz: 263a74be39a0ed6e0fee8d2e3c913f77767744fc54b4c0e7d5c469013ef1bd48cf134b5efbcb1c00aacd036a174a552ec2260a64f75156744f5bed1324a9089a
+  metadata.gz: 6f901c235ed7ef31b1693b1d99282ad86f7f8f37c1b18e9e62a4c67adafa29e10d5fc3b53e12adba5803ce4571958401bd78a65723e1a9e17736e52a7acf755a
+  data.tar.gz: 2aa36eb9d0a80795a230187894e1f05dd51b1125cf890cda3f740e8e66d1c0320f3f2637ac7d10e38b16698935cb313b3a5498283e7ccf26e1848df3c8620523

data/lib/rbbt/rest/client/adaptor.rb

@@ -61,6 +61,6 @@ class WorkflowRESTClient
     @asynchronous_exports = task_exports["asynchronous"].collect{|task| task.to_sym }
     @synchronous_exports = task_exports["synchronous"].collect{|task| task.to_sym }
     @exec_exports = task_exports["exec"].collect{|task| task.to_sym }
-    nil
+    @can_stream = task_exports["can_stream"]
   end
 end

data/lib/rbbt/rest/client/get.rb

@@ -57,11 +57,14 @@ class WorkflowRESTClient
     Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
     params = params.merge({ :_format => 'raw' })
     params = fix_params params
-    capture_exception do
+    res = capture_exception do
       Misc.insist(2, 0.5) do
-        RestClient.get(URI.encode(url), :params => params)
+        res = RestClient.get(URI.encode(url), :params => params)
+        raise TryAgain if res.code == 202
+        res
       end
     end
+    res
   end
  
   def self.get_json(url, params = {})
@@ -83,7 +86,6 @@ class WorkflowRESTClient
   end
 
   def self.post_jobname(url, params = {})
-    Log.stack caller
     Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
     params = params.merge({ :_format => 'jobname' })
     params = fix_params params
@@ -96,7 +98,7 @@ class WorkflowRESTClient
 
     name
   end
-  
+
   def self.post_json(url, params = {})
     if url =~ /_cache_type=:exec/
       JSON.parse(Open.open(url, :nocache => true))

data/lib/rbbt/rest/client/step.rb

@@ -1,7 +1,7 @@
 class WorkflowRESTClient
   class RemoteStep < Step
 
-    attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec
+    attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec, :stream_input
 
     def self.get_streams(inputs)
       new_inputs = {}
@@ -17,9 +17,10 @@ class WorkflowRESTClient
     end
 
 
-    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false)
+    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false, stream_input = nil)
       @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec = base_url, task, base_name, inputs, result_type, result_description, is_exec
       @mutex = Mutex.new
+      @stream_input = stream_input
       @inputs = RemoteStep.get_streams @inputs
     end
 
@@ -176,7 +177,34 @@ class WorkflowRESTClient
       load_res get
     end
     
+    def _stream_job(stream_input, cache_type = :exec)
+      require 'rbbt/util/misc/multipart_payload'
+      WorkflowRESTClient.capture_exception do
+        url = URI.encode(File.join(base_url, task.to_s))
+        Log.debug{ "RestClient stream: #{ url } #{stream_input} #{cache_type} - #{Misc.fingerprint inputs}" }
+        task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
+        res = RbbtMutiplartPayload.issue url, task_params, :mutations, nil, nil, true
+        type = res.gets
+        case type.strip
+        when "LOCATION"
+          url = res.gets
+          url.sub!(/\?.*/,'')
+          WorkflowRESTClient.get_raw(url)
+        when "STREAM"
+          res
+        when "BULK"
+          res.read
+        else
+          raise "What? " + type
+        end
+      end
+    end
+
     def _run_job(cache_type = :async)
+      #if cache_type == :stream and stream_input
+      if cache_type == :stream or cache_type == :exec and stream_input
+        return _stream_job(stream_input, cache_type) 
+      end
       WorkflowRESTClient.capture_exception do
         url = URI.encode(File.join(base_url, task.to_s))
         task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
@@ -205,10 +233,6 @@ class WorkflowRESTClient
           sin.close
           @done = true
         end
-        #nsout, nsin = Misc.pipe
-        #Misc.consume_stream(reader, true, nsin, true) do @done = true end
-        #iii :ret
-        #nsout
       end
     end
 

data/lib/rbbt/rest/client.rb

@@ -43,7 +43,9 @@ class WorkflowRESTClient
       end
     end
 
-    RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task))
+    
+    stream_input = @can_stream ? task_info(task)[:input_options].select{|k,o| o[:stream] }.collect{|k,o| k }.first : nil
+    RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task), stream_input)
   end
 
   def load_id(id)

data/lib/rbbt/tsv/accessor.rb

@@ -75,7 +75,8 @@ module TSV
 
   def setup_array(*args)
     res = NamedArray.setup(*args)
-    res.instance_variable_set(:@entity_templates, entity_templates)
+    return res if res.nil?
+    res.instance_variable_set(:@entity_templates, entity_templates) 
     res
   end
 

data/lib/rbbt/util/misc/inspect.rb

@@ -229,11 +229,15 @@ module Misc
           when (defined? Step and Step)
             "<Step #{obj.path}>"
           else
-            obj_ins = obj.inspect
-            if obj_ins =~ /:0x0/
-              obj_ins.gsub(/:0x[a-f0-9]+/,'')
+            if obj.respond_to? :filename
+              "<IO:" << obj.filename << ">"
             else
-              obj_ins
+              obj_ins = obj.inspect
+              if obj_ins =~ /:0x0/
+                obj_ins.gsub(/:0x[a-f0-9]+/,'')
+              else
+                obj_ins
+              end
             end
           end
 

data/lib/rbbt/util/misc/multipart_payload.rb

@@ -43,8 +43,11 @@ module RbbtMutiplartPayload
     header = input_header(name, filename)
     io.write "--" + BOUNDARY + EOL + header + EOL  + EOL
 
-    while c = content.read(1024)
-      io.write c
+    #while c = content.read(1024)
+    #  io.write c
+    #end
+    while line = content.gets
+      io.puts line
     end
     content.close
   end
@@ -56,9 +59,10 @@ module RbbtMutiplartPayload
   end
 
   def self.post_data_stream(inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil)
-    sout, sin = Misc.pipe
+    #sout, sin = Misc.pipe
 
-    Thread.new do
+    #Thread.new do
+    Misc.open_pipe do |sin|
       inputs.each do |input,content|
         input = input.to_s
         next if stream_input and input == stream_input.to_s
@@ -82,12 +86,11 @@ module RbbtMutiplartPayload
 
       RbbtMutiplartPayload.add_stream(sin, stream_input.to_s, stream_io, stream_filename) if stream_input
       RbbtMutiplartPayload.close_stream(sin)
+      sin.close unless sin.closed?
     end
-
-    sout
   end
 
-  def self.issue(url, inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil)
+  def self.issue(url, inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil, report_type = false)
 
     uri = URI(url)
     req = Net::HTTP::Post.new(uri.path)
@@ -100,7 +103,7 @@ module RbbtMutiplartPayload
                           when String
                             inputs[stream_input]
                           when File
-                            inputs[stream_input].filename
+                            inputs[stream_input].path
                           else
                             'file'
                           end
@@ -121,12 +124,15 @@ module RbbtMutiplartPayload
       Net::HTTP.start(uri.hostname, uri.port) do |http|
         http.request(req) do |res|
           if Net::HTTPRedirection === res
+            sin.puts "LOCATION" if report_type
             sin.write res["location"]
           elsif stream_input
+            sin.puts "STREAM" if report_type
             res.read_body do |c|
               sin.write c
             end
           else
+            sin.puts "BULK" if report_type
             sin.write res.body
           end
           sin.close

data/lib/rbbt/util/misc/pipes.rb

@@ -560,4 +560,13 @@ module Misc
   end
 
 
+  def self.add_stream_filename(io, filename)
+    if ! io.respond_to? :filename
+      class << io
+        attr_accessor :filename
+      end
+      io.filename = filename
+    end
+  end
+
 end

data/lib/rbbt/workflow/step/run.rb

@@ -172,11 +172,13 @@ class Step
         dependency.grace
 
         if dependency.aborted?
+          Log.warn "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} aborted (clean and retry) -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
           dependency.clean
           raise TryAgain
         end
 
         if dependency.error?
+          Log.error "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} error -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
           raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
         end
 
@@ -185,45 +187,14 @@ class Step
           next
         end
 
-        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
         begin
+          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
           dependency.join
           raise TryAgain unless dependency.done?
+          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
         rescue Aborted
           raise TryAgain
         end
-        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
-
-        #if not dependency.done?
-        #  if dependency.started?
-        #    dependency.grace
-        #    if dependency.error?
-        #      raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
-        #    elsif dependency.streaming?
-        #      Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} streaming -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
-        #    else
-        #      Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
-        #      begin
-        #        dependency.join unless dependency.streaming?
-        #      rescue Aborted
-        #        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} #{Log.color :red, "aborted"} -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
-        #        if dependency.aborted?
-        #          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} #{Log.color :red, "aborted cleaning"} -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}" 
-        #          dependency.clean
-        #          raise TryAgain
-        #        end
-        #      end
-        #      Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
-        #    end
-        #  else
-        #    Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} starting -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
-        #    dependency.run(:stream)
-        #    dependency.grace
-        #    dependency.join unless dependency.streaming?
-        #  end
-        #else
-        #  Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} done -- #{Log.color :blue, dependency.short_path}"
-        #end
 
       rescue TryAgain
         retry

data/share/rbbt_commands/app/start

@@ -58,7 +58,7 @@ Misc.in_dir(app_dir) do
   ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
 
   if options[:stream]
-    raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma|unico/
+    raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma/
     ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.19.36
+  version: 5.19.37
 platform: ruby
 authors:
 - Miguel Vazquez