rbbt-rest 1.8.9 → 1.8.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3efb056fd9e11310126a6cbec3dbdc9fea9ee093
-  data.tar.gz: e7fb374a165521846f5e10a2491f8dcceaaef8a5
+  metadata.gz: 1bbeba3d4ef508ff35ecfe2210b1d3b12d03a9d9
+  data.tar.gz: 809cd9b359f9658d5090a3a1ecdefbdcec706cde
 SHA512:
-  metadata.gz: a0e7197f529598c35dc7f2f0f33931d48cd208205f8b068df899007eaa905cf7550de75255d2e1c052f54b409acc483f5781ddf92a2ecf83003c0dd8217168d9
-  data.tar.gz: d73a4246be17d0b5056c80b686ac24776c51192fdfbc4bcaccbbd0360ef0647a374a4810435d104c2dfabf44879554c703f1f30ff0a3e7f549a84ced1eb2c2a1
+  metadata.gz: 4b72a79a9580da0803d84881d085861a47760be7d7cfce50a4e6e4322c1d3a3696a8cf0d4589fdc810fea0b76b6f4234ff41ed39f4b63c11e332876578e6bd81
+  data.tar.gz: 81f55b8dfeefd7294f41a9ea695e80c692fd35c6a2f9634f3f6bee3ccc84497a7ed9bd1ba313002242f94a6b574c9c58503a97b70d3460c7a1061e3bdacbd4ac

data/lib/rbbt/rest/workflow/stream_task.rb

@@ -48,7 +48,9 @@ class StreamWorkflowTask
   EOL = "\r\n"
   def read_chunk(sin, rest = "")
     parts = []
-    c = sin.gets
+    c = sin.read(1024)
+    #c = sin.gets
+    raise "Early connection close" if c.nil?
     c = rest << c unless rest.empty?
     c = c[2..-1] if c[0..1] == EOL
     index = c.index EOL
@@ -62,12 +64,11 @@ class StreamWorkflowTask
     [parts, rest]
   end
 
-
   def copy_chunked_stream(sin, sout, boundary)
 
     rest = ""
     done = false
-    content = true
+    content = false
 
     while not done
       parts, rest = read_chunk(sin, rest)
@@ -92,6 +93,38 @@ class StreamWorkflowTask
     sout.close
   end
 
+  def merge_chunks(sin, sout)
+
+    rest = ""
+    done = false
+    content = true
+
+    while not done
+      chunk_size_str = ""
+      while chunk_size_str.strip.empty?
+        chunk_size_str = sin.gets
+        raise "Error reading chunks" if chunk_size_str.nil?
+      end
+      size = chunk_size_str.strip.to_i(16)
+      break if size == 0
+      chunk = sin.read(size)
+      bound = sin.read(2)
+      raise "Size does not match" if false and chunk.length != size
+      sout.write chunk
+    end
+
+    sout.write rest
+    sout.close
+  end
+
+  def copy_until_boundary(sin, sout, boundary)
+    while line = sin.gets
+      break if line.include? boundary
+      sout.write line
+    end
+  end
+
+
   def call(env)
     if env["REQUEST_METHOD"] == "POST" and env["rack.hijack"] and env["CONTENT_TYPE"] and env["CONTENT_TYPE"].include? "Rbbt_Param_Stream" and env["HTTP_TRANSFER_ENCODING"] == 'chunked'
       Log.high "Hijacking post data"
@@ -99,8 +132,14 @@ class StreamWorkflowTask
       content_type = env["CONTENT_TYPE"]
       boundary = content_type.match(/boundary=([^\s;]*)/)[1]
       stream_input = content_type.match(/stream=([^\s;]*)/)[1]
-      post_stream = env["rack.hijack"].call
+      post_stream_chunked = env["rack.hijack"].call
+
+      job_url = nil
       begin
+        post_stream = Misc.open_pipe do |sin|
+          merge_chunks(post_stream_chunked, sin)
+        end
+
         inputs, filename = read_normal_inputs(post_stream, boundary, stream_input)
 
         input_stream_out, input_stream_in = Misc.pipe
@@ -110,6 +149,9 @@ class StreamWorkflowTask
         workflow, task = parse_uri(env)
         name = inputs.delete "jobname"
         job = workflow.job(task, name, inputs)
+        Log.high "Run job #{job.path} with inputs #{Misc.fingerprint(inputs)}"
+
+        job_url = File.join("/", workflow.to_s, task, job.name)
 
         task = task.to_sym
         execution_type = case
@@ -130,28 +172,40 @@ class StreamWorkflowTask
         when "exec", nil
           job.exec(:stream)
         when "sync", "synchronous", "async", "asynchronous"
-          job.run(:stream)
+          if job.done?
+            done_consumer = Thread.new do
+              while c = post_stream.read(1024)
+              end
+            end
+          else
+            job.run(:stream)
+          end
         else
           raise "Unknown execution_type: #{execution_type}"
         end
 
         t_in = Thread.new do 
           begin
-            copy_chunked_stream(post_stream, input_stream_in, boundary)
+            copy_until_boundary(post_stream, input_stream_in, boundary)
+            input_stream_in.close
+            post_stream.close_read
           rescue
             Log.exception $!
           end
-        end
+        end unless job.done?
 
         job_output = TSV.get_stream job
         t_out = Thread.new do 
           begin
-            post_stream.write "HTTP/1.1 200\r\n\r\n"
+            post_stream_chunked.write "HTTP/1.1 200\r\n"
+            post_stream_chunked.write "RBBT-STREAMING-JOB-URL: #{ job_url }\r\n"
+            post_stream_chunked.write "\r\n"
             while c = job_output.read(1024)
-              post_stream.write c
+              post_stream_chunked.write c
             end
             job_output.join if job_output.respond_to? :join
-            post_stream.close
+            post_stream_chunked.close_write
+            done_consumer.join if done_consumer
           rescue
             Log.exception $!
             job.abort
@@ -159,11 +213,100 @@ class StreamWorkflowTask
         end
 
       end
+
       [200, {}, nil]
     else
       Log.high "NOT Hijacking post data"
       @app.call(env)
     end
   end
+
+  #def call_old(env)
+  #  if env["REQUEST_METHOD"] == "POST" and env["rack.hijack"] and env["CONTENT_TYPE"] and env["CONTENT_TYPE"].include? "Rbbt_Param_Stream" and env["HTTP_TRANSFER_ENCODING"] == 'chunked'
+  #    Log.high "Hijacking post data"
+  #    inputs = {}
+  #    content_type = env["CONTENT_TYPE"]
+  #    boundary = content_type.match(/boundary=([^\s;]*)/)[1]
+  #    stream_input = content_type.match(/stream=([^\s;]*)/)[1]
+  #    post_stream = env["rack.hijack"].call
+  #    job_url = nil
+  #    begin
+  #      inputs, filename = read_normal_inputs(post_stream, boundary, stream_input)
+
+  #      input_stream_out, input_stream_in = Misc.pipe
+  #      Misc.add_stream_filename(input_stream_out, filename) if filename
+  #      inputs[stream_input] = input_stream_out
+
+  #      workflow, task = parse_uri(env)
+  #      name = inputs.delete "jobname"
+  #      job = workflow.job(task, name, inputs)
+  #      Log.high "Run job #{job.path} with inputs #{Misc.fingerprint(inputs)}"
+
+  #      job_url = File.join("/", workflow.to_s, task, job.name)
+
+  #      task = task.to_sym
+  #      execution_type = case
+  #                       when workflow.exec_exports.include?(task)
+  #                         "exec"
+  #                       when workflow.synchronous_exports.include?(task)
+  #                         "synchronous"
+  #                       when workflow.asynchronous_exports.include?(task)
+  #                         "asynchronous"
+  #                       else
+  #                         raise "No known export type for #{ workflow } #{ task }. Accesses denied"
+  #                       end 
+
+  #      execution_type = "exec" if inputs["_cache_type"] == 'exec'
+  #      Log.info "Streaming task with execution_type: #{ execution_type }"
+
+  #      case execution_type
+  #      when "exec", nil
+  #        job.exec(:stream)
+  #      when "sync", "synchronous", "async", "asynchronous"
+  #        if job.done?
+  #          done_consumer = Thread.new do
+  #            while c = post_stream.read(1024)
+  #            end
+  #          end
+  #        else
+  #          job.run(:stream)
+  #        end
+  #      else
+  #        raise "Unknown execution_type: #{execution_type}"
+  #      end
+
+  #      t_in = Thread.new do 
+  #        begin
+  #          copy_chunked_stream(post_stream, input_stream_in, boundary)
+  #        rescue
+  #          Log.exception $!
+  #        end
+  #      end unless job.done?
+
+  #      job_output = TSV.get_stream job
+  #      t_out = Thread.new do 
+  #        begin
+  #          post_stream.write "HTTP/1.1 200\r\n"
+  #          post_stream.write "RBBT-STREAMING-JOB-URL: #{ job_url }\r\n"
+  #          post_stream.write "\r\n"
+  #          while c = job_output.read(1024)
+  #            post_stream.write c
+  #          end
+  #          job_output.join if job_output.respond_to? :join
+  #          post_stream.close_write
+  #          done_consumer.join if done_consumer
+  #        rescue
+  #          Log.exception $!
+  #          job.abort
+  #        end
+  #      end
+
+  #    end
+  #    [200, {}, nil]
+  #  else
+  #    Log.high "NOT Hijacking post data"
+  #    @app.call(env)
+  #  end
+  #end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-rest
 version: !ruby/object:Gem::Version
-  version: 1.8.9
+  version: 1.8.10
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-13 00:00:00.000000000 Z
+date: 2016-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake