rbbt-rest 1.8.8 → 1.8.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cff29021a179c10bc6cc157b4b68c167f0d0cbce
-  data.tar.gz: bd58e81210656647a893ce423b3afacd1959f36a
+  metadata.gz: 3efb056fd9e11310126a6cbec3dbdc9fea9ee093
+  data.tar.gz: e7fb374a165521846f5e10a2491f8dcceaaef8a5
 SHA512:
-  metadata.gz: 3813b05fb8974c6b8ca3141bd0f2eb7108f8e979e68f932a888dc9ce8c9e505c978120037e11e2d9a4087bf2bcd1c21056aff5f95b941c9425b11c1de6d84d6a
-  data.tar.gz: eb7519b674f9c552f771d6fcd7f4ac75c8a244820a28820c80b5c42742d0b7fb1b0138727fd3a5a23439a88a9430884ac1c62f68279f363f77a277e7eb95814f
+  metadata.gz: a0e7197f529598c35dc7f2f0f33931d48cd208205f8b068df899007eaa905cf7550de75255d2e1c052f54b409acc483f5781ddf92a2ecf83003c0dd8217168d9
+  data.tar.gz: d73a4246be17d0b5056c80b686ac24776c51192fdfbc4bcaccbbd0360ef0647a374a4810435d104c2dfabf44879554c703f1f30ff0a3e7f549a84ced1eb2c2a1

data/lib/rbbt/rest/common/render.rb

@@ -109,9 +109,10 @@ module RbbtRESTHelpers
 
         pid = @step.child{
           begin
-            res = capture_haml &block
+            Log.low("Fragment started: #{ fragment_file } - #{Process.pid}")
+            res = capture_haml fragment_code, &block
             Open.write(fragment_file, res)
-            Log.low("Fragment done: #{ fragment_file }")
+            Log.low("Fragment done: #{ fragment_file } - #{Process.pid}")
           rescue Exception
             Open.write(fragment_file + '.error', [$!.class.to_s, $!.message] * ": ")
             Open.write(fragment_file + '.backtrace', $!.backtrace * "\n") if $!.backtrace
@@ -129,7 +130,11 @@ module RbbtRESTHelpers
         if link.nil?
           html_tag('a', "", :href => fragment_url, :class => 'fragment', "data-text" => text)
         else
-          if link =~ / href=/
+          if FalseClass === link
+            return fragment_code
+          elsif TrueClass === link
+            return fragment_url
+          elsif link =~ / href=/
             link = link.sub(/ href=('|")/," href='#{fragment_url}'")
           else
             link = link.sub(/<a /,"<a href='#{fragment_url}' ")

data/lib/rbbt/rest/file_server.rb

@@ -1,6 +1,5 @@
 require 'rbbt'
 require 'sinatra/base'
-require 'sinatra/streaming'
                                        
 
 module Sinatra
@@ -8,7 +7,6 @@ module Sinatra
     
     def self.registered(base)
       base.module_eval do
-        helpers Sinatra::Streaming
 
         get '/resource/:resource/get_directory' do
           directory, resource, create = params.values_at :directory, :resource, :create

data/lib/rbbt/rest/main.rb

@@ -8,8 +8,11 @@ require 'sinatra/base'
 require 'sinatra/cross_origin'
 require "sinatra/multi_route"
 require "sinatra/cookies"
+require 'sinatra/streaming'
 require 'json'
 
+require 'nakayoshi_fork'
+
 module Sinatra
   module RbbtRESTMain
 
@@ -46,6 +49,7 @@ module Sinatra
         helpers RbbtRESTHelpers
         register Sinatra::RbbtAuth
         helpers Sinatra::Cookies
+        helpers Sinatra::Streaming
 
         add_sass_load_path Rbbt.share.views.compass.find(:lib)
 
@@ -66,7 +70,7 @@ module Sinatra
         if production?
           set :haml, { :ugly => true }
           set :clean_trace, true
-          set :static_cache_control , [:public, {:max_age => 360000}]
+          set :static_cache_control , [:public, {:max_age => 360_000}]
         else
           set :static_cache_control , [:public, {:max_age => 0}]
         end
@@ -122,6 +126,7 @@ module Sinatra
             printer.print(:path => dir, :profile => 'profile')
             Log.info{ "Profile saved at #{ dir }: #{request.env["REQUEST_URI"]}" }
           end
+
           response.header["URI"] = request.env["REQUEST_URI"]
         end
 
@@ -239,14 +244,11 @@ module Sinatra
         end
 
         require 'rbbt/rest/monitor'
+
         register Sinatra::RbbtRESTMonitor
       end
     end
 
   end
-
-
-  #require 'rack/stream'
-  #use Rack::Stream
 end
 

data/lib/rbbt/rest/workflow.rb

@@ -46,7 +46,9 @@ module Sinatra
           workflow_render('tasks', workflow, nil, @clean_params)
         when :json
           content_type "application/json"
-          {:exec => workflow.exec_exports, :synchronous => workflow.synchronous_exports, :asynchronous => workflow.asynchronous_exports}.to_json
+
+          @can_stream = ENV["RBBT_WORKFLOW_TASK_STREAM"]  == 'true'
+          {:exec => workflow.exec_exports, :synchronous => workflow.synchronous_exports, :asynchronous => workflow.asynchronous_exports, :can_stream => !!@can_stream}.to_json
         else
           raise "Unsupported format specified: #{ format }"
         end
@@ -63,6 +65,7 @@ module Sinatra
           raise "Unsupported format specified: #{ format }"
         end
       end
+
       get "/#{workflow.to_s}/:task/info" do
         task     = consume_parameter(:task)
 
@@ -157,7 +160,7 @@ module Sinatra
                 raise RbbtRESTHelpers::Retry
               end
             else
-              halt 404, "Job not found"
+              halt 404, "Job not found: #{job.status}"
             end
           end
         rescue RbbtRESTHelpers::Retry
@@ -264,6 +267,15 @@ module Sinatra
     def self.registered(base)
       base.module_eval do
         helpers WorkflowRESTHelpers
+
+        if ENV["RBBT_WORKFLOW_TASK_STREAM"] == 'true'
+          require 'rbbt/rest/workflow/stream_task'
+          Log.info "Preparing server for streaming workflow tasks"
+          use StreamWorkflowTask if not @can_stream
+          @can_stream = true
+        else
+          @can_stream = false
+        end
       end
     end
   end

data/lib/rbbt/rest/workflow/jobs.rb

@@ -77,17 +77,20 @@ module WorkflowRESTHelpers
       halt 200, result.to_json
     when :tsv
       content_type "text/tab-separated-values"
-      halt 200, result.to_s
+      result = result.to_s unless String === result or result.respond_to? :gets
+      halt 200, result
     when :literal, :raw
       content_type "text/plain"
       case workflow.task_info(task)[:result_type]
       when :array
         halt 200, result * "\n"
       else
-        halt 200, result.to_s
+        result = result.to_s unless String === result or result.respond_to? :gets
+        halt 200, result
       end
     when :binary
       content_type "application/octet-stream"
+      result = result.to_s unless String === result or result.respond_to? :gets
       halt 200, result.to_s
     when :jobname
       halt 200, nil
@@ -146,6 +149,21 @@ module WorkflowRESTHelpers
     end
   end
 
+  def stream_job(job)
+    unless job.started? or job.done?
+      job.run(:stream) 
+      job.soft_grace
+    end
+
+    s = TSV.get_stream job
+
+    sout, sin = Misc.pipe
+
+    Misc.consume_stream(s, true, sin)
+
+    halt 200, sout
+  end
+
   def issue_job(workflow, task, jobname = nil, params = {})
     inputs = prepare_job_inputs(workflow, task, params)
     job = workflow.job(task, jobname, inputs)
@@ -157,7 +175,15 @@ module WorkflowRESTHelpers
 
     case execution_type.to_sym
     when :exec
-      show_exec_result job.exec, workflow, task
+      show_exec_result job.exec(:stream), workflow, task
+    when :stream
+      if update == :reload
+        job.abort
+        job.clean 
+      end
+
+      stream_job(job)
+
     when :synchronous, :sync
       if update == :reload
         job.abort
@@ -169,11 +195,10 @@ module WorkflowRESTHelpers
         job_url += "?_format=#{@format}" if @format
 
         if not job.started?
-          job.run true 
+          job.run
           job.join
         end
 
-        #halt 200, job.name if format == :jobname 
         if format == :jobname
           job.name
         else

data/lib/rbbt/rest/workflow/render.rb

@@ -22,7 +22,7 @@ module WorkflowRESTHelpers
     end
 
     if job 
-      locals[:job]      = job 
+      locals[:job] = job 
       @step = job
       cache_type = execution_type(workflow, task)
       cache_file = job.file('html')

data/lib/rbbt/rest/workflow/stream_task.rb

@@ -0,0 +1,169 @@
+class StreamWorkflowTask
+  def initialize(app)
+    @app = app
+  end
+
+  def read_normal_inputs(io, boundary, stream_input)
+    content = ""
+    content_start = false
+    variable = nil
+    filename = nil
+    inputs = {}
+
+    while line = io.gets
+      line.chomp!
+
+      if line == "--" + boundary
+        if variable
+          inputs[variable] = content
+        end
+        content_start = false
+        content = ""
+      elsif line =~ /^Content.* name="([^\s;"]*)"/
+        variable = $1
+        filename = line.match(/filename="([^"]+)"/)[1] if line =~ /filename/
+        content = ""
+      elsif line.empty?
+        content_start = true
+        break if variable == stream_input
+      else
+        content << line if content_start
+      end
+    end
+    
+    [inputs, filename]
+  end
+
+  def parse_uri(env)
+    uri = env["REQUEST_URI"]
+    _n, workflow, task = uri.split("/")
+    workflow = begin
+                 Kernel.const_get(workflow)
+               rescue
+                 raise "Could not accept task for workflow: #{ workflow }"
+               end
+    [workflow, task]
+  end
+
+  EOL = "\r\n"
+  def read_chunk(sin, rest = "")
+    parts = []
+    c = sin.gets
+    c = rest << c unless rest.empty?
+    c = c[2..-1] if c[0..1] == EOL
+    index = c.index EOL
+    while index
+      part = c[0..index-1]
+      parts << part
+      c = c[index+2..-1]
+      index = c.index EOL
+    end
+    rest = c
+    [parts, rest]
+  end
+
+
+  def copy_chunked_stream(sin, sout, boundary)
+
+    rest = ""
+    done = false
+    content = true
+
+    while not done
+      parts, rest = read_chunk(sin, rest)
+      while parts.any?
+        part = parts.shift
+        if content
+          part.split("\n").each do |line|
+            sout.puts line
+            if line.include? boundary
+              done = true
+              break
+            end
+          end
+          content = false
+        else
+          content = true
+        end
+      end
+    end
+
+    sout.write rest
+    sout.close
+  end
+
+  def call(env)
+    if env["REQUEST_METHOD"] == "POST" and env["rack.hijack"] and env["CONTENT_TYPE"] and env["CONTENT_TYPE"].include? "Rbbt_Param_Stream" and env["HTTP_TRANSFER_ENCODING"] == 'chunked'
+      Log.high "Hijacking post data"
+      inputs = {}
+      content_type = env["CONTENT_TYPE"]
+      boundary = content_type.match(/boundary=([^\s;]*)/)[1]
+      stream_input = content_type.match(/stream=([^\s;]*)/)[1]
+      post_stream = env["rack.hijack"].call
+      begin
+        inputs, filename = read_normal_inputs(post_stream, boundary, stream_input)
+
+        input_stream_out, input_stream_in = Misc.pipe
+        Misc.add_stream_filename(input_stream_out, filename) if filename
+        inputs[stream_input] = input_stream_out
+
+        workflow, task = parse_uri(env)
+        name = inputs.delete "jobname"
+        job = workflow.job(task, name, inputs)
+
+        task = task.to_sym
+        execution_type = case
+                         when workflow.exec_exports.include?(task)
+                           "exec"
+                         when workflow.synchronous_exports.include?(task)
+                           "synchronous"
+                         when workflow.asynchronous_exports.include?(task)
+                           "asynchronous"
+                         else
+                           raise "No known export type for #{ workflow } #{ task }. Accesses denied"
+                         end 
+
+        execution_type = "exec" if inputs["_cache_type"] == 'exec'
+        Log.info "Streaming task with execution_type: #{ execution_type }"
+
+        case execution_type
+        when "exec", nil
+          job.exec(:stream)
+        when "sync", "synchronous", "async", "asynchronous"
+          job.run(:stream)
+        else
+          raise "Unknown execution_type: #{execution_type}"
+        end
+
+        t_in = Thread.new do 
+          begin
+            copy_chunked_stream(post_stream, input_stream_in, boundary)
+          rescue
+            Log.exception $!
+          end
+        end
+
+        job_output = TSV.get_stream job
+        t_out = Thread.new do 
+          begin
+            post_stream.write "HTTP/1.1 200\r\n\r\n"
+            while c = job_output.read(1024)
+              post_stream.write c
+            end
+            job_output.join if job_output.respond_to? :join
+            post_stream.close
+          rescue
+            Log.exception $!
+            job.abort
+          end
+        end
+
+      end
+      [200, {}, nil]
+    else
+      Log.high "NOT Hijacking post data"
+      @app.call(env)
+    end
+  end
+end
+

data/share/views/error.haml

@@ -7,10 +7,10 @@
 
   - if defined? job.info_file and File.exists? job.info_file and request.env["REQUEST_URI"].include? job.name
     - clean_url = add_GET_param(request.env["REQUEST_URI"], "_update", "clean")
-    %a.ui.button.blue(href=clean_url) Clean
+    %a.ui.button.blue.clean(href=clean_url) Clean
   - else
     - clean_url = add_GET_param(request.env["REQUEST_URI"], "_update", "reload")
-    %a.ui.blue.button(href=clean_url) Reload
+    %a.ui.blue.button.reload(href=clean_url) Reload
 
 - backtrace = job.info[:backtrace]
 - if backtrace and backtrace.any?

data/share/views/job_info.haml

@@ -1 +1,18 @@
-= hash2dl(info)
+- new_info = {}
+- info.each{|k,v| new_info[k] = v unless k.to_s == 'dependencies'}
+= hash2dl(new_info)
+
+- if info[:dependencies] and info[:dependencies].any?
+  %h3 Dependencies
+  %ul
+
+    - task_exports = workflow.synchronous_exports + workflow.asynchronous_exports
+    - task_exports = task_exports.collect{|t| t.to_s}
+    - info[:dependencies].each do |task,name,path|
+      - url = "/" + [workflow, task, name] * "/"
+      %li
+        (#{workflow} #{task}) 
+        - if task_exports.include? task.to_s
+          %a(href=url) #{ name }  
+        - else
+          = name 

data/share/views/job_result/job_control.haml

@@ -6,7 +6,7 @@
       %a.ui.small.button(href="/#{[workflow.to_s, task, jobname, "info"] * "/"}") job info
       %a.ui.small.button(href="/#{[workflow.to_s, task, jobname] * "/"}?_format=raw") get raw result file
 
-  - files = job.files - ["html", "html.info"]
+  - files = job.files - ["html", "html.files", "html.info", "html.pid"]
   - if files.any?
     %dt.ui.header Files
     %dd.job_files

data/share/views/public/js/helpers/_ajax_replace.js

@@ -115,7 +115,8 @@ function replace_object(object, href, embedd, complete){
       error_message = error.sub(/\n\n.*/,'')
 
       error_span = $('<span>').html(error_message).addClass('error')
-      error = $('<div>').append(error_span)
+      error = $('<div>') //.append(error_span)
+      error.append(req.responseText)
       object.removeClass(_reloading_class).addClass("error").css('height', 0).html(error).css('height', 'auto').attr('target-href', clean_href);
     },
 

data/share/views/public/js/rbbt/actions.js

@@ -287,6 +287,10 @@ $.widget("rbbt.action_controller", {
          action_controller.addClass('loading');
 
          replace_object(action_div, href, true, this.options.complete);
+         var tool = this
+         action_div.on('click', '.reload', function(e){ 
+           update_embedded(action_div, true, tool.options.complete); return false
+         });
 
          return false
        }

data/share/views/wait.haml

@@ -1,5 +1,8 @@
 - @reload_page = true
 
+:sass
+  span.progress_time
+    color: grey
 .wait.ui.segment.info.message
   %h3.ui.header Waiting on #{format_name File.basename(job.name)}
 
@@ -10,10 +13,29 @@
     .footer.ui.basic.segment
       - progress = job.file(:progress).yaml
       - if progress[:last_percent]
+        - start = progress[:start]
+        - last_time = progress[:last_time]
+        - if progress[:last_percent].to_i > 0
+          - ellapsed = last_time - start
+          - if progress[:mean]
+            - missing = progress[:max].to_f - progress[:ticks].to_f
+            - eta = missing / progress[:mean].to_f
+          - else
+            - ratio = progress[:ticks].to_f/progress[:max].to_f
+            - eta = ellapsed * (1 - ratio)
+          - eta = Misc.format_seconds(eta)
+          - ellapsed = Misc.format_seconds(ellapsed)
+        - else
+          - ellapsed = '?'
+          - eta = '?'
         .ui.teal.step.percent.progress.indicating.meta(data-percent="#{progress[:last_percent]}" data-total=100)
           .bar
             .progress
-          .label= progress[:desc]
+          .label
+            = progress[:desc]
+            - if progress[:last_percent].to_i > 0
+              %span.progress_time
+                (#{ eta } ⇒ #{ellapsed})
           :deferjs
             $('.step.progress').progress()
       - else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-rest
 version: !ruby/object:Gem::Version
-  version: 1.8.8
+  version: 1.8.9
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-22 00:00:00.000000000 Z
+date: 2016-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -196,6 +196,7 @@ files:
 - lib/rbbt/rest/workflow/jobs.rb
 - lib/rbbt/rest/workflow/locate.rb
 - lib/rbbt/rest/workflow/render.rb
+- lib/rbbt/rest/workflow/stream_task.rb
 - share/views/association/Default.haml
 - share/views/compass/app.sass
 - share/views/compass/card_templates.sass