rbbt-util 5.19.35 → 5.19.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 71c4c5187485430e7ea4977a2a46002f2a5caa81
-  data.tar.gz: 60e13741b32e65b17e63621a0dcdd021b467b2ff
+  metadata.gz: 72a73b569032a1fce45e26a98b2e5e57c1abd83c
+  data.tar.gz: 1bb8e14d02dbc52654120389c6c1ff303ec25d05
 SHA512:
-  metadata.gz: 70760328e15502ce16d06483def0e5d6c7ee75d20d33e7891666e2a10c8cebb275a5a277be0afd0810fdecfa99f3d27ee48ddf93335b52caa94f629102fac250
-  data.tar.gz: 7310fddb61cc783289304aabd83b77bf47b2666d1f31ba90a7d15c91b701016b3272d2d2a7503b8b3697fd62a56278e420be3caf81efdfcf3f18c6bf629e138f
+  metadata.gz: 6126957c9cb71705cfa47105941752e32033b714e7972dac3b211179c0e23c3c7e7a7d693922f0c7bc141e9c4955b15936ebd1418daa80150fdab5ef950c07cb
+  data.tar.gz: 263a74be39a0ed6e0fee8d2e3c913f77767744fc54b4c0e7d5c469013ef1bd48cf134b5efbcb1c00aacd036a174a552ec2260a64f75156744f5bed1324a9089a

data/etc/app.d/base.rb

@@ -1,9 +1,13 @@
 #{{{ MODULES AND HELPERS
+register Sinatra::MultiRoute
 
 register Sinatra::RbbtRESTMain
 register Sinatra::RbbtRESTEntity
 register Sinatra::RbbtRESTFileServer # Remove to prevent serving files
 register Sinatra::RbbtRESTKnowledgeBase
+register Sinatra::RbbtRESTWorkflow
+
+helpers Sinatra::RbbtToolHelper
 helpers Sinatra::RbbtMiscHelpers
 
 #{{{ SESSIONS

data/etc/app.d/init.rb

@@ -5,3 +5,4 @@ require 'rbbt/rest/entity'
 require 'rbbt/rest/workflow'
 require 'rbbt/rest/file_server'
 require 'rbbt/rest/helpers'
+require 'rbbt/rest/web_tool'

data/lib/rbbt/annotations.rb

@@ -230,6 +230,7 @@ module Annotation
   end
 
   def clean_and_setup_hash(object, hash)
+    object.instance_variable_set(:@annotation_values, nil) if object.instance_variable_get(:@annotation_values).nil?
     annotation_values = object.instance_variable_get(:@annotation_values)
     annotation_values = annotation_values.nil? ? {} : annotation_values.dup
     annotation_values.instance_variable_set(:@annotation_md5, nil)

data/lib/rbbt/rest/client/get.rb

@@ -83,13 +83,18 @@ class WorkflowRESTClient
   end
 
   def self.post_jobname(url, params = {})
+    Log.stack caller
     Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
     params = params.merge({ :_format => 'jobname' })
     params = fix_params params
 
-    capture_exception do
+    name = capture_exception do
       RestClient.post(URI.encode(url), params)
     end
+
+    Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}: #{name}" }
+
+    name
   end
   
   def self.post_json(url, params = {})

data/lib/rbbt/rest/client/step.rb

@@ -1,30 +1,36 @@
-require 'excon'
 class WorkflowRESTClient
   class RemoteStep < Step
 
     attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec
 
     def self.get_streams(inputs)
+      new_inputs = {}
       inputs.each do |k,v|
-        if Step === v
-          stream = v.get_stream
-          inputs[k] = stream || v.run
+        if Step === v or RemoteStep ===  v
+          v.run
+          new_inputs[k] = v.load
+        else
+          new_inputs[k] = v
         end
       end
+      new_inputs
     end
 
+
     def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false)
       @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec = base_url, task, base_name, inputs, result_type, result_description, is_exec
       @mutex = Mutex.new
-      RemoteStep.get_streams @inputs
+      @inputs = RemoteStep.get_streams @inputs
     end
 
     def name
       return nil if @is_exec
+      return @path if @url.nil?
       (Array === @url ? @url.first : @url).split("/").last
     end
 
     def task_name
+      return task unless @url
       init_job
       (Array === @url ? @url.first : @url).split("/")[-2]
     end
@@ -41,6 +47,7 @@ class WorkflowRESTClient
     end
     
     def status
+      return nil if @url.nil?
       begin
         info[:status]
       ensure
@@ -82,8 +89,11 @@ class WorkflowRESTClient
     end
 
     def path
-      init_job
-      @url + '?_format=raw'
+      if @url
+        @url + '?_format=raw'
+      else
+        [base_url, task, Misc.fingerprint(inputs)] * "/"
+      end
     end
 
     def run(noload = false)
@@ -99,6 +109,7 @@ class WorkflowRESTClient
                       end
                     end
       end
+
       return @result if noload == :stream
       noload ? path + '?_format=raw' : @result
     end
@@ -186,7 +197,18 @@ class WorkflowRESTClient
           RestClient::Request.execute(:method => :post, :url => url, :payload => task_params, :block_response => bl)
         end
 
-        Zlib::GzipReader.new(sout)
+        reader = Zlib::GzipReader.new(sout)
+        Misc.open_pipe do |sin|
+          while c = reader.read(1015)
+            sin.write c
+          end
+          sin.close
+          @done = true
+        end
+        #nsout, nsin = Misc.pipe
+        #Misc.consume_stream(reader, true, nsin, true) do @done = true end
+        #iii :ret
+        #nsout
       end
     end
 

data/lib/rbbt/tsv/accessor.rb

@@ -19,6 +19,7 @@ module TSV
   end
 
   def entity_options
+    @entity_options ||= nil
     if @entity_options.nil?
       @entity_options = namespace ? {:namespace => namespace, :organism => namespace} : {}
       @entity_templates = nil
@@ -122,7 +123,7 @@ module TSV
   end
 
   def write?
-    @writable
+    @writable ||= false
   end
 
   def self._extended(data)

data/lib/rbbt/tsv/util.rb

@@ -133,13 +133,17 @@ module TSV
         StringIO.new file
       end
     when (defined? Step and Step)
-      file.grace
-      stream = file.get_stream
-      if stream
-        stream
+      if file.respond_to?(:base_url) and file.result
+        file.result
       else
-        file.join
-        get_stream(file.path)
+        file.grace
+        stream = file.get_stream
+        if stream
+          stream
+        else
+          file.join
+          get_stream(file.path)
+        end
       end
     when Array
       Misc.open_pipe do |sin|

data/lib/rbbt/util/R/plot.rb

@@ -10,6 +10,14 @@ module R
       sources = [:plot, Rbbt.share.Rlib["svg.R"].find(:lib), options[:source]].flatten.compact
       options.delete :source
 
+      fast = options[:fast]
+
+      if fast
+        save_method = "rbbt.SVG.save.fast"
+      else
+        save_method = "rbbt.SVG.save"
+      end
+
       if data
         data.each do |k,v|
           v = Array === v ? v : [v]
@@ -48,7 +56,7 @@ module R
           data.R <<-EOF, sources, options
   plot = { #{script} }
 
-  rbbt.SVG.save('#{tmpfile}', plot, width = #{R.ruby2R width}, height = #{R.ruby2R height})
+  #{save_method}('#{tmpfile}', plot, width = #{R.ruby2R width}, height = #{R.ruby2R height})
   data = NULL
           EOF
 
@@ -61,7 +69,7 @@ module R
           R.run <<-EOF, sources, options
   plot = { #{script} }
 
-  rbbt.SVG.save('#{tmpfile}', plot, width = #{R.ruby2R width}, height = #{R.ruby2R height})
+  #{save_method}('#{tmpfile}', plot, width = #{R.ruby2R width}, height = #{R.ruby2R height})
   data = NULL
           EOF
           Open.read(tmpfile).gsub(/(glyph\d+-\d+)/, '\1-' + File.basename(tmpfile))

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -70,7 +70,7 @@ class RbbtProcessQueue
 
         initial = Misc.memory_use(Process.pid)
         memory_cap = multiplier * initial
-        Log.medium "Worker #{Process.pid} started with #{@current} -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap}"
+        Log.medium "Worker for #{Process.pid} started with pid #{@current} -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap}"
 
         @monitor_thread = Thread.new do
           begin

data/lib/rbbt/util/log.rb

@@ -90,7 +90,7 @@ module Log
 
   def self.logfile(file=nil)
     if file.nil?
-      @logfile
+      @logfile ||= nil
     else
       case file
       when String

data/lib/rbbt/util/log/progress/report.rb

@@ -104,7 +104,7 @@ module Log
     end
 
     def save
-      info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max}
+      info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
       Open.write(@file, info.to_yaml)
     end
 

data/lib/rbbt/util/misc/exceptions.rb

@@ -8,6 +8,8 @@ class Aborted < Exception; end
 
 class RemoteServerError < Exception; end
 
+class DependencyError < Exception; end
+
 class KeepLocked < Exception
   attr_accessor :payload
   def initialize(payload)

data/lib/rbbt/util/misc/inspect.rb

@@ -208,7 +208,7 @@ module Misc
             'false'
           when Hash
             "{"<< obj.collect{|k,v| obj2str(k) + '=>' << obj2str(v)}*"," << "}"
-          when Symbol
+          when Symbol 
             obj.to_s
           when String
             if obj.length > HASH2MD5_MAX_STRING_LENGTH

data/lib/rbbt/util/misc/multipart_payload.rb

@@ -0,0 +1,138 @@
+require 'net/http'
+require 'rbbt-util'
+
+class Net::HTTPGenericRequest
+  alias old_send_request_with_body_stream send_request_with_body_stream
+
+  def send_request_with_body_stream(*args)
+    if chunked?
+      Thread.new do
+        old_send_request_with_body_stream(*args)
+      end
+    else
+      old_send_request_with_body_stream(*args)
+    end
+  end
+end
+
+module RbbtMutiplartPayload
+  BOUNDARY = "Rbbt_Param_Stream"
+  EOL = "\r\n"
+
+  def self.input_header(name, filename = nil)
+
+    if filename
+      head_text = 'Content-Disposition: form-data; name="' + name + '"; filename="' + filename + '"'
+    else
+      head_text = 'Content-Disposition: form-data; name="' + name + '"'
+    end
+
+    content_transfer_text = "Content-Transfer-Encoding: binary"
+
+    content_type_text = 'Content-Type: text/plain'
+
+    head_text + EOL + content_transfer_text + EOL + content_type_text + EOL
+  end
+
+  def self.add_input(name, content, filename = nil)
+    header = input_header(name, filename)
+    "--" + BOUNDARY + EOL + header + EOL + content + EOL
+  end
+
+  def self.add_stream(io, name, content, filename = nil)
+    header = input_header(name, filename)
+    io.write "--" + BOUNDARY + EOL + header + EOL  + EOL
+
+    while c = content.read(1024)
+      io.write c
+    end
+    content.close
+  end
+
+  def self.close_stream(io)
+    io.write "--" + BOUNDARY + "--" + EOL + EOL
+    io.write EOL
+    io.close
+  end
+
+  def self.post_data_stream(inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil)
+    sout, sin = Misc.pipe
+
+    Thread.new do
+      inputs.each do |input,content|
+        input = input.to_s
+        next if stream_input and input == stream_input.to_s
+        content_str = case content
+                      when String
+                        if Misc.is_filename?(content) and File.exists?(content)
+                          File.read(content)
+                        else
+                          content
+                        end
+                      when File, IO
+                        content.read
+                      when nil
+                        "nil"
+                      else
+                        content.to_s
+                      end
+        str = RbbtMutiplartPayload.add_input(input, content_str)
+        sin.write str
+      end
+
+      RbbtMutiplartPayload.add_stream(sin, stream_input.to_s, stream_io, stream_filename) if stream_input
+      RbbtMutiplartPayload.close_stream(sin)
+    end
+
+    sout
+  end
+
+  def self.issue(url, inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil)
+
+    uri = URI(url)
+    req = Net::HTTP::Post.new(uri.path)
+
+    IndiferentHash.setup(inputs)
+
+    if stream_input
+      stream_io ||= TSV.get_stream inputs[stream_input]
+      stream_filename ||= case inputs[stream_input]
+                          when String
+                            inputs[stream_input]
+                          when File
+                            inputs[stream_input].filename
+                          else
+                            'file'
+                          end
+    end
+
+    sout = RbbtMutiplartPayload.post_data_stream inputs, stream_input, stream_io, stream_filename
+
+    if stream_input
+      req.content_type = "multipart/form-data; boundary=" + RbbtMutiplartPayload::BOUNDARY + '; stream=' + stream_input.to_s
+      req.body_stream = sout
+      req.add_field "Transfer-Encoding", "chunked"
+    else
+      req.content_type = "multipart/form-data; boundary=" + RbbtMutiplartPayload::BOUNDARY
+      req.body = sout.read
+    end
+
+    Misc.open_pipe do |sin|
+      Net::HTTP.start(uri.hostname, uri.port) do |http|
+        http.request(req) do |res|
+          if Net::HTTPRedirection === res
+            sin.write res["location"]
+          elsif stream_input
+            res.read_body do |c|
+              sin.write c
+            end
+          else
+            sin.write res.body
+          end
+          sin.close
+        end
+      end
+    end
+  end
+
+end

data/lib/rbbt/util/misc/pipes.rb

@@ -159,7 +159,7 @@ module Misc
     str
   end
 
-  def self.consume_stream(io, in_thread = false, into = nil, into_close = true)
+  def self.consume_stream(io, in_thread = false, into = nil, into_close = true, &block)
     return if Path === io
     return unless io.respond_to? :read 
     if io.respond_to? :closed? and io.closed?
@@ -189,6 +189,7 @@ module Misc
         io.close unless io.closed?
         into.close if into and into_close and not into.closed?
         into.join if into and into_close and into.respond_to?(:joined?) and not into.joined?
+        block.call if block_given?
       rescue Aborted
         Log.medium "Consume stream aborted #{Misc.fingerprint io}"
         io.abort if io.respond_to? :abort

data/lib/rbbt/workflow/accessor.rb

@@ -38,6 +38,10 @@ class Step
     path.nil? ? nil : path + '.info'
   end
 
+  def self.pid_file(path)
+    path.nil? ? nil : path + '.pid'
+  end
+
   def self.step_info(path)
     begin
       Open.open(info_file(path)) do |f|
@@ -61,6 +65,10 @@ class Step
     path.sub(/.*\/#{Regexp.quote task.name.to_s}\/(.*)/, '\1')
   end
 
+  def short_path
+    [task_name, name] * "/"
+  end
+
   def task_name
     @task_name ||= task.name
   end
@@ -71,6 +79,10 @@ class Step
     @info_file ||= Step.info_file(path)
   end
 
+  def pid_file
+    @pid_file ||= Step.pid_file(path)
+  end
+
   def info_lock
     @info_lock = begin
                    path = Persist.persistence_path(info_file + '.lock', {:dir => Step.lock_dir})
@@ -118,6 +130,16 @@ class Step
     end
   end
 
+  def init_info
+    return nil if @exec or info_file.nil?
+    Open.lock(info_file, :lock => info_lock) do
+      i = {:status => :init}
+      @info_cache = i
+      Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true, :lock => false)
+      @info_cache_time = Time.now
+    end
+  end
+
   def set_info(key, value)
     return nil if @exec or info_file.nil?
     value = Annotated.purge value if defined? Annotated
@@ -286,7 +308,7 @@ class Step
   end
 
   def started?
-    Open.exists? info_file or Open.exists? path
+    Open.exists?(path) or Open.exists?(pid_file)
   end
 
   def dirty?
@@ -302,7 +324,7 @@ class Step
   end
 
   def running?
-    return nil if not Open.exists? info_file
+    return nil if not Open.exists? pid_file
     return nil if info[:pid].nil?
 
     pid = @pid || info[:pid]
@@ -468,13 +490,22 @@ module Workflow
                               deps.each do |dep| 
                                 case dep
                                 when Array
-                                  wf, t = dep
+                                  wf, t, o = dep
                                   wf.rec_dependencies(t).each do |d|
                                     if Array === d
-                                      all_deps << d
+                                      new = d
                                     else
-                                      all_deps << [dep.first, d]
+                                      new = [dep.first, d]
+                                    end
+                                    if Hash === o and not o.empty? 
+                                      if Hash === new.last
+                                        hash = new.last
+                                        o.each{|k,v| hash[k] ||= v}
+                                      else
+                                        new.push o
+                                      end
                                     end
+                                    all_deps << new
                                   end
                                 when String, Symbol
                                   all_deps.concat rec_dependencies(dep.to_sym)

data/lib/rbbt/workflow/definition.rb

@@ -59,7 +59,7 @@ module Workflow
         (defined? WorkflowRESTClient and WorkflowRESTClient === dependency.first) or
         Hash === dependency.last
 
-        dependency = ([self] + dependency) unless Module === dependency.first 
+        dependency = ([self] + dependency) unless Module === dependency.first or (defined? WorkflowRESTClient and WorkflowRESTClient === dependency.first)
         @dependencies << dependency
       else
         @dependencies.concat dependency

data/lib/rbbt/workflow/step.rb

@@ -201,8 +201,9 @@ class Step
 
   def self.clean(path)
     info_file = Step.info_file path
+    pid_file = Step.pid_file path
     files_dir = Step.files_dir path
-    if Open.exists?(path) or Open.exists?(info_file)
+    if Open.exists?(path) or Open.exists?(pid_file)
       begin
         self.abort if self.running?
       rescue Exception
@@ -213,10 +214,11 @@ class Step
 
       Misc.insist do
         Open.rm info_file if Open.exists? info_file
-        Open.rm info_file + '.lock' if Open.exists? info_file + '.lock'
+        #Open.rm info_file + '.lock' if Open.exists? info_file + '.lock'
         Open.rm path if Open.exists? path
-        Open.rm path + '.lock' if Open.exists? path + '.lock'
+        #Open.rm path + '.lock' if Open.exists? path + '.lock'
         Open.rm_rf files_dir if Open.exists? files_dir
+        Open.rm pid_file if Open.exists? pid_file
       end
     end
   end
@@ -236,7 +238,8 @@ class Step
 
     new_dependencies = []
     dependencies.each{|step| 
-      new_dependencies.concat step.rec_dependencies
+      r = step.rec_dependencies
+      new_dependencies.concat r
       new_dependencies << step
     }
     new_dependencies.uniq

data/lib/rbbt/workflow/step/run.rb

@@ -143,7 +143,11 @@ class Step
       next if (dependency.done? and not dependency.dirty?) or 
               (dependency.streaming? and dependency.running?) or 
               (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency and not (dependency.error? or dependency.aborted?))
-      dependency.clean if dependency.started? and dependency
+
+      dependency.clean if dependency.aborted? or (dependency.started? and not dependency.running? and not dependency.error?)
+
+      raise DependencyError, [dependency.short_path, dependency.messages.last] * ": " if dependency.error?
+
       dupping << dependency unless dependencies.include? dependency
     end
 
@@ -154,30 +158,83 @@ class Step
 
       begin
 
-        if not dependency.done?
-          if dependency.started?
-            if dependency.streaming?
-              Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} streaming -- #{Log.color :blue, dependency.path}"
-            else
-              Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path}"
-              dependency.join unless dependency.streaming?
-            end
-          else
-            Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} starting -- #{Log.color :blue, dependency.path}"
-            dependency.run(true).grace
-          end
-        else
-          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} done -- #{Log.color :blue, dependency.path}"
+        if dependency.done?
+          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} done -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+          next
+        end
+
+        if not dependency.started?
+          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} starting -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+          dependency.run(:stream)
+          raise TryAgain
+        end
+
+        dependency.grace
+
+        if dependency.aborted?
+          dependency.clean
+          raise TryAgain
+        end
+
+        if dependency.error?
+          raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
+        end
+
+        if dependency.streaming?
+          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} streaming -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+          next
         end
 
+        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+        begin
+          dependency.join
+          raise TryAgain unless dependency.done?
+        rescue Aborted
+          raise TryAgain
+        end
+        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
+
+        #if not dependency.done?
+        #  if dependency.started?
+        #    dependency.grace
+        #    if dependency.error?
+        #      raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
+        #    elsif dependency.streaming?
+        #      Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} streaming -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
+        #    else
+        #      Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
+        #      begin
+        #        dependency.join unless dependency.streaming?
+        #      rescue Aborted
+        #        Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} #{Log.color :red, "aborted"} -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
+        #        if dependency.aborted?
+        #          Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} #{Log.color :red, "aborted cleaning"} -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}" 
+        #          dependency.clean
+        #          raise TryAgain
+        #        end
+        #      end
+        #      Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
+        #    end
+        #  else
+        #    Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} starting -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
+        #    dependency.run(:stream)
+        #    dependency.grace
+        #    dependency.join unless dependency.streaming?
+        #  end
+        #else
+        #  Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} done -- #{Log.color :blue, dependency.short_path}"
+        #end
+
+      rescue TryAgain
+        retry
       rescue Aborted
-        Log.error "Aborted dep. #{Log.color :red, dependency.task.name.to_s}"
+        Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
         raise $!
       rescue Interrupt
-        Log.error "Interrupted while in dep. #{Log.color :red, dependency.task.name.to_s}"
+        Log.error "Interrupted while in dep. #{Log.color :red, dependency.task_name.to_s}"
         raise $!
       rescue Exception
-        Log.error "Exception in dep. #{ Log.color :red, dependency.task.name.to_s }"
+        Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s }"
         raise $!
       end
     end
@@ -189,13 +246,14 @@ class Step
     begin
       @mutex.synchronize do
         no_load = no_load ? :stream : false
-        result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do |lockfile|
+        result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do 
           if Step === Step.log_relay_step and not self == Step.log_relay_step
             relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
           end
-          @exec = false
 
-          Open.rm info_file if Open.exists? info_file
+          @exec = false
+          Open.write(pid_file, Process.pid.to_s)
+          init_info
 
           log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
 
@@ -204,16 +262,17 @@ class Step
             :issued => (issue_time = Time.now),
             :name => name,
             :clean_name => clean_name,
-            :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
           })
 
           dup_inputs
           begin
             run_dependencies
           rescue Exception
+            FileUtils.rm pid_file if File.exists?(pid_file)
             stop_dependencies
             raise $!
           end
+          set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
           set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
 
@@ -223,7 +282,6 @@ class Step
           begin
             result = _exec
           rescue Aborted
-            stop_dependencies
             log(:aborted, "Aborted")
             raise $!
           rescue Exception
@@ -231,9 +289,9 @@ class Step
 
             # HACK: This fixes an strange behaviour in 1.9.3 where some
             # backtrace strings are coded in ASCII-8BIT
+            backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
             set_info :backtrace, backtrace 
             log(:error, "#{$!.class}: #{$!.message}")
-            backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
             stop_dependencies
             raise $!
           end
@@ -266,6 +324,7 @@ class Step
                 Log.exception $!
               ensure
                 join
+                FileUtils.rm pid_file if File.exists?(pid_file)
               end
             end
             stream.abort_callback = Proc.new do
@@ -273,6 +332,8 @@ class Step
                 log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
               rescue
                 Log.exception $!
+                stop_dependencies
+                FileUtils.rm pid_file if File.exists?(pid_file)
               end
             end
           else
@@ -280,6 +341,7 @@ class Step
             set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
             set_info :time_elapsed, (time_elapsed = done_time - start_time)
             log :done, "#{Log.color :magenta, "Completed"} step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
+            FileUtils.rm pid_file if File.exists?(pid_file)
           end
 
           result
@@ -294,6 +356,7 @@ class Step
       end
     rescue Exception
       exception $!
+      stop_dependencies
       raise $!
     end
   end
@@ -302,10 +365,10 @@ class Step
     return self if done? and not dirty?
 
     if error? or aborted?
-      if force
+      if force or aborted?
         clean
       else
-        raise "Error in job: #{status}"
+        raise "Error in job: #{status} - #{self.path}"
       end
     end
 
@@ -326,6 +389,7 @@ class Step
         RbbtSemaphore.wait_semaphore(semaphore) if semaphore
         FileUtils.mkdir_p File.dirname(path) unless File.exists? File.dirname(path)
         begin
+          @forked = true
           res = run true
           set_info :forked, true
         rescue Aborted
@@ -386,13 +450,14 @@ class Step
       Log.medium "Could not abort #{path}: same process"
       false
     else
-      Log.medium "Aborting #{path}: #{ @pid }"
+      Log.medium "Aborting pid #{path}: #{ @pid }"
       begin
-        Process.kill("KILL", @pid)
+        Process.kill("INT", @pid)
         Process.waitpid @pid
       rescue Exception
         Log.debug("Aborted job #{@pid} was not killed: #{$!.message}")
       end
+      Log.medium "Aborted pid #{path}: #{ @pid }"
       true
     end
   end
@@ -422,7 +487,7 @@ class Step
     begin
       stop_dependencies
       abort_stream
-      abort_pid
+      abort_pid if defined? @forked and @forked
     rescue Aborted
       Log.medium{"#{Log.color :red, "Aborting ABORTED RETRY"} #{Log.color :blue, path}"}
       retry
@@ -463,7 +528,7 @@ class Step
   end
 
   def soft_grace
-    until Open.exists? info_file
+    until done? or File.exists?(info_file)
       sleep 1 
     end
     self
@@ -506,7 +571,7 @@ class Step
         pid = nil
         dependencies.each{|dep| dep.join }
       end
-      sleep 1 until path.exists?
+      sleep 1 until path.exists? or error? or aborted?
       self
     ensure
       set_info :joined, true

data/share/rbbt_commands/app/start

@@ -20,6 +20,8 @@ $ rbbt app start [options] <app_name>
 -R--Rserve_session* Rserve session to use, otherwise start new one
 -wd--workdir* Change the working directory of the workflow
 --views* Directory with view templates
+--stream Activate streaming of workflow tasks
+--options* Additional options for server (e.g. option1=value1;option2=value2)
 EOF
 
 if options[:help]
@@ -55,19 +57,32 @@ Misc.in_dir(app_dir) do
   ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
   ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
 
+  if options[:stream]
+    raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma|unico/
+    ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
+  end
+
   config_ru_file = File.exists?('./config.ru') ? './config.ru' : Rbbt.share['config.ru'].find
 
-  if server == 'unicorn'
-    `unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:port] || "2887"}`
+  if options[:options]
+    options[:options].split(";").each do |pair|
+      name, _sep, value = pair.partition("=")
+      name = name[1..-1].to_sym if name[0] == ':'
+      options[name] = value
+    end
+  end
+
+  case server
+  when 'unicorn'
+    system ENV, "unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:port] || "2887"}"
+  when 'puma_alt'
+    system ENV, "puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload"
   else
     options[:config] = config_ru_file
     Rack::Server.start(options)
   end
 end
 
-
-
-
 #!/usr/bin/env ruby
 
 require 'rbbt-util'

data/share/rbbt_commands/workflow/server

@@ -1,33 +1,28 @@
 #!/usr/bin/env ruby
 
+require 'rbbt-util'
 require 'rbbt/util/simpleopt'
-require 'rbbt/workflow'
 
-#require 'zurb-foundation'
-#require 'modular-scale'
-
-require 'rbbt/rest/main'
-require 'rbbt/rest/entity'
-require 'rbbt/rest/workflow'
-require 'rbbt/rest/file_server'
-require 'rbbt/rest/knowledge_base'
-require 'rbbt/rest/helpers'
-require 'rbbt/rest/web_tool'
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
 options = SOPT.setup <<EOF
 Start an rbbt app
 
-$ rbbt app start [options] <app_name>
+$ rbbt workflow server [options] <Workflow>
 
 -h--help Print this help
 -e--environment* Execution environment: production or development
--b--bind* Bind to a particular IP (net interface)
+-Ho--Host* Host name
+-B--Bind* Bind IP
 -p--port* TCP port
 -s--server* Server type:  thin, webrick, unicorn, etc
 -f--finder Start server with finder functionality
 -R--Rserve_session* Rserve session to use, otherwise start new one
+-wd--workdir* Change the working directory of the workflow
 -w--workflows* List of additional workflows to load
 --views* Directory with view templates
+--stream Activate streaming of workflow tasks
+--options* Additional options for server (e.g. option1=value1;option2=value2)
 EOF
 
 if options[:help]
@@ -39,57 +34,59 @@ if options[:help]
   exit 0
 end
 
-workflow = ARGV.first
-
-ENV["RBBT_FINDER"] = "true" if options.include?(:finder)
-ENV["RServe-session"] = options[:RServe_session] || workflow
-
-raise rbbt_usage unless workflow
-
-wf = Workflow.require_workflow workflow
-
-$title = wf.to_s
-
-load Rbbt.etc['app.d/init.rb'].find
-
-app = class WorkflowRest < Sinatra::Base; self end
-
-app.register Sinatra::RbbtRESTWorkflow
-app.register Sinatra::MultiRoute
-app.register Sinatra::RbbtRESTKnowledgeBase
-app.helpers Sinatra::RbbtToolHelper
-
-app.get '/' do
-  redirect to(File.join('/', wf.to_s))
-end
-app.instance_eval Rbbt.etc['app.d/base.rb'].read
-app.use Rack::Deflater
-
-load Rbbt.etc['app.d/resources.rb'].find
-
-app.class_eval do
-  eval Rbbt.etc['app.d/finder.rb'].read
+if options[:workdir]
+  require 'rbbt/workflow'
+  Workflow.workdir = options[:workdir]
 end
 
-WorkflowRest.add_workflow wf, :priority
-if options[:workflows]
-  options[:workflows].split(/[\s,]+/).each do |name|
-    _wf = Workflow.require_workflow name
-    WorkflowRest.add_workflow _wf, true
-  end
-end
+options[:Port] ||= options[:port]
+options[:Host] ||= "0.0.0.0"
+options[:Bind] ||= "0.0.0.0"
 
-load Rbbt.etc['app.d/requires.rb'].find
+workflow = ARGV.shift
+workflows = options[:workflows] || "" 
 
-load Rbbt.etc['app.d/entities.rb'].find
+workflow = File.expand_path(workflow) if File.exists?(workflow)
 
+ENV["RServe-session"] = options[:RServe_session] || workflow
 
-if options[:views] and not options[:views].empty?
-  Sinatra::RbbtRESTMain.add_resource_path(Path.setup(options[:views]), true)
+server = options[:server] || 'puma'
+
+TmpFile.with_file do |app_dir|
+  Misc.in_dir(app_dir) do
+    app_dir = Path.setup(app_dir.dup)
+    Open.write(app_dir.etc.target_workflow.find, workflow)
+    Open.write(app_dir.etc.workflows.find, workflows.split(/,\s/)*"\n") if workflows 
+
+    require 'rack'
+    ENV["RBBT_FINDER"] = "true" if options.include?(:finder)
+    ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
+    ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
+
+    if options[:stream]
+      raise "No streaming available for any server other than puma" unless options[:server] == 'puma'
+      ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
+    end
+
+    config_ru_file = File.exists?('./workflow_config.ru') ? './config.ru' : Rbbt.share['workflow_config.ru'].find
+
+
+    if options[:options]
+      options[:options].split(";").each do |pair|
+        name, _sep, value = pair.partition("=")
+        name = name[1..-1].to_sym if name[0] == ':'
+        options[name] = value
+      end
+    end
+
+    case server
+    when 'unicorn'
+      `unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:port] || "2887"}`
+    when 'puma_alt'
+      `puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload`
+    else
+      options[:config] = config_ru_file
+      Rack::Server.start(options)
+    end
+  end
 end
-
-WorkflowRest.port = options[:port] || 4567
-WorkflowRest.bind = options[:bind] || "0.0.0.0"
-WorkflowRest.environment = options[:environment] || "development"
-WorkflowRest.server = options[:server] if options[:server]
-WorkflowRest.run!

data/share/rbbt_commands/workflow/task

@@ -441,8 +441,8 @@ when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
     puts res.to_s
   end
 when Step
-  if step.streaming?
-    io = res.get_stream
+  if res.streaming?
+    io = TSV.get_stream res
     while block = io.read(2048) do
       out.write block
     end #unless io.closed? 

data/test/rbbt/util/test_misc.rb

@@ -493,6 +493,10 @@ eum fugiat quo voluptas nulla pariatur?"
     end
   end
 
+  def test_obj2md5_str_float
+    assert_equal Misc.obj2md5([1,2]), Misc.obj2md5(["1","2"])
+  end
+
   def test_sample_large_string
     str = "string" *  1000000
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.19.35
+  version: 5.19.36
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-27 00:00:00.000000000 Z
+date: 2016-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -261,6 +261,7 @@ files:
 - lib/rbbt/util/misc/lock.rb
 - lib/rbbt/util/misc/manipulation.rb
 - lib/rbbt/util/misc/math.rb
+- lib/rbbt/util/misc/multipart_payload.rb
 - lib/rbbt/util/misc/objects.rb
 - lib/rbbt/util/misc/omics.rb
 - lib/rbbt/util/misc/options.rb