rbbt-util 5.20.26 → 5.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a8c6536f28d6d8ac9e66e806944eb1b4295c7e6
-  data.tar.gz: 49c9c947c59b325e67d4b3f807606abb89852d2d
+  metadata.gz: 752166bf720d5525c25bf96a2a01915c5ee7430c
+  data.tar.gz: 973d98ccede9449355b494982789b8cb82c325e1
 SHA512:
-  metadata.gz: b0314ac49f73cc937c7732f5022fe5d6854bfdaf69ebd6e7d56ac858f0e56026a1e71806881ffc92e03076241295c6507d47b47e5108716dadff71a34e3f7bed
-  data.tar.gz: b8057ee7804c0dae6f86636ca53cdeabff58e20193108b1b96c6571710c50d3f7ec95dc07b27f0308be2e44556efe4689e37b64ac1086879a16b46b4821090a7
+  metadata.gz: 31ee0806440ed08fd319e7dfe9cbcd524ec7f40bbe10cfa8b56c411172d0e27d84397b8a64674000b6b340f433e79425fa8a779d640a802c3d835cc861e7df49
+  data.tar.gz: 7763048e9ee7786cb9270e9656b821093c5d67e2f2bdef8f1fc3448b11b0fa187fcf1f53a6918de4918a759bfec4ee4755b07313015efb6620895583cb4e9549

data/lib/rbbt/annotations.rb

@@ -69,7 +69,9 @@ module Annotated
   def info(masked = false)
 
     if @info.nil?
-      info = annotation_values.dup
+      annotation_values = self.annotation_values
+      annotation_values = annotation_values.dup unless annotation_values.nil?
+      info = annotation_values
       info[:annotation_types] = annotation_types
       info[:annotated_array] = true if AnnotatedArray === self
       @info = info

data/lib/rbbt/monitor.rb

@@ -10,16 +10,18 @@ module Rbbt
 
   SENSIBLE_WRITE_DIRS = Misc.sensiblewrite_dir.find_all
 
-  PERSIST_DIRS    = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all  
+  PERSIST_DIRS    = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all
 
   JOB_DIRS = Rbbt.var.jobs.find_all
 
+  MUTEX_FOR_THREAD_EXCLUSIVE = Mutex.new
+
   def self.dump_memory(file, obj = nil)
     Log.info "Dumping #{obj} objects into #{ file }"
     Thread.new do
       while true
         Open.write(file) do |f|
-          Thread.exclusive do
+          MUTEX_FOR_THREAD_EXCLUSIVE.synchronize do
             GC.start
             ObjectSpace.each_object(obj) do |o|
               f.puts "---"
@@ -60,9 +62,9 @@ module Rbbt
       lock_info[f] = {}
       begin
         lock_info[f].merge!(file_time(f))
-        if File.size(f) > 0 
+        if File.size(f) > 0
           info = Open.open(f) do |s|
-            YAML.load(s) 
+            YAML.load(s)
           end
           IndiferentHash.setup(info)
           lock_info[f][:pid] = info[:pid]
@@ -98,7 +100,7 @@ module Rbbt
   end
 
   # PERSISTS
-  
+
   def self.persists(dirs = PERSIST_DIRS)
     dirs.collect do |dir|
       next unless Open.exists? dir
@@ -145,7 +147,7 @@ module Rbbt
           files = `find "#{ taskdir }/" -not -type d -not -path "*/*.files/*" 2>/dev/null`.split("\n").sort
           _files = Set.new files
           TSV.traverse files, :type => :array, :into => jobs do |file|
-            if m = file.match(/(.*).info$/)
+            if m = file.match(/(.*).(info|pid)$/)
               file = m[1]
             end
 
@@ -161,7 +163,7 @@ module Rbbt
             if _files.include? file
               info = info.merge(file_time(file))
               info[:done] = true
-              info[:info_file] = File.exists?(info_file) ? info_file : nil
+              info[:info_file] = File.exist?(info_file) ? info_file : nil
             else
               info = info.merge({:info_file => info_file, :done => false})
             end
@@ -200,7 +202,7 @@ module Rbbt
             job = f.sub(/\.(info|files)/,'')
 
           jobs[workflow][task][job] ||= {}
-          if jobs[workflow][task][job][:status].nil? 
+          if jobs[workflow][task][job][:status].nil?
             status = nil
             status = :done if Open.exists? job
             if status.nil? and f=~/\.info/
@@ -208,7 +210,7 @@ module Rbbt
                        Step::INFO_SERIALIAZER.load(Open.read(f, :mode => 'rb'))
                      rescue
                        {}
-                     end 
+                     end
             status = info[:status]
             pid = info[:pid]
             end
@@ -224,7 +226,7 @@ module Rbbt
 
   def self.load_lock(lock)
     begin
-      info = Misc.insist 3 do 
+      info = Misc.insist 3 do
         YAML.load(Open.read(lock))
       end
       info.values_at "pid", "ppid", "time"

data/lib/rbbt/persist.rb

@@ -249,7 +249,7 @@ module Persist
 
         res = tee_stream(stream, path, type, callback, abort_callback, lockfile)
 
-        res.lockfile = lockfile
+        #res.lockfile = lockfile
 
         raise KeepLocked.new res 
       else

data/lib/rbbt/rest/client.rb

@@ -11,7 +11,7 @@ require 'rbbt/rest/client/step'
 class WorkflowRESTClient 
   include Workflow
 
-  attr_accessor :url, :name, :exec_exports, :asynchronous_exports, :synchronous_exports
+  attr_accessor :url, :name, :exec_exports, :synchronous_exports, :asynchronous_exports, :stream_exports
 
   def initialize(url, name)
     Log.debug{ "Loading remote workflow #{ name }: #{ url }" }
@@ -45,7 +45,7 @@ class WorkflowRESTClient
 
     
     stream_input = @can_stream ? task_info(task)[:input_options].select{|k,o| o[:stream] }.collect{|k,o| k }.first : nil
-    RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task), stream_input)
+    RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task), @stream_exports.include?(task), stream_input)
   end
 
   def load_id(id)

data/lib/rbbt/rest/client/adaptor.rb

@@ -61,6 +61,7 @@ class WorkflowRESTClient
     @asynchronous_exports = task_exports["asynchronous"].collect{|task| task.to_sym }
     @synchronous_exports = task_exports["synchronous"].collect{|task| task.to_sym }
     @exec_exports = task_exports["exec"].collect{|task| task.to_sym }
+    @stream_exports = task_exports["stream"].collect{|task| task.to_sym }
     @can_stream = task_exports["can_stream"]
   end
 end

data/lib/rbbt/rest/client/get.rb

@@ -54,11 +54,11 @@ class WorkflowRESTClient
   end
 
   def self.get_raw(url, params = {})
-    Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
     params = params.merge({ :_format => 'raw' })
     params = fix_params params
     res = capture_exception do
       Misc.insist(2, 0.5) do
+        Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
         res = RestClient.get(URI.encode(url), :params => params)
         raise TryAgain if res.code == 202
         res
@@ -119,36 +119,4 @@ class WorkflowRESTClient
     end
   end
 
-  def self.stream_job(task_url, task_params, stream_input, cache_type = :exec)
-    require 'rbbt/util/misc/multipart_payload'
-    WorkflowRESTClient.capture_exception do
-      Log.debug{ "RestClient stream #{Process.pid}: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint task_params}" }
-      res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
-      type = res.gets
-      out = case type.strip
-      when "LOCATION"
-        @url = res.gets
-        @url.sub!(/\?.*/,'')
-        WorkflowRESTClient.get_raw(@url)
-      when /STREAM: (.*)/
-        @url = $1.strip
-        res.callback = Proc.new do
-          Log.medium "Done streaming result from #{@url}"
-          @done = true
-        end
-        res
-      when "BULK"
-        begin
-          res.read
-        ensure
-          @done = true
-        end
-      else
-        raise "What? " + type
-      end
-      ConcurrentStream.setup(out, :filename => @url)
-      out
-    end
-  end
-
 end

data/lib/rbbt/rest/client/run.rb

@@ -0,0 +1,100 @@
+class WorkflowRESTClient::RemoteStep
+
+  def stream_job(task_url, task_params, stream_input, cache_type = :exec)
+    require 'rbbt/util/misc/multipart_payload'
+    WorkflowRESTClient.capture_exception do
+      @streaming = true
+
+      Log.debug{ "RestClient stream #{Process.pid}: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint task_params}" }
+      res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
+      type = res.gets
+
+      out = case type.strip
+            when "LOCATION"
+              @url = res.gets
+              @url.sub!(/\?.*/,'')
+              join
+              WorkflowRESTClient.get_raw(@url)
+              @done = true
+              @streaming = false
+            when /STREAM: (.*)/
+              @url = $1.strip
+              res.callback = Proc.new do
+                Log.medium "Done streaming result from #{@url}"
+                @done = true
+                @streaming = false
+              end
+              res
+            when "BULK"
+              begin
+                res.read
+              ensure
+                @done = true
+                @streaming = false
+              end
+            else
+              raise "What? " + type
+            end
+
+      ConcurrentStream.setup(out, :filename => @url)
+
+      out
+    end
+  end
+
+  def execute_job(task_url, task_params, cache_type)
+    WorkflowRESTClient.capture_exception do
+      task_url = URI.encode(File.join(base_url, task.to_s))
+
+      sout, sin = Misc.pipe
+
+      post_thread = Thread.new(Thread.current) do |parent|
+        bl = lambda do |rok|
+          if Net::HTTPOK === rok
+            _url = rok["RBBT-STREAMING-JOB-URL"]
+            @url = File.join(task_url, File.basename(_url)) if _url
+            rok.read_body do |c,_a, _b|
+              sin.write c
+            end
+            sin.close
+          else
+            parent.raise "Error in RestClient: " << rok.message
+          end
+        end
+
+        Log.debug{ "RestClient execute: #{ url } - #{Misc.fingerprint task_params}" }
+        RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
+      end
+
+      reader = Zlib::GzipReader.new(sout)
+      res_io = Misc.open_pipe do |sin|
+        while c = reader.read(Misc::BLOCK_SIZE)
+          sin.write c
+        end
+        sin.close
+        @done = true
+      end
+
+      ConcurrentStream.setup(res_io, :threads => [post_thread]) do
+        @done = true
+        @streaming = false
+      end
+    end
+  end
+
+  def _run_job(cache_type = :async)
+    get_streams
+
+    task_url = URI.encode(File.join(base_url, task.to_s))
+    WorkflowRESTClient.__prepare_inputs_for_restclient(inputs)
+    task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
+
+    if cache_type == :stream or cache_type == :exec and stream_input and inputs[stream_input]
+      io =  self.stream_job(task_url, task_params, stream_input, cache_type) 
+      return io
+    else
+      execute_job(task_url, task_params, cache_type)
+    end
+
+  end
+end

data/lib/rbbt/rest/client/step.rb

@@ -18,22 +18,34 @@ class WorkflowRESTClient
 
   class RemoteStep < Step
 
-    attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec, :stream_input
+    attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec, :is_stream, :stream_input
 
-    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false, stream_input = nil)
-      @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec, @stream_input = base_url, task, base_name, inputs, result_type, result_description, is_exec, stream_input
+    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false, is_stream = false, stream_input = nil)
+      @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec, @is_stream, @stream_input = base_url, task, base_name, inputs, result_type, result_description, is_exec, is_stream, stream_input
       @mutex = Mutex.new
     end
 
-    def run(noload = false)
+    def run(no_load = false)
+      no_load = @is_stream ? :stream : true if no_load
+
       @mutex.synchronize do
         @result ||= begin
                       if @is_exec
-                        exec(noload)
-                      elsif noload == :stream
+                        exec(no_load)
+                      elsif no_load == :stream
                         _run_job(:stream)
+                        #init_job 
+                        #join
+                        #Misc.open_pipe do |sin|
+                        #  body = get.body
+                        #  sin.write body
+                        #end
+                      elsif no_load
+                        init_job 
+                        nil
                       else
                         init_job 
+                        join
                         self.load
                       end
                     ensure
@@ -41,8 +53,8 @@ class WorkflowRESTClient
                     end
       end
 
-      return @result if noload == :stream
-      noload ? path + '?_format=raw' : @result
+      return @result if no_load == :stream
+      no_load ? path + '?_format=raw' : @result
     end
 
 
@@ -79,7 +91,7 @@ class WorkflowRESTClient
     end
 
     def abort
-      #WorkflowRESTClient.get_json(File.join(@url, '?_update=abort')) if @url
+      WorkflowRESTClient.get_json(@url + '?_update=abort') if @url and @name
     end
 
     def name
@@ -99,7 +111,7 @@ class WorkflowRESTClient
     end
 
     def info(check_lock=false)
-      @done = @info and @info[:status] and @info[:status].to_sym == :done
+      @done = @info && @info[:status] && @info[:status].to_sym == :done
       @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => !!@done) do
         init_job unless @url
         info = WorkflowRESTClient.get_json(File.join(@url, 'info'))
@@ -150,6 +162,7 @@ class WorkflowRESTClient
     end
 
     def grace
+      produce unless @started
       sleep 0.1 unless started?
       sleep 0.5 unless started?
       sleep 1 unless started?
@@ -259,72 +272,10 @@ class WorkflowRESTClient
 
     def load
       params = {}
+      join unless done? or streaming?
       load_res get
     end
 
-    def _run_job(cache_type = :async)
-      get_streams
-      if cache_type == :stream or cache_type == :exec and stream_input and inputs[stream_input]
-        task_url = URI.encode(File.join(base_url, task.to_s))
-        WorkflowRESTClient.__prepare_inputs_for_restclient(inputs)
-        task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
-        @streaming = true
-        io =  WorkflowRESTClient.stream_job(task_url, task_params, stream_input, cache_type) 
-        if IO === io
-          ConcurrentStream.setup(io)
-          io.add_callback do
-            @done = true
-            @streaming = false
-          end
-        else
-          @done = true
-          @streaming = false
-        end
-
-        @url = io.filename if io.filename
-        return io
-      end
-
-      WorkflowRESTClient.capture_exception do
-        @url = URI.encode(File.join(base_url, task.to_s))
-        task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
-
-        sout, sin = Misc.pipe
-        streamer = lambda do |c|
-          sin.write c
-        end
-
-        post_thread = Thread.new(Thread.current) do |parent|
-          bl = lambda do |rok|
-            if Net::HTTPOK === rok
-              _url = rok["RBBT-STREAMING-JOB-URL"]
-              @url = File.join(@url, File.basename(_url)) if _url
-              rok.read_body do |c,_a, _b|
-                sin.write c
-              end
-              sin.close
-            else
-              parent.raise "Error in RestClient: " << rok.message
-            end
-          end
-
-          Log.debug{ "RestClient execute: #{ url } - #{Misc.fingerprint task_params}" }
-          RestClient::Request.execute(:method => :post, :url => url, :payload => task_params, :block_response => bl)
-        end
-
-        reader = Zlib::GzipReader.new(sout)
-        res_io = Misc.open_pipe do |sin|
-          while c = reader.read(Misc::BLOCK_SIZE)
-            sin.write c
-          end
-          sin.close
-          @done = true
-        end
-
-        ConcurrentStream.setup(res_io, :threads => [post_thread])
-      end
-    end
-
     def exec_job
       res = _run_job(:exec)
       load_res res, result_type == :array ? :json : result_type
@@ -372,3 +323,5 @@ class WorkflowRESTClient
     end
   end
 end
+
+require 'rbbt/rest/client/run'