rbbt-util 5.26.157 → 5.26.158

data/lib/rbbt/workflow/remote/remote_step.rb

@@ -1,324 +0,0 @@
-class WorkflowRemoteClient
-  class RemoteStep < Step
-
-    attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec, :is_stream, :stream_input
-
-    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false, is_stream = false, stream_input = nil)
-      @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec, @is_stream, @stream_input = base_url, task, base_name, inputs, result_type, result_description, is_exec, is_stream, stream_input
-      @base_url = "http://" << @base_url unless @base_url =~ /^[a-z]+:\/\//
-      @mutex = Mutex.new
-      rest = base_url.include?('ssh:') ? false : true
-
-      if rest
-        @adaptor = WorkflowRESTClient
-      else
-        @adaptor = WorkflowSSHClient
-      end
-
-      self.extend @adaptor
-    end
-
-    def clean_name
-      @base_name
-    end
-
-    def cache_file
-      digest = Misc.obj2digest([base_url, task, base_name, inputs])
-      Rbbt.var.cache.REST[[task, clean_name, digest] * "."].find
-    end
-
-    def cache_files
-      Dir.glob(cache_file + '.*')
-    end
-
-    def run(no_load = false)
-      no_load = @is_stream ? :stream : true if no_load
-
-      @result ||= @mutex.synchronize do
-        begin
-          if @is_exec
-            exec(no_load)
-          elsif no_load == :stream
-            _run_job(:stream)
-          elsif no_load
-            init_job 
-            nil
-          else
-            if ! done?
-              init_job 
-              join
-            end
-            self.load
-          end
-        ensure
-          @started = true
-        end
-      end
-
-      return @result if no_load == :stream
-      no_load ? Misc.add_GET_param(path, "_format", "raw") : @result
-    end
-
-
-    def self.get_streams(inputs, stream_input = nil)
-      new_inputs = {}
-      inputs.each do |k,v|
-        stream = stream_input.to_s == k.to_s
-        if Step === v 
-          unless (v.done? or v.streaming?) # or RestClient::Step === v)
-            v.run(true) and v.grace 
-          end
-
-          begin
-            if stream
-              new_inputs[k] = TSV.get_stream(v)
-            else
-              new_inputs[k] = v.load
-            end
-          rescue Exception
-            raise $!
-          end
-        else
-          new_inputs[k] = v
-        end
-      end
-      new_inputs
-    end
-
-    def get_streams
-      return if @inputs_done
-      @inputs = RemoteStep.get_streams @inputs, @stream_input
-      @inputs_done = true
-      @inputs
-    end
-
-    def dup_inputs
-      return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
-      Log.low "Dupping inputs for #{path}"
-      dupped_inputs = {}
-      @inputs.collect do |k,input|
-        dupped_inputs[k] = Step.dup_stream input
-      end
-      @inputs = dupped_inputs
-      @dupped = true
-    end
-
-    def name
-      return nil if @is_exec
-      return @path if @url.nil?
-      (Array === @url ? @url.first : @url).split("/").last
-    end
-
-    def name=(name)
-      @url = [base_url,task, name] * "/"
-    end
-
-    def task_name
-      return task if task
-      init_job
-      (Array === @url ? @url.first : @url).split("/")[-2]
-    end
-
-    def nopid?
-      false
-    end
-
-    def info(check_lock=false)
-      @done = @info && @info[:status] && (@info[:status].to_sym == :done || @info[:status].to_sym == :error)
-
-      if !@done && (@last_info_time.nil? || (Time.now - @last_info_time) > 0.5)
-        update = true 
-      else
-        update = false
-      end
-
-      @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => true, :update => update) do
-        @last_info_time = Time.now
-        init_job unless @url
-        info = @adaptor.get_json(File.join(@url, 'info'))
-        info = @adaptor.fix_hash(info)
-        info[:status] = info[:status].to_sym if String === info[:status]
-        info
-      end
-      @info
-    end
-
-    def status
-      return :done if @done
-      return nil unless url or started?
-      #return :streaming if @streaming 
-      begin
-        status = info[:status]
-        @done = true if status and status.to_sym == :done
-        status
-      rescue
-        nil
-      ensure
-        @info = nil
-      end
-    end
-
-    def started?
-      @result != nil || @started || @streaming
-    end
-
-    def done?
-      return true if cache_files.any?
-      init_job unless @url
-      @done || status.to_s == 'done' || status.to_s == 'noinfo'
-    end
-
-    def files
-      @adaptor.get_json(File.join(url, 'files'))
-    end
-
-    def file(file)
-      @adaptor.get_raw(File.join(url, 'file', file))
-    end
-
-    def get_stream
-      case @result
-      when IO 
-        @result
-      when String
-        StringIO.new @result
-      else
-        nil
-      end
-    end
-
-    def grace
-      produce unless @started
-      sleep 0.1 unless started?
-      sleep 0.5 unless started?
-      sleep 1 unless started?
-      while not (done? or started?)
-        sleep 1 
-      end
-    end
-
-    #{{{ MANAGEMENT
-
-
-    def path
-      if @url
-        Misc.add_GET_param(@url, "_format", "raw")
-      elsif @base_name
-        [base_url, task, @base_name + '-' +  Misc.fingerprint(inputs)] * "/"
-      else
-        nil
-      end
-    end
-
-    def fork(noload=false, semaphore=nil)
-      init_job(:asynchronous)
-    end
-
-    def running?
-      ! %w(done error aborted noinfo).include? status.to_s
-    end
-
-    def exec(noload = false)
-      @result ||= begin
-                    if noload == :stream
-                      _run_job(:exec)
-                    else
-                      exec_job 
-                    end
-                  ensure
-                    @started = true
-                  end
-    end
-
-    def join
-      return true if cache_files.any?
-      init_job unless @url
-      Log.debug{ "Joining RestClient: #{path}" }
-      if IO === @result
-        res = @result
-        @result = nil
-        Misc.consume_stream(res, true) 
-      end
-
-      if not (self.done? || self.aborted? || self.error?)
-        self.info 
-        return self if self.done? || self.aborted? || self.error?
-        sleep 0.2 unless self.done? || self.aborted? || self.error?
-        sleep 1 unless self.done? || self.aborted? || self.error?
-        while not (self.done? || self.aborted? || self.error?)
-          sleep 3
-        end
-      end
-
-      self
-    end
-
-    def load_res(res, result_type = nil)
-
-      stream = true if res.respond_to? :read
-      join unless stream
-      result_type ||= self.result_type
-
-      case result_type.to_sym
-      when :string
-        stream ? res.read : res
-      when :boolean
-        (stream ? res.read : res) == 'true'
-      when :tsv
-        if stream
-          TSV.open(res, :monitor => true)
-        else
-          TSV.open(StringIO.new(res))
-        end
-      when :annotations
-        if stream
-          Annotated.load_tsv(TSV.open(res))
-        else
-          Annotated.load_tsv(TSV.open(StringIO.new(res)))
-        end
-      when :array
-        (stream ? res.read : res).split("\n")
-      else
-        json_text = if IO === res
-                      res.read
-                    else
-                      res
-                    end
-        begin
-          JSON.parse json_text
-        rescue
-          case
-          when json_text =~ /^\d+$/
-            json_text.to_i
-          when json_text =~ /^\d+\.\d/
-            json_text.to_f
-          else
-            raise $!
-          end
-        end
-      end
-    end
-
-    def _restart
-      @done = nil
-      @name = nil
-      @started = nil
-      @aborted = nil
-      new_inputs = {}
-      inputs.each do |k,i| 
-        if File === i 
-          new_inputs[k] = File.open(i.path)
-        else
-          new_inputs[k] = i
-        end
-      end
-      @inputs = new_inputs
-      @info = nil
-    end
-
-    def input_checks
-      []
-    end
-
-  end
-end

data/lib/rbbt/workflow/remote/rest/adaptor.rb

@@ -1,161 +0,0 @@
-require 'rest-client'
-
-module WorkflowRESTClient
-  def self.__prepare_inputs_for_restclient(inputs)
-    inputs.each do |k,v|
-      if v.respond_to? :path and not v.respond_to? :original_filename
-        class << v
-          def original_filename
-            File.expand_path(path)
-          end
-        end
-      end
-
-      if Array === v and v.empty?
-        inputs[k] = "EMPTY_ARRAY"
-      end
-    end
-  end
-
-  def workflow_description
-    WorkflowRESTClient.get_raw(File.join(url, 'description'))
-  end
-
-  def documentation
-    @documention ||= IndiferentHash.setup(WorkflowRESTClient.get_json(File.join(url, "documentation"),{}))
-  end
-
-  def self.task_info(url, task)
-    @@task_info ||= {}
-
-    key = [url, task] * "#"
-    @@task_info[key] ||= begin
-                           task_info = WorkflowRESTClient.get_json(File.join(url, task.to_s, 'info'))
-                           task_info = WorkflowRESTClient.fix_hash(task_info)
-
-                           task_info[:result_type] = task_info[:result_type].to_sym
-                           task_info[:export] = task_info[:export].to_sym
-                           task_info[:input_types] = WorkflowRESTClient.fix_hash(task_info[:input_types], true)
-                           task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
-
-                           @@task_info[key] = task_info
-                         end
-  end
-
-  def task_info(task)
-    WorkflowRESTClient.task_info(url, task)
-  end
-
-  def exported_tasks
-    (@asynchronous_exports  + @synchronous_exports + @exec_exports).compact.flatten
-  end
-
-  def tasks
-    @tasks ||= Hash.new do |hash,task_name| 
-      info = task_info(task_name)
-      task = Task.setup info do |*args|
-        raise "This is a remote task" 
-      end
-      task.name = task_name.to_sym
-      hash[task_name] = task
-    end
-  end
-
-  def load_tasks
-    exported_tasks.each{|name| tasks[name]}
-    nil
-  end
-
-  def task_dependencies
-    @task_dependencies ||= Hash.new do |hash,task| 
-      hash[task] = if exported_tasks.include? task
-        WorkflowRESTClient.get_json(File.join(url, task.to_s, 'dependencies'))
-      else
-        []
-      end
-    end
-  end
-
-  def init_remote_tasks
-    task_exports = WorkflowRESTClient.get_json(url)
-    @asynchronous_exports = (task_exports["asynchronous"] || []).collect{|task| task.to_sym }
-    @synchronous_exports = (task_exports["synchronous"] || []).collect{|task| task.to_sym }
-    @exec_exports = (task_exports["exec"] || []).collect{|task| task.to_sym }
-    @stream_exports = (task_exports["stream"] || []).collect{|task| task.to_sym }
-    @can_stream = task_exports["can_stream"]
-  end
-
-  def self.execute_job(base_url, task, task_params, cache_type)
-    self.capture_exception do
-      task_url = URI.encode(File.join(base_url, task.to_s))
-
-      sout, sin = Misc.pipe
-
-      post_thread = Thread.new(Thread.current) do |parent|
-        bl = lambda do |rok|
-          if Net::HTTPOK === rok
-            _url = rok["RBBT-STREAMING-JOB-URL"]
-            @url = File.join(task_url, File.basename(_url)) if _url
-            rok.read_body do |c,_a, _b|
-              sin.write c
-            end
-            sin.close
-          else
-            err = StringIO.new
-            rok.read_body do |c,_a, _b|
-              err.write c
-            end
-            text = begin
-                     reader = Zlib::GzipReader.new(err)
-                     reader.read
-                   rescue
-                     err.rewind
-                     err.read
-                   end
-            ne = @adaptor.parse_exception text
-            case ne
-            when String
-              parent.raise e.class, ne
-            when Exception
-              parent.raise ne
-            else
-              parent.raise "Error in RestClient: " << rok.message
-            end
-          end
-        end
-
-        task_params.each do |k,v|
-          task_params[k] = v.read if IO === v
-        end
-
-        Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
-        RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
-      end
-
-      # It seems like now response body are now decoded by Net::HTTP after 2.1
-      # https://github.com/rest-client/rest-client/blob/cf3e5a115bcdb8f3344aeac0e45b44d67fac1a42/history.md
-      decode = Gem.loaded_specs["rest-client"].version < Gem::Version.create('2.1')
-      if decode
-        reader = Zlib::GzipReader.new(sout)
-        res_io = Misc.open_pipe do |sin|
-          while c = reader.read(Misc::BLOCK_SIZE)
-            sin.write c
-          end
-          sin.close
-          @done = true
-        end
-        ConcurrentStream.setup(res_io, :threads => [post_thread]) do
-          @done = true
-          @streaming = false
-        end
-      else
-        ConcurrentStream.setup(sout, :threads => [post_thread]) do
-          @done = true
-          @streaming = false
-        end
-      end
-
-    end
-  end
-
-end