rbbt-util 5.26.157 → 5.26.158

data/lib/rbbt/rest/client/step.rb

@@ -1,397 +0,0 @@
-class WorkflowRESTClient
-
-  def self.__prepare_inputs_for_restclient(inputs)
-    inputs.each do |k,v|
-      if v.respond_to? :path and not v.respond_to? :original_filename
-        class << v
-          def original_filename
-            File.expand_path(path)
-          end
-        end
-      end
-
-      if Array === v and v.empty?
-        inputs[k] = "EMPTY_ARRAY"
-      end
-    end
-  end
-
-  class RemoteStep < Step
-
-    attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec, :is_stream, :stream_input
-
-    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false, is_stream = false, stream_input = nil)
-      @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec, @is_stream, @stream_input = base_url, task, base_name, inputs, result_type, result_description, is_exec, is_stream, stream_input
-      @base_url = "http://" << @base_url unless @base_url =~ /^https?:\/\//
-      @mutex = Mutex.new
-    end
-
-    def clean_name
-      @base_name
-    end
-
-    def run(no_load = false)
-      no_load = @is_stream ? :stream : true if no_load
-
-      @mutex.synchronize do
-        @result ||= begin
-                      if @is_exec
-                        exec(no_load)
-                      elsif no_load == :stream
-                        _run_job(:stream)
-                      elsif no_load
-                        init_job 
-                        nil
-                      else
-                        init_job 
-                        join
-                        self.load
-                      end
-                    ensure
-                      @started = true
-                    end
-      end
-
-      return @result if no_load == :stream
-      no_load ? Misc.add_GET_param(path, "_format", "raw") : @result
-    end
-
-
-    def self.get_streams(inputs, stream_input = nil)
-      new_inputs = {}
-      inputs.each do |k,v|
-        stream = stream_input.to_s == k.to_s
-        if Step === v 
-          unless (v.done? or v.streaming?) # or RestClient::Step === v)
-            v.run(true) and v.grace 
-          end
-
-          begin
-            if stream
-              new_inputs[k] = TSV.get_stream(v)
-            else
-              new_inputs[k] = v.load
-            end
-          rescue Exception
-            raise $!
-          end
-        else
-          new_inputs[k] = v
-        end
-      end
-      new_inputs
-    end
-
-    def get_streams
-      return if @inputs_done
-      @inputs = WorkflowRESTClient::RemoteStep.get_streams @inputs, @stream_input
-      @inputs_done = true
-      @inputs
-    end
-
-    def abort
-      return self if status == :done
-      WorkflowRESTClient.get_json(@url + '?_update=abort') if @url and @name
-      self
-    end
-
-    def dup_inputs
-      return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
-      Log.low "Dupping inputs for #{path}"
-      dupped_inputs = {}
-      @inputs.collect do |k,input|
-        dupped_inputs[k] = Step.dup_stream input
-      end
-      @inputs = dupped_inputs
-      @dupped = true
-    end
-
-    def name
-      return nil if @is_exec
-      return @path if @url.nil?
-      (Array === @url ? @url.first : @url).split("/").last
-    end
-
-    def name=(name)
-      @url = [base_url,task, name] * "/"
-    end
-
-    def task_name
-      return task if task
-      init_job
-      (Array === @url ? @url.first : @url).split("/")[-2]
-    end
-    
-    def nopid?
-      false
-    end
-
-    def info(check_lock=false)
-      @done = @info && @info[:status] && @info[:status].to_sym == :done
-
-      if !@done && (@last_info_time.nil? || (Time.now - @last_info_time) > 0.5)
-        update = true 
-      else
-        update = false
-      end
-
-      @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => true, :update => update) do
-        @last_info_time = Time.now
-        init_job unless @url
-        info = WorkflowRESTClient.get_json(File.join(@url, 'info'))
-        info = WorkflowRESTClient.fix_hash(info)
-        info[:status] = info[:status].to_sym if String === info[:status]
-        info
-      end
-    end
-
-    def status
-      return :done if @done
-      return nil unless url or started?
-      #return :streaming if @streaming 
-      begin
-        status = info[:status]
-        @done = true if status and status.to_sym == :done
-        status
-      rescue
-        nil
-      ensure
-        @info = nil
-      end
-    end
-
-    def started?
-      @result != nil || @started || @streaming
-    end
-
-    def done?
-      init_job unless @url
-      @done || status.to_s == 'done' || status.to_s == 'noinfo'
-    end
-
-    def files
-      WorkflowRESTClient.get_json(File.join(url, 'files'))
-    end
-
-    def file(file)
-      WorkflowRESTClient.get_raw(File.join(url, 'file', file))
-    end
-
-    def get_stream
-      case @result
-      when IO 
-        @result
-      when String
-        StringIO.new @result
-      else
-        nil
-      end
-    end
-
-    def grace
-      produce unless @started
-      sleep 0.1 unless started?
-      sleep 0.5 unless started?
-      sleep 1 unless started?
-      while not (done? or started?)
-        sleep 1 
-      end
-    end
-
-    #{{{ MANAGEMENT
-
-    def init_job(cache_type = nil, other_params = {})
-      cache_type = :asynchronous if cache_type.nil? and not @is_exec
-      cache_type = :exec if cache_type.nil?
-      @last_info_time = nil
-      @done = false
-      get_streams
-      @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
-        Misc.insist do
-          WorkflowRESTClient.post_jobname(File.join(base_url, task.to_s), inputs.merge(other_params).merge(:jobname => @name||@base_name, :_cache_type => cache_type))
-        end
-      end
-      if Open.remote? @name
-        @url = @name
-        @name = File.basename(@name)
-      else
-        @url = File.join(base_url, task.to_s, @name)
-      end
-      self
-    end
-
-
-    def fork(noload=false, semaphore=nil)
-      init_job(:asynchronous)
-    end
-
-    def running?
-      ! %w(done error aborted noinfo).include? status.to_s
-    end
-
-    def path
-      if @url
-        Misc.add_GET_param(@url, "_format", "raw")
-      else
-        [base_url, task, @base_name + '-' +  Misc.fingerprint(inputs)] * "/"
-      end
-    end
-
-    def exec(noload = false)
-      @result ||= begin
-                     if noload == :stream
-                       _run_job(:exec)
-                     else
-                       exec_job 
-                     end
-                    ensure
-                      @started = true
-                   end
-    end
-
-    def join
-      init_job unless @url
-      Log.debug{ "Joining RestClient: #{path}" }
-      if IO === @result
-        res = @result
-        @result = nil
-        Misc.consume_stream(res, true) 
-      end
-
-      if not (self.done? || self.aborted? || self.error?)
-        self.info 
-        return self if self.done? || self.aborted? || self.error?
-        sleep 0.2 unless self.done? || self.aborted? || self.error?
-        sleep 1 unless self.done? || self.aborted? || self.error?
-        while not (self.done? || self.aborted? || self.error?)
-          sleep 3
-        end
-      end
-
-      self
-    end
-
-    def get
-      params ||= {}
-      params = params.merge(:_format => [:string, :boolean, :tsv, :annotations,:array].include?(result_type.to_sym) ? :raw : :json )
-      Misc.insist 3, rand(2) + 1 do
-        begin
-          init_job if url.nil?
-          WorkflowRESTClient.get_raw(url, params)
-        rescue
-          Log.exception $!
-          raise $!
-        end
-      end
-    end
-
-    def load_res(res, result_type = nil)
-      stream = true if res.respond_to? :read
-      join unless stream
-      result_type ||= self.result_type
-      case result_type
-      when :string
-        stream ? res.read : res
-      when :boolean
-        (stream ? res.read : res) == 'true'
-      when :tsv
-        if stream
-          TSV.open(res, :monitor => true)
-        else
-          TSV.open(StringIO.new(res))
-        end
-      when :annotations
-        if stream
-          Annotated.load_tsv(TSV.open(res))
-        else
-          Annotated.load_tsv(TSV.open(StringIO.new(res)))
-        end
-      when :array
-        (stream ? res.read : res).split("\n")
-        res.split("\n")
-      else
-        json_text = if IO === res
-                      res.read
-                    else
-                      res
-                    end
-        begin
-          JSON.parse json_text
-        rescue
-          case
-          when json_text =~ /^\d+$/
-            json_text.to_i
-          when json_text =~ /^\d+\.\d/
-            json_text.to_f
-          else
-            raise $!
-          end
-        end
-      end
-    end
-
-    def load
-      params = {}
-      join unless done? or streaming?
-      raise get_exception if error? or aborted?
-      load_res get
-    end
-
-    def exec_job
-      res = _run_job(:exec)
-      load_res res, result_type == :array ? :json : result_type
-    end
-
-    def _restart
-      @done = nil
-      @name = nil
-      @started = nil
-      @aborted = nil
-      new_inputs = {}
-      inputs.each do |k,i| 
-        if File === i 
-          new_inputs[k] = File.open(i.path)
-        else
-          new_inputs[k] = i
-        end
-      end
-      @inputs = new_inputs
-      @info = nil
-    end
-
-    def recursive_clean
-      Log.warn "Not doing recursive cleans"
-      return
-      begin
-        _restart
-        params = {:_update => :recursive_clean}
-        WorkflowRESTClient.get_raw(url, params)
-      rescue Exception
-        Log.exception $!
-      end
-      self
-    end
-
-    def _clean
-      begin
-        _restart
-        params = {:_update => :clean}
-        WorkflowRESTClient.clean_url(url, params) if @url
-      rescue Exception
-        Log.exception $!
-      end
-    end
-
-    def clean
-      init_job
-      _clean
-      self
-    end
-
-    def input_checks
-      []
-    end
-  end
-end
-
-require 'rbbt/rest/client/run'

data/lib/rbbt/rest/client.rb

@@ -1,58 +0,0 @@
-require 'rest_client'
-require 'json'
-require 'rbbt/workflow'
-require 'rbbt/workflow/step'
-require 'rbbt/util/misc'
-
-require 'rbbt/rest/client/get'
-require 'rbbt/rest/client/adaptor'
-require 'rbbt/rest/client/step'
-
-class WorkflowRESTClient 
-  include Workflow
-
-  attr_accessor :url, :name, :exec_exports, :synchronous_exports, :asynchronous_exports, :stream_exports
-
-  def initialize(url, name)
-    Log.debug{ "Loading remote workflow #{ name }: #{ url }" }
-    @url, @name = url, name
-    init_remote_tasks
-  end
-  
-  def to_s
-    name
-  end
-
-  def job(task, name, inputs)
-    task_info = task_info(task)
-    fixed_inputs = {}
-    input_types = task_info[:input_types]
-
-    inputs.each do |k,v| 
-      k = k.to_sym
-      if TSV === v
-        fixed_inputs[k] = v.to_s
-      else
-        next if input_types[k].nil?
-        case input_types[k].to_sym
-        when :tsv, :array, :file, :text
-          fixed_inputs[k] = (String === v and Open.exists?(v)) ? Open.open(v) : v
-        else
-          fixed_inputs[k] = v
-        end
-      end
-    end
-
-    stream_input = @can_stream ? task_info(task)[:input_options].select{|k,o| o[:stream] }.collect{|k,o| k }.first : nil
-    RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task), @stream_exports.include?(task), stream_input)
-  end
-
-  def load_id(id)
-    task, name = id.split("/")
-    step = RemoteStep.new url, task, nil
-    step.name = name
-    step.result_type = task_info(task)[:result_type]
-    step.result_description = task_info(task)[:result_description]
-    step
-  end
-end