rbbt-util 5.26.77 → 5.26.78

data/lib/rbbt/workflow/remote/rest/get.rb

@@ -0,0 +1,305 @@
+module WorkflowRESTClient
+  def self.encode(url)
+    begin
+      URI.encode(url)
+    rescue
+      Log.warn $!.message
+      url
+    end
+  end
+
+  def self.fix_hash(hash, fix_values = false)
+    fixed = {}
+    hash.each do |key, value|
+      fixed[key.to_sym] = case value
+                          when TrueClass
+                            value
+                          when FalseClass
+                            value
+                          when Hash 
+                            fix_hash(value)  
+                          when (fix_values and String )
+                            value.to_sym
+                          when IO
+                            value.read
+                          when TSV::Dumper
+                            value.stream
+                          when Step
+                            stream = get_stream(value)
+                            stream || value.load
+                          else
+                            value
+                          end
+    end
+    fixed
+  end
+
+  def self.parse_exception(text)
+    klass, message = text.split " => "
+    begin
+      klass = Kernel.const_get klass
+      return klass.new message
+    rescue
+      message
+    end
+  end
+
+  def self.capture_exception
+    begin
+      yield
+    rescue Exception => e
+      raise e unless e.respond_to? :response
+      begin
+        ne = parse_exception e.response.to_s
+        case ne
+        when String
+          raise e.class, ne
+        when Exception
+          raise ne
+        else
+          raise
+        end
+      rescue
+        raise e
+      end
+      raise $!
+    end
+  end
+
+  def self.fix_params(params)
+    new_params = {}
+    params.each do |k,v|
+      if Array === v and v.empty?
+        new_params[k] = "EMPTY_ARRAY"
+      else
+        new_params[k] = v
+      end
+    end
+    new_params
+  end
+
+  def self.clean_url(url, params = {})
+    params = params.merge({ :_format => 'json', :update => 'clean' })
+    params = fix_params params
+    res = capture_exception do
+      Misc.insist(2, 0.5) do
+        Log.debug{ "RestClient clean: #{ url } - #{Misc.fingerprint params}" }
+        res = begin 
+                RestClient.get(self.encode(url), :params => params)
+              rescue RestClient::NotFound
+                return nil
+              end
+        raise TryAgain if res.code == 202
+        res
+      end
+    end
+    res
+  end
+ 
+  def self.get_raw(url, params = {})
+    params = params.merge({ :_format => 'raw' })
+    params = fix_params params
+    res = capture_exception do
+      Misc.insist(2, 0.5) do
+        raise "No url" if url.nil?
+        Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
+        res = RestClient.get(self.encode(url), :params => params)
+        raise TryAgain if res.code == 202
+        res.to_s
+      end
+    end
+    res
+  end
+ 
+  def self.get_json(url, params = {})
+    Log.debug{ "RestClient get_json: #{ url } - #{Misc.fingerprint params }" }
+    params = params.merge({ :_format => 'json' })
+    params = fix_params params
+
+    res = capture_exception do
+      Misc.insist(2, 0.5) do
+        RestClient.get(self.encode(url), :params => params)
+      end
+    end
+
+    begin
+      JSON.parse(res)
+    rescue
+      res
+    end
+  end
+
+  def self.post_jobname(url, params = {})
+    Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
+    params = params.merge({ :_format => 'jobname' })
+    params = fix_params params
+
+    WorkflowRESTClient.__prepare_inputs_for_restclient(params)
+    name = capture_exception do
+      RestClient.post(self.encode(url), params)
+    end
+
+    Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
+
+    name
+  end
+
+  def self.post_json(url, params = {})
+    if url =~ /_cache_type=:exec/
+      JSON.parse(Open.open(url, :nocache => true))
+    else
+      params = params.merge({ :_format => 'json' })
+      params = fix_params params
+
+      res = capture_exception do
+        RestClient.post(self.encode(url), params)
+      end
+
+      begin
+        JSON.parse(res)
+      rescue
+        res
+      end
+    end
+  end
+
+  def get
+    params ||= {}
+    params = params.merge(:_format => [:string, :boolean, :tsv, :annotations,:array].include?(result_type.to_sym) ? :raw : :json )
+    Misc.insist 3, rand(2) + 1 do
+      begin
+        init_job if url.nil?
+        @adaptor.get_raw(url, params)
+      rescue
+        Log.exception $!
+        raise $!
+      end
+    end
+  end
+
+  def load
+    params = {}
+    join unless done? or streaming?
+    raise get_exception if error? or aborted?
+    load_res get
+  end
+
+  def exec_job
+    res = _run_job(:exec)
+    load_res res, result_type == :array ? :json : result_type
+  end
+
+  def abort
+    return self if status == :done
+    @adaptor.get_json(@url + '?_update=abort') if @url and @name
+    self
+  end
+
+  def init_job(cache_type = nil, other_params = {})
+    cache_type = :asynchronous if cache_type.nil? and not @is_exec
+    cache_type = :exec if cache_type.nil?
+    @last_info_time = nil
+    @done = false
+    get_streams
+    @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
+      Misc.insist do
+        @adaptor.post_jobname(File.join(base_url, task.to_s), inputs.merge(other_params).merge(:jobname => @name||@base_name, :_cache_type => cache_type))
+      end
+    end
+    if Open.remote? @name
+      @url = @name
+      @name = File.basename(@name)
+    else
+      @url = File.join(base_url, task.to_s, @name)
+    end
+    self
+  end
+
+  def recursive_clean
+    Log.warn "Not doing recursive cleans"
+    return
+    begin
+      _restart
+      params = {:_update => :recursive_clean}
+      @adaptor.get_raw(url, params)
+    rescue Exception
+      Log.exception $!
+    end
+    self
+  end
+
+  def _clean
+    begin
+      _restart
+      params = {:_update => :clean}
+      @adaptor.clean_url(url, params) if @url
+    rescue Exception
+      Log.exception $!
+    end
+  end
+
+  def clean
+    init_job
+    _clean
+    self
+  end
+
+  def stream_job(task_url, task_params, stream_input, cache_type = :exec)
+    require 'rbbt/util/misc/multipart_payload'
+    WorkflowRESTClient.capture_exception do
+      @streaming = true
+
+      Log.debug{ "RestClient stream #{Process.pid}: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint task_params}" }
+      res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
+      type = res.gets
+
+      out = case type.strip
+            when "LOCATION"
+              @url = res.gets
+              @url.sub!(/\?.*/,'')
+              join
+              WorkflowRESTClient.get_raw(@url)
+              @done = true
+              @streaming = false
+            when /STREAM: (.*)/
+              @url = $1.strip
+              res.callback = Proc.new do
+                Log.medium "Done streaming result from #{@url}"
+                @done = true
+                @streaming = false
+              end
+              res
+            when "BULK"
+              begin
+                res.read
+              ensure
+                @done = true
+                @streaming = false
+              end
+            else
+              raise "What? " + type
+            end
+
+      ConcurrentStream.setup(out, :filename => @url)
+
+      out
+    end
+  end
+
+  def _run_job(cache_type = :async)
+    get_streams
+
+    task_url = URI.encode(File.join(base_url, task.to_s))
+    @adaptor.__prepare_inputs_for_restclient(inputs)
+    task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
+
+    if cache_type == :stream or cache_type == :exec and stream_input and inputs[stream_input]
+      io =  self.stream_job(task_url, task_params, stream_input, cache_type) 
+      return io
+    else
+      @adaptor.execute_job(base_url, task, task_params, cache_type)
+    end
+
+  end
+
+end

data/lib/rbbt/workflow/remote/ssh/adaptor.rb

@@ -0,0 +1,78 @@
+module WorkflowSSHClient
+  def self.__prepare_inputs_for_restclient(inputs)
+    inputs.each do |k,v|
+      if v.respond_to? :path and not v.respond_to? :original_filename
+        class << v
+          def original_filename
+            File.expand_path(path)
+          end
+        end
+      end
+
+      if Array === v and v.empty?
+        inputs[k] = "EMPTY_ARRAY"
+      end
+    end
+  end
+
+  def workflow_description
+    WorkflowSSHClient.get_raw(File.join(url, 'description'))
+  end
+
+  def documentation
+    @documention ||= IndiferentHash.setup(WorkflowSSHClient.get_json(File.join(url, "documentation")))
+    @documention
+  end
+
+  def task_info(task)
+    @task_info ||= IndiferentHash.setup({})
+    @task_info[task]
+    
+    if @task_info[task].nil?
+      task_info = WorkflowSSHClient.get_json(File.join(url, task.to_s, 'info'))
+      task_info = WorkflowSSHClient.fix_hash(task_info)
+
+      task_info[:result_type] = task_info[:result_type].to_sym
+      task_info[:export] = task_info[:export].to_sym
+      task_info[:input_types] = WorkflowSSHClient.fix_hash(task_info[:input_types], true)
+      task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
+
+      @task_info[task] = task_info
+    end
+    @task_info[task]
+  end
+
+  def tasks
+    @tasks ||= Hash.new do |hash,task_name| 
+      info = @task_info[task_name]
+      task = Task.setup info do |*args|
+        raise "This is a remote task" 
+      end
+      task.name = task_name.to_sym
+      hash[task_name] = task
+    end
+  end
+
+  def load_tasks
+    @task_info.keys.each{|name| tasks[name]}
+  end
+
+  def task_dependencies
+    @task_dependencies ||= Hash.new do |hash,task| 
+      hash[task] = if exported_tasks.include? task
+        WorkflowSSHClient.get_json(File.join(url, task.to_s, 'dependencies'))
+      else
+        []
+      end
+    end
+  end
+
+  def init_remote_tasks
+    @task_info = IndiferentHash.setup(WorkflowSSHClient.get_json(url))
+    @exec_exports = @stream_exports = @synchronous_exports = []
+    @asynchronous_exports = @task_info.keys
+  end
+
+  def self.execute_job(base_url, task, task_params, cache_type)
+  end
+end

data/lib/rbbt/workflow/remote/ssh/get.rb

@@ -0,0 +1,281 @@
+module SSHClient
+  def self.run(server, script)
+    Log.debug "Run ssh script in #{server}:\n#{script}"
+    CMD.cmd("ssh '#{server}' ruby ", :in => script).read.strip
+  end
+
+  def self.run_log(server, script)
+    Log.debug "Run and monitor ssh script in #{server}:\n#{script}"
+    CMD.cmd_log("ssh '#{server}' ruby ", :in => script)
+  end
+
+  def self.parse_url(url)
+    m = url.match(/ssh:\/\/([^:]+):(.*)/)
+    server = m.captures[0]
+    path = m.captures[1]
+    [server, path]
+  end
+
+  def self.path_script(path) 
+
+    workflow, task, job, *rest = path.split("/")
+    
+    script =<<-EOF
+require 'rbbt/workflow'
+wf = Workflow.require_workflow "#{workflow}"
+    EOF
+    case task
+    when nil
+      script +=<<-EOF
+task_info = {}
+wf.tasks.keys.each do |task|
+  task_info[task] = wf.task_info(task)
+end
+res = task_info
+      EOF
+    when 'documentation'
+      script +=<<-EOF
+res = documentation = wf.documentation
+      EOF
+    else
+      if job.nil?
+        script +=<<-EOF
+task = '#{task}'
+res = task_info = wf.task_info(task)
+        EOF
+      else
+        case rest.first
+        when nil
+          script +=<<-EOF
+task = '#{task}'
+jobname = '#{job}'
+res = job = wf.fast_load_id(File.join(task, jobname))
+          EOF
+        when "info"
+          script +=<<-EOF
+task = '#{task}'
+jobname = '#{job}'
+job = wf.fast_load_id(File.join(task, jobname))
+res = job_info = job.info
+          EOF
+        else
+          raise "Unkown path: #{[path, rest].inspect}"
+        end
+      end
+    end
+  end
+
+  def self.get_json(url, params)
+    server, path = parse_url(url)
+    script = path_script(path)
+
+    script +=<<-EOF
+puts res.to_json
+    EOF
+
+    JSON.parse(self.run(server, script))
+  end
+
+  def self.get_raw(url, params)
+    server, path = parse_url(url)
+    script = path_script(path)
+
+    script +=<<-EOF
+puts res
+    EOF
+
+    self.run(server, script)
+  end
+
+  def self.post_job(url, inputs, input_types, jobname = nil)
+    server, path = parse_url(url)
+    script = path_script(path)
+    
+    id = "inputs-" << rand(100000).to_s
+    CMD.cmd("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/ ", :in => script).read
+    TmpFile.with_file do |dir|
+      if Step.save_inputs(inputs, input_types, dir)
+        CMD.cmd("scp -r '#{dir}' .rbbt/tmp/tmp-ssh_job_inputs/#{id}")
+      end
+    end
+
+    script +=<<-EOF
+jobname = #{jobname.nil? ? 'nil' : "'#{jobname}'"}
+path = '.rbbt/tmp/tmp-ssh_job_inputs/#{id}'
+job_inputs = Workflow.load_inputs(path, task_info[:inputs], task_info[:input_types])
+job = wf.job(task, jobname, job_inputs)
+Log.severity = 10
+job.fork
+puts job.name
+    EOF
+    self.run(server, script)
+  end
+
+  def self.run_job(url, inputs, input_types, jobname = nil)
+    server, path = parse_url(url)
+    script = path_script(path)
+    
+    id = "inputs-" << rand(100000).to_s
+    CMD.cmd("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/ ", :in => script).read
+    TmpFile.with_file do |dir|
+      if Step.save_inputs(inputs, input_types, dir)
+        CMD.cmd("scp -r '#{dir}' .rbbt/tmp/tmp-ssh_job_inputs/#{id}")
+      end
+    end
+
+    script +=<<-EOF
+jobname = #{jobname.nil? ? 'nil' : "'#{jobname}'"}
+path = '.rbbt/tmp/tmp-ssh_job_inputs/#{id}'
+job_inputs = Workflow.load_inputs(path, task_info[:inputs], task_info[:input_types])
+job = wf.job(task, jobname, job_inputs)
+job.run
+    EOF
+    self.run_log(server, script)
+  end
+
+  def self.clean(url)
+    server, path = parse_url(url)
+    script = path_script(path)
+    
+    script +=<<-EOF
+job.clean
+    EOF
+    self.run(server, script)
+  end
+
+end
+
+module WorkflowSSHClient
+  def self.fix_hash(hash, fix_values = false)
+    fixed = {}
+    hash.each do |key, value|
+      fixed[key.to_sym] = case value
+                          when TrueClass
+                            value
+                          when FalseClass
+                            value
+                          when Hash 
+                            fix_hash(value)  
+                          when (fix_values and String )
+                            value.to_sym
+                          when IO
+                            value.read
+                          when TSV::Dumper
+                            value.stream
+                          when Step
+                            stream = get_stream(value)
+                            stream || value.load
+                          else
+                            value
+                          end
+    end
+    fixed
+  end
+
+  def self.parse_exception(text)
+    klass, message = text.split " => "
+    begin
+      klass = Kernel.const_get klass
+      return klass.new message
+    rescue
+      message
+    end
+  end
+
+  def self.capture_exception
+    begin
+      yield
+    rescue Exception => e
+      raise e unless e.respond_to? :response
+      begin
+        ne = parse_exception e.response.to_s
+        case ne
+        when String
+          raise e.class, ne
+        when Exception
+          raise ne
+        else
+          raise
+        end
+      rescue
+        raise e
+      end
+      raise $!
+    end
+  end
+
+  def self.fix_params(params)
+    new_params = {}
+    params.each do |k,v|
+      if Array === v and v.empty?
+        new_params[k] = "EMPTY_ARRAY"
+      else
+        new_params[k] = v
+      end
+    end
+    new_params
+  end
+
+  def self.get_json(url, params = {})
+    Log.debug{ "SSHClient get_json: #{ url } - #{Misc.fingerprint params }" }
+    params = params.merge({ :_format => 'json' })
+    params = fix_params params
+
+    res = capture_exception do
+      Misc.insist(2, 0.5) do
+        SSHClient.get_json(url, :params => params)
+      end
+    end
+
+    begin
+      JSON.parse(res)
+    rescue
+      res
+    end
+  end
+
+
+  def self.post_jobname(url, inputs, input_types)
+    SSHClient.post_job(url, inputs, input_types)
+  end
+
+  def init_job(cache_type = nil, other_params = {})
+    cache_type = :asynchronous if cache_type.nil? and not @is_exec
+    cache_type = :exec if cache_type.nil?
+    @last_info_time = nil
+    @done = false
+    get_streams
+    @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
+      Misc.insist do
+        #@adaptor.post_jobname(File.join(base_url, task.to_s), inputs.merge(other_params).merge(:jobname => @name||@base_name, :_cache_type => cache_type))
+        input_types = {}
+        @adaptor.post_jobname(File.join(base_url, task.to_s), inputs, input_types)
+      end
+    end
+    if Open.remote? @name
+      @url = @name
+      @name = File.basename(@name)
+    else
+      @url = File.join(base_url, task.to_s, @name)
+    end
+    self
+  end
+
+  def path
+    server, path = SSHClient.parse_url(url)
+    "ssh://" + server + ":" + info[:path]
+  end
+
+  def run(*args)
+    input_types = {}
+    SSHClient.run_job(File.join(base_url, task.to_s), inputs, input_types)
+  end
+
+
+  def clean
+    init_job
+    SSHClient.clean(@url) if done?
+    _restart
+  end
+
+end