rbbt-util 5.26.157 → 5.26.158

data/lib/rbbt/workflow/remote/rest/get.rb

@@ -1,310 +0,0 @@
-module WorkflowRESTClient
-  def self.encode(url)
-    begin
-      URI.encode(url)
-    rescue
-      Log.warn $!.message
-      url
-    end
-  end
-
-  def self.fix_hash(hash, fix_values = false)
-    fixed = {}
-    hash.each do |key, value|
-      fixed[key.to_sym] = case value
-                          when TrueClass
-                            value
-                          when FalseClass
-                            value
-                          when Hash 
-                            fix_hash(value)  
-                          when (fix_values and String )
-                            value.to_sym
-                          when IO
-                            value.read
-                          when TSV::Dumper
-                            value.stream
-                          when Step
-                            stream = get_stream(value)
-                            stream || value.load
-                          else
-                            value
-                          end
-    end
-    fixed
-  end
-
-  def self.parse_exception(text)
-    klass, message = text.split " => "
-    begin
-      klass = Kernel.const_get klass
-      return klass.new message
-    rescue
-      message
-    end
-  end
-
-  def self.capture_exception
-    begin
-      yield
-    rescue Exception => e
-      raise e unless e.respond_to? :response
-      begin
-        ne = parse_exception e.response.to_s
-        case ne
-        when String
-          raise e.class, ne
-        when Exception
-          raise ne
-        else
-          raise
-        end
-      rescue
-        raise e
-      end
-      raise $!
-    end
-  end
-
-  def self.fix_params(params)
-    new_params = {}
-    params.each do |k,v|
-      if Array === v and v.empty?
-        new_params[k] = "EMPTY_ARRAY"
-      else
-        new_params[k] = v
-      end
-    end
-    new_params
-  end
-
-  def self.clean_url(url, params = {})
-    params = params.merge({ :_format => 'json', :update => 'clean' })
-    params = fix_params params
-    res = capture_exception do
-      Misc.insist(2, 0.5) do
-        Log.debug{ "RestClient clean: #{ url } - #{Misc.fingerprint params}" }
-        res = begin 
-                RestClient.get(self.encode(url), :params => params)
-              rescue RestClient::NotFound
-                return nil
-              end
-        raise TryAgain if res.code == 202
-        res
-      end
-    end
-    res
-  end
- 
-  def self.get_raw(url, params = {})
-    params = params.merge({ :_format => 'raw' })
-    params = fix_params params
-    res = capture_exception do
-      Misc.insist(2, 0.5) do
-        raise "No url" if url.nil?
-        Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
-        res = RestClient.get(self.encode(url), :params => params)
-        raise TryAgain if res.code == 202
-        res.to_s
-      end
-    end
-    res
-  end
- 
-  def self.get_json(url, params = {})
-    Log.debug{ "RestClient get_json: #{ url } - #{Misc.fingerprint params }" }
-    params = params.merge({ :_format => 'json' })
-    params = fix_params params
-
-    res = capture_exception do
-      Misc.insist(2, 0.5) do
-        RestClient.get(self.encode(url), :params => params)
-      end
-    end
-
-    begin
-      JSON.parse(res)
-    rescue
-      res
-    end
-  end
-
-  def self.post_jobname(url, params = {})
-    Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
-    params = params.merge({ :_format => 'jobname' })
-    params = fix_params params
-
-    WorkflowRESTClient.__prepare_inputs_for_restclient(params)
-    name = capture_exception do
-      RestClient.post(self.encode(url), params)
-    end
-
-    Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
-
-    name
-  end
-
-  def self.post_json(url, params = {})
-    if url =~ /_cache_type=:exec/
-      JSON.parse(Open.open(url, :nocache => true))
-    else
-      params = params.merge({ :_format => 'json' })
-      params = fix_params params
-
-      res = capture_exception do
-        RestClient.post(self.encode(url), params)
-      end
-
-      begin
-        JSON.parse(res)
-      rescue
-        res
-      end
-    end
-  end
-
-  def get
-    params ||= {}
-    params = params.merge(:_format => [:string, :boolean, :tsv, :annotations, :array].include?(result_type.to_sym) ? :raw : :json )
-    @cache_result ||= Persist.persist("REST persist", :binary, :file => cache_file + "." + Misc.obj2digest(params)) do
-      Misc.insist 3, rand(2) + 1 do
-        begin
-          init_job if url.nil?
-          @adaptor.get_raw(url, params)
-        rescue
-          Log.exception $!
-          raise $!
-        end
-      end
-    end
-  end
-
-  def load
-    params = {}
-    join unless done? or streaming?
-    raise get_exception if error? or aborted?
-    load_res get
-  end
-
-  def exec_job
-    res = _run_job(:exec)
-    load_res res, result_type == :array ? :json : result_type
-  end
-
-  def abort
-    return self if status == :done
-    @adaptor.get_json(@url + '?_update=abort') if @url and @name
-    self
-  end
-
-  def init_job(cache_type = nil, other_params = {})
-    cache_type = :asynchronous if cache_type.nil? and not @is_exec
-    cache_type = :exec if cache_type.nil?
-    @last_info_time = nil
-    @done = false
-    get_streams
-    @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
-      Misc.insist do
-        @adaptor.post_jobname(File.join(base_url, task.to_s), inputs.merge(other_params).merge(:jobname => @name||@base_name, :_cache_type => cache_type))
-      end
-    end
-    if Open.remote? @name
-      @url = @name
-      @name = File.basename(@name)
-    else
-      @url = File.join(base_url, task.to_s, @name)
-    end
-    self
-  end
-
-  def recursive_clean
-    Log.warn "Not doing recursive cleans"
-    return
-    begin
-      _restart
-      params = {:_update => :recursive_clean}
-      @adaptor.get_raw(url, params)
-    rescue Exception
-      Log.exception $!
-    end
-    self
-  end
-
-  def _clean
-    begin
-      _restart
-      cache_files.each do |cache_file|
-        Open.rm cache_file
-      end
-      params = {:_update => :clean}
-      @adaptor.clean_url(url, params) if @url
-    rescue Exception
-      Log.exception $!
-    end
-  end
-
-  def clean
-    init_job
-    _clean
-    self
-  end
-
-  def stream_job(task_url, task_params, stream_input, cache_type = :exec)
-    require 'rbbt/util/misc/multipart_payload'
-    WorkflowRESTClient.capture_exception do
-      @streaming = true
-
-      Log.debug{ "RestClient stream #{Process.pid}: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint task_params}" }
-      res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
-      type = res.gets
-
-      out = case type.strip
-            when "LOCATION"
-              @url = res.gets
-              @url.sub!(/\?.*/,'')
-              join
-              WorkflowRESTClient.get_raw(@url)
-              @done = true
-              @streaming = false
-            when /STREAM: (.*)/
-              @url = $1.strip
-              res.callback = Proc.new do
-                Log.medium "Done streaming result from #{@url}"
-                @done = true
-                @streaming = false
-              end
-              res
-            when "BULK"
-              begin
-                res.read
-              ensure
-                @done = true
-                @streaming = false
-              end
-            else
-              raise "What? " + type
-            end
-
-      ConcurrentStream.setup(out, :filename => @url)
-
-      out
-    end
-  end
-
-  def _run_job(cache_type = :async)
-    get_streams
-
-    task_url = URI.encode(File.join(base_url, task.to_s))
-    @adaptor.__prepare_inputs_for_restclient(inputs)
-    task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
-
-    if cache_type == :stream or cache_type == :exec and stream_input and inputs[stream_input]
-      io =  self.stream_job(task_url, task_params, stream_input, cache_type) 
-      return io
-    else
-      @adaptor.execute_job(base_url, task, task_params, cache_type)
-    end
-
-  end
-
-end

data/lib/rbbt/workflow/remote/ssh/adaptor.rb

@@ -1,61 +0,0 @@
-require 'rbbt/workflow/remote/ssh/driver'
-
-module WorkflowSSHClient
-  def workflow_description
-    WorkflowSSHClient.get_raw(File.join(url, 'description'))
-  end
-
-  def documentation
-    @documention ||= IndiferentHash.setup(WorkflowSSHClient.get_json(File.join(url, "documentation")))
-    @documention
-  end
-
-  def task_info(task)
-    @task_info ||= IndiferentHash.setup({})
-    
-    if @task_info[task].nil?
-      task_info = WorkflowSSHClient.get_json(File.join(@base_url, task.to_s))
-      task_info = WorkflowSSHClient.fix_hash(task_info)
-
-      task_info[:result_type] = task_info[:result_type].to_sym if task_info[:result_type]
-      task_info[:export] = task_info[:export].to_sym if task_info[:export]
-      task_info[:input_types] = WorkflowSSHClient.fix_hash(task_info[:input_types], true)
-      task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
-
-      @task_info[task] = IndiferentHash.setup(task_info)
-    end
-
-    IndiferentHash.setup(@task_info[task])
-  end
-
-  def tasks
-    @tasks ||= Hash.new do |hash,task_name| 
-      info = @task_info[task_name]
-      task = Task.setup info do |*args|
-        raise "This is a remote task" 
-      end
-      task.name = task_name.to_sym
-      hash[task_name] = task
-    end
-  end
-
-  def load_tasks
-    @task_info.keys.each{|name| tasks[name]}
-  end
-
-  def task_dependencies
-    @task_dependencies ||= Hash.new do |hash,task| 
-      hash[task] = if exported_tasks.include? task
-        WorkflowSSHClient.get_json(File.join(url, task.to_s, 'dependencies'))
-      else
-        []
-      end
-    end
-  end
-
-  def init_remote_tasks
-    @task_info = IndiferentHash.setup(WorkflowSSHClient.get_json(url))
-    @exec_exports = @stream_exports = @synchronous_exports = []
-    @asynchronous_exports = @task_info.keys
-  end
-end

data/lib/rbbt/workflow/remote/ssh/driver.rb

@@ -1,147 +0,0 @@
-module SSHDriver
-  def self.run(server, script)
-    Log.debug "Run ssh script in #{server}:\n#{script}"
-    CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -i -c \"ruby\"' ", :in => script, :log => true).read
-  end
-
-  #def self.run_log(server, script)
-  #  Log.debug "Run and monitor ssh script in #{server}:\n#{script}"
-  #  CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -ic \"ruby\"' ", :in => script, :log => true)
-  #end
-
-  def self.parse_url(url)
-    m = url.match(/ssh:\/\/([^:]+):(.*)/)
-    server = m.captures[0]
-    path = m.captures[1]
-    [server, path]
-  end
-
-  def self.path_script(path) 
-
-    workflow, task, job, *rest = path.split("/")
-    
-    script =<<-EOF
-require 'rbbt/workflow'
-wf = Workflow.require_workflow "#{workflow}"
-    EOF
-
-    case task
-    when nil
-      script +=<<-EOF
-task_info = {}
-wf.tasks.keys.each do |task|
-  task_info[task] = wf.task_info(task)
-end
-res = task_info
-      EOF
-    when 'documentation'
-      script +=<<-EOF
-res = documentation = wf.documentation
-      EOF
-    else
-      if job.nil?
-        script +=<<-EOF
-task = '#{task}'
-res = task_info = wf.task_info(task)
-        EOF
-      else
-        case rest.first
-        when nil
-          script +=<<-EOF
-task = '#{task}'
-jobname = '#{job}'
-res = job = wf.fast_load_id(File.join(task, jobname))
-          EOF
-        when "info"
-          script +=<<-EOF
-task = '#{task}'
-jobname = '#{job}'
-job = wf.fast_load_id(File.join(task, jobname))
-res = job_info = job.info
-          EOF
-        else
-          raise "Unkown path: #{[path, rest].inspect}"
-        end
-      end
-    end
-  end
-
-  def self.job_script(inputs_id, jobname = nil)
-    script =<<-EOF
-jobname = #{jobname.nil? ? 'nil' : "'#{jobname}'"}
-path = File.join(ENV["HOME"], '.rbbt/tmp/tmp-ssh_job_inputs/#{inputs_id}')
-job_inputs = Workflow.load_inputs(path, task_info[:inputs], task_info[:input_types])
-job = wf.job(task, jobname, job_inputs)
-    EOF
-    script
-  end
-
-  def self.get_json(url, params)
-    server, path = parse_url(url)
-    script = path_script(path)
-
-    script +=<<-EOF
-STDOUT.write res.to_json
-    EOF
-
-    JSON.parse(self.run(server, script))
-  end
-
-  def self.get_raw(url, params)
-    server, path = parse_url(url)
-    script = path_script(path)
-
-    script +=<<-EOF
-STDOUT.write res
-    EOF
-
-    self.run(server, script)
-  end
-
-  def self.post_job(url, inputs_id, jobname = nil)
-    server, path = parse_url(url)
-
-    script = path_script(path)
-    script += job_script(inputs_id, jobname)
-    script +=<<-EOF
-job.init_info
-STDOUT.write job.name
-    EOF
-    @name = self.run(server, script)
-  end
-
-  def self.run_job(url, input_id, jobname = nil)
-    server, path = parse_url(url)
-    
-    script = path_script(path)
-    script += job_script(input_id, jobname)
-    script +=<<-EOF
-job.produce
-STDOUT.write job.path
-    EOF
-    self.run(server, script)
-  end
-
-  def self.run_slurm_job(url, input_id, jobname = nil)
-    server, path = parse_url(url)
-    
-    script = path_script(path)
-    script += job_script(input_id, jobname)
-    script +=<<-EOF
-job.produce
-STDOUT.write job.path
-    EOF
-    self.run(server, script)
-  end
-
-  def self.clean(url, input_id, jobname = nil)
-    server, path = parse_url(url)
-    
-    script = path_script(path)
-    script +=<<-EOF
-job.clean
-    EOF
-    self.run(server, script)
-  end
-
-end

data/lib/rbbt/workflow/remote/ssh/get.rb

@@ -1,193 +0,0 @@
-module WorkflowSSHClient
-  attr_accessor :override_dependencies
-
-  def self.fix_hash(hash, fix_values = false)
-    fixed = {}
-    hash.each do |key, value|
-      fixed[key.to_sym] = case value
-                          when TrueClass
-                            value
-                          when FalseClass
-                            value
-                          when Hash 
-                            fix_hash(value)  
-                          when (fix_values and String )
-                            value.to_sym
-                          when IO
-                            value.read
-                          when TSV::Dumper
-                            value.stream
-                          when Step
-                            stream = get_stream(value)
-                            stream || value.load
-                          else
-                            value
-                          end
-    end
-    fixed
-  end
-
-  def self.parse_exception(text)
-    klass, message = text.split " => "
-    begin
-      klass = Kernel.const_get klass
-      return klass.new message
-    rescue
-      message
-    end
-  end
-
-  def self.capture_exception
-    begin
-      yield
-    rescue Exception => e
-      raise e unless e.respond_to? :response
-      begin
-        ne = parse_exception e.response.to_s
-        case ne
-        when String
-          raise e.class, ne
-        when Exception
-          raise ne
-        else
-          raise
-        end
-      rescue
-        raise e
-      end
-      raise $!
-    end
-  end
-
-  def self.fix_params(params)
-    new_params = {}
-    params.each do |k,v|
-      if Array === v and v.empty?
-        new_params[k] = "EMPTY_ARRAY"
-      else
-        new_params[k] = v
-      end
-    end
-    new_params
-  end
-
-  def self.get_json(url, params = {})
-    Log.debug{ "SSHClient get_json: #{ url } - #{Misc.fingerprint params }" }
-    params = params.merge({ :_format => 'json' })
-    params = fix_params params
-
-    res = capture_exception do
-      Misc.insist(2, 0.5) do
-        SSHDriver.get_json(url, :params => params)
-      end
-    end
-
-    begin
-      JSON.parse(res)
-    rescue
-      res
-    end
-  end
-
-  def self.upload_inputs(server, inputs, input_types, input_id)
-    TmpFile.with_file do |dir|
-      if Step.save_inputs(inputs, input_types, dir)
-        CMD.cmd("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
-      end
-    end
-  end
-
-  #{{{ RUN
-  
-
-  def init_job(cache_type = nil, other_params = {})
-    cache_type = :asynchronous if cache_type.nil? and not @is_exec
-    cache_type = :exec if cache_type.nil?
-    @last_info_time = nil
-    @done = false
-    @server, @server_path = SSHDriver.parse_url base_url
-    @input_id ||= "inputs-" << rand(100000).to_s
-    @input_types = task_info(task)[:input_types]
-
-    if override_dependencies
-
-      if override_dependencies && override_dependencies.any?
-        override_dependencies.each do |od|
-          name, _sep, value = od.partition("=")
-          inputs[name] = value
-        end
-      end
-      iii inputs
-
-      WorkflowSSHClient.upload_inputs(@server, inputs, @input_types, @input_id)
-    else
-      WorkflowSSHClient.upload_inputs(@server, inputs, @input_types, @input_id)
-    end
-
-    @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
-      Misc.insist do
-        input_types = {}
-        SSHDriver.post_job(File.join(base_url, task.to_s), @input_id, @base_name)
-      end
-    end
-    if Open.remote? @name
-      @url = @name
-      @name = File.basename(@name)
-    else
-      @url = File.join(base_url, task.to_s, @name)
-    end
-    self
-  end
-
-  def path
-    @server, @server_path = SSHDriver.parse_url @base_url
-    "ssh://" + @server + ":" + @remote_path
-  end
-
-  def produce(*args)
-    input_types = {}
-    init_job
-    @remote_path = SSHDriver.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
-    while ! done?
-      sleep 1
-    end
-  end
-
-  def load
-    load_res Open.open(path)
-  end
-
-  def run(*args)
-    produce(*args)
-    self.load unless args.first
-  end
-
-  def clean
-    init_job
-    SSHDriver.clean(@url, @input_id, @base_name) if done?
-    _restart
-  end
-
-  def self.relay(workflow, task, jobname, inputs, server, options = {})
-    options = Misc.add_defaults options, :search_path => 'user'
-    search_path = options[:search_path]
-
-    job = workflow.job(task, jobname, inputs)
-
-    job.dependencies.each do |dep| 
-      dep.produce 
-    end
-
-    override_dependencies = job.dependencies.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
-
-    job.dependencies.each do |dep| 
-      Step.migrate(dep.path, search_path, :target => server)
-    end
-
-    remote = WorkflowRemoteClient.new("ssh://#{server}:#{workflow.to_s}", "#{workflow.to_s}")
-    rjob = remote.job(task, jobname, {})
-    rjob.override_dependencies = override_dependencies
-    rjob.run
-  end
-
-end