rbbt-util 5.26.157 → 5.26.158

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a7cb4caf03456a936ef3dab72d6ab957a2f3dce03324291fb10c0f4aabe527f
-  data.tar.gz: 8690112157a89e64313dc7fcec69c7192db507c210bac57cfd529271294b7174
+  metadata.gz: d372a2cf6d424504011197b2b99f84319e79522a5817d3e54d13f61370622b5a
+  data.tar.gz: e8914f26719c718b2885394034a688ec4280aaf59a2dca2bad7e4dabfa744b85
 SHA512:
-  metadata.gz: 68a347659e479c2cd5fdb09c15bbe09be216586c403f984689c6b72a360cb63f3c4815edabfb675362e4137440e67e5d364da836e4cca33c5ce200a5a7d19539
-  data.tar.gz: f77fa581d1b0cd8fd6d34f4ac63101a40162f0d7012e2ff9917c7f5b0de729a56f5c81a011528f676aac4cb59a30ebc09acdf47892bfadf1662d41a22ec2959c
+  metadata.gz: 4b5bbe62b81fdc7fb024b724d4c6c020839b54cb290d9ffe296f0d9841a233d951c44e0687faefc744cc04884672c10ea5d14ad0b32de9bdacfa152db54e4fbe
+  data.tar.gz: 99019bcc2bcedcf1205228277f41324d37731806f608f508a3f8772681c9eb0820e4f860430a1c0b4721f42f199fdf8d2876c8f5e19deac6b2792eb31e234c58

data/lib/rbbt/util/misc/development.rb

@@ -415,4 +415,11 @@ def self.add_libdir(dir=nil)
       CMD.cmd("unzip '#{zip_file}' -d '#{dir}'")
     end
   end
+
+
+  def self.ssh_run(server, script)
+    Log.debug "Run ssh script in #{server}:\n#{script}"
+    CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
+  end
+
 end

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -0,0 +1,230 @@
+require 'rest-client'
+class RemoteWorkflow
+  module REST
+
+    def self.__prepare_inputs_for_restclient(inputs)
+      inputs.each do |k,v|
+        if v.respond_to? :path and not v.respond_to? :original_filename
+          class << v
+            def original_filename
+              File.expand_path(path)
+            end
+          end
+        end
+
+        if Array === v and v.empty?
+          inputs[k] = "EMPTY_ARRAY"
+        end
+      end
+    end
+
+    def self.encode(url)
+      begin
+        URI.encode(url)
+      rescue
+        Log.warn $!.message
+        url
+      end
+    end
+
+    def self.clean_url(url, params = {})
+      params = params.merge({ :_format => 'json', :update => 'clean' })
+      params = fix_params params
+      res = RemoteWorkflow.capture_exception do
+        Misc.insist(2, 0.5) do
+          Log.debug{ "RestClient clean: #{ url } - #{Misc.fingerprint params}" }
+          res = begin 
+                  RestClient.get(self.encode(url), :params => params)
+                rescue RestClient::NotFound
+                  return nil
+                end
+          raise TryAgain if res.code == 202
+          res
+        end
+      end
+      res
+    end
+
+    def self.get_raw(url, params = {})
+      params = params.merge({ :_format => 'raw' })
+      params = fix_params params
+      res = RemoteWorkflow.capture_exception do
+        Misc.insist(2, 0.5) do
+          raise "No url" if url.nil?
+          Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
+          res = RestClient.get(self.encode(url), :params => params)
+          raise TryAgain if res.code == 202
+          res.to_s
+        end
+      end
+      res
+    end
+
+    def self.get_json(url, params = {})
+      Log.debug{ "RestClient get_json: #{ url } - #{Misc.fingerprint params }" }
+      params = params.merge({ :_format => 'json' })
+      params = RemoteWorkflow.fix_params params
+
+      res = RemoteWorkflow.capture_exception do
+        Misc.insist(2, 0.5) do
+          RestClient.get(self.encode(url), :params => params)
+        end
+      end
+
+      begin
+        JSON.parse(res)
+      rescue
+        res
+      end
+    end
+
+    def self.post_jobname(url, params = {})
+      Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
+      params = params.merge({ :_format => 'jobname' })
+      params = RemoteWorkflow.fix_params params
+
+      WorkflowRESTClient.__prepare_inputs_for_restclient(params)
+      name = RemoteWorkflow.capture_exception do
+        RestClient.post(self.encode(url), params)
+      end
+
+      Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
+
+      name
+    end
+
+    def self.post_json(url, params = {})
+      if url =~ /_cache_type=:exec/
+        JSON.parse(Open.open(url, :nocache => true))
+      else
+        params = params.merge({ :_format => 'json' })
+        params = fix_params params
+
+        res = RemoteWorkflow.capture_exception do
+          RestClient.post(self.encode(url), params)
+        end
+
+        begin
+          JSON.parse(res)
+        rescue
+          res
+        end
+      end
+    end
+
+    def self.task_info(url, task)
+      @@task_info ||= {}
+
+      key = [url, task] * "#"
+      @@task_info[key] ||= begin
+                             task_info = RemoteWorkflow::REST.get_json(File.join(url, task.to_s, 'info'))
+                             task_info = RemoteWorkflow.fix_hash(task_info)
+
+                             task_info[:result_type] = task_info[:result_type].to_sym
+                             task_info[:export] = task_info[:export].to_sym
+                             task_info[:input_types] = RemoteWorkflow.fix_hash(task_info[:input_types], true)
+                             task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
+
+                             task_info
+                           end
+    end
+
+    def self.execute_job(base_url, task, task_params, cache_type)
+      RemoteWorkflow.capture_exception do
+        task_url = URI.encode(File.join(base_url, task.to_s))
+
+        sout, sin = Misc.pipe
+
+        post_thread = Thread.new(Thread.current) do |parent|
+          bl = lambda do |rok|
+            if Net::HTTPOK === rok
+              _url = rok["RBBT-STREAMING-JOB-URL"]
+              @url = File.join(task_url, File.basename(_url)) if _url
+              rok.read_body do |c,_a, _b|
+                sin.write c
+              end
+              sin.close
+            else
+              err = StringIO.new
+              rok.read_body do |c,_a, _b|
+                err.write c
+              end
+              text = begin
+                       reader = Zlib::GzipReader.new(err)
+                       reader.read
+                     rescue
+                       err.rewind
+                       err.read
+                     end
+              ne = @adaptor.parse_exception text
+              case ne
+              when String
+                parent.raise e.class, ne
+              when Exception
+                parent.raise ne
+              else
+                parent.raise "Error in RestClient: " << rok.message
+              end
+            end
+          end
+
+          task_params.each do |k,v|
+            task_params[k] = v.read if IO === v
+          end
+
+          Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
+          RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
+        end
+
+        # It seems like now response body are now decoded by Net::HTTP after 2.1
+        # https://github.com/rest-client/rest-client/blob/cf3e5a115bcdb8f3344aeac0e45b44d67fac1a42/history.md
+        decode = Gem.loaded_specs["rest-client"].version < Gem::Version.create('2.1')
+        if decode
+          reader = Zlib::GzipReader.new(sout)
+          res_io = Misc.open_pipe do |sin|
+            while c = reader.read(Misc::BLOCK_SIZE)
+              sin.write c
+            end
+            sin.close
+            @done = true
+          end
+          ConcurrentStream.setup(res_io, :threads => [post_thread]) do
+            @done = true
+            @streaming = false
+          end
+        else
+          ConcurrentStream.setup(sout, :threads => [post_thread]) do
+            @done = true
+            @streaming = false
+          end
+        end
+
+      end
+    end
+
+    def tasks
+      @tasks ||= Hash.new do |hash,task_name| 
+        info = task_info(task_name)
+        task = Task.setup info do |*args|
+          raise "This is a remote task" 
+        end
+        task.name = task_name.to_sym
+        hash[task_name] = task
+      end
+    end
+
+
+    def task_info(task)
+      RemoteWorkflow::REST.task_info(url, task)
+    end
+
+    def init_remote_tasks
+      task_exports = RemoteWorkflow::REST.get_json(url)
+      @asynchronous_exports = (task_exports["asynchronous"] || []).collect{|task| task.to_sym }
+      @synchronous_exports = (task_exports["synchronous"] || []).collect{|task| task.to_sym }
+      @exec_exports = (task_exports["exec"] || []).collect{|task| task.to_sym }
+      @stream_exports = (task_exports["stream"] || []).collect{|task| task.to_sym }
+      @can_stream = task_exports["can_stream"]
+    end
+  end
+end

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -0,0 +1,237 @@
+class RemoteWorkflow
+  module SSH
+    #def self.run(server, script)
+    #  Log.debug "Run ssh script in #{server}:\n#{script}"
+    #  CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -i -c \"ruby\"' ", :in => script, :log => true).read
+    #end
+
+    #def self.run_log(server, script)
+    #  Log.debug "Run and monitor ssh script in #{server}:\n#{script}"
+    #  CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -ic \"ruby\"' ", :in => script, :log => true)
+    #end
+
+    def self.parse_url(url)
+      m = url.match(/ssh:\/\/([^:]+):(.*)/)
+      server = m.captures[0]
+      path = m.captures[1]
+      [server, path]
+    end
+
+    def self.path_script(path) 
+
+      workflow, task, job, *rest = path.split("/")
+
+      script =<<-EOF
+require 'rbbt/workflow'
+wf = Workflow.require_workflow "#{workflow}"
+      EOF
+
+      case task
+      when nil
+        script +=<<-EOF
+task_info = {}
+wf.tasks.keys.each do |task|
+  task_info[task] = wf.task_info(task)
+end
+res = task_info
+        EOF
+      when 'documentation'
+        script +=<<-EOF
+res = documentation = wf.documentation
+        EOF
+      else
+        if job.nil?
+          script +=<<-EOF
+task = '#{task}'
+res = task_info = wf.task_info(task)
+          EOF
+        else
+          case rest.first
+          when nil
+            script +=<<-EOF
+task = '#{task}'
+jobname = '#{job}'
+res = job = wf.fast_load_id(File.join(task, jobname))
+            EOF
+          when "info"
+            script +=<<-EOF
+task = '#{task}'
+jobname = '#{job}'
+job = wf.fast_load_id(File.join(task, jobname))
+res = job_info = job.info
+            EOF
+          else
+            raise "Unkown path: #{[path, rest].inspect}"
+          end
+        end
+      end
+    end
+
+    def self.job_script(inputs_id, jobname = nil)
+      script =<<-EOF
+jobname = #{jobname.nil? ? 'nil' : "'#{jobname}'"}
+path = File.join(ENV["HOME"], '.rbbt/tmp/tmp-ssh_job_inputs/#{inputs_id}')
+job_inputs = Workflow.load_inputs(path, task_info[:inputs], task_info[:input_types])
+job = wf.job(task, jobname, job_inputs)
+      EOF
+      script
+    end
+
+    def self.get_json(url, params = {})
+      server, path = parse_url(url)
+      script = path_script(path)
+
+      script +=<<-EOF
+STDOUT.write res.to_json
+      EOF
+
+      JSON.parse(Misc.ssh_run(server, script))
+    end
+
+    def self.get_raw(url, params)
+      server, path = parse_url(url)
+      script = path_script(path)
+
+      script +=<<-EOF
+STDOUT.write res
+      EOF
+
+      Misc.ssh_run(server, script)
+    end
+
+    def self.post_job(url, inputs_id, jobname = nil)
+      server, path = parse_url(url)
+
+      script = path_script(path)
+      script += job_script(inputs_id, jobname)
+      script +=<<-EOF
+job.init_info
+STDOUT.write job.name
+      EOF
+      @name = Misc.ssh_run(server, script)
+    end
+
+    def self.run_job(url, input_id, jobname = nil)
+      server, path = parse_url(url)
+
+      script = path_script(path)
+      script += job_script(input_id, jobname)
+      script +=<<-EOF
+job.produce
+STDOUT.write job.path
+      EOF
+      Misc.ssh_run(server, script)
+    end
+
+    def self.run_slurm_job(url, input_id, jobname = nil)
+      server, path = parse_url(url)
+
+      script = path_script(path)
+      script += job_script(input_id, jobname)
+      script +=<<-EOF
+job.produce
+STDOUT.write job.path
+      EOF
+      Misc.ssh_run(server, script)
+    end
+
+    def self.clean(url, input_id, jobname = nil)
+      server, path = parse_url(url)
+
+      script = path_script(path)
+      script +=<<-EOF
+job.clean
+      EOF
+      Misc.ssh_run(server, script)
+    end
+
+    def self.upload_inputs(server, inputs, input_types, input_id)
+      TmpFile.with_file do |dir|
+        if Step.save_inputs(inputs, input_types, dir)
+          CMD.cmd("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
+        end
+      end
+    end
+
+    def self.relay(workflow, task, jobname, inputs, server, options = {})
+      options = Misc.add_defaults options, :search_path => 'user'
+      search_path = options[:search_path]
+
+      job = workflow.job(task, jobname, inputs)
+
+      job.dependencies.each do |dep| 
+        dep.produce 
+      end
+
+      override_dependencies = job.dependencies.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
+
+      job.dependencies.each do |dep| 
+        Step.migrate(dep.path, search_path, :target => server)
+      end
+
+      remote = RemoteWorkflow.new("ssh://#{server}:#{workflow.to_s}", "#{workflow.to_s}")
+      rjob = remote.job(task, jobname, {})
+      rjob.override_dependencies = override_dependencies
+      rjob.run
+    end
+
+    def workflow_description
+      RemoteWorkflow::SSH.get_raw(File.join(url, 'description'))
+    end
+
+    def documentation
+      @documention ||= IndiferentHash.setup(RemoteWorkflow::SSH.get_json(File.join(url, "documentation")))
+      @documention
+    end
+
+    def task_info(task)
+      @task_info ||= IndiferentHash.setup({})
+
+      if @task_info[task].nil?
+        task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url, task.to_s))
+        task_info = RemoteWorkflow::SSH.fix_hash(task_info)
+
+        task_info[:result_type] = task_info[:result_type].to_sym if task_info[:result_type]
+        task_info[:export] = task_info[:export].to_sym if task_info[:export]
+        task_info[:input_types] = RemoteWorkflow::SSH.fix_hash(task_info[:input_types], true)
+        task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
+
+        @task_info[task] = IndiferentHash.setup(task_info)
+      end
+
+      IndiferentHash.setup(@task_info[task])
+    end
+
+    def tasks
+      @tasks ||= Hash.new do |hash,task_name| 
+        raise Workflow::TaskNotFoundException, "Task #{task_name} not found in workflow #{self.to_s}" unless @task_info.include?(task_name)
+        info = @task_info[task_name]
+        task = Task.setup info do |*args|
+          raise "This is a remote task" 
+        end
+        task.name = task_name.to_sym
+        hash[task_name] = task
+      end
+    end
+
+    def load_tasks
+      @task_info.keys.each{|name| tasks[name]}
+    end
+
+    def task_dependencies
+      @task_dependencies ||= Hash.new do |hash,task| 
+        hash[task] = if exported_tasks.include? task
+                       RemoteWorkflow::SSH.get_json(File.join(url, task.to_s, 'dependencies'))
+                     else
+                       []
+                     end
+      end
+    end
+
+    def init_remote_tasks
+      @task_info = IndiferentHash.setup(RemoteWorkflow::SSH.get_json(url))
+      @exec_exports = @stream_exports = @synchronous_exports = []
+      @asynchronous_exports = @task_info.keys
+    end
+  end
+end

data/lib/rbbt/workflow/remote_workflow/driver.rb

@@ -0,0 +1,75 @@
+require 'rbbt/workflow/remote_workflow/driver/rest'
+require 'rbbt/workflow/remote_workflow/driver/ssh'
+
+class RemoteWorkflow
+  def self.fix_hash(hash, fix_values = false)
+    fixed = {}
+    hash.each do |key, value|
+      fixed[key.to_sym] = case value
+                          when TrueClass
+                            value
+                          when FalseClass
+                            value
+                          when Hash 
+                            fix_hash(value)  
+                          when (fix_values and String )
+                            value.to_sym
+                          when IO
+                            value.read
+                          when TSV::Dumper
+                            value.stream
+                          when Step
+                            stream = get_stream(value)
+                            stream || value.load
+                          else
+                            value
+                          end
+    end
+    fixed
+  end
+
+  def self.parse_exception(text)
+    klass, message = text.split " => "
+    begin
+      klass = Kernel.const_get klass
+      return klass.new message
+    rescue
+      message
+    end
+  end
+
+  def self.capture_exception
+    begin
+      yield
+    rescue Exception => e
+      raise e unless e.respond_to? :response
+      begin
+        ne = parse_exception e.response.to_s
+        case ne
+        when String
+          raise e.class, ne
+        when Exception
+          raise ne
+        else
+          raise
+        end
+      rescue
+        raise e
+      end
+      raise $!
+    end
+  end
+
+  def self.fix_params(params)
+    new_params = {}
+    params.each do |k,v|
+      if Array === v and v.empty?
+        new_params[k] = "EMPTY_ARRAY"
+      else
+        new_params[k] = v
+      end
+    end
+    new_params
+  end
+
+end