rbbt-util 5.26.157 → 5.26.158

data/test/rbbt/workflow/test_remote_workflow.rb

@@ -0,0 +1,86 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/workflow'
+require 'rbbt/workflow/remote_workflow'
+
+class TestWFRest
+  extend Workflow
+
+  input :name, :string, "Name to greet", "World"
+  task :hi => :string do |name|
+    "Hello #{name}"
+  end
+
+  dep :hi
+  task :intro => :string do 
+    step(:hi).load + ", nice to meet you"
+  end
+end
+
+class TestRemoteWorkflow < Test::Unit::TestCase
+
+  def remote_workflow_server(workflow, options = {}, &block)
+    trap(:USR1){ raise TestServerLoaded}
+
+    begin
+      pid = Process.fork do 
+        TmpFile.with_file do |app_dir|
+          Misc.in_dir(app_dir) do
+            require 'rack'
+            ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
+
+            app_dir = Path.setup(app_dir.dup)
+            Open.write(app_dir.etc.target_workflow.find, workflow.to_s)
+
+            config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
+            options[:config] = config_ru_file
+            app = Rack::Server.new(options)
+            app.start do
+              Process.kill :USR1, Process.ppid
+            end
+          end
+        end
+      end
+
+      begin
+        sleep 1 while true
+      rescue TestServerLoaded
+      end
+
+      client = RemoteWorkflow.new "http://localhost:#{options[:Port] || 9292}/#{workflow.to_s}", workflow.to_s
+
+      yield client
+
+    rescue
+      Log.exception $!
+    ensure
+      Process.kill :INT, pid
+      Process.wait pid
+    end
+  end
+
+  def test_rest
+    Log.with_severity 0 do
+
+      remote_workflow_server(TestWFRest) do |client|
+        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
+        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
+        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
+      end
+
+      remote_workflow_server(TestWFRest, :Port => 1902) do |client|
+        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
+        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
+        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
+      end
+    end
+  end
+
+
+  def _test_ssh
+    Log.severity = 0
+    client = RemoteWorkflow.new "ssh://turbo:Translation", "Translation"
+    job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])
+    assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
+  end
+end
+

data/test/test_helper.rb

@@ -50,36 +50,39 @@ class Test::Unit::TestCase
   def workflow_server(workflow, options = {}, &block)
     trap(:USR1){ raise TestServerLoaded}
 
-    pid = Process.fork do 
-      TmpFile.with_file do |app_dir|
-        Misc.in_dir(app_dir) do
-          require 'rack'
-          ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
-
-          app_dir = Path.setup(app_dir.dup)
-          Open.write(app_dir.etc.target_workflow.find, workflow.to_s)
-
-          config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
-          options[:config] = config_ru_file
-          app = Rack::Server.new(options)
-          app.start do
-            Process.kill :USR1, Process.ppid
+    begin
+      pid = Process.fork do 
+        TmpFile.with_file do |app_dir|
+          Misc.in_dir(app_dir) do
+            require 'rack'
+            ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
+
+            app_dir = Path.setup(app_dir.dup)
+            Open.write(app_dir.etc.target_workflow.find, workflow.to_s)
+
+            config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
+            options[:config] = config_ru_file
+            app = Rack::Server.new(options)
+            app.start do
+              Process.kill :USR1, Process.ppid
+            end
           end
         end
       end
-    end
 
-    begin
-      sleep 1 while true
-    rescue TestServerLoaded
-    end
+      begin
+        sleep 1 while true
+      rescue TestServerLoaded
+      end
 
-    client = WorkflowRemoteClient.new "http://localhost:#{options[:Port] || 9292}/#{workflow.to_s}", workflow.to_s
+      client = WorkflowRemoteClient.new "http://localhost:#{options[:Port] || 9292}/#{workflow.to_s}", workflow.to_s
 
-    yield client
+      yield client
 
-    Process.kill :INT, pid
-    Process.wait pid
+    ensure
+      Process.kill :INT, pid
+      Process.wait pid
+    end
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.157
+  version: 5.26.158
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-24 00:00:00.000000000 Z
+date: 2020-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -236,11 +236,6 @@ files:
 - lib/rbbt/resource/rake.rb
 - lib/rbbt/resource/util.rb
 - lib/rbbt/resource/with_key.rb
-- lib/rbbt/rest/client.rb
-- lib/rbbt/rest/client/adaptor.rb
-- lib/rbbt/rest/client/get.rb
-- lib/rbbt/rest/client/run.rb
-- lib/rbbt/rest/client/step.rb
 - lib/rbbt/tsv.rb
 - lib/rbbt/tsv/accessor.rb
 - lib/rbbt/tsv/attach.rb
@@ -324,13 +319,13 @@ files:
 - lib/rbbt/workflow/examples.rb
 - lib/rbbt/workflow/integration/cromwell.rb
 - lib/rbbt/workflow/integration/nextflow.rb
-- lib/rbbt/workflow/remote/client.rb
-- lib/rbbt/workflow/remote/remote_step.rb
-- lib/rbbt/workflow/remote/rest/adaptor.rb
-- lib/rbbt/workflow/remote/rest/get.rb
-- lib/rbbt/workflow/remote/ssh/adaptor.rb
-- lib/rbbt/workflow/remote/ssh/driver.rb
-- lib/rbbt/workflow/remote/ssh/get.rb
+- lib/rbbt/workflow/remote_workflow.rb
+- lib/rbbt/workflow/remote_workflow/driver.rb
+- lib/rbbt/workflow/remote_workflow/driver/rest.rb
+- lib/rbbt/workflow/remote_workflow/driver/ssh.rb
+- lib/rbbt/workflow/remote_workflow/remote_step.rb
+- lib/rbbt/workflow/remote_workflow/remote_step/rest.rb
+- lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb
 - lib/rbbt/workflow/schedule.rb
 - lib/rbbt/workflow/soap.rb
 - lib/rbbt/workflow/step.rb
@@ -515,6 +510,7 @@ files:
 - test/rbbt/workflow/remote/test_client.rb
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_remote_workflow.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_task.rb
 - test/test_helper.rb
@@ -543,6 +539,7 @@ specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
 - test/rbbt/test_entity.rb
+- test/rbbt/workflow/test_remote_workflow.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/remote/test_client.rb

data/lib/rbbt/rest/client/adaptor.rb

@@ -1,67 +0,0 @@
-class WorkflowRESTClient
-
-  def workflow_description
-    WorkflowRESTClient.get_raw(File.join(url, 'description'))
-  end
-
-  def documentation
-    @documention ||= IndiferentHash.setup(WorkflowRESTClient.get_json(File.join(url, "documentation"),{}))
-  end
-
-  def task_info(task)
-    @task_info ||= {}
-    @task_info[task]
-    
-    if @task_info[task].nil?
-      task_info = WorkflowRESTClient.get_json(File.join(url, task.to_s, 'info'))
-      task_info = WorkflowRESTClient.fix_hash(task_info)
-
-      task_info[:result_type] = task_info[:result_type].to_sym
-      task_info[:export] = task_info[:export].to_sym
-      task_info[:input_types] = WorkflowRESTClient.fix_hash(task_info[:input_types], true)
-      task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
-
-      @task_info[task] = task_info
-    end
-    @task_info[task]
-  end
-
-  def exported_tasks
-    (@asynchronous_exports  + @synchronous_exports + @exec_exports).compact.flatten
-  end
-
-  def tasks
-    @tasks ||= Hash.new do |hash,task_name| 
-      info = task_info(task_name)
-      task = Task.setup info do |*args|
-        raise "This is a remote task" 
-      end
-      task.name = task_name.to_sym
-      hash[task_name] = task
-    end
-  end
-
-  def load_tasks
-    exported_tasks.each{|name| tasks[name]}
-    nil
-  end
-
-  def task_dependencies
-    @task_dependencies ||= Hash.new do |hash,task| 
-      hash[task] = if exported_tasks.include? task
-        WorkflowRESTClient.get_json(File.join(url, task.to_s, 'dependencies'))
-      else
-        []
-      end
-    end
-  end
-
-  def init_remote_tasks
-    task_exports = WorkflowRESTClient.get_json(url)
-    @asynchronous_exports = task_exports["asynchronous"].collect{|task| task.to_sym }
-    @synchronous_exports = task_exports["synchronous"].collect{|task| task.to_sym }
-    @exec_exports = task_exports["exec"].collect{|task| task.to_sym }
-    @stream_exports = task_exports["stream"].collect{|task| task.to_sym }
-    @can_stream = task_exports["can_stream"]
-  end
-end

data/lib/rbbt/rest/client/get.rb

@@ -1,166 +0,0 @@
-class WorkflowRESTClient
-  def self.encode(url)
-    begin
-      URI.encode(url)
-    rescue
-      Log.warn $!.message
-      url
-    end
-  end
-
-  def self.fix_hash(hash, fix_values = false)
-    fixed = {}
-    hash.each do |key, value|
-      fixed[key.to_sym] = case value
-                          when TrueClass
-                            value
-                          when FalseClass
-                            value
-                          when Hash 
-                            fix_hash(value)  
-                          when (fix_values and String )
-                            value.to_sym
-                          when IO
-                            value.read
-                          when TSV::Dumper
-                            value.stream
-                          when Step
-                            stream = get_stream(value)
-                            stream || value.load
-                          else
-                            value
-                          end
-    end
-    fixed
-  end
-
-  def self.parse_exception(text)
-    klass, message = text.split " => "
-    begin
-      klass = Kernel.const_get klass
-      return klass.new message
-    rescue
-      message
-    end
-  end
-
-  def self.capture_exception
-    begin
-      yield
-    rescue Exception => e
-      raise e unless e.respond_to? :response
-      begin
-        ne = parse_exception e.response.to_s
-        case ne
-        when String
-          raise e.class, ne
-        when Exception
-          raise ne
-        else
-          raise
-        end
-      rescue
-        raise e
-      end
-      raise $!
-    end
-  end
-
-  def self.fix_params(params)
-    new_params = {}
-    params.each do |k,v|
-      if Array === v and v.empty?
-        new_params[k] = "EMPTY_ARRAY"
-      else
-        new_params[k] = v
-      end
-    end
-    new_params
-  end
-
-  def self.clean_url(url, params = {})
-    params = params.merge({ :_format => 'json', :update => 'clean' })
-    params = fix_params params
-    res = capture_exception do
-      Misc.insist(2, 0.5) do
-        Log.debug{ "RestClient clean: #{ url } - #{Misc.fingerprint params}" }
-        res = begin 
-                RestClient.get(self.encode(url), :params => params)
-              rescue RestClient::NotFound
-                return nil
-              end
-        raise TryAgain if res.code == 202
-        res
-      end
-    end
-    res
-  end
- 
-  def self.get_raw(url, params = {})
-    params = params.merge({ :_format => 'raw' })
-    params = fix_params params
-    res = capture_exception do
-      Misc.insist(2, 0.5) do
-        raise "No url" if url.nil?
-        Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
-        res = RestClient.get(self.encode(url), :params => params)
-        raise TryAgain if res.code == 202
-        res.to_s
-      end
-    end
-    res
-  end
- 
-  def self.get_json(url, params = {})
-    Log.debug{ "RestClient get_json: #{ url } - #{Misc.fingerprint params }" }
-    params = params.merge({ :_format => 'json' })
-    params = fix_params params
-
-    res = capture_exception do
-      Misc.insist(2, 0.5) do
-        RestClient.get(self.encode(url), :params => params)
-      end
-    end
-
-    begin
-      JSON.parse(res)
-    rescue
-      res
-    end
-  end
-
-  def self.post_jobname(url, params = {})
-    Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
-    params = params.merge({ :_format => 'jobname' })
-    params = fix_params params
-
-    WorkflowRESTClient.__prepare_inputs_for_restclient(params)
-    name = capture_exception do
-      RestClient.post(self.encode(url), params)
-    end
-
-    Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
-
-    name
-  end
-
-  def self.post_json(url, params = {})
-    if url =~ /_cache_type=:exec/
-      JSON.parse(Open.open(url, :nocache => true))
-    else
-      params = params.merge({ :_format => 'json' })
-      params = fix_params params
-
-      res = capture_exception do
-        RestClient.post(self.encode(url), params)
-      end
-
-      begin
-        JSON.parse(res)
-      rescue
-        res
-      end
-    end
-  end
-
-end

data/lib/rbbt/rest/client/run.rb

@@ -1,133 +0,0 @@
-class WorkflowRESTClient::RemoteStep
-
-  def stream_job(task_url, task_params, stream_input, cache_type = :exec)
-    require 'rbbt/util/misc/multipart_payload'
-    WorkflowRESTClient.capture_exception do
-      @streaming = true
-
-      Log.debug{ "RestClient stream #{Process.pid}: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint task_params}" }
-      res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
-      type = res.gets
-
-      out = case type.strip
-            when "LOCATION"
-              @url = res.gets
-              @url.sub!(/\?.*/,'')
-              join
-              WorkflowRESTClient.get_raw(@url)
-              @done = true
-              @streaming = false
-            when /STREAM: (.*)/
-              @url = $1.strip
-              res.callback = Proc.new do
-                Log.medium "Done streaming result from #{@url}"
-                @done = true
-                @streaming = false
-              end
-              res
-            when "BULK"
-              begin
-                res.read
-              ensure
-                @done = true
-                @streaming = false
-              end
-            else
-              raise "What? " + type
-            end
-
-      ConcurrentStream.setup(out, :filename => @url)
-
-      out
-    end
-  end
-
-  def execute_job(task_url, task_params, cache_type)
-    WorkflowRESTClient.capture_exception do
-      task_url = URI.encode(File.join(base_url, task.to_s))
-
-      sout, sin = Misc.pipe
-
-      post_thread = Thread.new(Thread.current) do |parent|
-        bl = lambda do |rok|
-          if Net::HTTPOK === rok
-            _url = rok["RBBT-STREAMING-JOB-URL"]
-            @url = File.join(task_url, File.basename(_url)) if _url
-            rok.read_body do |c,_a, _b|
-              sin.write c
-            end
-            sin.close
-          else
-            err = StringIO.new
-            rok.read_body do |c,_a, _b|
-              err.write c
-            end
-            text = begin
-                     reader = Zlib::GzipReader.new(err)
-                     reader.read
-                   rescue
-                     err.rewind
-                     err.read
-                   end
-            ne = WorkflowRESTClient.parse_exception text
-            case ne
-            when String
-              parent.raise e.class, ne
-            when Exception
-              parent.raise ne
-            else
-              parent.raise "Error in RestClient: " << rok.message
-            end
-          end
-        end
-
-        task_params.each do |k,v|
-          task_params[k] = v.read if IO === v
-        end
-
-        Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
-        RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
-      end
-
-      # It seems like now response body are now decoded by Net::HTTP after 2.1
-      # https://github.com/rest-client/rest-client/blob/cf3e5a115bcdb8f3344aeac0e45b44d67fac1a42/history.md
-      decode = Gem.loaded_specs["rest-client"].version < Gem::Version.create('2.1')
-      if decode
-        reader = Zlib::GzipReader.new(sout)
-        res_io = Misc.open_pipe do |sin|
-          while c = reader.read(Misc::BLOCK_SIZE)
-            sin.write c
-          end
-          sin.close
-          @done = true
-        end
-        ConcurrentStream.setup(res_io, :threads => [post_thread]) do
-          @done = true
-          @streaming = false
-        end
-      else
-        ConcurrentStream.setup(sout, :threads => [post_thread]) do
-          @done = true
-          @streaming = false
-        end
-      end
-
-    end
-  end
-
-  def _run_job(cache_type = :async)
-    get_streams
-
-    task_url = URI.encode(File.join(base_url, task.to_s))
-    WorkflowRESTClient.__prepare_inputs_for_restclient(inputs)
-    task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
-
-    if cache_type == :stream or cache_type == :exec and stream_input and inputs[stream_input]
-      io =  self.stream_job(task_url, task_params, stream_input, cache_type) 
-      return io
-    else
-      execute_job(task_url, task_params, cache_type)
-    end
-
-  end
-end