rbbt-util 5.26.157 → 5.26.158

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,86 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt/workflow'
3
+ require 'rbbt/workflow/remote_workflow'
4
+
5
+ class TestWFRest
6
+ extend Workflow
7
+
8
+ input :name, :string, "Name to greet", "World"
9
+ task :hi => :string do |name|
10
+ "Hello #{name}"
11
+ end
12
+
13
+ dep :hi
14
+ task :intro => :string do
15
+ step(:hi).load + ", nice to meet you"
16
+ end
17
+ end
18
+
19
+ class TestRemoteWorkflow < Test::Unit::TestCase
20
+
21
+ def remote_workflow_server(workflow, options = {}, &block)
22
+ trap(:USR1){ raise TestServerLoaded}
23
+
24
+ begin
25
+ pid = Process.fork do
26
+ TmpFile.with_file do |app_dir|
27
+ Misc.in_dir(app_dir) do
28
+ require 'rack'
29
+ ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
30
+
31
+ app_dir = Path.setup(app_dir.dup)
32
+ Open.write(app_dir.etc.target_workflow.find, workflow.to_s)
33
+
34
+ config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
35
+ options[:config] = config_ru_file
36
+ app = Rack::Server.new(options)
37
+ app.start do
38
+ Process.kill :USR1, Process.ppid
39
+ end
40
+ end
41
+ end
42
+ end
43
+
44
+ begin
45
+ sleep 1 while true
46
+ rescue TestServerLoaded
47
+ end
48
+
49
+ client = RemoteWorkflow.new "http://localhost:#{options[:Port] || 9292}/#{workflow.to_s}", workflow.to_s
50
+
51
+ yield client
52
+
53
+ rescue
54
+ Log.exception $!
55
+ ensure
56
+ Process.kill :INT, pid
57
+ Process.wait pid
58
+ end
59
+ end
60
+
61
+ def test_rest
62
+ Log.with_severity 0 do
63
+
64
+ remote_workflow_server(TestWFRest) do |client|
65
+ assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
66
+ assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
67
+ assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
68
+ end
69
+
70
+ remote_workflow_server(TestWFRest, :Port => 1902) do |client|
71
+ assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
72
+ assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
73
+ assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
74
+ end
75
+ end
76
+ end
77
+
78
+
79
+ def _test_ssh
80
+ Log.severity = 0
81
+ client = RemoteWorkflow.new "ssh://turbo:Translation", "Translation"
82
+ job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])
83
+ assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
84
+ end
85
+ end
86
+
data/test/test_helper.rb CHANGED
@@ -50,36 +50,39 @@ class Test::Unit::TestCase
50
50
  def workflow_server(workflow, options = {}, &block)
51
51
  trap(:USR1){ raise TestServerLoaded}
52
52
 
53
- pid = Process.fork do
54
- TmpFile.with_file do |app_dir|
55
- Misc.in_dir(app_dir) do
56
- require 'rack'
57
- ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
58
-
59
- app_dir = Path.setup(app_dir.dup)
60
- Open.write(app_dir.etc.target_workflow.find, workflow.to_s)
61
-
62
- config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
63
- options[:config] = config_ru_file
64
- app = Rack::Server.new(options)
65
- app.start do
66
- Process.kill :USR1, Process.ppid
53
+ begin
54
+ pid = Process.fork do
55
+ TmpFile.with_file do |app_dir|
56
+ Misc.in_dir(app_dir) do
57
+ require 'rack'
58
+ ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
59
+
60
+ app_dir = Path.setup(app_dir.dup)
61
+ Open.write(app_dir.etc.target_workflow.find, workflow.to_s)
62
+
63
+ config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
64
+ options[:config] = config_ru_file
65
+ app = Rack::Server.new(options)
66
+ app.start do
67
+ Process.kill :USR1, Process.ppid
68
+ end
67
69
  end
68
70
  end
69
71
  end
70
- end
71
72
 
72
- begin
73
- sleep 1 while true
74
- rescue TestServerLoaded
75
- end
73
+ begin
74
+ sleep 1 while true
75
+ rescue TestServerLoaded
76
+ end
76
77
 
77
- client = WorkflowRemoteClient.new "http://localhost:#{options[:Port] || 9292}/#{workflow.to_s}", workflow.to_s
78
+ client = WorkflowRemoteClient.new "http://localhost:#{options[:Port] || 9292}/#{workflow.to_s}", workflow.to_s
78
79
 
79
- yield client
80
+ yield client
80
81
 
81
- Process.kill :INT, pid
82
- Process.wait pid
82
+ ensure
83
+ Process.kill :INT, pid
84
+ Process.wait pid
85
+ end
83
86
  end
84
87
 
85
88
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.26.157
4
+ version: 5.26.158
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-24 00:00:00.000000000 Z
11
+ date: 2020-03-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -236,11 +236,6 @@ files:
236
236
  - lib/rbbt/resource/rake.rb
237
237
  - lib/rbbt/resource/util.rb
238
238
  - lib/rbbt/resource/with_key.rb
239
- - lib/rbbt/rest/client.rb
240
- - lib/rbbt/rest/client/adaptor.rb
241
- - lib/rbbt/rest/client/get.rb
242
- - lib/rbbt/rest/client/run.rb
243
- - lib/rbbt/rest/client/step.rb
244
239
  - lib/rbbt/tsv.rb
245
240
  - lib/rbbt/tsv/accessor.rb
246
241
  - lib/rbbt/tsv/attach.rb
@@ -324,13 +319,13 @@ files:
324
319
  - lib/rbbt/workflow/examples.rb
325
320
  - lib/rbbt/workflow/integration/cromwell.rb
326
321
  - lib/rbbt/workflow/integration/nextflow.rb
327
- - lib/rbbt/workflow/remote/client.rb
328
- - lib/rbbt/workflow/remote/remote_step.rb
329
- - lib/rbbt/workflow/remote/rest/adaptor.rb
330
- - lib/rbbt/workflow/remote/rest/get.rb
331
- - lib/rbbt/workflow/remote/ssh/adaptor.rb
332
- - lib/rbbt/workflow/remote/ssh/driver.rb
333
- - lib/rbbt/workflow/remote/ssh/get.rb
322
+ - lib/rbbt/workflow/remote_workflow.rb
323
+ - lib/rbbt/workflow/remote_workflow/driver.rb
324
+ - lib/rbbt/workflow/remote_workflow/driver/rest.rb
325
+ - lib/rbbt/workflow/remote_workflow/driver/ssh.rb
326
+ - lib/rbbt/workflow/remote_workflow/remote_step.rb
327
+ - lib/rbbt/workflow/remote_workflow/remote_step/rest.rb
328
+ - lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb
334
329
  - lib/rbbt/workflow/schedule.rb
335
330
  - lib/rbbt/workflow/soap.rb
336
331
  - lib/rbbt/workflow/step.rb
@@ -515,6 +510,7 @@ files:
515
510
  - test/rbbt/workflow/remote/test_client.rb
516
511
  - test/rbbt/workflow/step/test_dependencies.rb
517
512
  - test/rbbt/workflow/test_doc.rb
513
+ - test/rbbt/workflow/test_remote_workflow.rb
518
514
  - test/rbbt/workflow/test_step.rb
519
515
  - test/rbbt/workflow/test_task.rb
520
516
  - test/test_helper.rb
@@ -543,6 +539,7 @@ specification_version: 4
543
539
  summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
544
540
  test_files:
545
541
  - test/rbbt/test_entity.rb
542
+ - test/rbbt/workflow/test_remote_workflow.rb
546
543
  - test/rbbt/workflow/test_doc.rb
547
544
  - test/rbbt/workflow/test_step.rb
548
545
  - test/rbbt/workflow/remote/test_client.rb
@@ -1,67 +0,0 @@
1
- class WorkflowRESTClient
2
-
3
- def workflow_description
4
- WorkflowRESTClient.get_raw(File.join(url, 'description'))
5
- end
6
-
7
- def documentation
8
- @documention ||= IndiferentHash.setup(WorkflowRESTClient.get_json(File.join(url, "documentation"),{}))
9
- end
10
-
11
- def task_info(task)
12
- @task_info ||= {}
13
- @task_info[task]
14
-
15
- if @task_info[task].nil?
16
- task_info = WorkflowRESTClient.get_json(File.join(url, task.to_s, 'info'))
17
- task_info = WorkflowRESTClient.fix_hash(task_info)
18
-
19
- task_info[:result_type] = task_info[:result_type].to_sym
20
- task_info[:export] = task_info[:export].to_sym
21
- task_info[:input_types] = WorkflowRESTClient.fix_hash(task_info[:input_types], true)
22
- task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
23
-
24
- @task_info[task] = task_info
25
- end
26
- @task_info[task]
27
- end
28
-
29
- def exported_tasks
30
- (@asynchronous_exports + @synchronous_exports + @exec_exports).compact.flatten
31
- end
32
-
33
- def tasks
34
- @tasks ||= Hash.new do |hash,task_name|
35
- info = task_info(task_name)
36
- task = Task.setup info do |*args|
37
- raise "This is a remote task"
38
- end
39
- task.name = task_name.to_sym
40
- hash[task_name] = task
41
- end
42
- end
43
-
44
- def load_tasks
45
- exported_tasks.each{|name| tasks[name]}
46
- nil
47
- end
48
-
49
- def task_dependencies
50
- @task_dependencies ||= Hash.new do |hash,task|
51
- hash[task] = if exported_tasks.include? task
52
- WorkflowRESTClient.get_json(File.join(url, task.to_s, 'dependencies'))
53
- else
54
- []
55
- end
56
- end
57
- end
58
-
59
- def init_remote_tasks
60
- task_exports = WorkflowRESTClient.get_json(url)
61
- @asynchronous_exports = task_exports["asynchronous"].collect{|task| task.to_sym }
62
- @synchronous_exports = task_exports["synchronous"].collect{|task| task.to_sym }
63
- @exec_exports = task_exports["exec"].collect{|task| task.to_sym }
64
- @stream_exports = task_exports["stream"].collect{|task| task.to_sym }
65
- @can_stream = task_exports["can_stream"]
66
- end
67
- end
@@ -1,166 +0,0 @@
1
- class WorkflowRESTClient
2
- def self.encode(url)
3
- begin
4
- URI.encode(url)
5
- rescue
6
- Log.warn $!.message
7
- url
8
- end
9
- end
10
-
11
- def self.fix_hash(hash, fix_values = false)
12
- fixed = {}
13
- hash.each do |key, value|
14
- fixed[key.to_sym] = case value
15
- when TrueClass
16
- value
17
- when FalseClass
18
- value
19
- when Hash
20
- fix_hash(value)
21
- when (fix_values and String )
22
- value.to_sym
23
- when IO
24
- value.read
25
- when TSV::Dumper
26
- value.stream
27
- when Step
28
- stream = get_stream(value)
29
- stream || value.load
30
- else
31
- value
32
- end
33
- end
34
- fixed
35
- end
36
-
37
- def self.parse_exception(text)
38
- klass, message = text.split " => "
39
- begin
40
- klass = Kernel.const_get klass
41
- return klass.new message
42
- rescue
43
- message
44
- end
45
- end
46
-
47
- def self.capture_exception
48
- begin
49
- yield
50
- rescue Exception => e
51
- raise e unless e.respond_to? :response
52
- begin
53
- ne = parse_exception e.response.to_s
54
- case ne
55
- when String
56
- raise e.class, ne
57
- when Exception
58
- raise ne
59
- else
60
- raise
61
- end
62
- rescue
63
- raise e
64
- end
65
- raise $!
66
- end
67
- end
68
-
69
- def self.fix_params(params)
70
- new_params = {}
71
- params.each do |k,v|
72
- if Array === v and v.empty?
73
- new_params[k] = "EMPTY_ARRAY"
74
- else
75
- new_params[k] = v
76
- end
77
- end
78
- new_params
79
- end
80
-
81
- def self.clean_url(url, params = {})
82
- params = params.merge({ :_format => 'json', :update => 'clean' })
83
- params = fix_params params
84
- res = capture_exception do
85
- Misc.insist(2, 0.5) do
86
- Log.debug{ "RestClient clean: #{ url } - #{Misc.fingerprint params}" }
87
- res = begin
88
- RestClient.get(self.encode(url), :params => params)
89
- rescue RestClient::NotFound
90
- return nil
91
- end
92
- raise TryAgain if res.code == 202
93
- res
94
- end
95
- end
96
- res
97
- end
98
-
99
- def self.get_raw(url, params = {})
100
- params = params.merge({ :_format => 'raw' })
101
- params = fix_params params
102
- res = capture_exception do
103
- Misc.insist(2, 0.5) do
104
- raise "No url" if url.nil?
105
- Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
106
- res = RestClient.get(self.encode(url), :params => params)
107
- raise TryAgain if res.code == 202
108
- res.to_s
109
- end
110
- end
111
- res
112
- end
113
-
114
- def self.get_json(url, params = {})
115
- Log.debug{ "RestClient get_json: #{ url } - #{Misc.fingerprint params }" }
116
- params = params.merge({ :_format => 'json' })
117
- params = fix_params params
118
-
119
- res = capture_exception do
120
- Misc.insist(2, 0.5) do
121
- RestClient.get(self.encode(url), :params => params)
122
- end
123
- end
124
-
125
- begin
126
- JSON.parse(res)
127
- rescue
128
- res
129
- end
130
- end
131
-
132
- def self.post_jobname(url, params = {})
133
- Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
134
- params = params.merge({ :_format => 'jobname' })
135
- params = fix_params params
136
-
137
- WorkflowRESTClient.__prepare_inputs_for_restclient(params)
138
- name = capture_exception do
139
- RestClient.post(self.encode(url), params)
140
- end
141
-
142
- Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
143
-
144
- name
145
- end
146
-
147
- def self.post_json(url, params = {})
148
- if url =~ /_cache_type=:exec/
149
- JSON.parse(Open.open(url, :nocache => true))
150
- else
151
- params = params.merge({ :_format => 'json' })
152
- params = fix_params params
153
-
154
- res = capture_exception do
155
- RestClient.post(self.encode(url), params)
156
- end
157
-
158
- begin
159
- JSON.parse(res)
160
- rescue
161
- res
162
- end
163
- end
164
- end
165
-
166
- end
@@ -1,133 +0,0 @@
1
- class WorkflowRESTClient::RemoteStep
2
-
3
- def stream_job(task_url, task_params, stream_input, cache_type = :exec)
4
- require 'rbbt/util/misc/multipart_payload'
5
- WorkflowRESTClient.capture_exception do
6
- @streaming = true
7
-
8
- Log.debug{ "RestClient stream #{Process.pid}: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint task_params}" }
9
- res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
10
- type = res.gets
11
-
12
- out = case type.strip
13
- when "LOCATION"
14
- @url = res.gets
15
- @url.sub!(/\?.*/,'')
16
- join
17
- WorkflowRESTClient.get_raw(@url)
18
- @done = true
19
- @streaming = false
20
- when /STREAM: (.*)/
21
- @url = $1.strip
22
- res.callback = Proc.new do
23
- Log.medium "Done streaming result from #{@url}"
24
- @done = true
25
- @streaming = false
26
- end
27
- res
28
- when "BULK"
29
- begin
30
- res.read
31
- ensure
32
- @done = true
33
- @streaming = false
34
- end
35
- else
36
- raise "What? " + type
37
- end
38
-
39
- ConcurrentStream.setup(out, :filename => @url)
40
-
41
- out
42
- end
43
- end
44
-
45
- def execute_job(task_url, task_params, cache_type)
46
- WorkflowRESTClient.capture_exception do
47
- task_url = URI.encode(File.join(base_url, task.to_s))
48
-
49
- sout, sin = Misc.pipe
50
-
51
- post_thread = Thread.new(Thread.current) do |parent|
52
- bl = lambda do |rok|
53
- if Net::HTTPOK === rok
54
- _url = rok["RBBT-STREAMING-JOB-URL"]
55
- @url = File.join(task_url, File.basename(_url)) if _url
56
- rok.read_body do |c,_a, _b|
57
- sin.write c
58
- end
59
- sin.close
60
- else
61
- err = StringIO.new
62
- rok.read_body do |c,_a, _b|
63
- err.write c
64
- end
65
- text = begin
66
- reader = Zlib::GzipReader.new(err)
67
- reader.read
68
- rescue
69
- err.rewind
70
- err.read
71
- end
72
- ne = WorkflowRESTClient.parse_exception text
73
- case ne
74
- when String
75
- parent.raise e.class, ne
76
- when Exception
77
- parent.raise ne
78
- else
79
- parent.raise "Error in RestClient: " << rok.message
80
- end
81
- end
82
- end
83
-
84
- task_params.each do |k,v|
85
- task_params[k] = v.read if IO === v
86
- end
87
-
88
- Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
89
- RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
90
- end
91
-
92
- # It seems like now response body are now decoded by Net::HTTP after 2.1
93
- # https://github.com/rest-client/rest-client/blob/cf3e5a115bcdb8f3344aeac0e45b44d67fac1a42/history.md
94
- decode = Gem.loaded_specs["rest-client"].version < Gem::Version.create('2.1')
95
- if decode
96
- reader = Zlib::GzipReader.new(sout)
97
- res_io = Misc.open_pipe do |sin|
98
- while c = reader.read(Misc::BLOCK_SIZE)
99
- sin.write c
100
- end
101
- sin.close
102
- @done = true
103
- end
104
- ConcurrentStream.setup(res_io, :threads => [post_thread]) do
105
- @done = true
106
- @streaming = false
107
- end
108
- else
109
- ConcurrentStream.setup(sout, :threads => [post_thread]) do
110
- @done = true
111
- @streaming = false
112
- end
113
- end
114
-
115
- end
116
- end
117
-
118
- def _run_job(cache_type = :async)
119
- get_streams
120
-
121
- task_url = URI.encode(File.join(base_url, task.to_s))
122
- WorkflowRESTClient.__prepare_inputs_for_restclient(inputs)
123
- task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
124
-
125
- if cache_type == :stream or cache_type == :exec and stream_input and inputs[stream_input]
126
- io = self.stream_job(task_url, task_params, stream_input, cache_type)
127
- return io
128
- else
129
- execute_job(task_url, task_params, cache_type)
130
- end
131
-
132
- end
133
- end