rbbt-util 5.26.157 → 5.26.158
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/util/misc/development.rb +7 -0
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +230 -0
- data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +237 -0
- data/lib/rbbt/workflow/remote_workflow/driver.rb +75 -0
- data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +148 -0
- data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +73 -0
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +329 -0
- data/lib/rbbt/workflow/{remote/client.rb → remote_workflow.rb} +9 -16
- data/lib/rbbt/workflow/util/archive.rb +2 -3
- data/lib/rbbt/workflow.rb +4 -4
- data/share/rbbt_commands/workflow/remote/add +28 -2
- data/share/rbbt_commands/workflow/task +4 -3
- data/test/rbbt/workflow/test_remote_workflow.rb +86 -0
- data/test/test_helper.rb +26 -23
- metadata +11 -14
- data/lib/rbbt/rest/client/adaptor.rb +0 -67
- data/lib/rbbt/rest/client/get.rb +0 -166
- data/lib/rbbt/rest/client/run.rb +0 -133
- data/lib/rbbt/rest/client/step.rb +0 -397
- data/lib/rbbt/rest/client.rb +0 -58
- data/lib/rbbt/workflow/remote/remote_step.rb +0 -324
- data/lib/rbbt/workflow/remote/rest/adaptor.rb +0 -161
- data/lib/rbbt/workflow/remote/rest/get.rb +0 -310
- data/lib/rbbt/workflow/remote/ssh/adaptor.rb +0 -61
- data/lib/rbbt/workflow/remote/ssh/driver.rb +0 -147
- data/lib/rbbt/workflow/remote/ssh/get.rb +0 -193
@@ -0,0 +1,86 @@
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
|
2
|
+
require 'rbbt/workflow'
|
3
|
+
require 'rbbt/workflow/remote_workflow'
|
4
|
+
|
5
|
+
class TestWFRest
|
6
|
+
extend Workflow
|
7
|
+
|
8
|
+
input :name, :string, "Name to greet", "World"
|
9
|
+
task :hi => :string do |name|
|
10
|
+
"Hello #{name}"
|
11
|
+
end
|
12
|
+
|
13
|
+
dep :hi
|
14
|
+
task :intro => :string do
|
15
|
+
step(:hi).load + ", nice to meet you"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
class TestRemoteWorkflow < Test::Unit::TestCase
|
20
|
+
|
21
|
+
def remote_workflow_server(workflow, options = {}, &block)
|
22
|
+
trap(:USR1){ raise TestServerLoaded}
|
23
|
+
|
24
|
+
begin
|
25
|
+
pid = Process.fork do
|
26
|
+
TmpFile.with_file do |app_dir|
|
27
|
+
Misc.in_dir(app_dir) do
|
28
|
+
require 'rack'
|
29
|
+
ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
|
30
|
+
|
31
|
+
app_dir = Path.setup(app_dir.dup)
|
32
|
+
Open.write(app_dir.etc.target_workflow.find, workflow.to_s)
|
33
|
+
|
34
|
+
config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
|
35
|
+
options[:config] = config_ru_file
|
36
|
+
app = Rack::Server.new(options)
|
37
|
+
app.start do
|
38
|
+
Process.kill :USR1, Process.ppid
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
begin
|
45
|
+
sleep 1 while true
|
46
|
+
rescue TestServerLoaded
|
47
|
+
end
|
48
|
+
|
49
|
+
client = RemoteWorkflow.new "http://localhost:#{options[:Port] || 9292}/#{workflow.to_s}", workflow.to_s
|
50
|
+
|
51
|
+
yield client
|
52
|
+
|
53
|
+
rescue
|
54
|
+
Log.exception $!
|
55
|
+
ensure
|
56
|
+
Process.kill :INT, pid
|
57
|
+
Process.wait pid
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def test_rest
|
62
|
+
Log.with_severity 0 do
|
63
|
+
|
64
|
+
remote_workflow_server(TestWFRest) do |client|
|
65
|
+
assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
|
66
|
+
assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
|
67
|
+
assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
|
68
|
+
end
|
69
|
+
|
70
|
+
remote_workflow_server(TestWFRest, :Port => 1902) do |client|
|
71
|
+
assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
|
72
|
+
assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
|
73
|
+
assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
def _test_ssh
|
80
|
+
Log.severity = 0
|
81
|
+
client = RemoteWorkflow.new "ssh://turbo:Translation", "Translation"
|
82
|
+
job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])
|
83
|
+
assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
data/test/test_helper.rb
CHANGED
@@ -50,36 +50,39 @@ class Test::Unit::TestCase
|
|
50
50
|
def workflow_server(workflow, options = {}, &block)
|
51
51
|
trap(:USR1){ raise TestServerLoaded}
|
52
52
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
53
|
+
begin
|
54
|
+
pid = Process.fork do
|
55
|
+
TmpFile.with_file do |app_dir|
|
56
|
+
Misc.in_dir(app_dir) do
|
57
|
+
require 'rack'
|
58
|
+
ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
|
59
|
+
|
60
|
+
app_dir = Path.setup(app_dir.dup)
|
61
|
+
Open.write(app_dir.etc.target_workflow.find, workflow.to_s)
|
62
|
+
|
63
|
+
config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
|
64
|
+
options[:config] = config_ru_file
|
65
|
+
app = Rack::Server.new(options)
|
66
|
+
app.start do
|
67
|
+
Process.kill :USR1, Process.ppid
|
68
|
+
end
|
67
69
|
end
|
68
70
|
end
|
69
71
|
end
|
70
|
-
end
|
71
72
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
73
|
+
begin
|
74
|
+
sleep 1 while true
|
75
|
+
rescue TestServerLoaded
|
76
|
+
end
|
76
77
|
|
77
|
-
|
78
|
+
client = WorkflowRemoteClient.new "http://localhost:#{options[:Port] || 9292}/#{workflow.to_s}", workflow.to_s
|
78
79
|
|
79
|
-
|
80
|
+
yield client
|
80
81
|
|
81
|
-
|
82
|
-
|
82
|
+
ensure
|
83
|
+
Process.kill :INT, pid
|
84
|
+
Process.wait pid
|
85
|
+
end
|
83
86
|
end
|
84
87
|
|
85
88
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.26.
|
4
|
+
version: 5.26.158
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-03-
|
11
|
+
date: 2020-03-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -236,11 +236,6 @@ files:
|
|
236
236
|
- lib/rbbt/resource/rake.rb
|
237
237
|
- lib/rbbt/resource/util.rb
|
238
238
|
- lib/rbbt/resource/with_key.rb
|
239
|
-
- lib/rbbt/rest/client.rb
|
240
|
-
- lib/rbbt/rest/client/adaptor.rb
|
241
|
-
- lib/rbbt/rest/client/get.rb
|
242
|
-
- lib/rbbt/rest/client/run.rb
|
243
|
-
- lib/rbbt/rest/client/step.rb
|
244
239
|
- lib/rbbt/tsv.rb
|
245
240
|
- lib/rbbt/tsv/accessor.rb
|
246
241
|
- lib/rbbt/tsv/attach.rb
|
@@ -324,13 +319,13 @@ files:
|
|
324
319
|
- lib/rbbt/workflow/examples.rb
|
325
320
|
- lib/rbbt/workflow/integration/cromwell.rb
|
326
321
|
- lib/rbbt/workflow/integration/nextflow.rb
|
327
|
-
- lib/rbbt/workflow/
|
328
|
-
- lib/rbbt/workflow/
|
329
|
-
- lib/rbbt/workflow/
|
330
|
-
- lib/rbbt/workflow/
|
331
|
-
- lib/rbbt/workflow/
|
332
|
-
- lib/rbbt/workflow/
|
333
|
-
- lib/rbbt/workflow/
|
322
|
+
- lib/rbbt/workflow/remote_workflow.rb
|
323
|
+
- lib/rbbt/workflow/remote_workflow/driver.rb
|
324
|
+
- lib/rbbt/workflow/remote_workflow/driver/rest.rb
|
325
|
+
- lib/rbbt/workflow/remote_workflow/driver/ssh.rb
|
326
|
+
- lib/rbbt/workflow/remote_workflow/remote_step.rb
|
327
|
+
- lib/rbbt/workflow/remote_workflow/remote_step/rest.rb
|
328
|
+
- lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb
|
334
329
|
- lib/rbbt/workflow/schedule.rb
|
335
330
|
- lib/rbbt/workflow/soap.rb
|
336
331
|
- lib/rbbt/workflow/step.rb
|
@@ -515,6 +510,7 @@ files:
|
|
515
510
|
- test/rbbt/workflow/remote/test_client.rb
|
516
511
|
- test/rbbt/workflow/step/test_dependencies.rb
|
517
512
|
- test/rbbt/workflow/test_doc.rb
|
513
|
+
- test/rbbt/workflow/test_remote_workflow.rb
|
518
514
|
- test/rbbt/workflow/test_step.rb
|
519
515
|
- test/rbbt/workflow/test_task.rb
|
520
516
|
- test/test_helper.rb
|
@@ -543,6 +539,7 @@ specification_version: 4
|
|
543
539
|
summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
|
544
540
|
test_files:
|
545
541
|
- test/rbbt/test_entity.rb
|
542
|
+
- test/rbbt/workflow/test_remote_workflow.rb
|
546
543
|
- test/rbbt/workflow/test_doc.rb
|
547
544
|
- test/rbbt/workflow/test_step.rb
|
548
545
|
- test/rbbt/workflow/remote/test_client.rb
|
@@ -1,67 +0,0 @@
|
|
1
|
-
class WorkflowRESTClient
|
2
|
-
|
3
|
-
def workflow_description
|
4
|
-
WorkflowRESTClient.get_raw(File.join(url, 'description'))
|
5
|
-
end
|
6
|
-
|
7
|
-
def documentation
|
8
|
-
@documention ||= IndiferentHash.setup(WorkflowRESTClient.get_json(File.join(url, "documentation"),{}))
|
9
|
-
end
|
10
|
-
|
11
|
-
def task_info(task)
|
12
|
-
@task_info ||= {}
|
13
|
-
@task_info[task]
|
14
|
-
|
15
|
-
if @task_info[task].nil?
|
16
|
-
task_info = WorkflowRESTClient.get_json(File.join(url, task.to_s, 'info'))
|
17
|
-
task_info = WorkflowRESTClient.fix_hash(task_info)
|
18
|
-
|
19
|
-
task_info[:result_type] = task_info[:result_type].to_sym
|
20
|
-
task_info[:export] = task_info[:export].to_sym
|
21
|
-
task_info[:input_types] = WorkflowRESTClient.fix_hash(task_info[:input_types], true)
|
22
|
-
task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
|
23
|
-
|
24
|
-
@task_info[task] = task_info
|
25
|
-
end
|
26
|
-
@task_info[task]
|
27
|
-
end
|
28
|
-
|
29
|
-
def exported_tasks
|
30
|
-
(@asynchronous_exports + @synchronous_exports + @exec_exports).compact.flatten
|
31
|
-
end
|
32
|
-
|
33
|
-
def tasks
|
34
|
-
@tasks ||= Hash.new do |hash,task_name|
|
35
|
-
info = task_info(task_name)
|
36
|
-
task = Task.setup info do |*args|
|
37
|
-
raise "This is a remote task"
|
38
|
-
end
|
39
|
-
task.name = task_name.to_sym
|
40
|
-
hash[task_name] = task
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
def load_tasks
|
45
|
-
exported_tasks.each{|name| tasks[name]}
|
46
|
-
nil
|
47
|
-
end
|
48
|
-
|
49
|
-
def task_dependencies
|
50
|
-
@task_dependencies ||= Hash.new do |hash,task|
|
51
|
-
hash[task] = if exported_tasks.include? task
|
52
|
-
WorkflowRESTClient.get_json(File.join(url, task.to_s, 'dependencies'))
|
53
|
-
else
|
54
|
-
[]
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
def init_remote_tasks
|
60
|
-
task_exports = WorkflowRESTClient.get_json(url)
|
61
|
-
@asynchronous_exports = task_exports["asynchronous"].collect{|task| task.to_sym }
|
62
|
-
@synchronous_exports = task_exports["synchronous"].collect{|task| task.to_sym }
|
63
|
-
@exec_exports = task_exports["exec"].collect{|task| task.to_sym }
|
64
|
-
@stream_exports = task_exports["stream"].collect{|task| task.to_sym }
|
65
|
-
@can_stream = task_exports["can_stream"]
|
66
|
-
end
|
67
|
-
end
|
data/lib/rbbt/rest/client/get.rb
DELETED
@@ -1,166 +0,0 @@
|
|
1
|
-
class WorkflowRESTClient
|
2
|
-
def self.encode(url)
|
3
|
-
begin
|
4
|
-
URI.encode(url)
|
5
|
-
rescue
|
6
|
-
Log.warn $!.message
|
7
|
-
url
|
8
|
-
end
|
9
|
-
end
|
10
|
-
|
11
|
-
def self.fix_hash(hash, fix_values = false)
|
12
|
-
fixed = {}
|
13
|
-
hash.each do |key, value|
|
14
|
-
fixed[key.to_sym] = case value
|
15
|
-
when TrueClass
|
16
|
-
value
|
17
|
-
when FalseClass
|
18
|
-
value
|
19
|
-
when Hash
|
20
|
-
fix_hash(value)
|
21
|
-
when (fix_values and String )
|
22
|
-
value.to_sym
|
23
|
-
when IO
|
24
|
-
value.read
|
25
|
-
when TSV::Dumper
|
26
|
-
value.stream
|
27
|
-
when Step
|
28
|
-
stream = get_stream(value)
|
29
|
-
stream || value.load
|
30
|
-
else
|
31
|
-
value
|
32
|
-
end
|
33
|
-
end
|
34
|
-
fixed
|
35
|
-
end
|
36
|
-
|
37
|
-
def self.parse_exception(text)
|
38
|
-
klass, message = text.split " => "
|
39
|
-
begin
|
40
|
-
klass = Kernel.const_get klass
|
41
|
-
return klass.new message
|
42
|
-
rescue
|
43
|
-
message
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def self.capture_exception
|
48
|
-
begin
|
49
|
-
yield
|
50
|
-
rescue Exception => e
|
51
|
-
raise e unless e.respond_to? :response
|
52
|
-
begin
|
53
|
-
ne = parse_exception e.response.to_s
|
54
|
-
case ne
|
55
|
-
when String
|
56
|
-
raise e.class, ne
|
57
|
-
when Exception
|
58
|
-
raise ne
|
59
|
-
else
|
60
|
-
raise
|
61
|
-
end
|
62
|
-
rescue
|
63
|
-
raise e
|
64
|
-
end
|
65
|
-
raise $!
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
def self.fix_params(params)
|
70
|
-
new_params = {}
|
71
|
-
params.each do |k,v|
|
72
|
-
if Array === v and v.empty?
|
73
|
-
new_params[k] = "EMPTY_ARRAY"
|
74
|
-
else
|
75
|
-
new_params[k] = v
|
76
|
-
end
|
77
|
-
end
|
78
|
-
new_params
|
79
|
-
end
|
80
|
-
|
81
|
-
def self.clean_url(url, params = {})
|
82
|
-
params = params.merge({ :_format => 'json', :update => 'clean' })
|
83
|
-
params = fix_params params
|
84
|
-
res = capture_exception do
|
85
|
-
Misc.insist(2, 0.5) do
|
86
|
-
Log.debug{ "RestClient clean: #{ url } - #{Misc.fingerprint params}" }
|
87
|
-
res = begin
|
88
|
-
RestClient.get(self.encode(url), :params => params)
|
89
|
-
rescue RestClient::NotFound
|
90
|
-
return nil
|
91
|
-
end
|
92
|
-
raise TryAgain if res.code == 202
|
93
|
-
res
|
94
|
-
end
|
95
|
-
end
|
96
|
-
res
|
97
|
-
end
|
98
|
-
|
99
|
-
def self.get_raw(url, params = {})
|
100
|
-
params = params.merge({ :_format => 'raw' })
|
101
|
-
params = fix_params params
|
102
|
-
res = capture_exception do
|
103
|
-
Misc.insist(2, 0.5) do
|
104
|
-
raise "No url" if url.nil?
|
105
|
-
Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
|
106
|
-
res = RestClient.get(self.encode(url), :params => params)
|
107
|
-
raise TryAgain if res.code == 202
|
108
|
-
res.to_s
|
109
|
-
end
|
110
|
-
end
|
111
|
-
res
|
112
|
-
end
|
113
|
-
|
114
|
-
def self.get_json(url, params = {})
|
115
|
-
Log.debug{ "RestClient get_json: #{ url } - #{Misc.fingerprint params }" }
|
116
|
-
params = params.merge({ :_format => 'json' })
|
117
|
-
params = fix_params params
|
118
|
-
|
119
|
-
res = capture_exception do
|
120
|
-
Misc.insist(2, 0.5) do
|
121
|
-
RestClient.get(self.encode(url), :params => params)
|
122
|
-
end
|
123
|
-
end
|
124
|
-
|
125
|
-
begin
|
126
|
-
JSON.parse(res)
|
127
|
-
rescue
|
128
|
-
res
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
def self.post_jobname(url, params = {})
|
133
|
-
Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
|
134
|
-
params = params.merge({ :_format => 'jobname' })
|
135
|
-
params = fix_params params
|
136
|
-
|
137
|
-
WorkflowRESTClient.__prepare_inputs_for_restclient(params)
|
138
|
-
name = capture_exception do
|
139
|
-
RestClient.post(self.encode(url), params)
|
140
|
-
end
|
141
|
-
|
142
|
-
Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
|
143
|
-
|
144
|
-
name
|
145
|
-
end
|
146
|
-
|
147
|
-
def self.post_json(url, params = {})
|
148
|
-
if url =~ /_cache_type=:exec/
|
149
|
-
JSON.parse(Open.open(url, :nocache => true))
|
150
|
-
else
|
151
|
-
params = params.merge({ :_format => 'json' })
|
152
|
-
params = fix_params params
|
153
|
-
|
154
|
-
res = capture_exception do
|
155
|
-
RestClient.post(self.encode(url), params)
|
156
|
-
end
|
157
|
-
|
158
|
-
begin
|
159
|
-
JSON.parse(res)
|
160
|
-
rescue
|
161
|
-
res
|
162
|
-
end
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
|
-
end
|
data/lib/rbbt/rest/client/run.rb
DELETED
@@ -1,133 +0,0 @@
|
|
1
|
-
class WorkflowRESTClient::RemoteStep
|
2
|
-
|
3
|
-
def stream_job(task_url, task_params, stream_input, cache_type = :exec)
|
4
|
-
require 'rbbt/util/misc/multipart_payload'
|
5
|
-
WorkflowRESTClient.capture_exception do
|
6
|
-
@streaming = true
|
7
|
-
|
8
|
-
Log.debug{ "RestClient stream #{Process.pid}: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint task_params}" }
|
9
|
-
res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
|
10
|
-
type = res.gets
|
11
|
-
|
12
|
-
out = case type.strip
|
13
|
-
when "LOCATION"
|
14
|
-
@url = res.gets
|
15
|
-
@url.sub!(/\?.*/,'')
|
16
|
-
join
|
17
|
-
WorkflowRESTClient.get_raw(@url)
|
18
|
-
@done = true
|
19
|
-
@streaming = false
|
20
|
-
when /STREAM: (.*)/
|
21
|
-
@url = $1.strip
|
22
|
-
res.callback = Proc.new do
|
23
|
-
Log.medium "Done streaming result from #{@url}"
|
24
|
-
@done = true
|
25
|
-
@streaming = false
|
26
|
-
end
|
27
|
-
res
|
28
|
-
when "BULK"
|
29
|
-
begin
|
30
|
-
res.read
|
31
|
-
ensure
|
32
|
-
@done = true
|
33
|
-
@streaming = false
|
34
|
-
end
|
35
|
-
else
|
36
|
-
raise "What? " + type
|
37
|
-
end
|
38
|
-
|
39
|
-
ConcurrentStream.setup(out, :filename => @url)
|
40
|
-
|
41
|
-
out
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def execute_job(task_url, task_params, cache_type)
|
46
|
-
WorkflowRESTClient.capture_exception do
|
47
|
-
task_url = URI.encode(File.join(base_url, task.to_s))
|
48
|
-
|
49
|
-
sout, sin = Misc.pipe
|
50
|
-
|
51
|
-
post_thread = Thread.new(Thread.current) do |parent|
|
52
|
-
bl = lambda do |rok|
|
53
|
-
if Net::HTTPOK === rok
|
54
|
-
_url = rok["RBBT-STREAMING-JOB-URL"]
|
55
|
-
@url = File.join(task_url, File.basename(_url)) if _url
|
56
|
-
rok.read_body do |c,_a, _b|
|
57
|
-
sin.write c
|
58
|
-
end
|
59
|
-
sin.close
|
60
|
-
else
|
61
|
-
err = StringIO.new
|
62
|
-
rok.read_body do |c,_a, _b|
|
63
|
-
err.write c
|
64
|
-
end
|
65
|
-
text = begin
|
66
|
-
reader = Zlib::GzipReader.new(err)
|
67
|
-
reader.read
|
68
|
-
rescue
|
69
|
-
err.rewind
|
70
|
-
err.read
|
71
|
-
end
|
72
|
-
ne = WorkflowRESTClient.parse_exception text
|
73
|
-
case ne
|
74
|
-
when String
|
75
|
-
parent.raise e.class, ne
|
76
|
-
when Exception
|
77
|
-
parent.raise ne
|
78
|
-
else
|
79
|
-
parent.raise "Error in RestClient: " << rok.message
|
80
|
-
end
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
task_params.each do |k,v|
|
85
|
-
task_params[k] = v.read if IO === v
|
86
|
-
end
|
87
|
-
|
88
|
-
Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
|
89
|
-
RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
|
90
|
-
end
|
91
|
-
|
92
|
-
# It seems like now response body are now decoded by Net::HTTP after 2.1
|
93
|
-
# https://github.com/rest-client/rest-client/blob/cf3e5a115bcdb8f3344aeac0e45b44d67fac1a42/history.md
|
94
|
-
decode = Gem.loaded_specs["rest-client"].version < Gem::Version.create('2.1')
|
95
|
-
if decode
|
96
|
-
reader = Zlib::GzipReader.new(sout)
|
97
|
-
res_io = Misc.open_pipe do |sin|
|
98
|
-
while c = reader.read(Misc::BLOCK_SIZE)
|
99
|
-
sin.write c
|
100
|
-
end
|
101
|
-
sin.close
|
102
|
-
@done = true
|
103
|
-
end
|
104
|
-
ConcurrentStream.setup(res_io, :threads => [post_thread]) do
|
105
|
-
@done = true
|
106
|
-
@streaming = false
|
107
|
-
end
|
108
|
-
else
|
109
|
-
ConcurrentStream.setup(sout, :threads => [post_thread]) do
|
110
|
-
@done = true
|
111
|
-
@streaming = false
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
end
|
116
|
-
end
|
117
|
-
|
118
|
-
def _run_job(cache_type = :async)
|
119
|
-
get_streams
|
120
|
-
|
121
|
-
task_url = URI.encode(File.join(base_url, task.to_s))
|
122
|
-
WorkflowRESTClient.__prepare_inputs_for_restclient(inputs)
|
123
|
-
task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
|
124
|
-
|
125
|
-
if cache_type == :stream or cache_type == :exec and stream_input and inputs[stream_input]
|
126
|
-
io = self.stream_job(task_url, task_params, stream_input, cache_type)
|
127
|
-
return io
|
128
|
-
else
|
129
|
-
execute_job(task_url, task_params, cache_type)
|
130
|
-
end
|
131
|
-
|
132
|
-
end
|
133
|
-
end
|