rbbt-util 5.15.3 → 5.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/rest/client.rb +56 -0
- data/lib/rbbt/rest/client/adaptor.rb +66 -0
- data/lib/rbbt/rest/client/get.rb +96 -0
- data/lib/rbbt/rest/client/step.rb +164 -0
- metadata +5 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7eb0331120fec20040b7d6d0c2572e5d09b2a423
|
4
|
+
data.tar.gz: 689c858bc997cfc925e2be00a3a71a8c1d708e85
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 976840c06b1e1c277beaf3852920c1fc6f7be6f25d1eb0b9f41fbbe920e140649ab23bc32e6c081567d7236fe37b6cc3339de4094de407e6a77da023cd591a94
|
7
|
+
data.tar.gz: ad29facb7a20e308c8045ef1ef7134fdd425568408acff53e20db772416b7d7a78989307197427ec163aa96c8aeac83e7f1cb77823bc20d9ea2394ee182d7091
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'rest_client'
|
2
|
+
require 'json'
|
3
|
+
require 'rbbt/workflow'
|
4
|
+
require 'rbbt/workflow/step'
|
5
|
+
require 'rbbt/util/misc'
|
6
|
+
|
7
|
+
require 'rbbt/rest/client/get'
|
8
|
+
require 'rbbt/rest/client/adaptor'
|
9
|
+
require 'rbbt/rest/client/step'
|
10
|
+
|
11
|
+
class WorkflowRESTClient
|
12
|
+
include Workflow
|
13
|
+
|
14
|
+
attr_accessor :url, :name, :exec_exports, :asynchronous_exports, :synchronous_exports
|
15
|
+
|
16
|
+
def initialize(url, name)
|
17
|
+
Log.debug{ "Loading remote workflow #{ name }: #{ url }" }
|
18
|
+
@url, @name = url, name
|
19
|
+
init_remote_tasks
|
20
|
+
end
|
21
|
+
|
22
|
+
def to_s
|
23
|
+
name
|
24
|
+
end
|
25
|
+
|
26
|
+
def job(task, name, inputs)
|
27
|
+
task_info = task_info(task)
|
28
|
+
fixed_inputs = {}
|
29
|
+
input_types = task_info[:input_types]
|
30
|
+
|
31
|
+
inputs.each do |k,v|
|
32
|
+
k = k.to_sym
|
33
|
+
if TSV === v
|
34
|
+
fixed_inputs[k] = v.to_s
|
35
|
+
else
|
36
|
+
case input_types[k].to_sym
|
37
|
+
when :tsv, :array, :file, :text
|
38
|
+
fixed_inputs[k] = (String === v and Open.exists?(v)) ? Open.open(v) : v
|
39
|
+
else
|
40
|
+
fixed_inputs[k] = v
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task))
|
46
|
+
end
|
47
|
+
|
48
|
+
def load_id(id)
|
49
|
+
task, name = id.split("/")
|
50
|
+
step = RemoteStep.new url, task, nil
|
51
|
+
step.name = name
|
52
|
+
step.result_type = task_info(task)[:result_type]
|
53
|
+
step.result_description = task_info(task)[:result_description]
|
54
|
+
step
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
class WorkflowRESTClient
|
2
|
+
|
3
|
+
def workflow_description
|
4
|
+
WorkflowRESTClient.get_raw(File.join(url, 'description'))
|
5
|
+
end
|
6
|
+
|
7
|
+
def documentation
|
8
|
+
@documention ||= IndiferentHash.setup(WorkflowRESTClient.get_json(File.join(url, "documentation"),{}))
|
9
|
+
end
|
10
|
+
|
11
|
+
def task_info(task)
|
12
|
+
@task_info ||= {}
|
13
|
+
@task_info[task]
|
14
|
+
|
15
|
+
if @task_info[task].nil?
|
16
|
+
task_info = WorkflowRESTClient.get_json(File.join(url, task.to_s, 'info'))
|
17
|
+
task_info = WorkflowRESTClient.fix_hash(task_info)
|
18
|
+
|
19
|
+
task_info[:result_type] = task_info[:result_type].to_sym
|
20
|
+
task_info[:export] = task_info[:export].to_sym
|
21
|
+
task_info[:input_types] = WorkflowRESTClient.fix_hash(task_info[:input_types], true)
|
22
|
+
task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
|
23
|
+
|
24
|
+
@task_info[task] = task_info
|
25
|
+
end
|
26
|
+
@task_info[task]
|
27
|
+
end
|
28
|
+
|
29
|
+
def exported_tasks
|
30
|
+
(@asynchronous_exports + @synchronous_exports + @exec_exports).compact.flatten
|
31
|
+
end
|
32
|
+
|
33
|
+
def tasks
|
34
|
+
@tasks ||= Hash.new do |hash,task_name|
|
35
|
+
info = task_info(task_name)
|
36
|
+
task = Task.setup info do |*args|
|
37
|
+
raise "This is a remote task"
|
38
|
+
end
|
39
|
+
task.name = task_name.to_sym
|
40
|
+
hash[task_name] = task
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def load_tasks
|
45
|
+
exported_tasks.each{|name| tasks[name]}
|
46
|
+
nil
|
47
|
+
end
|
48
|
+
|
49
|
+
def task_dependencies
|
50
|
+
@task_dependencies ||= Hash.new do |hash,task|
|
51
|
+
hash[task] = if exported_tasks.include? task
|
52
|
+
WorkflowRESTClient.get_json(File.join(url, task.to_s, 'dependencies'))
|
53
|
+
else
|
54
|
+
[]
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def init_remote_tasks
|
60
|
+
task_exports = WorkflowRESTClient.get_json(url)
|
61
|
+
@asynchronous_exports = task_exports["asynchronous"].collect{|task| task.to_sym }
|
62
|
+
@synchronous_exports = task_exports["synchronous"].collect{|task| task.to_sym }
|
63
|
+
@exec_exports = task_exports["exec"].collect{|task| task.to_sym }
|
64
|
+
nil
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
class WorkflowRESTClient
|
2
|
+
def self.fix_hash(hash, fix_values = false)
|
3
|
+
fixed = {}
|
4
|
+
hash.each do |key, value|
|
5
|
+
fixed[key.to_sym] = case value
|
6
|
+
when TrueClass
|
7
|
+
value
|
8
|
+
when FalseClass
|
9
|
+
value
|
10
|
+
when Hash
|
11
|
+
fix_hash(value)
|
12
|
+
when (fix_values and String )
|
13
|
+
value.to_sym
|
14
|
+
when IO
|
15
|
+
value.read
|
16
|
+
when TSV::Dumper
|
17
|
+
value.stream
|
18
|
+
when Step
|
19
|
+
stream = get_stream(value)
|
20
|
+
stream || value.load
|
21
|
+
else
|
22
|
+
value
|
23
|
+
end
|
24
|
+
end
|
25
|
+
fixed
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.capture_exception
|
29
|
+
begin
|
30
|
+
yield
|
31
|
+
rescue Exception => e
|
32
|
+
raise e unless e.respond_to? :response
|
33
|
+
begin
|
34
|
+
klass, message = e.response.to_s.split " => "
|
35
|
+
klass = Kernel.const_get klass
|
36
|
+
raise klass.new message
|
37
|
+
rescue
|
38
|
+
raise e
|
39
|
+
end
|
40
|
+
raise $!
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.get_raw(url, params = {})
|
45
|
+
Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
|
46
|
+
params = params.merge({ :_format => 'raw' })
|
47
|
+
capture_exception do
|
48
|
+
Misc.insist(2, 0.5) do
|
49
|
+
RestClient.get(URI.encode(url), :params => params)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.post_jobname(url, params = {})
|
55
|
+
Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
|
56
|
+
params = params.merge({ :_format => 'jobname' })
|
57
|
+
|
58
|
+
capture_exception do
|
59
|
+
RestClient.post(URI.encode(url), params)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.get_json(url, params = {})
|
64
|
+
Log.debug{ "RestClient get_json: #{ url } - #{Misc.fingerprint params }" }
|
65
|
+
params = params.merge({ :_format => 'json' })
|
66
|
+
|
67
|
+
res = capture_exception do
|
68
|
+
RestClient.get(URI.encode(url), :params => params)
|
69
|
+
end
|
70
|
+
|
71
|
+
begin
|
72
|
+
JSON.parse(res)
|
73
|
+
rescue
|
74
|
+
res
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def self.post_json(url, params = {})
|
79
|
+
if url =~ /_cache_type=:exec/
|
80
|
+
JSON.parse(Open.open(url, :nocache => true))
|
81
|
+
else
|
82
|
+
params = params.merge({ :_format => 'json' })
|
83
|
+
|
84
|
+
res = capture_exception do
|
85
|
+
RestClient.post(URI.encode(url), params)
|
86
|
+
end
|
87
|
+
|
88
|
+
begin
|
89
|
+
JSON.parse(res)
|
90
|
+
rescue
|
91
|
+
res
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
@@ -0,0 +1,164 @@
|
|
1
|
+
class WorkflowRESTClient
|
2
|
+
class RemoteStep < Step
|
3
|
+
|
4
|
+
attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec
|
5
|
+
|
6
|
+
def self.get_streams(inputs)
|
7
|
+
inputs.each do |k,v|
|
8
|
+
if Step === v
|
9
|
+
stream = v.get_stream
|
10
|
+
inputs[k] = stream || v.load
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false)
|
15
|
+
@base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec = base_url, task, base_name, inputs, result_type, result_description, is_exec
|
16
|
+
@mutex = Mutex.new
|
17
|
+
RemoteStep.get_streams @inputs
|
18
|
+
end
|
19
|
+
|
20
|
+
def name
|
21
|
+
return nil if @is_exec
|
22
|
+
(Array === @url ? @url.first : @url).split("/").last
|
23
|
+
end
|
24
|
+
|
25
|
+
def task_name
|
26
|
+
(Array === @url ? @url.first : @url).split("/")[-2]
|
27
|
+
end
|
28
|
+
|
29
|
+
def info(check_lock=false)
|
30
|
+
@info ||= begin
|
31
|
+
init_job unless url
|
32
|
+
info = WorkflowRESTClient.get_json(File.join(url, 'info'))
|
33
|
+
info = WorkflowRESTClient.fix_hash(info)
|
34
|
+
info[:status] = info[:status].to_sym if String === info[:status]
|
35
|
+
info
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def status
|
40
|
+
begin
|
41
|
+
info[:status]
|
42
|
+
ensure
|
43
|
+
@info = nil
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def done?
|
48
|
+
@done || status.to_s == 'done'
|
49
|
+
end
|
50
|
+
|
51
|
+
def files
|
52
|
+
WorkflowRESTClient.get_json(File.join(url, 'files'))
|
53
|
+
end
|
54
|
+
|
55
|
+
def file(file)
|
56
|
+
WorkflowRESTClient.get_raw(File.join(url, 'file', file))
|
57
|
+
end
|
58
|
+
|
59
|
+
#{{{ MANAGEMENT
|
60
|
+
|
61
|
+
def init_job(cache_type = :asynchronous)
|
62
|
+
@name ||= Persist.memory("RemoteSteps", :jobname => @name, :inputs => inputs) do
|
63
|
+
WorkflowRESTClient.post_jobname(File.join(base_url, task.to_s), inputs.merge(:jobname => @name||@base_name, :_cache_type => cache_type))
|
64
|
+
end
|
65
|
+
@url = File.join(base_url, task.to_s, @name)
|
66
|
+
nil
|
67
|
+
end
|
68
|
+
|
69
|
+
def load_res(res, result_type = nil)
|
70
|
+
result_type ||= self.result_type
|
71
|
+
case result_type
|
72
|
+
when :string
|
73
|
+
res
|
74
|
+
when :boolean
|
75
|
+
res == "true"
|
76
|
+
when :tsv
|
77
|
+
TSV.open(StringIO.new(res))
|
78
|
+
when :annotations
|
79
|
+
Annotated.load_tsv(TSV.open(StringIO.new(res)))
|
80
|
+
when :array
|
81
|
+
res.split("\n")
|
82
|
+
else
|
83
|
+
JSON.parse res
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def get
|
88
|
+
params ||= {}
|
89
|
+
params = params.merge(:_format => [:string, :boolean, :tsv, :annotations,:array].include?(result_type.to_sym) ? :raw : :json )
|
90
|
+
Misc.insist 3, rand(2) + 1 do
|
91
|
+
begin
|
92
|
+
WorkflowRESTClient.get_raw(url, params)
|
93
|
+
rescue
|
94
|
+
Log.exception $!
|
95
|
+
raise $!
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def load
|
101
|
+
params = {}
|
102
|
+
load_res get
|
103
|
+
end
|
104
|
+
|
105
|
+
def exec_job
|
106
|
+
res = WorkflowRESTClient.capture_exception do
|
107
|
+
RestClient.post(URI.encode(File.join(base_url, task.to_s)), inputs.merge(:_cache_type => :exec, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json))
|
108
|
+
end
|
109
|
+
load_res res, result_type == :array ? :json : result_type
|
110
|
+
end
|
111
|
+
|
112
|
+
def fork
|
113
|
+
init_job(:asynchronous)
|
114
|
+
end
|
115
|
+
|
116
|
+
def running?
|
117
|
+
! %w(done error aborted).include? status.to_s
|
118
|
+
end
|
119
|
+
|
120
|
+
def path
|
121
|
+
url
|
122
|
+
end
|
123
|
+
|
124
|
+
def run(noload = false)
|
125
|
+
@mutex.synchronize do
|
126
|
+
@result ||= begin
|
127
|
+
if @is_exec
|
128
|
+
exec_job
|
129
|
+
else
|
130
|
+
init_job(:synchronous)
|
131
|
+
self.load
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
noload ? path : @result
|
136
|
+
end
|
137
|
+
|
138
|
+
def exec(*args)
|
139
|
+
exec_job
|
140
|
+
end
|
141
|
+
|
142
|
+
def join
|
143
|
+
return if self.done?
|
144
|
+
self.load
|
145
|
+
self
|
146
|
+
end
|
147
|
+
|
148
|
+
def recursive_clean
|
149
|
+
begin
|
150
|
+
inputs[:_update] = :recursive_clean
|
151
|
+
rescue Exception
|
152
|
+
end
|
153
|
+
self
|
154
|
+
end
|
155
|
+
|
156
|
+
def clean
|
157
|
+
begin
|
158
|
+
inputs[:_update] = :clean
|
159
|
+
rescue Exception
|
160
|
+
end
|
161
|
+
self
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.16.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
@@ -174,6 +174,10 @@ files:
|
|
174
174
|
- lib/rbbt/resource/rake.rb
|
175
175
|
- lib/rbbt/resource/util.rb
|
176
176
|
- lib/rbbt/resource/with_key.rb
|
177
|
+
- lib/rbbt/rest/client.rb
|
178
|
+
- lib/rbbt/rest/client/adaptor.rb
|
179
|
+
- lib/rbbt/rest/client/get.rb
|
180
|
+
- lib/rbbt/rest/client/step.rb
|
177
181
|
- lib/rbbt/tsv.rb
|
178
182
|
- lib/rbbt/tsv/accessor.rb
|
179
183
|
- lib/rbbt/tsv/attach.rb
|