rbbt-util 5.15.3 → 5.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 63ec9663b0d7998f633493f92b70a30854fe604e
4
- data.tar.gz: feaceed53691a6b261e7c0eb17d97e0f70d05a39
3
+ metadata.gz: 7eb0331120fec20040b7d6d0c2572e5d09b2a423
4
+ data.tar.gz: 689c858bc997cfc925e2be00a3a71a8c1d708e85
5
5
  SHA512:
6
- metadata.gz: bf81be699f8921c98657fc085f9295a3895d64d452c8df9bf3f87746ee3ec1487fc3c12ed37bfb9d88b1613b32b8c826cbce5c267e206cf54fd323e10810a24c
7
- data.tar.gz: 1161f4eb4c9d9856b258e6d0f0fd2be7aa6d18d9a6b42ade8eff3a143334012bd138d1874fb4e0391a430357a2f0845e96b5ad0a43293df7cfb0dd59bdcdf6a4
6
+ metadata.gz: 976840c06b1e1c277beaf3852920c1fc6f7be6f25d1eb0b9f41fbbe920e140649ab23bc32e6c081567d7236fe37b6cc3339de4094de407e6a77da023cd591a94
7
+ data.tar.gz: ad29facb7a20e308c8045ef1ef7134fdd425568408acff53e20db772416b7d7a78989307197427ec163aa96c8aeac83e7f1cb77823bc20d9ea2394ee182d7091
@@ -0,0 +1,56 @@
1
+ require 'rest_client'
2
+ require 'json'
3
+ require 'rbbt/workflow'
4
+ require 'rbbt/workflow/step'
5
+ require 'rbbt/util/misc'
6
+
7
+ require 'rbbt/rest/client/get'
8
+ require 'rbbt/rest/client/adaptor'
9
+ require 'rbbt/rest/client/step'
10
+
11
+ class WorkflowRESTClient
12
+ include Workflow
13
+
14
+ attr_accessor :url, :name, :exec_exports, :asynchronous_exports, :synchronous_exports
15
+
16
+ def initialize(url, name)
17
+ Log.debug{ "Loading remote workflow #{ name }: #{ url }" }
18
+ @url, @name = url, name
19
+ init_remote_tasks
20
+ end
21
+
22
+ def to_s
23
+ name
24
+ end
25
+
26
+ def job(task, name, inputs)
27
+ task_info = task_info(task)
28
+ fixed_inputs = {}
29
+ input_types = task_info[:input_types]
30
+
31
+ inputs.each do |k,v|
32
+ k = k.to_sym
33
+ if TSV === v
34
+ fixed_inputs[k] = v.to_s
35
+ else
36
+ case input_types[k].to_sym
37
+ when :tsv, :array, :file, :text
38
+ fixed_inputs[k] = (String === v and Open.exists?(v)) ? Open.open(v) : v
39
+ else
40
+ fixed_inputs[k] = v
41
+ end
42
+ end
43
+ end
44
+
45
+ RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task))
46
+ end
47
+
48
+ def load_id(id)
49
+ task, name = id.split("/")
50
+ step = RemoteStep.new url, task, nil
51
+ step.name = name
52
+ step.result_type = task_info(task)[:result_type]
53
+ step.result_description = task_info(task)[:result_description]
54
+ step
55
+ end
56
+ end
@@ -0,0 +1,66 @@
1
+ class WorkflowRESTClient
2
+
3
+ def workflow_description
4
+ WorkflowRESTClient.get_raw(File.join(url, 'description'))
5
+ end
6
+
7
+ def documentation
8
+ @documention ||= IndiferentHash.setup(WorkflowRESTClient.get_json(File.join(url, "documentation"),{}))
9
+ end
10
+
11
+ def task_info(task)
12
+ @task_info ||= {}
13
+ @task_info[task]
14
+
15
+ if @task_info[task].nil?
16
+ task_info = WorkflowRESTClient.get_json(File.join(url, task.to_s, 'info'))
17
+ task_info = WorkflowRESTClient.fix_hash(task_info)
18
+
19
+ task_info[:result_type] = task_info[:result_type].to_sym
20
+ task_info[:export] = task_info[:export].to_sym
21
+ task_info[:input_types] = WorkflowRESTClient.fix_hash(task_info[:input_types], true)
22
+ task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
23
+
24
+ @task_info[task] = task_info
25
+ end
26
+ @task_info[task]
27
+ end
28
+
29
+ def exported_tasks
30
+ (@asynchronous_exports + @synchronous_exports + @exec_exports).compact.flatten
31
+ end
32
+
33
+ def tasks
34
+ @tasks ||= Hash.new do |hash,task_name|
35
+ info = task_info(task_name)
36
+ task = Task.setup info do |*args|
37
+ raise "This is a remote task"
38
+ end
39
+ task.name = task_name.to_sym
40
+ hash[task_name] = task
41
+ end
42
+ end
43
+
44
+ def load_tasks
45
+ exported_tasks.each{|name| tasks[name]}
46
+ nil
47
+ end
48
+
49
+ def task_dependencies
50
+ @task_dependencies ||= Hash.new do |hash,task|
51
+ hash[task] = if exported_tasks.include? task
52
+ WorkflowRESTClient.get_json(File.join(url, task.to_s, 'dependencies'))
53
+ else
54
+ []
55
+ end
56
+ end
57
+ end
58
+
59
+ def init_remote_tasks
60
+ task_exports = WorkflowRESTClient.get_json(url)
61
+ @asynchronous_exports = task_exports["asynchronous"].collect{|task| task.to_sym }
62
+ @synchronous_exports = task_exports["synchronous"].collect{|task| task.to_sym }
63
+ @exec_exports = task_exports["exec"].collect{|task| task.to_sym }
64
+ nil
65
+ end
66
+ end
@@ -0,0 +1,96 @@
1
+ class WorkflowRESTClient
2
+ def self.fix_hash(hash, fix_values = false)
3
+ fixed = {}
4
+ hash.each do |key, value|
5
+ fixed[key.to_sym] = case value
6
+ when TrueClass
7
+ value
8
+ when FalseClass
9
+ value
10
+ when Hash
11
+ fix_hash(value)
12
+ when (fix_values and String )
13
+ value.to_sym
14
+ when IO
15
+ value.read
16
+ when TSV::Dumper
17
+ value.stream
18
+ when Step
19
+ stream = get_stream(value)
20
+ stream || value.load
21
+ else
22
+ value
23
+ end
24
+ end
25
+ fixed
26
+ end
27
+
28
+ def self.capture_exception
29
+ begin
30
+ yield
31
+ rescue Exception => e
32
+ raise e unless e.respond_to? :response
33
+ begin
34
+ klass, message = e.response.to_s.split " => "
35
+ klass = Kernel.const_get klass
36
+ raise klass.new message
37
+ rescue
38
+ raise e
39
+ end
40
+ raise $!
41
+ end
42
+ end
43
+
44
+ def self.get_raw(url, params = {})
45
+ Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
46
+ params = params.merge({ :_format => 'raw' })
47
+ capture_exception do
48
+ Misc.insist(2, 0.5) do
49
+ RestClient.get(URI.encode(url), :params => params)
50
+ end
51
+ end
52
+ end
53
+
54
+ def self.post_jobname(url, params = {})
55
+ Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
56
+ params = params.merge({ :_format => 'jobname' })
57
+
58
+ capture_exception do
59
+ RestClient.post(URI.encode(url), params)
60
+ end
61
+ end
62
+
63
+ def self.get_json(url, params = {})
64
+ Log.debug{ "RestClient get_json: #{ url } - #{Misc.fingerprint params }" }
65
+ params = params.merge({ :_format => 'json' })
66
+
67
+ res = capture_exception do
68
+ RestClient.get(URI.encode(url), :params => params)
69
+ end
70
+
71
+ begin
72
+ JSON.parse(res)
73
+ rescue
74
+ res
75
+ end
76
+ end
77
+
78
+ def self.post_json(url, params = {})
79
+ if url =~ /_cache_type=:exec/
80
+ JSON.parse(Open.open(url, :nocache => true))
81
+ else
82
+ params = params.merge({ :_format => 'json' })
83
+
84
+ res = capture_exception do
85
+ RestClient.post(URI.encode(url), params)
86
+ end
87
+
88
+ begin
89
+ JSON.parse(res)
90
+ rescue
91
+ res
92
+ end
93
+ end
94
+ end
95
+
96
+ end
@@ -0,0 +1,164 @@
1
+ class WorkflowRESTClient
2
+ class RemoteStep < Step
3
+
4
+ attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec
5
+
6
+ def self.get_streams(inputs)
7
+ inputs.each do |k,v|
8
+ if Step === v
9
+ stream = v.get_stream
10
+ inputs[k] = stream || v.load
11
+ end
12
+ end
13
+ end
14
+ def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false)
15
+ @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec = base_url, task, base_name, inputs, result_type, result_description, is_exec
16
+ @mutex = Mutex.new
17
+ RemoteStep.get_streams @inputs
18
+ end
19
+
20
+ def name
21
+ return nil if @is_exec
22
+ (Array === @url ? @url.first : @url).split("/").last
23
+ end
24
+
25
+ def task_name
26
+ (Array === @url ? @url.first : @url).split("/")[-2]
27
+ end
28
+
29
+ def info(check_lock=false)
30
+ @info ||= begin
31
+ init_job unless url
32
+ info = WorkflowRESTClient.get_json(File.join(url, 'info'))
33
+ info = WorkflowRESTClient.fix_hash(info)
34
+ info[:status] = info[:status].to_sym if String === info[:status]
35
+ info
36
+ end
37
+ end
38
+
39
+ def status
40
+ begin
41
+ info[:status]
42
+ ensure
43
+ @info = nil
44
+ end
45
+ end
46
+
47
+ def done?
48
+ @done || status.to_s == 'done'
49
+ end
50
+
51
+ def files
52
+ WorkflowRESTClient.get_json(File.join(url, 'files'))
53
+ end
54
+
55
+ def file(file)
56
+ WorkflowRESTClient.get_raw(File.join(url, 'file', file))
57
+ end
58
+
59
+ #{{{ MANAGEMENT
60
+
61
+ def init_job(cache_type = :asynchronous)
62
+ @name ||= Persist.memory("RemoteSteps", :jobname => @name, :inputs => inputs) do
63
+ WorkflowRESTClient.post_jobname(File.join(base_url, task.to_s), inputs.merge(:jobname => @name||@base_name, :_cache_type => cache_type))
64
+ end
65
+ @url = File.join(base_url, task.to_s, @name)
66
+ nil
67
+ end
68
+
69
+ def load_res(res, result_type = nil)
70
+ result_type ||= self.result_type
71
+ case result_type
72
+ when :string
73
+ res
74
+ when :boolean
75
+ res == "true"
76
+ when :tsv
77
+ TSV.open(StringIO.new(res))
78
+ when :annotations
79
+ Annotated.load_tsv(TSV.open(StringIO.new(res)))
80
+ when :array
81
+ res.split("\n")
82
+ else
83
+ JSON.parse res
84
+ end
85
+ end
86
+
87
+ def get
88
+ params ||= {}
89
+ params = params.merge(:_format => [:string, :boolean, :tsv, :annotations,:array].include?(result_type.to_sym) ? :raw : :json )
90
+ Misc.insist 3, rand(2) + 1 do
91
+ begin
92
+ WorkflowRESTClient.get_raw(url, params)
93
+ rescue
94
+ Log.exception $!
95
+ raise $!
96
+ end
97
+ end
98
+ end
99
+
100
+ def load
101
+ params = {}
102
+ load_res get
103
+ end
104
+
105
+ def exec_job
106
+ res = WorkflowRESTClient.capture_exception do
107
+ RestClient.post(URI.encode(File.join(base_url, task.to_s)), inputs.merge(:_cache_type => :exec, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json))
108
+ end
109
+ load_res res, result_type == :array ? :json : result_type
110
+ end
111
+
112
+ def fork
113
+ init_job(:asynchronous)
114
+ end
115
+
116
+ def running?
117
+ ! %w(done error aborted).include? status.to_s
118
+ end
119
+
120
+ def path
121
+ url
122
+ end
123
+
124
+ def run(noload = false)
125
+ @mutex.synchronize do
126
+ @result ||= begin
127
+ if @is_exec
128
+ exec_job
129
+ else
130
+ init_job(:synchronous)
131
+ self.load
132
+ end
133
+ end
134
+ end
135
+ noload ? path : @result
136
+ end
137
+
138
+ def exec(*args)
139
+ exec_job
140
+ end
141
+
142
+ def join
143
+ return if self.done?
144
+ self.load
145
+ self
146
+ end
147
+
148
+ def recursive_clean
149
+ begin
150
+ inputs[:_update] = :recursive_clean
151
+ rescue Exception
152
+ end
153
+ self
154
+ end
155
+
156
+ def clean
157
+ begin
158
+ inputs[:_update] = :clean
159
+ rescue Exception
160
+ end
161
+ self
162
+ end
163
+ end
164
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.15.3
4
+ version: 5.16.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
@@ -174,6 +174,10 @@ files:
174
174
  - lib/rbbt/resource/rake.rb
175
175
  - lib/rbbt/resource/util.rb
176
176
  - lib/rbbt/resource/with_key.rb
177
+ - lib/rbbt/rest/client.rb
178
+ - lib/rbbt/rest/client/adaptor.rb
179
+ - lib/rbbt/rest/client/get.rb
180
+ - lib/rbbt/rest/client/step.rb
177
181
  - lib/rbbt/tsv.rb
178
182
  - lib/rbbt/tsv/accessor.rb
179
183
  - lib/rbbt/tsv/attach.rb