rbbt-util 5.20.26 → 5.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/annotations.rb +3 -1
- data/lib/rbbt/monitor.rb +12 -10
- data/lib/rbbt/persist.rb +1 -1
- data/lib/rbbt/rest/client.rb +2 -2
- data/lib/rbbt/rest/client/adaptor.rb +1 -0
- data/lib/rbbt/rest/client/get.rb +1 -33
- data/lib/rbbt/rest/client/run.rb +100 -0
- data/lib/rbbt/rest/client/step.rb +26 -73
- data/lib/rbbt/tsv/util.rb +7 -2
- data/lib/rbbt/util/log/progress/report.rb +1 -0
- data/lib/rbbt/util/misc/concurrent_stream.rb +6 -2
- data/lib/rbbt/util/misc/development.rb +18 -14
- data/lib/rbbt/util/misc/exceptions.rb +16 -7
- data/lib/rbbt/util/misc/inspect.rb +3 -2
- data/lib/rbbt/util/misc/pipes.rb +10 -7
- data/lib/rbbt/util/task/job.rb +6 -6
- data/lib/rbbt/workflow.rb +29 -2
- data/lib/rbbt/workflow/accessor.rb +14 -1
- data/lib/rbbt/workflow/definition.rb +23 -4
- data/lib/rbbt/workflow/soap.rb +1 -0
- data/lib/rbbt/workflow/step/dependencies.rb +5 -3
- data/lib/rbbt/workflow/step/run.rb +23 -5
- data/share/rbbt_commands/workflow/server +16 -3
- data/share/rbbt_commands/workflow/task +9 -0
- data/share/workflow_config.ru +35 -7
- data/test/rbbt/test_workflow.rb +24 -0
- data/test/rbbt/tsv/test_stream.rb +16 -0
- data/test/rbbt/util/misc/test_bgzf.rb +1 -1
- data/test/rbbt/util/test_misc.rb +29 -12
- metadata +64 -63
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -108,8 +108,13 @@ module TSV
|
|
108
108
|
StringIO.new file
|
109
109
|
end
|
110
110
|
when (defined? Step and Step)
|
111
|
-
if file.respond_to?(:base_url)
|
112
|
-
file.result
|
111
|
+
if file.respond_to?(:base_url)
|
112
|
+
if file.result and IO === file.result
|
113
|
+
file.result
|
114
|
+
else
|
115
|
+
file.join
|
116
|
+
get_stream(file.path, open_options.merge(:nocache => true))
|
117
|
+
end
|
113
118
|
else
|
114
119
|
file.grace
|
115
120
|
stream = file.get_stream
|
@@ -105,6 +105,7 @@ module Log
|
|
105
105
|
|
106
106
|
def save
|
107
107
|
info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
|
108
|
+
info.delete_if{|k,v| v.nil?}
|
108
109
|
Open.write(@file, info.to_yaml)
|
109
110
|
end
|
110
111
|
|
@@ -104,7 +104,11 @@ module ConcurrentStream
|
|
104
104
|
|
105
105
|
def join_callback
|
106
106
|
if @callback and not joined?
|
107
|
-
|
107
|
+
begin
|
108
|
+
@callback.call
|
109
|
+
rescue Exception
|
110
|
+
Log.exception $!
|
111
|
+
end
|
108
112
|
@callback = nil
|
109
113
|
end
|
110
114
|
end
|
@@ -162,7 +166,7 @@ module ConcurrentStream
|
|
162
166
|
|
163
167
|
def abort(exception = nil)
|
164
168
|
if @aborted
|
165
|
-
Log.medium "
|
169
|
+
Log.medium "Already aborted stream #{Misc.fingerprint self} [#{@aborted}]"
|
166
170
|
return
|
167
171
|
else
|
168
172
|
Log.medium "Aborting stream #{Misc.fingerprint self} [#{@aborted}]"
|
@@ -1,20 +1,20 @@
|
|
1
1
|
module Misc
|
2
2
|
|
3
|
-
|
3
|
+
def self.add_libdir(dir=nil)
|
4
4
|
dir ||= File.join(Path.caller_lib_dir(caller.first), 'lib')
|
5
5
|
$LOAD_PATH.unshift(dir) unless $LOAD_PATH.include? dir
|
6
6
|
end
|
7
7
|
|
8
8
|
def self.pre_fork
|
9
|
-
Persist::CONNECTIONS.values.each do |db|
|
10
|
-
db.close if db.write?
|
9
|
+
Persist::CONNECTIONS.values.each do |db|
|
10
|
+
db.close if db.write?
|
11
11
|
end
|
12
12
|
Log::ProgressBar::BARS.clear
|
13
|
-
ObjectSpace.each_object(Mutex) do |m|
|
14
|
-
begin
|
15
|
-
m.unlock
|
13
|
+
ObjectSpace.each_object(Mutex) do |m|
|
14
|
+
begin
|
15
|
+
m.unlock
|
16
16
|
rescue ThreadError
|
17
|
-
end if m.locked?
|
17
|
+
end if m.locked?
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
@@ -146,6 +146,8 @@ module Misc
|
|
146
146
|
rescue TryAgain
|
147
147
|
sleep sleep
|
148
148
|
retry
|
149
|
+
rescue StopInsist
|
150
|
+
raise $!.exception
|
149
151
|
rescue Aborted, Interrupt
|
150
152
|
if msg
|
151
153
|
Log.warn("Not Insisting after Aborted: #{$!.message} -- #{msg}")
|
@@ -158,7 +160,7 @@ module Misc
|
|
158
160
|
Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
|
159
161
|
else
|
160
162
|
Log.warn("Insisting after exception: #{$!.class} #{$!.message}")
|
161
|
-
end
|
163
|
+
end
|
162
164
|
|
163
165
|
if sleep and try > 0
|
164
166
|
sleep sleep
|
@@ -216,14 +218,14 @@ module Misc
|
|
216
218
|
else
|
217
219
|
v, n = template[pos], template[-1]
|
218
220
|
template.pop
|
219
|
-
template[pos] = n
|
221
|
+
template[pos] = n
|
220
222
|
end
|
221
223
|
p << v
|
222
224
|
end
|
223
225
|
else
|
224
|
-
size.times do
|
226
|
+
size.times do
|
225
227
|
pos = nil
|
226
|
-
while pos.nil?
|
228
|
+
while pos.nil?
|
227
229
|
pos = (rand * total).floor
|
228
230
|
if p.include? pos
|
229
231
|
pos = nil
|
@@ -245,9 +247,11 @@ module Misc
|
|
245
247
|
end
|
246
248
|
end
|
247
249
|
|
250
|
+
MUTEX_FOR_THREAD_EXCLUSIVE = Mutex.new
|
251
|
+
|
248
252
|
def self.object_delta(*args)
|
249
253
|
res, delta = nil, nil
|
250
|
-
|
254
|
+
MUTEX_FOR_THREAD_EXCLUSIVE.synchronize do
|
251
255
|
pre = Set.new
|
252
256
|
delta = Set.new
|
253
257
|
|
@@ -264,7 +268,7 @@ module Misc
|
|
264
268
|
end
|
265
269
|
|
266
270
|
end
|
267
|
-
Log.info "Delta: #{delta.inspect}"
|
271
|
+
Log.info "Delta: #{delta.inspect}"
|
268
272
|
res
|
269
273
|
end
|
270
274
|
|
@@ -362,7 +366,7 @@ module Misc
|
|
362
366
|
|
363
367
|
PUSHBULLET_KEY=begin
|
364
368
|
if ENV["PUSHBULLET_KEY"]
|
365
|
-
ENV["PUSHBULLET_KEY"]
|
369
|
+
ENV["PUSHBULLET_KEY"]
|
366
370
|
else
|
367
371
|
config_api = File.join(ENV['HOME'], 'config/apps/pushbullet/apikey')
|
368
372
|
if File.exist? config_api
|
@@ -1,12 +1,13 @@
|
|
1
|
-
class
|
2
|
-
class
|
3
|
-
class
|
4
|
-
class
|
1
|
+
class RbbtException < StandardError; end
|
2
|
+
class ParameterException < RbbtException; end
|
3
|
+
class FieldNotFoundError < RbbtException;end
|
4
|
+
class TryAgain < RbbtException; end
|
5
|
+
class ClosedStream < RbbtException; end
|
5
6
|
|
6
|
-
class ProcessFailed <
|
7
|
-
class Aborted <
|
7
|
+
class ProcessFailed < RbbtException; end
|
8
|
+
class Aborted < RbbtException; end
|
8
9
|
|
9
|
-
class RemoteServerError <
|
10
|
+
class RemoteServerError < RbbtException; end
|
10
11
|
|
11
12
|
class DependencyError < Aborted
|
12
13
|
def initialize(msg)
|
@@ -35,3 +36,11 @@ class KeepBar < Exception
|
|
35
36
|
@payload = payload
|
36
37
|
end
|
37
38
|
end
|
39
|
+
|
40
|
+
class StopInsist < Exception
|
41
|
+
attr_accessor :exception
|
42
|
+
def initialize(exception)
|
43
|
+
@exception = exception
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
@@ -182,8 +182,9 @@ module Misc
|
|
182
182
|
end
|
183
183
|
end
|
184
184
|
|
185
|
-
if defined? Annotated and Annotated === _v and not (defined? AssociationItem and AssociationItem === _v)
|
186
|
-
info =
|
185
|
+
if _v and defined? Annotated and Annotated === _v and not (defined? AssociationItem and AssociationItem === _v)
|
186
|
+
info = _v.info
|
187
|
+
info = Annotated.purge(info)
|
187
188
|
str << "_" << hash2md5(info)
|
188
189
|
end
|
189
190
|
end
|
data/lib/rbbt/util/misc/pipes.rb
CHANGED
@@ -23,12 +23,14 @@ module Misc
|
|
23
23
|
OPEN_PIPE_IN = []
|
24
24
|
def self.pipe
|
25
25
|
OPEN_PIPE_IN.delete_if{|pipe| pipe.closed? }
|
26
|
-
PIPE_MUTEX.synchronize do
|
26
|
+
res = PIPE_MUTEX.synchronize do
|
27
27
|
sout, sin = IO.pipe
|
28
28
|
OPEN_PIPE_IN << sin
|
29
29
|
|
30
30
|
[sout, sin]
|
31
31
|
end
|
32
|
+
Log.debug{"Creating pipe #{[res.last.inspect,res.first.inspect] * " => "}"}
|
33
|
+
res
|
32
34
|
end
|
33
35
|
|
34
36
|
def self.release_pipes(*pipes)
|
@@ -188,6 +190,7 @@ module Misc
|
|
188
190
|
end
|
189
191
|
|
190
192
|
stream.join if stream.respond_to? :join
|
193
|
+
|
191
194
|
rescue Aborted, Interrupt
|
192
195
|
stream.abort if stream.respond_to? :abort
|
193
196
|
out_pipes.each do |sout|
|
@@ -206,14 +209,14 @@ module Misc
|
|
206
209
|
end
|
207
210
|
|
208
211
|
out_pipes.each do |sout|
|
209
|
-
ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename
|
212
|
+
ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :_pair => stream
|
210
213
|
end
|
211
214
|
|
212
|
-
abort_callback = Proc.new do
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
end
|
215
|
+
#abort_callback = Proc.new do
|
216
|
+
# out_pipes.each do |s|
|
217
|
+
# s.abort if s.respond_to? :abort
|
218
|
+
# end
|
219
|
+
#end
|
217
220
|
|
218
221
|
out_pipes
|
219
222
|
end
|
data/lib/rbbt/util/task/job.rb
CHANGED
@@ -77,7 +77,7 @@ class Task
|
|
77
77
|
end
|
78
78
|
|
79
79
|
def info
|
80
|
-
return {} if not File.
|
80
|
+
return {} if not File.exist?(info_file)
|
81
81
|
info = YAML.load(File.open(info_file)) || {}
|
82
82
|
info.extend IndiferentHash
|
83
83
|
end
|
@@ -146,7 +146,7 @@ class Task
|
|
146
146
|
end
|
147
147
|
|
148
148
|
def run_dependencies
|
149
|
-
required_files.each do |file| file.produce unless File.
|
149
|
+
required_files.each do |file| file.produce unless File.exist? file end unless required_files.nil?
|
150
150
|
previous_jobs.each do |job|
|
151
151
|
if not job.recursive_done?
|
152
152
|
job.clean if job.error?
|
@@ -222,7 +222,7 @@ class Task
|
|
222
222
|
def run
|
223
223
|
return self if recursive_done?
|
224
224
|
begin
|
225
|
-
FileUtils.rm info_file if File.
|
225
|
+
FileUtils.rm info_file if File.exist? info_file
|
226
226
|
step(:started)
|
227
227
|
start
|
228
228
|
step(:done)
|
@@ -293,9 +293,9 @@ class Task
|
|
293
293
|
end
|
294
294
|
|
295
295
|
def clean
|
296
|
-
FileUtils.rm path if File.
|
297
|
-
FileUtils.rm info_file if File.
|
298
|
-
FileUtils.rm_rf path + '.files' if File.
|
296
|
+
FileUtils.rm path if File.exist? path
|
297
|
+
FileUtils.rm info_file if File.exist? info_file
|
298
|
+
FileUtils.rm_rf path + '.files' if File.exist? path + '.files'
|
299
299
|
self
|
300
300
|
end
|
301
301
|
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -181,7 +181,7 @@ module Workflow
|
|
181
181
|
attr_accessor :libdir, :workdir
|
182
182
|
attr_accessor :helpers, :tasks
|
183
183
|
attr_accessor :task_dependencies, :task_description, :last_task
|
184
|
-
attr_accessor :asynchronous_exports, :synchronous_exports, :exec_exports
|
184
|
+
attr_accessor :stream_exports, :asynchronous_exports, :synchronous_exports, :exec_exports
|
185
185
|
attr_accessor :step_cache
|
186
186
|
attr_accessor :remote_tasks
|
187
187
|
|
@@ -237,6 +237,10 @@ module Workflow
|
|
237
237
|
@task_description ||= {}
|
238
238
|
end
|
239
239
|
|
240
|
+
def stream_exports
|
241
|
+
@stream_exports ||= []
|
242
|
+
end
|
243
|
+
|
240
244
|
def asynchronous_exports
|
241
245
|
@asynchronous_exports ||= []
|
242
246
|
end
|
@@ -250,7 +254,7 @@ module Workflow
|
|
250
254
|
end
|
251
255
|
|
252
256
|
def all_exports
|
253
|
-
@all_exports ||= asynchronous_exports + synchronous_exports + exec_exports
|
257
|
+
@all_exports ||= asynchronous_exports + synchronous_exports + exec_exports + stream_exports
|
254
258
|
end
|
255
259
|
|
256
260
|
# {{{ JOB MANAGEMENT
|
@@ -404,4 +408,27 @@ module Workflow
|
|
404
408
|
self.workdir = saved
|
405
409
|
end
|
406
410
|
end
|
411
|
+
|
412
|
+
def add_remote_tasks(remote_tasks)
|
413
|
+
remote_tasks.each do |remote, tasks|
|
414
|
+
tasks.each do |task|
|
415
|
+
self.remote_tasks[task.to_f] = remote
|
416
|
+
end
|
417
|
+
end
|
418
|
+
end
|
419
|
+
|
420
|
+
def self.process_remote_tasks(remote_tasks)
|
421
|
+
require 'rbbt/rest/client'
|
422
|
+
remote_tasks.each do |workflow, info|
|
423
|
+
wf = Workflow.require_workflow workflow
|
424
|
+
wf.remote_tasks ||= {}
|
425
|
+
info.each do |remote, tasks|
|
426
|
+
remote_wf = WorkflowRESTClient.new remote, workflow
|
427
|
+
tasks.each do |task|
|
428
|
+
wf.remote_tasks[task.to_sym] = remote_wf
|
429
|
+
end
|
430
|
+
end
|
431
|
+
end
|
432
|
+
end
|
433
|
+
|
407
434
|
end
|
@@ -315,6 +315,17 @@ class Step
|
|
315
315
|
self._abort
|
316
316
|
end
|
317
317
|
|
318
|
+
def recoverable_error?
|
319
|
+
return true if aborted?
|
320
|
+
return false unless error?
|
321
|
+
begin
|
322
|
+
klass = Kernel.const_get(info[:exception][:class])
|
323
|
+
not RbbtException === klass
|
324
|
+
rescue Exception
|
325
|
+
false
|
326
|
+
end
|
327
|
+
end
|
328
|
+
|
318
329
|
def started?
|
319
330
|
Open.exists?(path) or Open.exists?(pid_file)
|
320
331
|
end
|
@@ -469,6 +480,8 @@ module Workflow
|
|
469
480
|
:asynchronous
|
470
481
|
when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
|
471
482
|
:exec
|
483
|
+
when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
|
484
|
+
:stream
|
472
485
|
else
|
473
486
|
:none
|
474
487
|
end
|
@@ -772,7 +785,7 @@ module Workflow
|
|
772
785
|
end
|
773
786
|
|
774
787
|
def task_exports
|
775
|
-
[exec_exports, synchronous_exports, asynchronous_exports].compact.flatten.uniq
|
788
|
+
[exec_exports, synchronous_exports, asynchronous_exports, stream_exports].compact.flatten.uniq
|
776
789
|
end
|
777
790
|
|
778
791
|
end
|
@@ -92,23 +92,42 @@ module Workflow
|
|
92
92
|
tasks[name] = task
|
93
93
|
task_dependencies[name] = consume_dependencies
|
94
94
|
end
|
95
|
+
|
96
|
+
def unexport(*names)
|
97
|
+
names = names.collect{|n| n.to_s} + names.collect{|n| n.to_sym}
|
98
|
+
names.uniq!
|
99
|
+
exec_exports.replace exec_exports - names if exec_exports
|
100
|
+
synchronous_exports.replace synchronous_exports - names if synchronous_exports
|
101
|
+
asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
|
102
|
+
stream_exports.replace stream_exports - names if stream_exports
|
103
|
+
end
|
95
104
|
|
96
105
|
def export_exec(*names)
|
106
|
+
unexport *names
|
97
107
|
exec_exports.concat names
|
98
108
|
exec_exports.uniq!
|
99
109
|
exec_exports
|
100
110
|
end
|
101
111
|
|
112
|
+
def export_synchronous(*names)
|
113
|
+
unexport *names
|
114
|
+
synchronous_exports.concat names
|
115
|
+
synchronous_exports.uniq!
|
116
|
+
synchronous_exports
|
117
|
+
end
|
118
|
+
|
102
119
|
def export_asynchronous(*names)
|
120
|
+
unexport *names
|
103
121
|
asynchronous_exports.concat names
|
104
122
|
asynchronous_exports.uniq!
|
105
123
|
asynchronous_exports
|
106
124
|
end
|
107
125
|
|
108
|
-
def
|
109
|
-
|
110
|
-
|
111
|
-
|
126
|
+
def export_stream(*names)
|
127
|
+
unexport *names
|
128
|
+
stream_exports.concat names
|
129
|
+
stream_exports.uniq!
|
130
|
+
stream_exports
|
112
131
|
end
|
113
132
|
|
114
133
|
alias export export_asynchronous
|
data/lib/rbbt/workflow/soap.rb
CHANGED
@@ -19,6 +19,7 @@ class WorkflowSOAP < SimpleWS
|
|
19
19
|
@workflow = workflow
|
20
20
|
@workflow.synchronous_exports.each do |name| synchronous name end
|
21
21
|
@workflow.asynchronous_exports.each do |name| asynchronous name end
|
22
|
+
@workflow.stream_exports.each do |name| asynchronous name end
|
22
23
|
|
23
24
|
desc "Job management: Check the status of a job"
|
24
25
|
param_desc :jobid => "Job identifier", :return => "Status code. Special status codes are: 'done' and 'error'"
|
@@ -114,7 +114,7 @@ class Step
|
|
114
114
|
|
115
115
|
if not dependency.started?
|
116
116
|
log_dependency_exec(dependency, :starting)
|
117
|
-
dependency.run(
|
117
|
+
dependency.run(true)
|
118
118
|
raise TryAgain
|
119
119
|
end
|
120
120
|
|
@@ -154,6 +154,7 @@ class Step
|
|
154
154
|
raise $!
|
155
155
|
rescue Exception
|
156
156
|
Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s } -- #{$!.message}"
|
157
|
+
Log.exception $!
|
157
158
|
raise $!
|
158
159
|
end
|
159
160
|
end
|
@@ -187,7 +188,7 @@ class Step
|
|
187
188
|
Misc.insist do
|
188
189
|
begin
|
189
190
|
step.produce
|
190
|
-
rescue
|
191
|
+
rescue Exception
|
191
192
|
step.abort
|
192
193
|
raise $!
|
193
194
|
end
|
@@ -203,12 +204,13 @@ class Step
|
|
203
204
|
Misc.insist do
|
204
205
|
begin
|
205
206
|
dep.produce
|
207
|
+
Log.warn "Error producing #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
|
206
208
|
rescue Aborted
|
207
209
|
dep.abort
|
208
210
|
raise $!
|
209
211
|
rescue Exception
|
210
212
|
dep.exception $!
|
211
|
-
raise $!
|
213
|
+
raise StopInsist.new($!)
|
212
214
|
end
|
213
215
|
end
|
214
216
|
nil
|