rbbt-util 5.28.0 → 5.28.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc.rb +5 -2
- data/lib/rbbt/resource/path.rb +4 -2
- data/lib/rbbt/tsv/change_id.rb +2 -2
- data/lib/rbbt/tsv/parallel/traverse.rb +7 -8
- data/lib/rbbt/tsv/util.rb +20 -12
- data/lib/rbbt/util/misc/inspect.rb +3 -3
- data/lib/rbbt/workflow.rb +1 -0
- data/lib/rbbt/workflow/accessor.rb +94 -93
- data/lib/rbbt/workflow/integration/cromwell.rb +10 -3
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +5 -1
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +5 -0
- data/lib/rbbt/workflow/step/accessor.rb +10 -7
- data/lib/rbbt/workflow/step/dependencies.rb +2 -2
- data/lib/rbbt/workflow/usage.rb +1 -1
- data/share/rbbt_commands/app/start +2 -2
- data/share/rbbt_commands/workflow/server +2 -0
- data/test/rbbt/tsv/parallel/test_traverse.rb +21 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 900106fb1799c857a482187c7cb7d18a40c35dbeb5dd231bf8e1851235e28c63
|
4
|
+
data.tar.gz: e5ec2271b3ada0b282963f4ffc7e2b623d8bb84fcb72867a6848b93a80052463
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1a418d5b4fe4369f7c25dd063c60a65c2b6c0bd7e3dbf9fd156fe875f85ef1b2eeb4d2eb1a03472e191de98bbfbae20abdb86877d381193f86fe4c8325d2f391
|
7
|
+
data.tar.gz: 5225d6dbdbab08d0ed5239eba91bbe3d2a8dd16c92c24fe177b66ed3b9d4baef4e24818057de8bf694edf715aabd72b481926a2466c76492c27b75be25634b27
|
data/lib/rbbt/hpc.rb
CHANGED
@@ -451,12 +451,15 @@ EOF
|
|
451
451
|
out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists?(fout) and not tail == :STDERR
|
452
452
|
err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exists?(ferr)
|
453
453
|
|
454
|
-
Misc.consume_stream(err, true, STDERR) if err
|
455
|
-
Misc.consume_stream(out, true, STDOUT) if out
|
454
|
+
terr = Misc.consume_stream(err, true, STDERR) if err
|
455
|
+
tout = Misc.consume_stream(out, true, STDOUT) if out
|
456
456
|
|
457
457
|
sleep 3 while CMD.cmd("squeue --job #{job}").read.include? job.to_s
|
458
|
+
rescue Aborted
|
458
459
|
ensure
|
459
460
|
begin
|
461
|
+
terr.exit if terr
|
462
|
+
tout.exit if tout
|
460
463
|
err.close if err
|
461
464
|
err.join if err
|
462
465
|
rescue Exception
|
data/lib/rbbt/resource/path.rb
CHANGED
@@ -76,7 +76,7 @@ module Path
|
|
76
76
|
return [] unless self.exists?
|
77
77
|
found = self.find
|
78
78
|
exp = File.join(found, pattern)
|
79
|
-
paths = Dir.glob(exp).collect{|f|
|
79
|
+
paths = Dir.glob(exp).collect{|f| self.annotate(f) }
|
80
80
|
|
81
81
|
paths.each do |p|
|
82
82
|
p.original = File.join(found.original, p.sub(/^#{found}/, ''))
|
@@ -412,7 +412,9 @@ module Path
|
|
412
412
|
end
|
413
413
|
|
414
414
|
def replace_extension(new_extension = nil, multiple = false)
|
415
|
-
if multiple
|
415
|
+
if String === multiple
|
416
|
+
new_path = self.sub(/(\.[^\.\/]{1,5})(.#{multiple})?$/,'')
|
417
|
+
elsif multiple
|
416
418
|
new_path = self.sub(/(\.[^\.\/]{1,5})+$/,'')
|
417
419
|
else
|
418
420
|
new_path = self.sub(/\.[^\.\/]{1,5}$/,'')
|
data/lib/rbbt/tsv/change_id.rb
CHANGED
@@ -58,13 +58,13 @@ module TSV
|
|
58
58
|
|
59
59
|
identifiers, persist_input, compact = Misc.process_options options, :identifiers, :persist, :compact
|
60
60
|
identifiers = tsv.identifier_files.first if identifiers.nil?
|
61
|
-
identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil?
|
61
|
+
identifiers = Organism.identifiers(tsv.namespace) if defined?(Organism) && identifiers.nil? && tsv.namespace && Organism.identifiers(tsv.namespace).exists?
|
62
62
|
identifiers.namespace ||= tsv.namespace
|
63
63
|
|
64
64
|
fields = (identifiers and identifiers.all_fields.include?(field))? [field] : nil
|
65
65
|
#index = identifiers.index :target => format, :fields => fields, :persist => persist_input, :order => true
|
66
66
|
|
67
|
-
grep = Organism.blacklist_genes(tsv.namespace).list if identifiers.namespace
|
67
|
+
grep = Organism.blacklist_genes(tsv.namespace).list if defined?(Organism) && identifiers.namespace && Organism.blacklist_genes(tsv.namespace).exists?
|
68
68
|
if fields.nil?
|
69
69
|
index = identifiers.index(:data_tsv_grep => grep, :data_invert_grep => true, :target => format, :persist => true, :order => true, :unnamed => true, :data_persist => true)
|
70
70
|
else
|
@@ -101,8 +101,8 @@ module TSV
|
|
101
101
|
error = true
|
102
102
|
raise $!
|
103
103
|
ensure
|
104
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
105
104
|
join.call(error) if join
|
105
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
106
106
|
end
|
107
107
|
end
|
108
108
|
|
@@ -137,8 +137,8 @@ module TSV
|
|
137
137
|
error = true
|
138
138
|
raise $!
|
139
139
|
ensure
|
140
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
141
140
|
join.call(error) if join
|
141
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
@@ -177,8 +177,8 @@ module TSV
|
|
177
177
|
error = true
|
178
178
|
raise $!
|
179
179
|
ensure
|
180
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
181
180
|
join.call(error) if join
|
181
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
182
182
|
end
|
183
183
|
end
|
184
184
|
|
@@ -219,8 +219,8 @@ module TSV
|
|
219
219
|
error = true
|
220
220
|
raise $!
|
221
221
|
ensure
|
222
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
223
222
|
join.call(error) if join
|
223
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
224
224
|
end
|
225
225
|
end
|
226
226
|
|
@@ -273,8 +273,8 @@ module TSV
|
|
273
273
|
error = true
|
274
274
|
raise $!
|
275
275
|
ensure
|
276
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
277
276
|
join.call(error) if join
|
277
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
278
278
|
end
|
279
279
|
end
|
280
280
|
|
@@ -318,8 +318,8 @@ module TSV
|
|
318
318
|
error = true
|
319
319
|
raise $!
|
320
320
|
ensure
|
321
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
322
321
|
join.call(error) if join
|
322
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
323
323
|
end
|
324
324
|
end
|
325
325
|
|
@@ -377,6 +377,7 @@ module TSV
|
|
377
377
|
obj.run(true) unless obj.done? || obj.started? || obj.result
|
378
378
|
|
379
379
|
stream = obj.get_stream
|
380
|
+
options = {:type => :array}.merge(options) if obj.result_type == :array
|
380
381
|
|
381
382
|
if stream
|
382
383
|
traverse_obj(stream, options, &block)
|
@@ -727,8 +728,6 @@ module TSV
|
|
727
728
|
Log.exception $!
|
728
729
|
abort_stream(into, $!)
|
729
730
|
raise $!
|
730
|
-
ensure
|
731
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
732
731
|
end
|
733
732
|
|
734
733
|
into
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -316,22 +316,30 @@ module TSV
|
|
316
316
|
|
317
317
|
def to_single
|
318
318
|
new = {}
|
319
|
-
|
320
|
-
|
321
|
-
through do |k,v|
|
322
|
-
new[k] = v.first.first
|
323
|
-
end
|
324
|
-
when :flat
|
319
|
+
|
320
|
+
if block_given?
|
325
321
|
through do |k,v|
|
326
|
-
new[k] = v
|
322
|
+
new[k] = yield v
|
327
323
|
end
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
324
|
+
else
|
325
|
+
case type
|
326
|
+
when :double
|
327
|
+
through do |k,v|
|
328
|
+
new[k] = v.first.first
|
329
|
+
end
|
330
|
+
when :flat
|
331
|
+
through do |k,v|
|
332
|
+
new[k] = v.first
|
333
|
+
end
|
334
|
+
when :single
|
335
|
+
return self
|
336
|
+
when :list
|
337
|
+
through do |k,v|
|
338
|
+
new[k] = v.first
|
339
|
+
end
|
333
340
|
end
|
334
341
|
end
|
342
|
+
|
335
343
|
self.annotate(new)
|
336
344
|
new.type = :single
|
337
345
|
new.fields = [new.fields.first] if new.fields.length > 1
|
@@ -290,9 +290,9 @@ module Misc
|
|
290
290
|
if obj.exists?
|
291
291
|
if obj.directory?
|
292
292
|
files = obj.glob("**/*")
|
293
|
-
"directory: #{files}"
|
293
|
+
"directory: #{Misc.fingerprint(files)}"
|
294
294
|
else
|
295
|
-
"file: " << obj << "--" << mtime_str(obj)
|
295
|
+
"file: " << Open.realpath(obj) << "--" << mtime_str(obj)
|
296
296
|
end
|
297
297
|
else
|
298
298
|
obj + " (file missing)"
|
@@ -401,7 +401,7 @@ module Misc
|
|
401
401
|
|
402
402
|
def self.scan_version_text(text, cmd = nil)
|
403
403
|
cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
|
404
|
-
m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
404
|
+
m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
405
405
|
return nil if m.nil?
|
406
406
|
m[1]
|
407
407
|
end
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -364,6 +364,7 @@ module Workflow
|
|
364
364
|
raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
|
365
365
|
end
|
366
366
|
|
367
|
+
# jobname => true sets the value of the input to the name of the job
|
367
368
|
if task.input_options
|
368
369
|
jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
|
369
370
|
else
|
@@ -26,105 +26,106 @@ module Workflow
|
|
26
26
|
raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
|
27
27
|
id = File.join(self.to_s, name.to_s)
|
28
28
|
@task_info ||= {}
|
29
|
-
@task_info[id] ||= begin
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
29
|
+
@task_info[id] ||= begin
|
30
|
+
description = task.description
|
31
|
+
result_description = task.result_description
|
32
|
+
result_type = task.result_type
|
33
|
+
inputs = rec_inputs(name).uniq
|
34
|
+
input_types = rec_input_types(name)
|
35
|
+
input_descriptions = rec_input_descriptions(name)
|
36
|
+
input_use = rec_input_use(name)
|
37
|
+
input_defaults = rec_input_defaults(name)
|
38
|
+
input_options = rec_input_options(name)
|
39
|
+
extension = task.extension
|
40
|
+
export = case
|
41
|
+
when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
|
42
|
+
:synchronous
|
43
|
+
when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
|
44
|
+
:asynchronous
|
45
|
+
when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
|
46
|
+
:exec
|
47
|
+
when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
|
48
|
+
:stream
|
49
|
+
else
|
50
|
+
:none
|
51
|
+
end
|
51
52
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
53
|
+
dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
|
54
|
+
{ :id => id,
|
55
|
+
:description => description,
|
56
|
+
:export => export,
|
57
|
+
:inputs => inputs,
|
58
|
+
:input_types => input_types,
|
59
|
+
:input_descriptions => input_descriptions,
|
60
|
+
:input_defaults => input_defaults,
|
61
|
+
:input_options => input_options,
|
62
|
+
:input_use => input_use,
|
63
|
+
:result_type => result_type,
|
64
|
+
:result_description => result_description,
|
65
|
+
:dependencies => dependencies,
|
66
|
+
:extension => extension
|
67
|
+
}
|
68
|
+
end
|
68
69
|
end
|
69
70
|
|
70
71
|
def rec_dependencies(taskname)
|
71
72
|
@rec_dependencies ||= {}
|
72
73
|
@rec_dependencies[taskname] ||= begin
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
74
|
+
if task_dependencies.include? taskname
|
75
|
+
|
76
|
+
deps = task_dependencies[taskname]
|
77
|
+
|
78
|
+
#all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
|
79
|
+
|
80
|
+
all_deps = []
|
81
|
+
deps.each do |dep|
|
82
|
+
if DependencyBlock === dep
|
83
|
+
all_deps << dep.dependency if dep.dependency
|
84
|
+
else
|
85
|
+
all_deps << dep unless Proc === dep
|
86
|
+
end
|
87
|
+
|
88
|
+
begin
|
89
|
+
case dep
|
90
|
+
when Array
|
91
|
+
wf, t, o = dep
|
92
|
+
|
93
|
+
wf.rec_dependencies(t.to_sym).each do |d|
|
94
|
+
if Array === d
|
95
|
+
new = d.dup
|
96
|
+
else
|
97
|
+
new = [dep.first, d]
|
98
|
+
end
|
99
|
+
|
100
|
+
if Hash === o and not o.empty?
|
101
|
+
if Hash === new.last
|
102
|
+
hash = new.last.dup
|
103
|
+
o.each{|k,v| hash[k] ||= v}
|
104
|
+
new[new.length-1] = hash
|
105
|
+
else
|
106
|
+
new.push o.dup
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
all_deps << new
|
111
|
+
end if wf && t
|
112
|
+
|
113
|
+
when String, Symbol
|
114
|
+
rec_deps = rec_dependencies(dep.to_sym)
|
115
|
+
all_deps.concat rec_deps
|
116
|
+
when DependencyBlock
|
117
|
+
dep = dep.dependency
|
118
|
+
raise TryAgain
|
119
|
+
end
|
120
|
+
rescue TryAgain
|
121
|
+
retry
|
122
|
+
end
|
106
123
|
end
|
124
|
+
all_deps.uniq
|
125
|
+
else
|
126
|
+
[]
|
107
127
|
end
|
108
|
-
|
109
|
-
all_deps << new
|
110
|
-
end if wf && t
|
111
|
-
|
112
|
-
when String, Symbol
|
113
|
-
rec_deps = rec_dependencies(dep.to_sym)
|
114
|
-
all_deps.concat rec_deps
|
115
|
-
when DependencyBlock
|
116
|
-
dep = dep.dependency
|
117
|
-
raise TryAgain
|
118
|
-
end
|
119
|
-
rescue TryAgain
|
120
|
-
retry
|
121
|
-
end
|
122
|
-
end
|
123
|
-
all_deps.uniq
|
124
|
-
else
|
125
|
-
[]
|
126
|
-
end
|
127
|
-
end
|
128
|
+
end
|
128
129
|
end
|
129
130
|
|
130
131
|
def task_from_dep(dep)
|
@@ -182,7 +183,7 @@ module Workflow
|
|
182
183
|
acc
|
183
184
|
}.tap{|h| IndiferentHash.setup(h)}
|
184
185
|
end
|
185
|
-
|
186
|
+
|
186
187
|
def rec_input_use(taskname)
|
187
188
|
task = task_from_dep(taskname)
|
188
189
|
deps = rec_dependencies(taskname)
|
@@ -474,7 +475,7 @@ module Workflow
|
|
474
475
|
else
|
475
476
|
workdir_find = workdir
|
476
477
|
end
|
477
|
-
|
478
|
+
|
478
479
|
workdir_find = File.expand_path(workdir_find)
|
479
480
|
path = File.expand_path(path)
|
480
481
|
dir = File.dirname(path)
|
@@ -1,13 +1,20 @@
|
|
1
|
-
module
|
1
|
+
module Cromwell
|
2
2
|
|
3
3
|
Rbbt.claim Rbbt.software.opt.jar["cromwell.jar"], :url, "https://github.com/broadinstitute/cromwell/releases/download/48/cromwell-48.jar"
|
4
4
|
Rbbt.claim Rbbt.software.opt.jar["wdltool.jar"], :url, "https://github.com/broadinstitute/wdltool/releases/download/0.14/wdltool-0.14.jar"
|
5
5
|
|
6
|
-
def run_cromwell(file, work_dir, options = {})
|
6
|
+
def self.run_cromwell(file, work_dir, options = {})
|
7
|
+
cromwell_inputs_file = Misc.process_options options, :cromwell_inputs_file
|
7
8
|
jar = Rbbt.software.opt.jar["cromwell.jar"].produce.find
|
8
|
-
|
9
|
+
if cromwell_inputs_file
|
10
|
+
CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}' -i #{cromwell_inputs_file}", options.merge("add_option_dashes" => true))
|
11
|
+
else
|
12
|
+
CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}'", options.merge("add_option_dashes" => true))
|
13
|
+
end
|
9
14
|
end
|
15
|
+
end
|
10
16
|
|
17
|
+
module Workflow
|
11
18
|
def load_cromwell(file)
|
12
19
|
jar = Rbbt.software.opt.jar["wdltool.jar"].produce.find
|
13
20
|
inputs = JSON.load(CMD.cmd("java -jar '#{jar}' inputs '#{file}'"))
|
@@ -85,7 +85,11 @@ class RemoteWorkflow
|
|
85
85
|
|
86
86
|
RemoteWorkflow::REST.__prepare_inputs_for_restclient(params)
|
87
87
|
name = RemoteWorkflow.capture_exception do
|
88
|
-
|
88
|
+
begin
|
89
|
+
RestClient.post(self.encode(url), params)
|
90
|
+
rescue RestClient::MovedPermanently, RestClient::Found, RestClient::TemporaryRedirect
|
91
|
+
raise RbbtException, "REST end-point moved to: #{$!.response.headers[:location]}"
|
92
|
+
end
|
89
93
|
end
|
90
94
|
|
91
95
|
Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
|
@@ -340,6 +340,11 @@ class RemoteStep < Step
|
|
340
340
|
@inputs = new_inputs
|
341
341
|
@info = nil
|
342
342
|
end
|
343
|
+
|
344
|
+
def init_info(*args)
|
345
|
+
i = {:status => :waiting, :pid => Process.pid, :path => path}
|
346
|
+
i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
|
347
|
+
end
|
343
348
|
end
|
344
349
|
|
345
350
|
require 'rbbt/workflow/remote_workflow/remote_step/rest'
|
@@ -93,8 +93,8 @@ class Step
|
|
93
93
|
else
|
94
94
|
Open.write(path + '.read', value.to_s)
|
95
95
|
end
|
96
|
-
when Step ===
|
97
|
-
|
96
|
+
when Step === value
|
97
|
+
value = value.produce.load
|
98
98
|
else
|
99
99
|
Open.write(path, value.to_s)
|
100
100
|
end
|
@@ -110,12 +110,14 @@ class Step
|
|
110
110
|
task_info = workflow.task_info(task_name)
|
111
111
|
input_types = task_info[:input_types]
|
112
112
|
task_inputs = task_info[:inputs]
|
113
|
+
input_defaults = task_info[:input_defaults]
|
113
114
|
|
114
115
|
inputs = {}
|
115
116
|
job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
|
116
117
|
next unless task_inputs.include? name.to_sym
|
117
118
|
next if options and ! options.include?(name)
|
118
119
|
next if value.nil?
|
120
|
+
next if input_defaults[name] == value
|
119
121
|
inputs[name] = value
|
120
122
|
end
|
121
123
|
|
@@ -125,7 +127,7 @@ class Step
|
|
125
127
|
end
|
126
128
|
save_inputs(inputs, input_types, dir)
|
127
129
|
|
128
|
-
inputs.
|
130
|
+
inputs.keys
|
129
131
|
end
|
130
132
|
|
131
133
|
def name
|
@@ -208,7 +210,7 @@ class Step
|
|
208
210
|
end
|
209
211
|
|
210
212
|
def init_info(force = false)
|
211
|
-
return nil if @exec
|
213
|
+
return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
|
212
214
|
Open.lock(info_file, :lock => info_lock) do
|
213
215
|
i = {:status => :waiting, :pid => Process.pid, :path => path}
|
214
216
|
i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
|
@@ -437,11 +439,12 @@ class Step
|
|
437
439
|
rec_dependencies = self.rec_dependencies
|
438
440
|
return [] if rec_dependencies.empty?
|
439
441
|
canfail_paths = self.canfail_paths
|
442
|
+
dep = rec_dependencies.select{|d| d.task_name.to_s == 'contamination'}.first
|
440
443
|
dirty_files = rec_dependencies.reject{|dep|
|
441
444
|
(defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dep) ||
|
442
445
|
! Open.exists?(dep.info_file) ||
|
443
446
|
(dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) ||
|
444
|
-
((dep.error? || dep.aborted?
|
447
|
+
((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
|
445
448
|
}
|
446
449
|
end
|
447
450
|
|
@@ -508,12 +511,12 @@ class Step
|
|
508
511
|
|
509
512
|
def nopid?
|
510
513
|
pid = info[:pid] || Open.exists?(pid_file)
|
511
|
-
! pid && ! (status.nil? || status == :aborted || status == :done || status == :error)
|
514
|
+
! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
|
512
515
|
end
|
513
516
|
|
514
517
|
def aborted?
|
515
518
|
status = self.status
|
516
|
-
status == :aborted || ((status != :noinfo && status != :setup && status != :noinfo) && nopid?)
|
519
|
+
status == :aborted || ((status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
|
517
520
|
end
|
518
521
|
|
519
522
|
# {{{ INFO
|
@@ -24,7 +24,7 @@ class Step
|
|
24
24
|
Log.medium "Not duplicating stream #{stream_key}"
|
25
25
|
STREAM_CACHE[stream_key] = stream
|
26
26
|
when File
|
27
|
-
if Open.exists?
|
27
|
+
if Open.exists?(current.path)
|
28
28
|
Log.medium "Reopening file #{stream_key}"
|
29
29
|
Open.open(current.path)
|
30
30
|
else
|
@@ -97,7 +97,7 @@ class Step
|
|
97
97
|
end
|
98
98
|
|
99
99
|
job.dup_inputs unless status == 'done' or job.started?
|
100
|
-
job.init_info unless status == '
|
100
|
+
job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
|
101
101
|
|
102
102
|
canfail = ComputeDependency === job && job.canfail?
|
103
103
|
end
|
data/lib/rbbt/workflow/usage.rb
CHANGED
@@ -54,9 +54,9 @@ app = ARGV.shift
|
|
54
54
|
|
55
55
|
ENV["RServe-session"] = options[:RServe_session] || app
|
56
56
|
|
57
|
-
app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps
|
57
|
+
app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps
|
58
58
|
|
59
|
-
app_dir = app_dir[app]
|
59
|
+
app_dir = app_dir[app].find
|
60
60
|
|
61
61
|
server = options[:server] || 'puma'
|
62
62
|
Misc.in_dir(app_dir) do
|
@@ -25,6 +25,7 @@ $ rbbt workflow server [options] <Workflow>
|
|
25
25
|
--stream Activate streaming of workflow tasks
|
26
26
|
-fs--file_server Activate file serving for resources
|
27
27
|
-mj--monitor_jobs Monitor jobs (UNSAFE)
|
28
|
+
-a--app_dir* Application execution directory
|
28
29
|
--export_all Export all workflow tasks (use with caution!)
|
29
30
|
--export* Export workflow tasks (asynchronous)
|
30
31
|
--export_asynchronous* Export workflow tasks as asynchronous
|
@@ -74,6 +75,7 @@ sync_exports = options[:export_synchronous].split(/\s*,/) if options[:export_syn
|
|
74
75
|
exec_exports = options[:export_exec].split(/\s*,/) if options[:export_exec]
|
75
76
|
|
76
77
|
TmpFile.with_file do |app_dir|
|
78
|
+
app_dir = options[:app_dir] if options[:app_dir]
|
77
79
|
Misc.in_dir(app_dir) do
|
78
80
|
app_dir = Path.setup(app_dir.dup)
|
79
81
|
Open.write(app_dir.etc.target_workflow.find, workflow)
|
@@ -202,7 +202,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
202
202
|
stream = datafile_test('identifiers').open
|
203
203
|
dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
|
204
204
|
dumper.init
|
205
|
-
TSV.traverse stream, :head => head, :into => dumper do |k,v|
|
205
|
+
TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
|
206
206
|
k = k.first
|
207
207
|
[k,v]
|
208
208
|
end
|
@@ -212,6 +212,26 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
212
212
|
assert_equal head, res.size
|
213
213
|
end
|
214
214
|
|
215
|
+
def test_traverse_into_dumper_error_bar
|
216
|
+
require 'rbbt/sources/organism'
|
217
|
+
|
218
|
+
head = 2_000
|
219
|
+
|
220
|
+
stream = datafile_test('identifiers').open
|
221
|
+
dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
|
222
|
+
dumper.init
|
223
|
+
TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
|
224
|
+
k = k.first
|
225
|
+
raise
|
226
|
+
[k,v]
|
227
|
+
end
|
228
|
+
|
229
|
+
assert_raise do
|
230
|
+
res = TSV.open(dumper.stream)
|
231
|
+
end
|
232
|
+
|
233
|
+
end
|
234
|
+
|
215
235
|
def test_traverse_into_dumper_threads
|
216
236
|
require 'rbbt/sources/organism'
|
217
237
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.28.
|
4
|
+
version: 5.28.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|