rbbt-util 5.28.0 → 5.28.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc.rb +5 -2
- data/lib/rbbt/resource/path.rb +4 -2
- data/lib/rbbt/tsv/change_id.rb +2 -2
- data/lib/rbbt/tsv/parallel/traverse.rb +7 -8
- data/lib/rbbt/tsv/util.rb +20 -12
- data/lib/rbbt/util/misc/inspect.rb +3 -3
- data/lib/rbbt/workflow.rb +1 -0
- data/lib/rbbt/workflow/accessor.rb +94 -93
- data/lib/rbbt/workflow/integration/cromwell.rb +10 -3
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +5 -1
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +5 -0
- data/lib/rbbt/workflow/step/accessor.rb +10 -7
- data/lib/rbbt/workflow/step/dependencies.rb +2 -2
- data/lib/rbbt/workflow/usage.rb +1 -1
- data/share/rbbt_commands/app/start +2 -2
- data/share/rbbt_commands/workflow/server +2 -0
- data/test/rbbt/tsv/parallel/test_traverse.rb +21 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 900106fb1799c857a482187c7cb7d18a40c35dbeb5dd231bf8e1851235e28c63
|
4
|
+
data.tar.gz: e5ec2271b3ada0b282963f4ffc7e2b623d8bb84fcb72867a6848b93a80052463
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1a418d5b4fe4369f7c25dd063c60a65c2b6c0bd7e3dbf9fd156fe875f85ef1b2eeb4d2eb1a03472e191de98bbfbae20abdb86877d381193f86fe4c8325d2f391
|
7
|
+
data.tar.gz: 5225d6dbdbab08d0ed5239eba91bbe3d2a8dd16c92c24fe177b66ed3b9d4baef4e24818057de8bf694edf715aabd72b481926a2466c76492c27b75be25634b27
|
data/lib/rbbt/hpc.rb
CHANGED
@@ -451,12 +451,15 @@ EOF
|
|
451
451
|
out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists?(fout) and not tail == :STDERR
|
452
452
|
err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exists?(ferr)
|
453
453
|
|
454
|
-
Misc.consume_stream(err, true, STDERR) if err
|
455
|
-
Misc.consume_stream(out, true, STDOUT) if out
|
454
|
+
terr = Misc.consume_stream(err, true, STDERR) if err
|
455
|
+
tout = Misc.consume_stream(out, true, STDOUT) if out
|
456
456
|
|
457
457
|
sleep 3 while CMD.cmd("squeue --job #{job}").read.include? job.to_s
|
458
|
+
rescue Aborted
|
458
459
|
ensure
|
459
460
|
begin
|
461
|
+
terr.exit if terr
|
462
|
+
tout.exit if tout
|
460
463
|
err.close if err
|
461
464
|
err.join if err
|
462
465
|
rescue Exception
|
data/lib/rbbt/resource/path.rb
CHANGED
@@ -76,7 +76,7 @@ module Path
|
|
76
76
|
return [] unless self.exists?
|
77
77
|
found = self.find
|
78
78
|
exp = File.join(found, pattern)
|
79
|
-
paths = Dir.glob(exp).collect{|f|
|
79
|
+
paths = Dir.glob(exp).collect{|f| self.annotate(f) }
|
80
80
|
|
81
81
|
paths.each do |p|
|
82
82
|
p.original = File.join(found.original, p.sub(/^#{found}/, ''))
|
@@ -412,7 +412,9 @@ module Path
|
|
412
412
|
end
|
413
413
|
|
414
414
|
def replace_extension(new_extension = nil, multiple = false)
|
415
|
-
if multiple
|
415
|
+
if String === multiple
|
416
|
+
new_path = self.sub(/(\.[^\.\/]{1,5})(.#{multiple})?$/,'')
|
417
|
+
elsif multiple
|
416
418
|
new_path = self.sub(/(\.[^\.\/]{1,5})+$/,'')
|
417
419
|
else
|
418
420
|
new_path = self.sub(/\.[^\.\/]{1,5}$/,'')
|
data/lib/rbbt/tsv/change_id.rb
CHANGED
@@ -58,13 +58,13 @@ module TSV
|
|
58
58
|
|
59
59
|
identifiers, persist_input, compact = Misc.process_options options, :identifiers, :persist, :compact
|
60
60
|
identifiers = tsv.identifier_files.first if identifiers.nil?
|
61
|
-
identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil?
|
61
|
+
identifiers = Organism.identifiers(tsv.namespace) if defined?(Organism) && identifiers.nil? && tsv.namespace && Organism.identifiers(tsv.namespace).exists?
|
62
62
|
identifiers.namespace ||= tsv.namespace
|
63
63
|
|
64
64
|
fields = (identifiers and identifiers.all_fields.include?(field))? [field] : nil
|
65
65
|
#index = identifiers.index :target => format, :fields => fields, :persist => persist_input, :order => true
|
66
66
|
|
67
|
-
grep = Organism.blacklist_genes(tsv.namespace).list if identifiers.namespace
|
67
|
+
grep = Organism.blacklist_genes(tsv.namespace).list if defined?(Organism) && identifiers.namespace && Organism.blacklist_genes(tsv.namespace).exists?
|
68
68
|
if fields.nil?
|
69
69
|
index = identifiers.index(:data_tsv_grep => grep, :data_invert_grep => true, :target => format, :persist => true, :order => true, :unnamed => true, :data_persist => true)
|
70
70
|
else
|
@@ -101,8 +101,8 @@ module TSV
|
|
101
101
|
error = true
|
102
102
|
raise $!
|
103
103
|
ensure
|
104
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
105
104
|
join.call(error) if join
|
105
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
106
106
|
end
|
107
107
|
end
|
108
108
|
|
@@ -137,8 +137,8 @@ module TSV
|
|
137
137
|
error = true
|
138
138
|
raise $!
|
139
139
|
ensure
|
140
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
141
140
|
join.call(error) if join
|
141
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
@@ -177,8 +177,8 @@ module TSV
|
|
177
177
|
error = true
|
178
178
|
raise $!
|
179
179
|
ensure
|
180
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
181
180
|
join.call(error) if join
|
181
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
182
182
|
end
|
183
183
|
end
|
184
184
|
|
@@ -219,8 +219,8 @@ module TSV
|
|
219
219
|
error = true
|
220
220
|
raise $!
|
221
221
|
ensure
|
222
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
223
222
|
join.call(error) if join
|
223
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
224
224
|
end
|
225
225
|
end
|
226
226
|
|
@@ -273,8 +273,8 @@ module TSV
|
|
273
273
|
error = true
|
274
274
|
raise $!
|
275
275
|
ensure
|
276
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
277
276
|
join.call(error) if join
|
277
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
278
278
|
end
|
279
279
|
end
|
280
280
|
|
@@ -318,8 +318,8 @@ module TSV
|
|
318
318
|
error = true
|
319
319
|
raise $!
|
320
320
|
ensure
|
321
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
322
321
|
join.call(error) if join
|
322
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
323
323
|
end
|
324
324
|
end
|
325
325
|
|
@@ -377,6 +377,7 @@ module TSV
|
|
377
377
|
obj.run(true) unless obj.done? || obj.started? || obj.result
|
378
378
|
|
379
379
|
stream = obj.get_stream
|
380
|
+
options = {:type => :array}.merge(options) if obj.result_type == :array
|
380
381
|
|
381
382
|
if stream
|
382
383
|
traverse_obj(stream, options, &block)
|
@@ -727,8 +728,6 @@ module TSV
|
|
727
728
|
Log.exception $!
|
728
729
|
abort_stream(into, $!)
|
729
730
|
raise $!
|
730
|
-
ensure
|
731
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
732
731
|
end
|
733
732
|
|
734
733
|
into
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -316,22 +316,30 @@ module TSV
|
|
316
316
|
|
317
317
|
def to_single
|
318
318
|
new = {}
|
319
|
-
|
320
|
-
|
321
|
-
through do |k,v|
|
322
|
-
new[k] = v.first.first
|
323
|
-
end
|
324
|
-
when :flat
|
319
|
+
|
320
|
+
if block_given?
|
325
321
|
through do |k,v|
|
326
|
-
new[k] = v
|
322
|
+
new[k] = yield v
|
327
323
|
end
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
324
|
+
else
|
325
|
+
case type
|
326
|
+
when :double
|
327
|
+
through do |k,v|
|
328
|
+
new[k] = v.first.first
|
329
|
+
end
|
330
|
+
when :flat
|
331
|
+
through do |k,v|
|
332
|
+
new[k] = v.first
|
333
|
+
end
|
334
|
+
when :single
|
335
|
+
return self
|
336
|
+
when :list
|
337
|
+
through do |k,v|
|
338
|
+
new[k] = v.first
|
339
|
+
end
|
333
340
|
end
|
334
341
|
end
|
342
|
+
|
335
343
|
self.annotate(new)
|
336
344
|
new.type = :single
|
337
345
|
new.fields = [new.fields.first] if new.fields.length > 1
|
@@ -290,9 +290,9 @@ module Misc
|
|
290
290
|
if obj.exists?
|
291
291
|
if obj.directory?
|
292
292
|
files = obj.glob("**/*")
|
293
|
-
"directory: #{files}"
|
293
|
+
"directory: #{Misc.fingerprint(files)}"
|
294
294
|
else
|
295
|
-
"file: " << obj << "--" << mtime_str(obj)
|
295
|
+
"file: " << Open.realpath(obj) << "--" << mtime_str(obj)
|
296
296
|
end
|
297
297
|
else
|
298
298
|
obj + " (file missing)"
|
@@ -401,7 +401,7 @@ module Misc
|
|
401
401
|
|
402
402
|
def self.scan_version_text(text, cmd = nil)
|
403
403
|
cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
|
404
|
-
m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
404
|
+
m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
405
405
|
return nil if m.nil?
|
406
406
|
m[1]
|
407
407
|
end
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -364,6 +364,7 @@ module Workflow
|
|
364
364
|
raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
|
365
365
|
end
|
366
366
|
|
367
|
+
# jobname => true sets the value of the input to the name of the job
|
367
368
|
if task.input_options
|
368
369
|
jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
|
369
370
|
else
|
@@ -26,105 +26,106 @@ module Workflow
|
|
26
26
|
raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
|
27
27
|
id = File.join(self.to_s, name.to_s)
|
28
28
|
@task_info ||= {}
|
29
|
-
@task_info[id] ||= begin
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
29
|
+
@task_info[id] ||= begin
|
30
|
+
description = task.description
|
31
|
+
result_description = task.result_description
|
32
|
+
result_type = task.result_type
|
33
|
+
inputs = rec_inputs(name).uniq
|
34
|
+
input_types = rec_input_types(name)
|
35
|
+
input_descriptions = rec_input_descriptions(name)
|
36
|
+
input_use = rec_input_use(name)
|
37
|
+
input_defaults = rec_input_defaults(name)
|
38
|
+
input_options = rec_input_options(name)
|
39
|
+
extension = task.extension
|
40
|
+
export = case
|
41
|
+
when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
|
42
|
+
:synchronous
|
43
|
+
when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
|
44
|
+
:asynchronous
|
45
|
+
when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
|
46
|
+
:exec
|
47
|
+
when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
|
48
|
+
:stream
|
49
|
+
else
|
50
|
+
:none
|
51
|
+
end
|
51
52
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
53
|
+
dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
|
54
|
+
{ :id => id,
|
55
|
+
:description => description,
|
56
|
+
:export => export,
|
57
|
+
:inputs => inputs,
|
58
|
+
:input_types => input_types,
|
59
|
+
:input_descriptions => input_descriptions,
|
60
|
+
:input_defaults => input_defaults,
|
61
|
+
:input_options => input_options,
|
62
|
+
:input_use => input_use,
|
63
|
+
:result_type => result_type,
|
64
|
+
:result_description => result_description,
|
65
|
+
:dependencies => dependencies,
|
66
|
+
:extension => extension
|
67
|
+
}
|
68
|
+
end
|
68
69
|
end
|
69
70
|
|
70
71
|
def rec_dependencies(taskname)
|
71
72
|
@rec_dependencies ||= {}
|
72
73
|
@rec_dependencies[taskname] ||= begin
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
74
|
+
if task_dependencies.include? taskname
|
75
|
+
|
76
|
+
deps = task_dependencies[taskname]
|
77
|
+
|
78
|
+
#all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
|
79
|
+
|
80
|
+
all_deps = []
|
81
|
+
deps.each do |dep|
|
82
|
+
if DependencyBlock === dep
|
83
|
+
all_deps << dep.dependency if dep.dependency
|
84
|
+
else
|
85
|
+
all_deps << dep unless Proc === dep
|
86
|
+
end
|
87
|
+
|
88
|
+
begin
|
89
|
+
case dep
|
90
|
+
when Array
|
91
|
+
wf, t, o = dep
|
92
|
+
|
93
|
+
wf.rec_dependencies(t.to_sym).each do |d|
|
94
|
+
if Array === d
|
95
|
+
new = d.dup
|
96
|
+
else
|
97
|
+
new = [dep.first, d]
|
98
|
+
end
|
99
|
+
|
100
|
+
if Hash === o and not o.empty?
|
101
|
+
if Hash === new.last
|
102
|
+
hash = new.last.dup
|
103
|
+
o.each{|k,v| hash[k] ||= v}
|
104
|
+
new[new.length-1] = hash
|
105
|
+
else
|
106
|
+
new.push o.dup
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
all_deps << new
|
111
|
+
end if wf && t
|
112
|
+
|
113
|
+
when String, Symbol
|
114
|
+
rec_deps = rec_dependencies(dep.to_sym)
|
115
|
+
all_deps.concat rec_deps
|
116
|
+
when DependencyBlock
|
117
|
+
dep = dep.dependency
|
118
|
+
raise TryAgain
|
119
|
+
end
|
120
|
+
rescue TryAgain
|
121
|
+
retry
|
122
|
+
end
|
106
123
|
end
|
124
|
+
all_deps.uniq
|
125
|
+
else
|
126
|
+
[]
|
107
127
|
end
|
108
|
-
|
109
|
-
all_deps << new
|
110
|
-
end if wf && t
|
111
|
-
|
112
|
-
when String, Symbol
|
113
|
-
rec_deps = rec_dependencies(dep.to_sym)
|
114
|
-
all_deps.concat rec_deps
|
115
|
-
when DependencyBlock
|
116
|
-
dep = dep.dependency
|
117
|
-
raise TryAgain
|
118
|
-
end
|
119
|
-
rescue TryAgain
|
120
|
-
retry
|
121
|
-
end
|
122
|
-
end
|
123
|
-
all_deps.uniq
|
124
|
-
else
|
125
|
-
[]
|
126
|
-
end
|
127
|
-
end
|
128
|
+
end
|
128
129
|
end
|
129
130
|
|
130
131
|
def task_from_dep(dep)
|
@@ -182,7 +183,7 @@ module Workflow
|
|
182
183
|
acc
|
183
184
|
}.tap{|h| IndiferentHash.setup(h)}
|
184
185
|
end
|
185
|
-
|
186
|
+
|
186
187
|
def rec_input_use(taskname)
|
187
188
|
task = task_from_dep(taskname)
|
188
189
|
deps = rec_dependencies(taskname)
|
@@ -474,7 +475,7 @@ module Workflow
|
|
474
475
|
else
|
475
476
|
workdir_find = workdir
|
476
477
|
end
|
477
|
-
|
478
|
+
|
478
479
|
workdir_find = File.expand_path(workdir_find)
|
479
480
|
path = File.expand_path(path)
|
480
481
|
dir = File.dirname(path)
|
@@ -1,13 +1,20 @@
|
|
1
|
-
module
|
1
|
+
module Cromwell
|
2
2
|
|
3
3
|
Rbbt.claim Rbbt.software.opt.jar["cromwell.jar"], :url, "https://github.com/broadinstitute/cromwell/releases/download/48/cromwell-48.jar"
|
4
4
|
Rbbt.claim Rbbt.software.opt.jar["wdltool.jar"], :url, "https://github.com/broadinstitute/wdltool/releases/download/0.14/wdltool-0.14.jar"
|
5
5
|
|
6
|
-
def run_cromwell(file, work_dir, options = {})
|
6
|
+
def self.run_cromwell(file, work_dir, options = {})
|
7
|
+
cromwell_inputs_file = Misc.process_options options, :cromwell_inputs_file
|
7
8
|
jar = Rbbt.software.opt.jar["cromwell.jar"].produce.find
|
8
|
-
|
9
|
+
if cromwell_inputs_file
|
10
|
+
CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}' -i #{cromwell_inputs_file}", options.merge("add_option_dashes" => true))
|
11
|
+
else
|
12
|
+
CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}'", options.merge("add_option_dashes" => true))
|
13
|
+
end
|
9
14
|
end
|
15
|
+
end
|
10
16
|
|
17
|
+
module Workflow
|
11
18
|
def load_cromwell(file)
|
12
19
|
jar = Rbbt.software.opt.jar["wdltool.jar"].produce.find
|
13
20
|
inputs = JSON.load(CMD.cmd("java -jar '#{jar}' inputs '#{file}'"))
|
@@ -85,7 +85,11 @@ class RemoteWorkflow
|
|
85
85
|
|
86
86
|
RemoteWorkflow::REST.__prepare_inputs_for_restclient(params)
|
87
87
|
name = RemoteWorkflow.capture_exception do
|
88
|
-
|
88
|
+
begin
|
89
|
+
RestClient.post(self.encode(url), params)
|
90
|
+
rescue RestClient::MovedPermanently, RestClient::Found, RestClient::TemporaryRedirect
|
91
|
+
raise RbbtException, "REST end-point moved to: #{$!.response.headers[:location]}"
|
92
|
+
end
|
89
93
|
end
|
90
94
|
|
91
95
|
Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
|
@@ -340,6 +340,11 @@ class RemoteStep < Step
|
|
340
340
|
@inputs = new_inputs
|
341
341
|
@info = nil
|
342
342
|
end
|
343
|
+
|
344
|
+
def init_info(*args)
|
345
|
+
i = {:status => :waiting, :pid => Process.pid, :path => path}
|
346
|
+
i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
|
347
|
+
end
|
343
348
|
end
|
344
349
|
|
345
350
|
require 'rbbt/workflow/remote_workflow/remote_step/rest'
|
@@ -93,8 +93,8 @@ class Step
|
|
93
93
|
else
|
94
94
|
Open.write(path + '.read', value.to_s)
|
95
95
|
end
|
96
|
-
when Step ===
|
97
|
-
|
96
|
+
when Step === value
|
97
|
+
value = value.produce.load
|
98
98
|
else
|
99
99
|
Open.write(path, value.to_s)
|
100
100
|
end
|
@@ -110,12 +110,14 @@ class Step
|
|
110
110
|
task_info = workflow.task_info(task_name)
|
111
111
|
input_types = task_info[:input_types]
|
112
112
|
task_inputs = task_info[:inputs]
|
113
|
+
input_defaults = task_info[:input_defaults]
|
113
114
|
|
114
115
|
inputs = {}
|
115
116
|
job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
|
116
117
|
next unless task_inputs.include? name.to_sym
|
117
118
|
next if options and ! options.include?(name)
|
118
119
|
next if value.nil?
|
120
|
+
next if input_defaults[name] == value
|
119
121
|
inputs[name] = value
|
120
122
|
end
|
121
123
|
|
@@ -125,7 +127,7 @@ class Step
|
|
125
127
|
end
|
126
128
|
save_inputs(inputs, input_types, dir)
|
127
129
|
|
128
|
-
inputs.
|
130
|
+
inputs.keys
|
129
131
|
end
|
130
132
|
|
131
133
|
def name
|
@@ -208,7 +210,7 @@ class Step
|
|
208
210
|
end
|
209
211
|
|
210
212
|
def init_info(force = false)
|
211
|
-
return nil if @exec
|
213
|
+
return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
|
212
214
|
Open.lock(info_file, :lock => info_lock) do
|
213
215
|
i = {:status => :waiting, :pid => Process.pid, :path => path}
|
214
216
|
i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
|
@@ -437,11 +439,12 @@ class Step
|
|
437
439
|
rec_dependencies = self.rec_dependencies
|
438
440
|
return [] if rec_dependencies.empty?
|
439
441
|
canfail_paths = self.canfail_paths
|
442
|
+
dep = rec_dependencies.select{|d| d.task_name.to_s == 'contamination'}.first
|
440
443
|
dirty_files = rec_dependencies.reject{|dep|
|
441
444
|
(defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dep) ||
|
442
445
|
! Open.exists?(dep.info_file) ||
|
443
446
|
(dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) ||
|
444
|
-
((dep.error? || dep.aborted?
|
447
|
+
((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
|
445
448
|
}
|
446
449
|
end
|
447
450
|
|
@@ -508,12 +511,12 @@ class Step
|
|
508
511
|
|
509
512
|
def nopid?
|
510
513
|
pid = info[:pid] || Open.exists?(pid_file)
|
511
|
-
! pid && ! (status.nil? || status == :aborted || status == :done || status == :error)
|
514
|
+
! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
|
512
515
|
end
|
513
516
|
|
514
517
|
def aborted?
|
515
518
|
status = self.status
|
516
|
-
status == :aborted || ((status != :noinfo && status != :setup && status != :noinfo) && nopid?)
|
519
|
+
status == :aborted || ((status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
|
517
520
|
end
|
518
521
|
|
519
522
|
# {{{ INFO
|
@@ -24,7 +24,7 @@ class Step
|
|
24
24
|
Log.medium "Not duplicating stream #{stream_key}"
|
25
25
|
STREAM_CACHE[stream_key] = stream
|
26
26
|
when File
|
27
|
-
if Open.exists?
|
27
|
+
if Open.exists?(current.path)
|
28
28
|
Log.medium "Reopening file #{stream_key}"
|
29
29
|
Open.open(current.path)
|
30
30
|
else
|
@@ -97,7 +97,7 @@ class Step
|
|
97
97
|
end
|
98
98
|
|
99
99
|
job.dup_inputs unless status == 'done' or job.started?
|
100
|
-
job.init_info unless status == '
|
100
|
+
job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
|
101
101
|
|
102
102
|
canfail = ComputeDependency === job && job.canfail?
|
103
103
|
end
|
data/lib/rbbt/workflow/usage.rb
CHANGED
@@ -54,9 +54,9 @@ app = ARGV.shift
|
|
54
54
|
|
55
55
|
ENV["RServe-session"] = options[:RServe_session] || app
|
56
56
|
|
57
|
-
app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps
|
57
|
+
app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps
|
58
58
|
|
59
|
-
app_dir = app_dir[app]
|
59
|
+
app_dir = app_dir[app].find
|
60
60
|
|
61
61
|
server = options[:server] || 'puma'
|
62
62
|
Misc.in_dir(app_dir) do
|
@@ -25,6 +25,7 @@ $ rbbt workflow server [options] <Workflow>
|
|
25
25
|
--stream Activate streaming of workflow tasks
|
26
26
|
-fs--file_server Activate file serving for resources
|
27
27
|
-mj--monitor_jobs Monitor jobs (UNSAFE)
|
28
|
+
-a--app_dir* Application execution directory
|
28
29
|
--export_all Export all workflow tasks (use with caution!)
|
29
30
|
--export* Export workflow tasks (asynchronous)
|
30
31
|
--export_asynchronous* Export workflow tasks as asynchronous
|
@@ -74,6 +75,7 @@ sync_exports = options[:export_synchronous].split(/\s*,/) if options[:export_syn
|
|
74
75
|
exec_exports = options[:export_exec].split(/\s*,/) if options[:export_exec]
|
75
76
|
|
76
77
|
TmpFile.with_file do |app_dir|
|
78
|
+
app_dir = options[:app_dir] if options[:app_dir]
|
77
79
|
Misc.in_dir(app_dir) do
|
78
80
|
app_dir = Path.setup(app_dir.dup)
|
79
81
|
Open.write(app_dir.etc.target_workflow.find, workflow)
|
@@ -202,7 +202,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
202
202
|
stream = datafile_test('identifiers').open
|
203
203
|
dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
|
204
204
|
dumper.init
|
205
|
-
TSV.traverse stream, :head => head, :into => dumper do |k,v|
|
205
|
+
TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
|
206
206
|
k = k.first
|
207
207
|
[k,v]
|
208
208
|
end
|
@@ -212,6 +212,26 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
212
212
|
assert_equal head, res.size
|
213
213
|
end
|
214
214
|
|
215
|
+
def test_traverse_into_dumper_error_bar
|
216
|
+
require 'rbbt/sources/organism'
|
217
|
+
|
218
|
+
head = 2_000
|
219
|
+
|
220
|
+
stream = datafile_test('identifiers').open
|
221
|
+
dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
|
222
|
+
dumper.init
|
223
|
+
TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
|
224
|
+
k = k.first
|
225
|
+
raise
|
226
|
+
[k,v]
|
227
|
+
end
|
228
|
+
|
229
|
+
assert_raise do
|
230
|
+
res = TSV.open(dumper.stream)
|
231
|
+
end
|
232
|
+
|
233
|
+
end
|
234
|
+
|
215
235
|
def test_traverse_into_dumper_threads
|
216
236
|
require 'rbbt/sources/organism'
|
217
237
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.28.
|
4
|
+
version: 5.28.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|