rbbt-util 5.32.13 → 5.32.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/annotations/util.rb +1 -0
- data/lib/rbbt/entity.rb +6 -1
- data/lib/rbbt/hpc/batch.rb +3 -2
- data/lib/rbbt/hpc/orchestrate.rb +1 -0
- data/lib/rbbt/hpc/slurm.rb +2 -0
- data/lib/rbbt/resource/path.rb +6 -6
- data/lib/rbbt/tsv.rb +5 -0
- data/lib/rbbt/tsv/parallel/traverse.rb +9 -12
- data/lib/rbbt/util/log.rb +15 -4
- data/lib/rbbt/util/log/progress/util.rb +2 -1
- data/lib/rbbt/util/misc/inspect.rb +3 -1
- data/lib/rbbt/util/misc/omics.rb +2 -2
- data/lib/rbbt/util/misc/system.rb +2 -2
- data/lib/rbbt/util/open.rb +1 -0
- data/lib/rbbt/util/python.rb +39 -0
- data/lib/rbbt/workflow.rb +54 -2
- data/lib/rbbt/workflow/accessor.rb +15 -242
- data/lib/rbbt/workflow/definition.rb +2 -1
- data/lib/rbbt/workflow/dependencies.rb +195 -0
- data/lib/rbbt/workflow/step.rb +10 -158
- data/lib/rbbt/workflow/step/accessor.rb +1 -311
- data/lib/rbbt/workflow/step/dependencies.rb +43 -3
- data/lib/rbbt/workflow/step/info.rb +294 -0
- data/lib/rbbt/workflow/step/status.rb +146 -0
- data/lib/rbbt/workflow/usage.rb +4 -2
- data/lib/rbbt/workflow/util/data.rb +5 -1
- data/lib/rbbt/workflow/util/orchestrator.rb +1 -1
- data/lib/rbbt/workflow/util/provenance.rb +40 -9
- data/python/rbbt.py +5 -1
- data/share/rbbt_commands/hpc/orchestrate +4 -1
- data/share/rbbt_commands/hpc/task +2 -0
- data/share/rbbt_commands/lsf/orchestrate +4 -1
- data/share/rbbt_commands/lsf/task +2 -0
- data/share/rbbt_commands/slurm/orchestrate +4 -1
- data/share/rbbt_commands/slurm/task +2 -0
- data/share/rbbt_commands/system/clean +1 -0
- data/share/rbbt_commands/workflow/prov +1 -1
- data/test/rbbt/workflow/util/test_data.rb +15 -2
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: adee22216a6bbce0a4292f20e89f3c620fc47fcb28aaaf15082cd3ee54d2c093
|
4
|
+
data.tar.gz: 920f61f95dce1acbecc279ae09731dab297d0eb57cffe7a08734297e917436f8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f7fb0024e458db313ab31a46c4142092121d76efc5ca79c85876a8dc412e6483c12cf3058a1f9f5e6300a31ce672e87b60864eff46c2af8662df411e5680ac39
|
7
|
+
data.tar.gz: 4216a058f4329d25ed573129c0b40408921566218267922f1b3831108ab2745fc36279fdf7ecedd580d71c0163fae44a8d36adfc2223fecad76ac5864ce90c5f
|
@@ -152,6 +152,7 @@ module Annotated
|
|
152
152
|
tsv = TSV.setup({}, :key_field => "List", :fields => fields, :type => :list, :unnamed => true)
|
153
153
|
|
154
154
|
annot_id = annotations.id
|
155
|
+
annot_id = annot_id * "," if Array === annot_id
|
155
156
|
tsv[annot_id] = annotations.tsv_values(*fields).dup
|
156
157
|
|
157
158
|
when Array === annotations
|
data/lib/rbbt/entity.rb
CHANGED
@@ -197,7 +197,7 @@ module Entity
|
|
197
197
|
if self.instance_variable_get("@multiple_result_" + name.to_s)
|
198
198
|
return self.instance_variable_get("@multiple_result_" + name.to_s)
|
199
199
|
end
|
200
|
-
raise MultipleEntity, "Entity #{name} runs with multiple entities"
|
200
|
+
raise MultipleEntity, "Entity property #{name} runs with multiple entities"
|
201
201
|
end
|
202
202
|
|
203
203
|
define_method name do |*args|
|
@@ -225,10 +225,12 @@ module Entity
|
|
225
225
|
case res
|
226
226
|
when Array
|
227
227
|
missing.zip(res).each do |o,res|
|
228
|
+
raise "Multiple function #{name} result nil for element #{o}" if res.nil?
|
228
229
|
o.instance_variable_set("@multiple_result_" + name.to_s, res)
|
229
230
|
end
|
230
231
|
when Hash
|
231
232
|
res.each do |o,res|
|
233
|
+
raise "Multiple function #{name} result nil for element #{o}" if res.nil?
|
232
234
|
o.instance_variable_set("@multiple_result_" + name.to_s, res)
|
233
235
|
end
|
234
236
|
end
|
@@ -254,7 +256,10 @@ module Entity
|
|
254
256
|
|
255
257
|
orig_method_name = method_name
|
256
258
|
multi_name = "_multiple_" + method_name.to_s
|
259
|
+
single_name = "_single_" + method_name.to_s
|
260
|
+
|
257
261
|
method_name = multi_name if self.instance_methods.include?(multi_name.to_sym)
|
262
|
+
method_name = single_name if self.instance_methods.include?(single_name.to_sym)
|
258
263
|
|
259
264
|
orig_name = UNPERSISTED_PREFIX + method_name.to_s
|
260
265
|
alias_method orig_name, method_name unless self.instance_methods.include? orig_name.to_sym
|
data/lib/rbbt/hpc/batch.rb
CHANGED
@@ -139,7 +139,7 @@ EOF
|
|
139
139
|
|
140
140
|
keys = [
|
141
141
|
:batch_dir,
|
142
|
-
:
|
142
|
+
:lua_modules,
|
143
143
|
:batch_name,
|
144
144
|
:contain,
|
145
145
|
:contain_and_sync,
|
@@ -157,6 +157,7 @@ EOF
|
|
157
157
|
:singularity_ruby_inline,
|
158
158
|
:sync,
|
159
159
|
:task_cpus,
|
160
|
+
:gres,
|
160
161
|
:mem,
|
161
162
|
:mem_per_cpu,
|
162
163
|
:licenses,
|
@@ -289,7 +290,7 @@ let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
|
|
289
290
|
end
|
290
291
|
|
291
292
|
def prepare_environment(options = {})
|
292
|
-
modules = options[:
|
293
|
+
modules = options[:lua_modules]
|
293
294
|
|
294
295
|
prepare_environment = ""
|
295
296
|
|
data/lib/rbbt/hpc/orchestrate.rb
CHANGED
data/lib/rbbt/hpc/slurm.rb
CHANGED
@@ -30,6 +30,7 @@ export BATCH_SYSTEM=SLURM
|
|
30
30
|
highmem = Misc.process_options options, :highmem
|
31
31
|
licenses = Misc.process_options options, :licenses
|
32
32
|
constraint = Misc.process_options options, :constraint
|
33
|
+
gres = Misc.process_options options, :gres
|
33
34
|
|
34
35
|
mem = Misc.process_options options, :mem
|
35
36
|
mem_per_cpu = Misc.process_options options, :mem_per_cpu
|
@@ -50,6 +51,7 @@ export BATCH_SYSTEM=SLURM
|
|
50
51
|
"time" => time,
|
51
52
|
"exclusive" => exclusive,
|
52
53
|
"licenses" => licenses,
|
54
|
+
"gres" => gres,
|
53
55
|
"mem" => mem,
|
54
56
|
"mem-per-cpu" => mem_per_cpu,
|
55
57
|
}
|
data/lib/rbbt/resource/path.rb
CHANGED
@@ -159,6 +159,12 @@ module Path
|
|
159
159
|
end
|
160
160
|
|
161
161
|
def find(where = nil, caller_lib = nil, paths = nil)
|
162
|
+
|
163
|
+
if located?
|
164
|
+
self.original ||= self
|
165
|
+
return self
|
166
|
+
end
|
167
|
+
|
162
168
|
@path ||= {}
|
163
169
|
rsearch_paths = (resource and resource.respond_to?(:search_paths)) ? resource.search_paths : nil
|
164
170
|
key_elems = [where, caller_lib, rsearch_paths, paths]
|
@@ -167,12 +173,6 @@ module Path
|
|
167
173
|
|
168
174
|
return @path[key] if @path[key]
|
169
175
|
|
170
|
-
if located?
|
171
|
-
@path[key] = self
|
172
|
-
self.original ||= self
|
173
|
-
return self
|
174
|
-
end
|
175
|
-
|
176
176
|
@path[key] ||= begin
|
177
177
|
paths = [paths, rsearch_paths, self.search_paths, SEARCH_PATHS].reverse.compact.inject({}){|acc,h| acc.merge! h; acc }
|
178
178
|
where = paths[:default] if where == :default
|
data/lib/rbbt/tsv.rb
CHANGED
@@ -98,6 +98,11 @@ module TSV
|
|
98
98
|
stream = get_stream source, options.merge(open_options)
|
99
99
|
parse stream, data, options.merge(:tsv_grep => tsv_grep)
|
100
100
|
|
101
|
+
if ! open_options[:noclose]
|
102
|
+
stream.close unless stream.closed?
|
103
|
+
stream.join if stream.respond_to?(:join)
|
104
|
+
end
|
105
|
+
|
101
106
|
data.filename = filename.to_s unless filename.nil?
|
102
107
|
|
103
108
|
if data.identifiers.nil? and Path === filename and filename.identifier_file_path
|
@@ -97,12 +97,11 @@ module TSV
|
|
97
97
|
end
|
98
98
|
end
|
99
99
|
rescue
|
100
|
-
Log.exception $!
|
101
100
|
error = true
|
102
101
|
raise $!
|
103
102
|
ensure
|
104
103
|
join.call(error) if join
|
105
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
104
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
106
105
|
end
|
107
106
|
end
|
108
107
|
|
@@ -138,7 +137,7 @@ module TSV
|
|
138
137
|
raise $!
|
139
138
|
ensure
|
140
139
|
join.call(error) if join
|
141
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
140
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
142
141
|
end
|
143
142
|
end
|
144
143
|
|
@@ -178,7 +177,7 @@ module TSV
|
|
178
177
|
raise $!
|
179
178
|
ensure
|
180
179
|
join.call(error) if join
|
181
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
180
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
182
181
|
end
|
183
182
|
end
|
184
183
|
|
@@ -220,7 +219,7 @@ module TSV
|
|
220
219
|
raise $!
|
221
220
|
ensure
|
222
221
|
join.call(error) if join
|
223
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
222
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
224
223
|
end
|
225
224
|
end
|
226
225
|
|
@@ -274,7 +273,7 @@ module TSV
|
|
274
273
|
raise $!
|
275
274
|
ensure
|
276
275
|
join.call(error) if join
|
277
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
276
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
278
277
|
end
|
279
278
|
end
|
280
279
|
|
@@ -319,7 +318,7 @@ module TSV
|
|
319
318
|
raise $!
|
320
319
|
ensure
|
321
320
|
join.call(error) if join
|
322
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
321
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
323
322
|
end
|
324
323
|
end
|
325
324
|
|
@@ -333,7 +332,7 @@ module TSV
|
|
333
332
|
Log.low{"Traversing #{name} #{Log.color :green, "->"} #{stream_name(options[:into])}"}
|
334
333
|
begin
|
335
334
|
case obj
|
336
|
-
when (defined? FastContainers
|
335
|
+
when (defined? FastContainers && FastContainers::PriorityQueue === obj)
|
337
336
|
traverse_priority_queue(obj, options, &block)
|
338
337
|
when TSV
|
339
338
|
traverse_tsv(obj, options, &block)
|
@@ -390,7 +389,7 @@ module TSV
|
|
390
389
|
when Set
|
391
390
|
traverse_array(obj.to_a, options, &block)
|
392
391
|
when String
|
393
|
-
if Open.remote?
|
392
|
+
if Open.remote?(obj) or Misc.is_filename?(obj)
|
394
393
|
Open.open(obj) do |s|
|
395
394
|
traverse_obj(s, options, &block)
|
396
395
|
end
|
@@ -491,9 +490,7 @@ module TSV
|
|
491
490
|
q.join
|
492
491
|
raise $!
|
493
492
|
ensure
|
494
|
-
if bar
|
495
|
-
Log::ProgressBar.remove_bar(bar, error)
|
496
|
-
end
|
493
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
497
494
|
end
|
498
495
|
end
|
499
496
|
|
data/lib/rbbt/util/log.rb
CHANGED
@@ -322,8 +322,13 @@ module Log
|
|
322
322
|
|
323
323
|
def self.exception(e)
|
324
324
|
stack = caller
|
325
|
-
|
326
|
-
|
325
|
+
if ENV["RBBT_ORIGINAL_STACK"] == 'true'
|
326
|
+
error([e.class.to_s, e.message].compact * ": " )
|
327
|
+
error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
|
328
|
+
else
|
329
|
+
error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
|
330
|
+
error([e.class.to_s, e.message].compact * ": " )
|
331
|
+
end
|
327
332
|
end
|
328
333
|
|
329
334
|
def self.deprecated(m)
|
@@ -370,11 +375,17 @@ module Log
|
|
370
375
|
|
371
376
|
def self.stack(stack)
|
372
377
|
LOG_MUTEX.synchronize do
|
373
|
-
|
374
|
-
|
378
|
+
if ENV["RBBT_ORIGINAL_STACK"] == 'true'
|
379
|
+
STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
|
375
380
|
color_stack(stack).each do |line|
|
376
381
|
STDERR.puts line
|
377
382
|
end
|
383
|
+
else
|
384
|
+
STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
|
385
|
+
color_stack(stack.reverse).each do |line|
|
386
|
+
STDERR.puts line
|
387
|
+
end
|
388
|
+
end
|
378
389
|
end
|
379
390
|
end
|
380
391
|
|
@@ -320,7 +320,9 @@ module Misc
|
|
320
320
|
end
|
321
321
|
end
|
322
322
|
when String
|
323
|
-
|
323
|
+
good_filename = Misc.is_filename?(obj, false) && ! %w(. ..).include?(obj) && %w(. /).include?(obj[0])
|
324
|
+
is_path = Path === obj
|
325
|
+
if good_filename && ! is_path
|
324
326
|
obj2str Path.setup(obj.dup)
|
325
327
|
else
|
326
328
|
obj = obj.chomp if String === obj
|
data/lib/rbbt/util/misc/omics.rb
CHANGED
@@ -511,9 +511,9 @@ module Misc
|
|
511
511
|
|
512
512
|
case chr_prefix.to_s.downcase
|
513
513
|
when "remove"
|
514
|
-
sort_order = sort_order.collect{|chr| "chr" + chr } unless sort_order.first.include?('chr')
|
515
|
-
when "true", "add"
|
516
514
|
sort_order = sort_order.collect{|chr| chr.sub('chr', '') } if sort_order.first.include?('chr')
|
515
|
+
when "true", "add"
|
516
|
+
sort_order = sort_order.collect{|chr| "chr" + chr } unless sort_order.first.include?('chr')
|
517
517
|
end
|
518
518
|
|
519
519
|
sort_genomic_locations_by_contig(mutations, sort_order)
|
@@ -112,10 +112,10 @@ end
|
|
112
112
|
res
|
113
113
|
end
|
114
114
|
|
115
|
-
def self.is_filename?(string)
|
115
|
+
def self.is_filename?(string, need_to_exists = true)
|
116
116
|
return true if defined? Path and Path === string
|
117
117
|
return true if string.respond_to? :exists
|
118
|
-
return true if String === string and string.length < 265 and File.exist?(string)
|
118
|
+
return true if String === string and string.length < 265 and (File.exist?(string) || ! need_to_exists)
|
119
119
|
return false
|
120
120
|
end
|
121
121
|
|
data/lib/rbbt/util/open.rb
CHANGED
data/lib/rbbt/util/python.rb
CHANGED
@@ -8,6 +8,45 @@ module RbbtPython
|
|
8
8
|
PyCall.exec(script)
|
9
9
|
end
|
10
10
|
|
11
|
+
def self.iterate(iterator, options = {})
|
12
|
+
bar = options[:bar]
|
13
|
+
|
14
|
+
case bar
|
15
|
+
when TrueClass
|
16
|
+
bar = Log::ProgressBar.new nil, :desc => "RbbtPython iterate"
|
17
|
+
when String
|
18
|
+
bar = Log::ProgressBar.new nil, :desc => bar
|
19
|
+
end
|
20
|
+
|
21
|
+
while true
|
22
|
+
begin
|
23
|
+
yield iterator.__next__
|
24
|
+
bar.tick if bar
|
25
|
+
rescue PyCall::PyError
|
26
|
+
if $!.type.to_s == "<class 'StopIteration'>"
|
27
|
+
break
|
28
|
+
else
|
29
|
+
raise $!
|
30
|
+
end
|
31
|
+
rescue
|
32
|
+
bar.error if bar
|
33
|
+
raise $!
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
Log::ProgressBar.remove_bar bar if bar
|
38
|
+
nil
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.collect(iterator, options = {}, &block)
|
42
|
+
acc = []
|
43
|
+
self.iterate(iterator, options) do |elem|
|
44
|
+
res = block.call elem
|
45
|
+
acc << res
|
46
|
+
end
|
47
|
+
acc
|
48
|
+
end
|
49
|
+
|
11
50
|
def self.run(mod = nil, imports = nil, &block)
|
12
51
|
if mod
|
13
52
|
if Array === imports
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'rbbt/workflow/definition'
|
2
|
+
require 'rbbt/workflow/dependencies'
|
2
3
|
require 'rbbt/workflow/task'
|
3
4
|
require 'rbbt/workflow/step'
|
4
5
|
require 'rbbt/workflow/accessor'
|
@@ -228,6 +229,47 @@ module Workflow
|
|
228
229
|
end
|
229
230
|
end
|
230
231
|
|
232
|
+
TAG = ENV["RBBT_INPUT_JOBNAME"] == "true" ? :inputs : :hash
|
233
|
+
DEBUG_JOB_HASH = ENV["RBBT_DEBUG_JOB_HASH"] == 'true'
|
234
|
+
def step_path(taskname, jobname, inputs, dependencies, extension = nil)
|
235
|
+
raise "Jobname makes an invalid path: #{ jobname }" if jobname.include? '..'
|
236
|
+
if inputs.length > 0 or dependencies.any?
|
237
|
+
tagged_jobname = case TAG
|
238
|
+
when :hash
|
239
|
+
clean_inputs = Annotated.purge(inputs)
|
240
|
+
clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
|
241
|
+
deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (d.overriden? ? d.path : d.short_path) : d }
|
242
|
+
key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
|
243
|
+
key_str = Misc.obj2str(key_obj)
|
244
|
+
hash_str = Misc.digest(key_str)
|
245
|
+
Log.debug "Hash for '#{[taskname, jobname] * "/"}' #{hash_str} for #{key_str}" if DEBUG_JOB_HASH
|
246
|
+
jobname + '_' << hash_str
|
247
|
+
when :inputs
|
248
|
+
all_inputs = {}
|
249
|
+
inputs.zip(self.task_info(taskname)[:inputs]) do |i,f|
|
250
|
+
all_inputs[f] = i
|
251
|
+
end
|
252
|
+
dependencies.each do |dep|
|
253
|
+
ri = dep.recursive_inputs
|
254
|
+
ri.zip(ri.fields).each do |i,f|
|
255
|
+
all_inputs[f] = i
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
all_inputs.any? ? jobname + '_' << Misc.obj2str(all_inputs) : jobname
|
260
|
+
else
|
261
|
+
jobname
|
262
|
+
end
|
263
|
+
else
|
264
|
+
tagged_jobname = jobname
|
265
|
+
end
|
266
|
+
|
267
|
+
if extension and not extension.empty?
|
268
|
+
tagged_jobname = tagged_jobname + ('.' << extension.to_s)
|
269
|
+
end
|
270
|
+
|
271
|
+
workdir[taskname][tagged_jobname].find
|
272
|
+
end
|
231
273
|
def import_task(workflow, orig, new)
|
232
274
|
orig_task = workflow.tasks[orig]
|
233
275
|
new_task = orig_task.dup
|
@@ -411,12 +453,22 @@ module Workflow
|
|
411
453
|
|
412
454
|
overriden = has_overriden_inputs || overriden_deps.any?
|
413
455
|
|
456
|
+
extension = task.extension
|
457
|
+
|
458
|
+
if extension == :dep_task
|
459
|
+
extension = nil
|
460
|
+
if dependencies.any?
|
461
|
+
dep_basename = File.basename(dependencies.last.path)
|
462
|
+
extension = dep_basename.split(".").last if dep_basename.include?('.')
|
463
|
+
end
|
464
|
+
end
|
465
|
+
|
414
466
|
if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden
|
415
|
-
step_path = step_path taskname, jobname, [], [],
|
467
|
+
step_path = step_path taskname, jobname, [], [], extension
|
416
468
|
input_values = task.take_input_values(inputs)
|
417
469
|
else
|
418
470
|
input_values = task.take_input_values(inputs)
|
419
|
-
step_path = step_path taskname, jobname, input_values, dependencies,
|
471
|
+
step_path = step_path taskname, jobname, input_values, dependencies, extension
|
420
472
|
end
|
421
473
|
|
422
474
|
job = get_job_step step_path, task, input_values, dependencies
|