rbbt-util 5.32.13 → 5.32.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/annotations/util.rb +1 -0
- data/lib/rbbt/entity.rb +6 -1
- data/lib/rbbt/hpc/batch.rb +3 -2
- data/lib/rbbt/hpc/orchestrate.rb +1 -0
- data/lib/rbbt/hpc/slurm.rb +2 -0
- data/lib/rbbt/tsv/parallel/traverse.rb +7 -10
- data/lib/rbbt/util/log/progress/util.rb +2 -1
- data/lib/rbbt/util/misc/omics.rb +2 -2
- data/lib/rbbt/util/misc/system.rb +2 -2
- data/lib/rbbt/util/python.rb +39 -0
- data/lib/rbbt/workflow/util/data.rb +5 -1
- data/python/rbbt.py +5 -1
- data/share/rbbt_commands/hpc/orchestrate +4 -1
- data/share/rbbt_commands/hpc/task +2 -0
- data/share/rbbt_commands/lsf/orchestrate +4 -1
- data/share/rbbt_commands/lsf/task +2 -0
- data/share/rbbt_commands/slurm/orchestrate +4 -1
- data/share/rbbt_commands/slurm/task +2 -0
- data/test/rbbt/workflow/util/test_data.rb +15 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 54a6869616f33a2c526e35ddddf90c1e7c29e79c5df9fabaa51d9711e0aba249
|
4
|
+
data.tar.gz: 7a6c7290ff27c00ba5911a609897eb9104b357adec7a38b892ad5a8c01267e26
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1d7f69b1680ffc9cc8d30435da870ed50411d02ac285ab8b98f98f4fd8f9b3e07bf3b228904f5dc5ea03a987e4b37de398e99d50e8679f833682a0ecba8f632a
|
7
|
+
data.tar.gz: 35ff2212eccbb3fbb2141f7aa7ec1c6d6de4ae77f3e2289ce8d99b11ff5d657d967a25367e5710e0f1fa13f0e7cff2a6f525b13ce68b82b933a7113b16146b10
|
@@ -152,6 +152,7 @@ module Annotated
|
|
152
152
|
tsv = TSV.setup({}, :key_field => "List", :fields => fields, :type => :list, :unnamed => true)
|
153
153
|
|
154
154
|
annot_id = annotations.id
|
155
|
+
annot_id = annot_id * "," if Array === annot_id
|
155
156
|
tsv[annot_id] = annotations.tsv_values(*fields).dup
|
156
157
|
|
157
158
|
when Array === annotations
|
data/lib/rbbt/entity.rb
CHANGED
@@ -197,7 +197,7 @@ module Entity
|
|
197
197
|
if self.instance_variable_get("@multiple_result_" + name.to_s)
|
198
198
|
return self.instance_variable_get("@multiple_result_" + name.to_s)
|
199
199
|
end
|
200
|
-
raise MultipleEntity, "Entity #{name} runs with multiple entities"
|
200
|
+
raise MultipleEntity, "Entity property #{name} runs with multiple entities"
|
201
201
|
end
|
202
202
|
|
203
203
|
define_method name do |*args|
|
@@ -225,10 +225,12 @@ module Entity
|
|
225
225
|
case res
|
226
226
|
when Array
|
227
227
|
missing.zip(res).each do |o,res|
|
228
|
+
raise "Multiple function #{name} result nil for element #{o}" if res.nil?
|
228
229
|
o.instance_variable_set("@multiple_result_" + name.to_s, res)
|
229
230
|
end
|
230
231
|
when Hash
|
231
232
|
res.each do |o,res|
|
233
|
+
raise "Multiple function #{name} result nil for element #{o}" if res.nil?
|
232
234
|
o.instance_variable_set("@multiple_result_" + name.to_s, res)
|
233
235
|
end
|
234
236
|
end
|
@@ -254,7 +256,10 @@ module Entity
|
|
254
256
|
|
255
257
|
orig_method_name = method_name
|
256
258
|
multi_name = "_multiple_" + method_name.to_s
|
259
|
+
single_name = "_single_" + method_name.to_s
|
260
|
+
|
257
261
|
method_name = multi_name if self.instance_methods.include?(multi_name.to_sym)
|
262
|
+
method_name = single_name if self.instance_methods.include?(single_name.to_sym)
|
258
263
|
|
259
264
|
orig_name = UNPERSISTED_PREFIX + method_name.to_s
|
260
265
|
alias_method orig_name, method_name unless self.instance_methods.include? orig_name.to_sym
|
data/lib/rbbt/hpc/batch.rb
CHANGED
@@ -139,7 +139,7 @@ EOF
|
|
139
139
|
|
140
140
|
keys = [
|
141
141
|
:batch_dir,
|
142
|
-
:
|
142
|
+
:lua_modules,
|
143
143
|
:batch_name,
|
144
144
|
:contain,
|
145
145
|
:contain_and_sync,
|
@@ -157,6 +157,7 @@ EOF
|
|
157
157
|
:singularity_ruby_inline,
|
158
158
|
:sync,
|
159
159
|
:task_cpus,
|
160
|
+
:gres,
|
160
161
|
:mem,
|
161
162
|
:mem_per_cpu,
|
162
163
|
:licenses,
|
@@ -289,7 +290,7 @@ let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
|
|
289
290
|
end
|
290
291
|
|
291
292
|
def prepare_environment(options = {})
|
292
|
-
modules = options[:
|
293
|
+
modules = options[:lua_modules]
|
293
294
|
|
294
295
|
prepare_environment = ""
|
295
296
|
|
data/lib/rbbt/hpc/orchestrate.rb
CHANGED
data/lib/rbbt/hpc/slurm.rb
CHANGED
@@ -30,6 +30,7 @@ export BATCH_SYSTEM=SLURM
|
|
30
30
|
highmem = Misc.process_options options, :highmem
|
31
31
|
licenses = Misc.process_options options, :licenses
|
32
32
|
constraint = Misc.process_options options, :constraint
|
33
|
+
gres = Misc.process_options options, :gres
|
33
34
|
|
34
35
|
mem = Misc.process_options options, :mem
|
35
36
|
mem_per_cpu = Misc.process_options options, :mem_per_cpu
|
@@ -50,6 +51,7 @@ export BATCH_SYSTEM=SLURM
|
|
50
51
|
"time" => time,
|
51
52
|
"exclusive" => exclusive,
|
52
53
|
"licenses" => licenses,
|
54
|
+
"gres" => gres,
|
53
55
|
"mem" => mem,
|
54
56
|
"mem-per-cpu" => mem_per_cpu,
|
55
57
|
}
|
@@ -97,12 +97,11 @@ module TSV
|
|
97
97
|
end
|
98
98
|
end
|
99
99
|
rescue
|
100
|
-
Log.exception $!
|
101
100
|
error = true
|
102
101
|
raise $!
|
103
102
|
ensure
|
104
103
|
join.call(error) if join
|
105
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
104
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
106
105
|
end
|
107
106
|
end
|
108
107
|
|
@@ -138,7 +137,7 @@ module TSV
|
|
138
137
|
raise $!
|
139
138
|
ensure
|
140
139
|
join.call(error) if join
|
141
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
140
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
142
141
|
end
|
143
142
|
end
|
144
143
|
|
@@ -178,7 +177,7 @@ module TSV
|
|
178
177
|
raise $!
|
179
178
|
ensure
|
180
179
|
join.call(error) if join
|
181
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
180
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
182
181
|
end
|
183
182
|
end
|
184
183
|
|
@@ -220,7 +219,7 @@ module TSV
|
|
220
219
|
raise $!
|
221
220
|
ensure
|
222
221
|
join.call(error) if join
|
223
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
222
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
224
223
|
end
|
225
224
|
end
|
226
225
|
|
@@ -274,7 +273,7 @@ module TSV
|
|
274
273
|
raise $!
|
275
274
|
ensure
|
276
275
|
join.call(error) if join
|
277
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
276
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
278
277
|
end
|
279
278
|
end
|
280
279
|
|
@@ -319,7 +318,7 @@ module TSV
|
|
319
318
|
raise $!
|
320
319
|
ensure
|
321
320
|
join.call(error) if join
|
322
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
321
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
323
322
|
end
|
324
323
|
end
|
325
324
|
|
@@ -491,9 +490,7 @@ module TSV
|
|
491
490
|
q.join
|
492
491
|
raise $!
|
493
492
|
ensure
|
494
|
-
if bar
|
495
|
-
Log::ProgressBar.remove_bar(bar, error)
|
496
|
-
end
|
493
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
497
494
|
end
|
498
495
|
end
|
499
496
|
|
data/lib/rbbt/util/misc/omics.rb
CHANGED
@@ -511,9 +511,9 @@ module Misc
|
|
511
511
|
|
512
512
|
case chr_prefix.to_s.downcase
|
513
513
|
when "remove"
|
514
|
-
sort_order = sort_order.collect{|chr| "chr" + chr } unless sort_order.first.include?('chr')
|
515
|
-
when "true", "add"
|
516
514
|
sort_order = sort_order.collect{|chr| chr.sub('chr', '') } if sort_order.first.include?('chr')
|
515
|
+
when "true", "add"
|
516
|
+
sort_order = sort_order.collect{|chr| "chr" + chr } unless sort_order.first.include?('chr')
|
517
517
|
end
|
518
518
|
|
519
519
|
sort_genomic_locations_by_contig(mutations, sort_order)
|
@@ -112,10 +112,10 @@ end
|
|
112
112
|
res
|
113
113
|
end
|
114
114
|
|
115
|
-
def self.is_filename?(string)
|
115
|
+
def self.is_filename?(string, need_to_exists = true)
|
116
116
|
return true if defined? Path and Path === string
|
117
117
|
return true if string.respond_to? :exists
|
118
|
-
return true if String === string and string.length < 265 and File.exist?(string)
|
118
|
+
return true if String === string and string.length < 265 and (File.exist?(string) || ! need_to_exists)
|
119
119
|
return false
|
120
120
|
end
|
121
121
|
|
data/lib/rbbt/util/python.rb
CHANGED
@@ -8,6 +8,45 @@ module RbbtPython
|
|
8
8
|
PyCall.exec(script)
|
9
9
|
end
|
10
10
|
|
11
|
+
def self.iterate(iterator, options = {})
|
12
|
+
bar = options[:bar]
|
13
|
+
|
14
|
+
case bar
|
15
|
+
when TrueClass
|
16
|
+
bar = Log::ProgressBar.new nil, :desc => "RbbtPython iterate"
|
17
|
+
when String
|
18
|
+
bar = Log::ProgressBar.new nil, :desc => bar
|
19
|
+
end
|
20
|
+
|
21
|
+
while true
|
22
|
+
begin
|
23
|
+
yield iterator.__next__
|
24
|
+
bar.tick if bar
|
25
|
+
rescue PyCall::PyError
|
26
|
+
if $!.type.to_s == "<class 'StopIteration'>"
|
27
|
+
break
|
28
|
+
else
|
29
|
+
raise $!
|
30
|
+
end
|
31
|
+
rescue
|
32
|
+
bar.error if bar
|
33
|
+
raise $!
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
Log::ProgressBar.remove_bar bar if bar
|
38
|
+
nil
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.collect(iterator, options = {}, &block)
|
42
|
+
acc = []
|
43
|
+
self.iterate(iterator, options) do |elem|
|
44
|
+
res = block.call elem
|
45
|
+
acc << res
|
46
|
+
end
|
47
|
+
acc
|
48
|
+
end
|
49
|
+
|
11
50
|
def self.run(mod = nil, imports = nil, &block)
|
12
51
|
if mod
|
13
52
|
if Array === imports
|
@@ -23,7 +23,11 @@ module Workflow
|
|
23
23
|
task_info = workflow.task_info(oname)
|
24
24
|
dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
|
25
25
|
data_options = block.call data_dir, dir_options, task_info
|
26
|
-
|
26
|
+
if data_options.include?(:inputs)
|
27
|
+
data_options
|
28
|
+
else
|
29
|
+
{:inputs => data_options.merge(options)}
|
30
|
+
end
|
27
31
|
end
|
28
32
|
end
|
29
33
|
end
|
data/python/rbbt.py
CHANGED
@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
|
|
28
28
|
-t--task_cpus* Tasks
|
29
29
|
-tm--time* Time
|
30
30
|
-m--mem* SLURM minimum memory
|
31
|
+
--gres* SLURM Generic resources
|
31
32
|
-mcpu--mem_per_cpu* SLURM minimum memory per CPU
|
32
33
|
-lin--licenses* SLURM licenses
|
33
34
|
-cons--constraint* SLURM constraint
|
34
35
|
-W--workflows* Additional workflows
|
35
|
-
-OR--orchestration_rules* Orchestration rules
|
36
36
|
-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
|
37
|
+
-lmod--lua_modules* Lua Modules to load
|
38
|
+
-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
|
39
|
+
-OR--orchestration_rules* Orchestration rules
|
37
40
|
EOF
|
38
41
|
|
39
42
|
batch_system = $slurm_options.delete :batch_system
|
@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
|
|
27
27
|
-t--task_cpus* Tasks
|
28
28
|
-tm--time* Time
|
29
29
|
-m--mem* SLURM minimum memory
|
30
|
+
--gres* SLURM Generic resources
|
30
31
|
-mcpu--mem_per_cpu* SLURM minimum memory per CPU
|
31
32
|
-lin--licenses* SLURM licenses
|
32
33
|
-cons--constraint* SLURM constraint
|
33
34
|
-W--workflows* Additional workflows
|
34
35
|
-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
|
36
|
+
-lmod--lua_modules* Lua Modules to load
|
35
37
|
-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
|
36
38
|
EOF
|
37
39
|
|
@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
|
|
28
28
|
-t--task_cpus* Tasks
|
29
29
|
-tm--time* Time
|
30
30
|
-m--mem* SLURM minimum memory
|
31
|
+
--gres* SLURM Generic resources
|
31
32
|
-mcpu--mem_per_cpu* SLURM minimum memory per CPU
|
32
33
|
-lin--licenses* SLURM licenses
|
33
34
|
-cons--constraint* SLURM constraint
|
34
35
|
-W--workflows* Additional workflows
|
35
|
-
-OR--orchestration_rules* Orchestration rules
|
36
36
|
-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
|
37
|
+
-lmod--lua_modules* Lua Modules to load
|
38
|
+
-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
|
39
|
+
-OR--orchestration_rules* Orchestration rules
|
37
40
|
EOF
|
38
41
|
|
39
42
|
batch_system = $slurm_options.delete :batch_system
|
@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
|
|
27
27
|
-t--task_cpus* Tasks
|
28
28
|
-tm--time* Time
|
29
29
|
-m--mem* SLURM minimum memory
|
30
|
+
--gres* SLURM Generic resources
|
30
31
|
-mcpu--mem_per_cpu* SLURM minimum memory per CPU
|
31
32
|
-lin--licenses* SLURM licenses
|
32
33
|
-cons--constraint* SLURM constraint
|
33
34
|
-W--workflows* Additional workflows
|
34
35
|
-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
|
36
|
+
-lmod--lua_modules* Lua Modules to load
|
35
37
|
-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
|
36
38
|
EOF
|
37
39
|
|
@@ -28,12 +28,15 @@ $slurm_options = SOPT.get <<EOF
|
|
28
28
|
-t--task_cpus* Tasks
|
29
29
|
-tm--time* Time
|
30
30
|
-m--mem* SLURM minimum memory
|
31
|
+
--gres* SLURM Generic resources
|
31
32
|
-mcpu--mem_per_cpu* SLURM minimum memory per CPU
|
32
33
|
-lin--licenses* SLURM licenses
|
33
34
|
-cons--constraint* SLURM constraint
|
34
35
|
-W--workflows* Additional workflows
|
35
|
-
-OR--orchestration_rules* Orchestration rules
|
36
36
|
-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
|
37
|
+
-lmod--lua_modules* Lua Modules to load
|
38
|
+
-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
|
39
|
+
-OR--orchestration_rules* Orchestration rules
|
37
40
|
EOF
|
38
41
|
|
39
42
|
batch_system = $slurm_options.delete :batch_system
|
@@ -27,11 +27,13 @@ $slurm_options = SOPT.get <<EOF
|
|
27
27
|
-t--task_cpus* Tasks
|
28
28
|
-tm--time* Time
|
29
29
|
-m--mem* SLURM minimum memory
|
30
|
+
--gres* SLURM Generic resources
|
30
31
|
-mcpu--mem_per_cpu* SLURM minimum memory per CPU
|
31
32
|
-lin--licenses* SLURM licenses
|
32
33
|
-cons--constraint* SLURM constraint
|
33
34
|
-W--workflows* Additional workflows
|
34
35
|
-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
|
36
|
+
-lmod--lua_modules* Lua Modules to load
|
35
37
|
-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
|
36
38
|
EOF
|
37
39
|
|
@@ -5,15 +5,24 @@ module TestDataWF
|
|
5
5
|
extend Workflow
|
6
6
|
extend Workflow::Data
|
7
7
|
|
8
|
+
input :salutation, :string
|
9
|
+
task :salute_luis => :string do |name,salutation|
|
10
|
+
"Hi Luis: #{salutation}"
|
11
|
+
end
|
12
|
+
|
8
13
|
input :name, :string
|
9
14
|
input :salutation, :string
|
10
15
|
task :salute => :string do |name,salutation|
|
11
16
|
"Hi #{name}: #{salutation}"
|
12
17
|
end
|
13
18
|
|
14
|
-
data_task :salute_data, TestDataWF, :salute do |directory,options|
|
19
|
+
data_task :salute_data, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
|
15
20
|
options.merge({:salutation => directory.salutation.read})
|
16
21
|
end
|
22
|
+
|
23
|
+
data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
|
24
|
+
{:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
|
25
|
+
end
|
17
26
|
end
|
18
27
|
|
19
28
|
class TestWorkflowData < Test::Unit::TestCase
|
@@ -28,7 +37,11 @@ class TestWorkflowData < Test::Unit::TestCase
|
|
28
37
|
|
29
38
|
job = TestDataWF.job(:salute_data, "TestDir")
|
30
39
|
job.recursive_clean.run
|
31
|
-
|
40
|
+
assert job.run.include? "Miguel"
|
41
|
+
|
42
|
+
job = TestDataWF.job(:salute_data2, "TestDir")
|
43
|
+
job.recursive_clean.run
|
44
|
+
assert job.run.include? "Luis"
|
32
45
|
end
|
33
46
|
end
|
34
47
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.32.
|
4
|
+
version: 5.32.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-06-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|