rbbt-util 5.31.10 → 5.31.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc/batch.rb +5 -0
- data/lib/rbbt/hpc/slurm.rb +2 -0
- data/lib/rbbt/util/cmd.rb +9 -6
- data/lib/rbbt/util/misc/inspect.rb +14 -2
- data/lib/rbbt/util/misc/options.rb +2 -2
- data/lib/rbbt/util/misc/pipes.rb +2 -1
- data/lib/rbbt/workflow/definition.rb +8 -2
- data/lib/rbbt/workflow/step/accessor.rb +2 -2
- data/lib/rbbt/workflow/usage.rb +1 -1
- data/share/rbbt_commands/hpc/orchestrate +1 -0
- data/share/rbbt_commands/hpc/task +1 -0
- data/share/rbbt_commands/workflow/trace +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c451106f276fa0ba5bf8cacce6f4867439b756f41ea439f8ce1f7f9e926f15c8
|
4
|
+
data.tar.gz: ef03fd45181215040380e9e0344a0fd86b4552e41daf7f5e2feb6ff32be325cf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b0efd21d9ea3ec27fb0264fd3e4a163178e8b3b2b9430150e2822bd41ad01b729a3ca9193b779f7956545334442accc207a163e1c02ec125b17930e9a7d88dd3
|
7
|
+
data.tar.gz: abf50fd62c965161f5c13230d35aae45502df0934c82d69c59ca034e33c01154fa5105b2a65d8725463bb94352b0c3c97de4f5e6955d44581e3b31d4343be5cd
|
data/lib/rbbt/hpc/batch.rb
CHANGED
@@ -121,6 +121,9 @@ EOF
|
|
121
121
|
:nodes,
|
122
122
|
:queue,
|
123
123
|
:singularity,
|
124
|
+
:singularity_img,
|
125
|
+
:singularity_opt_dir,
|
126
|
+
:singularity_ruby_inline,
|
124
127
|
:sync,
|
125
128
|
:task_cpus,
|
126
129
|
:time,
|
@@ -163,6 +166,8 @@ EOF
|
|
163
166
|
batch_options[:scratch_group_dir] = File.join('/gpfs/scratch/', group)
|
164
167
|
batch_options[:projects_group_dir] = File.join('/gpfs/projects/', group)
|
165
168
|
|
169
|
+
batch_options[:singularity] = true if batch_options[:singularity_img]
|
170
|
+
|
166
171
|
if batch_options[:contain_and_sync]
|
167
172
|
if batch_options[:contain].nil?
|
168
173
|
contain_base = Rbbt::Config.get(:contain_base_dir, :batch_contain, :batch, :default => "/scratch/tmp/rbbt-[USER]")
|
data/lib/rbbt/hpc/slurm.rb
CHANGED
@@ -23,6 +23,7 @@ BATCH_SYSTEM=SLURM
|
|
23
23
|
nodes = Misc.process_options options, :nodes
|
24
24
|
workdir = Misc.process_options options, :workdir
|
25
25
|
exclusive = Misc.process_options options, :exclusive
|
26
|
+
highmem = Misc.process_options options, :highmem
|
26
27
|
|
27
28
|
batch_dir = Misc.process_options options, :batch_dir
|
28
29
|
batch_name = Misc.process_options options, :batch_name
|
@@ -45,6 +46,7 @@ BATCH_SYSTEM=SLURM
|
|
45
46
|
EOF
|
46
47
|
|
47
48
|
header << "#SBATCH --exclusive" << "\n" if exclusive
|
49
|
+
header << "#SBATCH --constraint=highmem" << "\n" if highmem
|
48
50
|
|
49
51
|
header
|
50
52
|
end
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -38,16 +38,18 @@ module CMD
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
version_txt = ""
|
41
|
-
|
41
|
+
version = nil
|
42
|
+
["--version", "-version", "--help", ""].each do |f|
|
42
43
|
begin
|
43
|
-
version_txt += CMD.cmd("#{tool} #{f}").read
|
44
|
-
|
44
|
+
version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
|
45
|
+
ppp version_txt
|
46
|
+
version = Misc.scan_version_text(version_txt, tool)
|
47
|
+
break if version
|
45
48
|
rescue
|
49
|
+
Log.exception $!
|
46
50
|
end
|
47
51
|
end
|
48
52
|
|
49
|
-
version = Misc.scan_version_text(version_txt, tool)
|
50
|
-
|
51
53
|
@@init_cmd_tool[tool] = version || true
|
52
54
|
|
53
55
|
return cmd if cmd
|
@@ -75,7 +77,8 @@ module CMD
|
|
75
77
|
string = ""
|
76
78
|
options.each do |option, value|
|
77
79
|
raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=]+$/i
|
78
|
-
raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
|
80
|
+
#raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
|
81
|
+
value = value.gsub("'","\\'") if value.to_s.include? "'"
|
79
82
|
|
80
83
|
option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
|
81
84
|
|
@@ -431,8 +431,20 @@ module Misc
|
|
431
431
|
|
432
432
|
def self.scan_version_text(text, cmd = nil)
|
433
433
|
cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
|
434
|
+
text.split("\n").each do |line|
|
435
|
+
next unless line =~ /\W#{cmd}\W/i
|
436
|
+
m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
437
|
+
return m[1] if m
|
438
|
+
m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
|
439
|
+
return m[1] if m
|
440
|
+
next unless line =~ /\Wversion\W/i
|
441
|
+
m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
442
|
+
return m[1] if m
|
443
|
+
end
|
434
444
|
m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
435
|
-
return
|
436
|
-
m
|
445
|
+
return m[1] if m
|
446
|
+
m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
|
447
|
+
return m[1] if m
|
448
|
+
nil
|
437
449
|
end
|
438
450
|
end
|
@@ -46,8 +46,8 @@ module Misc
|
|
46
46
|
def self.hash2string(hash)
|
47
47
|
hash.sort_by{|k,v| k.to_s}.collect{|k,v|
|
48
48
|
next unless %w(Symbol String Float Fixnum Integer Numeric TrueClass FalseClass Module Class Object).include? v.class.to_s
|
49
|
-
[ Symbol === k ? ":" << k.to_s : k,
|
50
|
-
Symbol === v ? ":" << v.to_s : v] * "="
|
49
|
+
[ Symbol === k ? ":" << k.to_s : k.to_s.chomp,
|
50
|
+
Symbol === v ? ":" << v.to_s : v.to_s.chomp] * "="
|
51
51
|
}.compact * "#"
|
52
52
|
end
|
53
53
|
|
data/lib/rbbt/util/misc/pipes.rb
CHANGED
@@ -35,10 +35,11 @@ module Misc
|
|
35
35
|
res
|
36
36
|
end
|
37
37
|
|
38
|
-
def self.with_fifo(path = nil, &block)
|
38
|
+
def self.with_fifo(path = nil, clean = true, &block)
|
39
39
|
begin
|
40
40
|
erase = path.nil?
|
41
41
|
path = TmpFile.tmp_file if path.nil?
|
42
|
+
File.rm path if clean && File.exists?(path)
|
42
43
|
File.mkfifo path
|
43
44
|
yield path
|
44
45
|
ensure
|
@@ -76,8 +76,10 @@ module Workflow
|
|
76
76
|
extension workflow.tasks[oname].extension if workflow.tasks.include?(oname) unless @extension
|
77
77
|
task name do
|
78
78
|
raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
|
79
|
-
dep = dependencies.last
|
79
|
+
dep = dependencies.last
|
80
|
+
dep.join
|
80
81
|
raise dep.get_exception if dep.error?
|
82
|
+
raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
|
81
83
|
set_info :result_type, dep.info[:result_type]
|
82
84
|
forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
|
83
85
|
if forget
|
@@ -103,7 +105,11 @@ module Workflow
|
|
103
105
|
Open.rm_rf self.files_dir
|
104
106
|
Open.link dep.files_dir, self.files_dir
|
105
107
|
end
|
106
|
-
|
108
|
+
if defined?(RemoteStep) && RemoteStep === dep
|
109
|
+
Open.write(self.tmp_path, Open.read(dep.path))
|
110
|
+
else
|
111
|
+
Open.link dep.path, self.path
|
112
|
+
end
|
107
113
|
end
|
108
114
|
nil
|
109
115
|
end
|
@@ -368,7 +368,7 @@ class Step
|
|
368
368
|
end
|
369
369
|
|
370
370
|
def self.log_progress(status, options = {}, path = nil, &block)
|
371
|
-
options = Misc.add_defaults options, :severity => Log::INFO, :file => path
|
371
|
+
options = Misc.add_defaults options, :severity => Log::INFO, :file => (@exec ? nil : path)
|
372
372
|
max = Misc.process_options options, :max
|
373
373
|
Log::ProgressBar.with_bar(max, options) do |bar|
|
374
374
|
begin
|
@@ -394,7 +394,7 @@ class Step
|
|
394
394
|
options = {} if options.nil?
|
395
395
|
|
396
396
|
max = options[:max]
|
397
|
-
Log::ProgressBar.new_bar(max, {:desc => msg, :file => file(:progress)}.merge(options))
|
397
|
+
Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
|
398
398
|
end
|
399
399
|
|
400
400
|
def self.log(status, message, path, &block)
|
data/lib/rbbt/workflow/usage.rb
CHANGED
@@ -215,7 +215,7 @@ module Workflow
|
|
215
215
|
|
216
216
|
dep_tree = {[self, task_name] => dep_tree(task_name)}
|
217
217
|
prov_tree = prov_tree(dep_tree)
|
218
|
-
if prov_tree && ! prov_tree.empty?
|
218
|
+
if prov_tree && ! prov_tree.empty? && prov_tree.split("\n").length > 2
|
219
219
|
|
220
220
|
puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
|
221
221
|
puts
|
@@ -12,6 +12,7 @@ $slurm_options = SOPT.get <<EOF
|
|
12
12
|
-cj--clean_job Clean job
|
13
13
|
--drbbt* Use development version of rbbt
|
14
14
|
-sing--singularity Use Singularity
|
15
|
+
-si--singularity_img* Singularity image to use
|
15
16
|
-ug--user_group* Use alternative user group for group project directory
|
16
17
|
-c--contain* Contain in directory (using Singularity)
|
17
18
|
-s--sync* Contain in directory and sync jobs
|
@@ -11,6 +11,7 @@ $slurm_options = SOPT.get <<EOF
|
|
11
11
|
-cj--clean_job Clean job
|
12
12
|
--drbbt* Use development version of rbbt
|
13
13
|
-sing--singularity Use Singularity
|
14
|
+
-si--singularity_img* Singularity image to use
|
14
15
|
-ug--user_group* Use alternative user group for group project directory
|
15
16
|
-c--contain* Contain in directory (using Singularity)
|
16
17
|
-s--sync* Contain in directory and sync jobs
|
@@ -23,7 +23,7 @@ $ rbbt workflow trace <job-result>
|
|
23
23
|
-h--help Help
|
24
24
|
-w--width* Image Width
|
25
25
|
-h--height* Image Height
|
26
|
-
-p--plot Plot file
|
26
|
+
-p--plot* Plot file
|
27
27
|
-s--size* Image Size (Height and Width)
|
28
28
|
-fg--fix_gap Remove execution gaps
|
29
29
|
-pd--plot_data Print plot data
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.31.
|
4
|
+
version: 5.31.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-03-
|
11
|
+
date: 2021-03-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|