rbbt-util 5.29.4 → 5.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc/slurm.rb +17 -6
- data/lib/rbbt/persist/tsv/adapter.rb +48 -13
- data/lib/rbbt/util/cmd.rb +6 -1
- data/lib/rbbt/util/misc/options.rb +0 -42
- data/lib/rbbt/util/procpath.rb +49 -0
- data/lib/rbbt/workflow/step/accessor.rb +3 -4
- data/lib/rbbt/workflow/step/run.rb +2 -4
- data/lib/rbbt/workflow/usage.rb +1 -1
- data/share/rbbt_commands/slurm/list +56 -1
- data/share/rbbt_commands/slurm/orchestrate +1 -0
- data/share/rbbt_commands/slurm/task +1 -0
- data/share/rbbt_commands/tsv/slice +3 -3
- data/share/rbbt_commands/workflow/info +1 -1
- data/share/rbbt_commands/workflow/task +8 -1
- data/share/rbbt_commands/workflow/write_info +52 -0
- data/test/rbbt/util/test_procpath.rb +23 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b0e0e6d9165c16d73d1122a8d85e8038befd335215646bc3779400f5747b4b07
|
4
|
+
data.tar.gz: 7c0e92d49399111157fae43149c7ec8e598aa64510591fc2f74314bbefc89b2d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6f1ecdb8ea4481b12138f6b63d2508f888ad60d42b56cc187ac74e3d705bb40db76d52f7e615bbd3b7fba74e49298325dcf7e5477f5527e55aa4a42e6744586d
|
7
|
+
data.tar.gz: 5b3761d1300940911383fc0659a4335b422db217a9118a0fd41ebd2374cec87884775564f003d5a0a14555eff68fcac7b70c68c3cea72c6d802908a48b8b514c
|
data/lib/rbbt/hpc/slurm.rb
CHANGED
@@ -33,7 +33,8 @@ module HPC
|
|
33
33
|
group = File.basename(File.dirname(ENV['HOME']))
|
34
34
|
|
35
35
|
if contain_and_sync
|
36
|
-
|
36
|
+
random_file = TmpFile.random_name
|
37
|
+
contain = "/scratch/tmp/rbbt-#{user}/#{random_file}" if contain.nil?
|
37
38
|
sync = "~/.rbbt/var/jobs" if sync.nil?
|
38
39
|
wipe_container = "post" if wipe_container.nil?
|
39
40
|
end
|
@@ -243,7 +244,7 @@ EOF
|
|
243
244
|
end
|
244
245
|
|
245
246
|
if contain
|
246
|
-
rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}
|
247
|
+
rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}/workdir')
|
247
248
|
end
|
248
249
|
end
|
249
250
|
|
@@ -251,6 +252,10 @@ EOF
|
|
251
252
|
cmd =<<-EOF
|
252
253
|
#{exec_cmd} \\
|
253
254
|
#{rbbt_cmd}
|
255
|
+
EOF
|
256
|
+
annotate_cmd =<<-EOF
|
257
|
+
#{exec_cmd} \\
|
258
|
+
workflow write_info --recursive --force=false --check_pid "$step_path" slurm_job $SLURM_JOB_ID
|
254
259
|
EOF
|
255
260
|
|
256
261
|
header +=<<-EOF
|
@@ -260,11 +265,14 @@ EOF
|
|
260
265
|
run +=<<-EOF
|
261
266
|
|
262
267
|
# Run command
|
263
|
-
#{cmd}
|
268
|
+
step_path=$(#{cmd})
|
264
269
|
|
265
270
|
# Save exit status
|
266
271
|
exit_status=$?
|
267
272
|
|
273
|
+
# Annotate info with SLURM job_info
|
274
|
+
#{annotate_cmd}
|
275
|
+
|
268
276
|
EOF
|
269
277
|
|
270
278
|
# CODA
|
@@ -286,7 +294,7 @@ EOF
|
|
286
294
|
sync = sync.strip
|
287
295
|
source = File.join(File.expand_path(contain), source)
|
288
296
|
else
|
289
|
-
source = File.join(File.expand_path(contain), '
|
297
|
+
source = File.join(File.expand_path(contain), 'workdir/var/jobs')
|
290
298
|
end
|
291
299
|
|
292
300
|
target = File.expand_path(sync)
|
@@ -516,6 +524,7 @@ EOF
|
|
516
524
|
dry_run = options.delete :dry_run
|
517
525
|
tail = options.delete :tail
|
518
526
|
dependencies = options.delete :slurm_dependencies
|
527
|
+
procpath = options.delete :SLURM_procpath
|
519
528
|
options[:jobname] = job.clean_name
|
520
529
|
|
521
530
|
workflow = job.workflow
|
@@ -543,13 +552,15 @@ EOF
|
|
543
552
|
|
544
553
|
if saved && saved.any?
|
545
554
|
options[:inputs_dir] = inputs_dir
|
546
|
-
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '
|
555
|
+
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
|
547
556
|
else
|
548
|
-
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '
|
557
|
+
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--log', (options[:log] || Log.severity).to_s]
|
549
558
|
end
|
550
559
|
|
551
560
|
cmd << "--override_deps='#{override_deps.gsub("'", '\'')}'" if override_deps and not override_deps.empty?
|
552
561
|
|
562
|
+
cmd << "--procpath_performance='#{tmp_directory}/procpath##{procpath.gsub(',', '#')}'" if procpath
|
563
|
+
|
553
564
|
template = self.template(cmd, options)
|
554
565
|
jobid = self.issue_template(template, options.merge(:slurm_basedir => slurm_basedir, :dry_run => dry_run, :slurm_dependencies => dependencies))
|
555
566
|
|
@@ -104,9 +104,6 @@ module Persist
|
|
104
104
|
write(true) if closed? || ! write?
|
105
105
|
res = begin
|
106
106
|
yield
|
107
|
-
rescue Exception
|
108
|
-
Log.exception $!
|
109
|
-
raise $!
|
110
107
|
ensure
|
111
108
|
close
|
112
109
|
end
|
@@ -115,7 +112,6 @@ module Persist
|
|
115
112
|
end
|
116
113
|
|
117
114
|
def read_and_close
|
118
|
-
#return yield if @locked
|
119
115
|
if read? || write?
|
120
116
|
begin
|
121
117
|
return yield
|
@@ -134,6 +130,45 @@ module Persist
|
|
134
130
|
end
|
135
131
|
end
|
136
132
|
|
133
|
+
def read_lock
|
134
|
+
read if closed?
|
135
|
+
if read?
|
136
|
+
begin
|
137
|
+
return yield
|
138
|
+
ensure
|
139
|
+
close
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
lock do
|
144
|
+
close
|
145
|
+
read true
|
146
|
+
begin
|
147
|
+
yield
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def write_lock
|
153
|
+
write if closed?
|
154
|
+
if write?
|
155
|
+
begin
|
156
|
+
return yield
|
157
|
+
ensure
|
158
|
+
close
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
lock do
|
163
|
+
close
|
164
|
+
write true
|
165
|
+
begin
|
166
|
+
yield
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
|
137
172
|
def merge!(hash)
|
138
173
|
hash.each do |key,values|
|
139
174
|
self[key] = values
|
@@ -141,38 +176,38 @@ module Persist
|
|
141
176
|
end
|
142
177
|
|
143
178
|
def range(*args)
|
144
|
-
self.
|
179
|
+
self.read_lock do
|
145
180
|
super(*args)
|
146
181
|
end
|
147
182
|
end
|
148
183
|
|
149
184
|
def include?(*args)
|
150
|
-
self.
|
185
|
+
self.read_lock do
|
151
186
|
super(*args) #- TSV::ENTRY_KEYS.to_a
|
152
187
|
end
|
153
188
|
end
|
154
189
|
|
155
190
|
def [](*args)
|
156
|
-
self.
|
191
|
+
self.read_lock do
|
157
192
|
super(*args) #- TSV::ENTRY_KEYS.to_a
|
158
193
|
end
|
159
194
|
end
|
160
195
|
|
161
196
|
def []=(*args)
|
162
|
-
self.
|
197
|
+
self.write_lock do
|
163
198
|
super(*args) #- TSV::ENTRY_KEYS.to_a
|
164
199
|
end
|
165
200
|
end
|
166
201
|
|
167
202
|
def keys(*args)
|
168
|
-
self.
|
203
|
+
self.read_lock do
|
169
204
|
super(*args)
|
170
205
|
end
|
171
206
|
end
|
172
207
|
|
173
208
|
|
174
209
|
def prefix(key)
|
175
|
-
self.
|
210
|
+
self.read_lock do
|
176
211
|
range(key, 1, key + MAX_CHAR, 1)
|
177
212
|
end
|
178
213
|
end
|
@@ -184,13 +219,13 @@ module Persist
|
|
184
219
|
|
185
220
|
|
186
221
|
def size(*args)
|
187
|
-
self.
|
222
|
+
self.read_lock do
|
188
223
|
super(*args)
|
189
224
|
end
|
190
225
|
end
|
191
226
|
|
192
227
|
def each(*args, &block)
|
193
|
-
self.
|
228
|
+
self.read_lock do
|
194
229
|
super(*args, &block)
|
195
230
|
end
|
196
231
|
end
|
@@ -208,7 +243,7 @@ module Persist
|
|
208
243
|
end
|
209
244
|
|
210
245
|
def values_at(*keys)
|
211
|
-
self.
|
246
|
+
self.read_lock do
|
212
247
|
keys.collect do |k|
|
213
248
|
self[k]
|
214
249
|
end
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -217,7 +217,7 @@ module CMD
|
|
217
217
|
end
|
218
218
|
end
|
219
219
|
|
220
|
-
def self.
|
220
|
+
def self.cmd_pid(*args)
|
221
221
|
all_args = *args
|
222
222
|
|
223
223
|
all_args << {} unless Hash === all_args.last
|
@@ -248,4 +248,9 @@ module CMD
|
|
248
248
|
nil
|
249
249
|
end
|
250
250
|
|
251
|
+
def self.cmd_log(*args)
|
252
|
+
cmd_pid(*args)
|
253
|
+
nil
|
254
|
+
end
|
255
|
+
|
251
256
|
end
|
@@ -242,48 +242,6 @@ module Misc
|
|
242
242
|
|
243
243
|
return options
|
244
244
|
|
245
|
-
options = {}
|
246
|
-
string.split(/#/).each do |str|
|
247
|
-
if str.match(/(.*)=(.*)/)
|
248
|
-
option, value = $1, $2
|
249
|
-
else
|
250
|
-
option, value = str, true
|
251
|
-
end
|
252
|
-
|
253
|
-
option = option.sub(":",'').to_sym if option.chars.first == ':'
|
254
|
-
value = value.sub(":",'').to_sym if String === value and value.chars.first == ':'
|
255
|
-
|
256
|
-
if value == true
|
257
|
-
options[option] = option.to_s.chars.first != '!'
|
258
|
-
else
|
259
|
-
options[option] = Thread.start do
|
260
|
-
$SAFE = 0;
|
261
|
-
case
|
262
|
-
when value =~ /^(?:true|T)$/i
|
263
|
-
true
|
264
|
-
when value =~ /^(?:false|F)$/i
|
265
|
-
false
|
266
|
-
when Symbol === value
|
267
|
-
value
|
268
|
-
when (String === value and value =~ /^\/(.*)\/$/)
|
269
|
-
Regexp.new /#{$1}/
|
270
|
-
else
|
271
|
-
begin
|
272
|
-
Kernel.const_get value
|
273
|
-
rescue
|
274
|
-
begin
|
275
|
-
raise if value =~ /[a-z]/ and defined? value
|
276
|
-
eval(value)
|
277
|
-
rescue Exception
|
278
|
-
value
|
279
|
-
end
|
280
|
-
end
|
281
|
-
end
|
282
|
-
end.value
|
283
|
-
end
|
284
|
-
end
|
285
|
-
|
286
|
-
options
|
287
245
|
end
|
288
246
|
|
289
247
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'rbbt/util/cmd'
|
2
|
+
module ProcPath
|
3
|
+
CMD.tool :procpath do
|
4
|
+
'pip install procpath'
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.record(pid, path, options = {})
|
8
|
+
IndiferentHash.setup(options)
|
9
|
+
options = Misc.add_defaults options, "interval" => 30
|
10
|
+
|
11
|
+
cmd_options = %w(interval recnum reevalnum).inject({}){|acc,k| acc[k] = options[k]; acc}
|
12
|
+
|
13
|
+
Log.debug "ProcPath recording #{pid} in #{path} (#{Misc.fingerprint options})"
|
14
|
+
procpath_thread = Thread.new do
|
15
|
+
begin
|
16
|
+
procpath_pid = CMD.cmd_pid(:procpath, "record --database-file '#{path}' '$..children[?(@.stat.pid == #{pid})]'", cmd_options.merge(:nofail => true, :add_option_dashes => true))
|
17
|
+
rescue Exception
|
18
|
+
Log.exceptions $!
|
19
|
+
Process.kill "INT", procpath_pid
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
procpath_thread.report_on_exception = false
|
24
|
+
|
25
|
+
Process.wait pid.to_i
|
26
|
+
procpath_thread.raise Interrupt
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.plot(path, output, options = {})
|
30
|
+
IndiferentHash.setup(options)
|
31
|
+
options = Misc.add_defaults options, "query-name" => 'rss', 'epsilon' => 0.5, "moving-average-window" => 10
|
32
|
+
|
33
|
+
cmd_options = %w(query-name epsilon monitor-average-window title logarithmic after before custom-query-file custom-value-expr).inject({}){|acc,k| acc[k] = options[k]; acc}
|
34
|
+
CMD.cmd_log(:procpath, "plot --database-file '#{path}' --plot-file '#{output}' ", cmd_options.merge(:nofail => true, :add_option_dashes => true))
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.monitor(pid, path)
|
38
|
+
database, options_str = path.split("#")
|
39
|
+
options = options_str.nil? ? {} : Misc.string2hash(options_str)
|
40
|
+
|
41
|
+
database = File.expand_path database
|
42
|
+
Log.low "ProcPath monitor #{pid} in #{database} (#{Misc.fingerprint options})"
|
43
|
+
|
44
|
+
ProcPath.record(pid, database + '.sqlite3', options)
|
45
|
+
ProcPath.plot(database + '.sqlite3', database + '.cpu.svg', options.merge("query-name" => 'cpu'))
|
46
|
+
ProcPath.plot(database + '.sqlite3', database + '.rss.svg', options.merge("query-name" => 'rss'))
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
@@ -505,8 +505,8 @@ class Step
|
|
505
505
|
|
506
506
|
def running?
|
507
507
|
return false if ! (started? || status == :ending)
|
508
|
-
|
509
|
-
|
508
|
+
return nil unless Open.exist?(self.pid_file)
|
509
|
+
pid = Open.read(self.pid_file).to_i
|
510
510
|
|
511
511
|
return false if done? or error? or aborted?
|
512
512
|
|
@@ -530,8 +530,7 @@ class Step
|
|
530
530
|
end
|
531
531
|
|
532
532
|
def nopid?
|
533
|
-
|
534
|
-
! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
|
533
|
+
! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
|
535
534
|
end
|
536
535
|
|
537
536
|
def aborted?
|
@@ -373,7 +373,6 @@ class Step
|
|
373
373
|
Log.exception $!
|
374
374
|
ensure
|
375
375
|
Step.purge_stream_cache
|
376
|
-
set_info :pid, nil
|
377
376
|
Open.rm pid_file if Open.exist?(pid_file)
|
378
377
|
end
|
379
378
|
end
|
@@ -388,7 +387,6 @@ class Step
|
|
388
387
|
_clean_finished
|
389
388
|
rescue
|
390
389
|
stop_dependencies
|
391
|
-
set_info :pid, nil
|
392
390
|
Open.rm pid_file if Open.exist?(pid_file)
|
393
391
|
end
|
394
392
|
end
|
@@ -449,7 +447,7 @@ class Step
|
|
449
447
|
ensure
|
450
448
|
no_load = false unless IO === result
|
451
449
|
Open.rm pid_file if Open.exist?(pid_file) unless no_load
|
452
|
-
set_info :pid, nil unless no_load
|
450
|
+
#set_info :pid, nil unless no_load
|
453
451
|
end
|
454
452
|
end
|
455
453
|
|
@@ -559,7 +557,7 @@ class Step
|
|
559
557
|
RbbtSemaphore.post_semaphore(semaphore) if semaphore
|
560
558
|
Kernel.exit! -1
|
561
559
|
end
|
562
|
-
set_info :pid, nil
|
560
|
+
#set_info :pid, nil
|
563
561
|
ensure
|
564
562
|
RbbtSemaphore.post_semaphore(semaphore) if semaphore
|
565
563
|
end
|
data/lib/rbbt/workflow/usage.rb
CHANGED
@@ -57,7 +57,7 @@ module Task
|
|
57
57
|
puts Log.color(:magenta, "Input select options")
|
58
58
|
puts
|
59
59
|
selects.collect{|p| p}.uniq.each do |input,options|
|
60
|
-
puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| o.to_s} * ", ") << "\n"
|
60
|
+
puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
|
61
61
|
puts unless Log.compact
|
62
62
|
end
|
63
63
|
puts
|
@@ -21,6 +21,8 @@ $ rbbt mnl [options]
|
|
21
21
|
-s--search* Regular expression
|
22
22
|
-t--tail* Show the last lines of the STDERR
|
23
23
|
-SBP--sbatch_parameters show sbatch parameters
|
24
|
+
-PERF--procpath_performance show Procpath performance summary
|
25
|
+
-sacct--sacct_peformance show sacct performance summary
|
24
26
|
EOF
|
25
27
|
|
26
28
|
if options[:help]
|
@@ -140,9 +142,62 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
140
142
|
|
141
143
|
if options[:sbatch_parameters]
|
142
144
|
puts Log.color(:magenta, "SBATCH parameters: ")
|
143
|
-
|
145
|
+
text = CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
|
146
|
+
lines = text.split("\n").collect{|line| header, _sep, value = line.partition(/\s+/); Log.color(:yellow, header + ": ") + value}
|
147
|
+
puts Log.color :yellow, lines * "\n"
|
144
148
|
end
|
145
149
|
|
150
|
+
fprocpath = File.join(dir, 'procpath.sqlite3')
|
151
|
+
if options[:procpath_performance] && Open.exists?(fprocpath)
|
152
|
+
puts Log.color(:magenta, "Procpath summary: ")
|
153
|
+
require 'rbbt/tsv/csv'
|
154
|
+
meta = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from meta;' "))
|
155
|
+
perf = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from record;' "))
|
156
|
+
|
157
|
+
page_size = meta["page_size"].first.to_f
|
158
|
+
clock_ticks = meta["clock_ticks"].first.to_f
|
159
|
+
|
160
|
+
cpu_average = {}
|
161
|
+
rss_average = {}
|
162
|
+
perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
|
163
|
+
time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
|
164
|
+
time = time.to_f
|
165
|
+
|
166
|
+
cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
|
167
|
+
cpu_average[stat_pid] ||= {}
|
168
|
+
cpu_average[stat_pid][time] ||= []
|
169
|
+
cpu_average[stat_pid][time] << cpu.to_f
|
170
|
+
rss_average[time] ||= []
|
171
|
+
rss_average[time] << rss.to_f * page_size
|
172
|
+
end
|
173
|
+
|
174
|
+
ticks = 0
|
175
|
+
cpu_average.each do |stat_pid, cpu_average_pid|
|
176
|
+
start = cpu_average_pid.keys.sort.first
|
177
|
+
eend = cpu_average_pid.keys.sort.last
|
178
|
+
ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
|
179
|
+
end
|
180
|
+
start = rss_average.keys.sort.first
|
181
|
+
eend = rss_average.keys.sort.last
|
182
|
+
time_elapsed = eend - start
|
183
|
+
puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
|
184
|
+
puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s
|
185
|
+
|
186
|
+
end
|
187
|
+
|
188
|
+
if options[:sacct_peformance]
|
189
|
+
begin
|
190
|
+
tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
|
191
|
+
values = tsv[tsv.keys.first]
|
192
|
+
if values.compact.any?
|
193
|
+
puts Log.color(:magenta, "SACCT performance: ")
|
194
|
+
puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ") + v.to_s } * "\n"
|
195
|
+
end
|
196
|
+
rescue
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
|
146
201
|
if tail && File.exists?(File.join(dir, 'std.err'))
|
147
202
|
if exit_status && exit_status != 0
|
148
203
|
puts Log.color(:magenta, "First error or exception found: ")
|
@@ -21,6 +21,7 @@ $slurm_options = SOPT.get <<EOF
|
|
21
21
|
-CS--contain_and_sync Contain and sync to default locations
|
22
22
|
-ci--copy_image When using a container directory, copy image there
|
23
23
|
-t--tail Tail the logs
|
24
|
+
-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
|
24
25
|
-q--queue* Queue
|
25
26
|
-t--task_cpus* Tasks
|
26
27
|
-W--workflows* Additional workflows
|
@@ -20,6 +20,7 @@ $slurm_options = SOPT.get <<EOF
|
|
20
20
|
-CS--contain_and_sync Contain and sync to default locations
|
21
21
|
-ci--copy_image When using a container directory, copy image there
|
22
22
|
-t--tail Tail the logs
|
23
|
+
-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
|
23
24
|
-q--queue* Queue
|
24
25
|
-t--task_cpus* Tasks
|
25
26
|
-W--workflows* Additional workflows
|
@@ -35,7 +35,7 @@ file = case file
|
|
35
35
|
fields = options[:fields]
|
36
36
|
raise ParameterException, "Please specify the fields to slice" if fields.nil?
|
37
37
|
|
38
|
-
options[:header_hash]
|
38
|
+
options[:header_hash] ||= options["header_hash"]
|
39
39
|
|
40
40
|
case
|
41
41
|
when options[:tokyocabinet]
|
@@ -45,8 +45,8 @@ when options[:tokyocabinet_bd]
|
|
45
45
|
tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
|
46
46
|
puts tsv.summary
|
47
47
|
else
|
48
|
-
stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :
|
49
|
-
|
48
|
+
stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :fields => fields.split(","), :unnamed => true) do |k,fields,names|
|
49
|
+
[k,fields].flatten * "\t"
|
50
50
|
end
|
51
51
|
puts stream.read
|
52
52
|
exit 0
|
@@ -86,7 +86,7 @@ messages = info[:messages]
|
|
86
86
|
backtrace = info[:backtrace]
|
87
87
|
pid = info[:pid]
|
88
88
|
exception = info[:exception]
|
89
|
-
rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :
|
89
|
+
rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :archived_info]
|
90
90
|
|
91
91
|
|
92
92
|
puts Log.color(:magenta, "File") << ": " << step.path
|
@@ -203,6 +203,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
|
|
203
203
|
-prec--prepare_cpus* Number of dependencies prepared in parallel
|
204
204
|
-rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
|
205
205
|
-od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
|
206
|
+
-PERF--procpath_performance* Measure performance using procpath
|
206
207
|
EOF
|
207
208
|
|
208
209
|
workflow = ARGV.shift
|
@@ -407,6 +408,13 @@ begin
|
|
407
408
|
exit 0
|
408
409
|
end
|
409
410
|
|
411
|
+
if options[:procpath_performance]
|
412
|
+
require 'rbbt/util/procpath'
|
413
|
+
job.fork
|
414
|
+
job.soft_grace
|
415
|
+
pid = job.info[:pid]
|
416
|
+
ProcPath.monitor(pid, options[:procpath_performance])
|
417
|
+
end
|
410
418
|
|
411
419
|
if do_fork
|
412
420
|
ENV["RBBT_NO_PROGRESS"] = "true"
|
@@ -423,7 +431,6 @@ begin
|
|
423
431
|
res = job
|
424
432
|
end
|
425
433
|
|
426
|
-
|
427
434
|
if options.delete(:printpath)
|
428
435
|
job.join
|
429
436
|
raise job.messages.last if (job.error? || job.aborted?) && job.messages
|
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rbbt/workflow'
|
4
|
+
|
5
|
+
require 'rbbt-util'
|
6
|
+
require 'rbbt-util'
|
7
|
+
require 'rbbt/util/simpleopt'
|
8
|
+
|
9
|
+
$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
|
10
|
+
|
11
|
+
options = SOPT.setup <<EOF
|
12
|
+
Examine the info of a job result
|
13
|
+
|
14
|
+
$ rbbt workflow info <job-result> <key> <value>
|
15
|
+
|
16
|
+
-h--help Help
|
17
|
+
-f--force Write info even if key is already present
|
18
|
+
-r--recursive Write info for all dependencies as well
|
19
|
+
-p--check_pid Check that recursive jobs where created by the same process
|
20
|
+
EOF
|
21
|
+
|
22
|
+
SOPT.usage if options[:help]
|
23
|
+
|
24
|
+
file, key, value = ARGV
|
25
|
+
|
26
|
+
force, recursive, check_pid = options.values_at :force, :recursive, :check_pid
|
27
|
+
|
28
|
+
def get_step(file)
|
29
|
+
file = file.sub(/\.(info|files)/,'')
|
30
|
+
step = Workflow.load_step file
|
31
|
+
step
|
32
|
+
end
|
33
|
+
|
34
|
+
raise ParameterException if key.nil? || value.nil?
|
35
|
+
|
36
|
+
if %w(DELETE nil).include? value
|
37
|
+
value = nil
|
38
|
+
force = true
|
39
|
+
end
|
40
|
+
|
41
|
+
step = get_step file
|
42
|
+
|
43
|
+
step.set_info key, value if force || ! step.info.include?(key)
|
44
|
+
|
45
|
+
pid = step.info[:pid]
|
46
|
+
host = step.info[:pid_hostname]
|
47
|
+
|
48
|
+
step.rec_dependencies.each do |dep|
|
49
|
+
dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
|
50
|
+
rescue
|
51
|
+
Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
|
52
|
+
end if recursive
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
|
2
|
+
require 'rbbt/util/procpath'
|
3
|
+
|
4
|
+
class TestProcPath < Test::Unit::TestCase
|
5
|
+
def test_record_and_plot
|
6
|
+
Log.with_severity 0 do
|
7
|
+
pid = Process.fork do
|
8
|
+
a = ""
|
9
|
+
(0..1000).each do
|
10
|
+
a << (0..rand(10000).to_i).to_a.collect{|i| "TEST #{i}" } * " "
|
11
|
+
sleep 0.1
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
TmpFile.with_file(nil, false) do |db|
|
16
|
+
|
17
|
+
ProcPath.record(pid, db, :interval => '1', "recnum" => 100)
|
18
|
+
ProcPath.plot(db, db + '.svg', "moving-average-window" => 1 )
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.30.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-01
|
11
|
+
date: 2021-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -300,6 +300,7 @@ files:
|
|
300
300
|
- lib/rbbt/util/misc/system.rb
|
301
301
|
- lib/rbbt/util/named_array.rb
|
302
302
|
- lib/rbbt/util/open.rb
|
303
|
+
- lib/rbbt/util/procpath.rb
|
303
304
|
- lib/rbbt/util/python.rb
|
304
305
|
- lib/rbbt/util/semaphore.rb
|
305
306
|
- lib/rbbt/util/simpleDSL.rb
|
@@ -438,6 +439,7 @@ files:
|
|
438
439
|
- share/rbbt_commands/workflow/server
|
439
440
|
- share/rbbt_commands/workflow/task
|
440
441
|
- share/rbbt_commands/workflow/trace
|
442
|
+
- share/rbbt_commands/workflow/write_info
|
441
443
|
- share/unicorn.rb
|
442
444
|
- share/workflow_config.ru
|
443
445
|
- test/rbbt/annotations/test_util.rb
|
@@ -516,6 +518,7 @@ files:
|
|
516
518
|
- test/rbbt/util/test_log.rb
|
517
519
|
- test/rbbt/util/test_misc.rb
|
518
520
|
- test/rbbt/util/test_open.rb
|
521
|
+
- test/rbbt/util/test_procpath.rb
|
519
522
|
- test/rbbt/util/test_python.rb
|
520
523
|
- test/rbbt/util/test_semaphore.rb
|
521
524
|
- test/rbbt/util/test_simpleDSL.rb
|
@@ -563,6 +566,7 @@ test_files:
|
|
563
566
|
- test/rbbt/workflow/test_task.rb
|
564
567
|
- test/rbbt/resource/test_path.rb
|
565
568
|
- test/rbbt/util/test_colorize.rb
|
569
|
+
- test/rbbt/util/test_procpath.rb
|
566
570
|
- test/rbbt/util/misc/test_omics.rb
|
567
571
|
- test/rbbt/util/misc/test_pipes.rb
|
568
572
|
- test/rbbt/util/misc/test_format.rb
|