rbbt-util 5.29.4 → 5.30.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc/slurm.rb +17 -6
- data/lib/rbbt/persist/tsv/adapter.rb +48 -13
- data/lib/rbbt/util/cmd.rb +6 -1
- data/lib/rbbt/util/misc/options.rb +0 -42
- data/lib/rbbt/util/procpath.rb +49 -0
- data/lib/rbbt/workflow/step/accessor.rb +3 -4
- data/lib/rbbt/workflow/step/run.rb +2 -4
- data/lib/rbbt/workflow/usage.rb +1 -1
- data/share/rbbt_commands/slurm/list +56 -1
- data/share/rbbt_commands/slurm/orchestrate +1 -0
- data/share/rbbt_commands/slurm/task +1 -0
- data/share/rbbt_commands/tsv/slice +3 -3
- data/share/rbbt_commands/workflow/info +1 -1
- data/share/rbbt_commands/workflow/task +8 -1
- data/share/rbbt_commands/workflow/write_info +52 -0
- data/test/rbbt/util/test_procpath.rb +23 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b0e0e6d9165c16d73d1122a8d85e8038befd335215646bc3779400f5747b4b07
|
4
|
+
data.tar.gz: 7c0e92d49399111157fae43149c7ec8e598aa64510591fc2f74314bbefc89b2d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6f1ecdb8ea4481b12138f6b63d2508f888ad60d42b56cc187ac74e3d705bb40db76d52f7e615bbd3b7fba74e49298325dcf7e5477f5527e55aa4a42e6744586d
|
7
|
+
data.tar.gz: 5b3761d1300940911383fc0659a4335b422db217a9118a0fd41ebd2374cec87884775564f003d5a0a14555eff68fcac7b70c68c3cea72c6d802908a48b8b514c
|
data/lib/rbbt/hpc/slurm.rb
CHANGED
@@ -33,7 +33,8 @@ module HPC
|
|
33
33
|
group = File.basename(File.dirname(ENV['HOME']))
|
34
34
|
|
35
35
|
if contain_and_sync
|
36
|
-
|
36
|
+
random_file = TmpFile.random_name
|
37
|
+
contain = "/scratch/tmp/rbbt-#{user}/#{random_file}" if contain.nil?
|
37
38
|
sync = "~/.rbbt/var/jobs" if sync.nil?
|
38
39
|
wipe_container = "post" if wipe_container.nil?
|
39
40
|
end
|
@@ -243,7 +244,7 @@ EOF
|
|
243
244
|
end
|
244
245
|
|
245
246
|
if contain
|
246
|
-
rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}
|
247
|
+
rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}/workdir')
|
247
248
|
end
|
248
249
|
end
|
249
250
|
|
@@ -251,6 +252,10 @@ EOF
|
|
251
252
|
cmd =<<-EOF
|
252
253
|
#{exec_cmd} \\
|
253
254
|
#{rbbt_cmd}
|
255
|
+
EOF
|
256
|
+
annotate_cmd =<<-EOF
|
257
|
+
#{exec_cmd} \\
|
258
|
+
workflow write_info --recursive --force=false --check_pid "$step_path" slurm_job $SLURM_JOB_ID
|
254
259
|
EOF
|
255
260
|
|
256
261
|
header +=<<-EOF
|
@@ -260,11 +265,14 @@ EOF
|
|
260
265
|
run +=<<-EOF
|
261
266
|
|
262
267
|
# Run command
|
263
|
-
#{cmd}
|
268
|
+
step_path=$(#{cmd})
|
264
269
|
|
265
270
|
# Save exit status
|
266
271
|
exit_status=$?
|
267
272
|
|
273
|
+
# Annotate info with SLURM job_info
|
274
|
+
#{annotate_cmd}
|
275
|
+
|
268
276
|
EOF
|
269
277
|
|
270
278
|
# CODA
|
@@ -286,7 +294,7 @@ EOF
|
|
286
294
|
sync = sync.strip
|
287
295
|
source = File.join(File.expand_path(contain), source)
|
288
296
|
else
|
289
|
-
source = File.join(File.expand_path(contain), '
|
297
|
+
source = File.join(File.expand_path(contain), 'workdir/var/jobs')
|
290
298
|
end
|
291
299
|
|
292
300
|
target = File.expand_path(sync)
|
@@ -516,6 +524,7 @@ EOF
|
|
516
524
|
dry_run = options.delete :dry_run
|
517
525
|
tail = options.delete :tail
|
518
526
|
dependencies = options.delete :slurm_dependencies
|
527
|
+
procpath = options.delete :SLURM_procpath
|
519
528
|
options[:jobname] = job.clean_name
|
520
529
|
|
521
530
|
workflow = job.workflow
|
@@ -543,13 +552,15 @@ EOF
|
|
543
552
|
|
544
553
|
if saved && saved.any?
|
545
554
|
options[:inputs_dir] = inputs_dir
|
546
|
-
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '
|
555
|
+
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
|
547
556
|
else
|
548
|
-
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '
|
557
|
+
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--log', (options[:log] || Log.severity).to_s]
|
549
558
|
end
|
550
559
|
|
551
560
|
cmd << "--override_deps='#{override_deps.gsub("'", '\'')}'" if override_deps and not override_deps.empty?
|
552
561
|
|
562
|
+
cmd << "--procpath_performance='#{tmp_directory}/procpath##{procpath.gsub(',', '#')}'" if procpath
|
563
|
+
|
553
564
|
template = self.template(cmd, options)
|
554
565
|
jobid = self.issue_template(template, options.merge(:slurm_basedir => slurm_basedir, :dry_run => dry_run, :slurm_dependencies => dependencies))
|
555
566
|
|
@@ -104,9 +104,6 @@ module Persist
|
|
104
104
|
write(true) if closed? || ! write?
|
105
105
|
res = begin
|
106
106
|
yield
|
107
|
-
rescue Exception
|
108
|
-
Log.exception $!
|
109
|
-
raise $!
|
110
107
|
ensure
|
111
108
|
close
|
112
109
|
end
|
@@ -115,7 +112,6 @@ module Persist
|
|
115
112
|
end
|
116
113
|
|
117
114
|
def read_and_close
|
118
|
-
#return yield if @locked
|
119
115
|
if read? || write?
|
120
116
|
begin
|
121
117
|
return yield
|
@@ -134,6 +130,45 @@ module Persist
|
|
134
130
|
end
|
135
131
|
end
|
136
132
|
|
133
|
+
def read_lock
|
134
|
+
read if closed?
|
135
|
+
if read?
|
136
|
+
begin
|
137
|
+
return yield
|
138
|
+
ensure
|
139
|
+
close
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
lock do
|
144
|
+
close
|
145
|
+
read true
|
146
|
+
begin
|
147
|
+
yield
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def write_lock
|
153
|
+
write if closed?
|
154
|
+
if write?
|
155
|
+
begin
|
156
|
+
return yield
|
157
|
+
ensure
|
158
|
+
close
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
lock do
|
163
|
+
close
|
164
|
+
write true
|
165
|
+
begin
|
166
|
+
yield
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
|
137
172
|
def merge!(hash)
|
138
173
|
hash.each do |key,values|
|
139
174
|
self[key] = values
|
@@ -141,38 +176,38 @@ module Persist
|
|
141
176
|
end
|
142
177
|
|
143
178
|
def range(*args)
|
144
|
-
self.
|
179
|
+
self.read_lock do
|
145
180
|
super(*args)
|
146
181
|
end
|
147
182
|
end
|
148
183
|
|
149
184
|
def include?(*args)
|
150
|
-
self.
|
185
|
+
self.read_lock do
|
151
186
|
super(*args) #- TSV::ENTRY_KEYS.to_a
|
152
187
|
end
|
153
188
|
end
|
154
189
|
|
155
190
|
def [](*args)
|
156
|
-
self.
|
191
|
+
self.read_lock do
|
157
192
|
super(*args) #- TSV::ENTRY_KEYS.to_a
|
158
193
|
end
|
159
194
|
end
|
160
195
|
|
161
196
|
def []=(*args)
|
162
|
-
self.
|
197
|
+
self.write_lock do
|
163
198
|
super(*args) #- TSV::ENTRY_KEYS.to_a
|
164
199
|
end
|
165
200
|
end
|
166
201
|
|
167
202
|
def keys(*args)
|
168
|
-
self.
|
203
|
+
self.read_lock do
|
169
204
|
super(*args)
|
170
205
|
end
|
171
206
|
end
|
172
207
|
|
173
208
|
|
174
209
|
def prefix(key)
|
175
|
-
self.
|
210
|
+
self.read_lock do
|
176
211
|
range(key, 1, key + MAX_CHAR, 1)
|
177
212
|
end
|
178
213
|
end
|
@@ -184,13 +219,13 @@ module Persist
|
|
184
219
|
|
185
220
|
|
186
221
|
def size(*args)
|
187
|
-
self.
|
222
|
+
self.read_lock do
|
188
223
|
super(*args)
|
189
224
|
end
|
190
225
|
end
|
191
226
|
|
192
227
|
def each(*args, &block)
|
193
|
-
self.
|
228
|
+
self.read_lock do
|
194
229
|
super(*args, &block)
|
195
230
|
end
|
196
231
|
end
|
@@ -208,7 +243,7 @@ module Persist
|
|
208
243
|
end
|
209
244
|
|
210
245
|
def values_at(*keys)
|
211
|
-
self.
|
246
|
+
self.read_lock do
|
212
247
|
keys.collect do |k|
|
213
248
|
self[k]
|
214
249
|
end
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -217,7 +217,7 @@ module CMD
|
|
217
217
|
end
|
218
218
|
end
|
219
219
|
|
220
|
-
def self.
|
220
|
+
def self.cmd_pid(*args)
|
221
221
|
all_args = *args
|
222
222
|
|
223
223
|
all_args << {} unless Hash === all_args.last
|
@@ -248,4 +248,9 @@ module CMD
|
|
248
248
|
nil
|
249
249
|
end
|
250
250
|
|
251
|
+
def self.cmd_log(*args)
|
252
|
+
cmd_pid(*args)
|
253
|
+
nil
|
254
|
+
end
|
255
|
+
|
251
256
|
end
|
@@ -242,48 +242,6 @@ module Misc
|
|
242
242
|
|
243
243
|
return options
|
244
244
|
|
245
|
-
options = {}
|
246
|
-
string.split(/#/).each do |str|
|
247
|
-
if str.match(/(.*)=(.*)/)
|
248
|
-
option, value = $1, $2
|
249
|
-
else
|
250
|
-
option, value = str, true
|
251
|
-
end
|
252
|
-
|
253
|
-
option = option.sub(":",'').to_sym if option.chars.first == ':'
|
254
|
-
value = value.sub(":",'').to_sym if String === value and value.chars.first == ':'
|
255
|
-
|
256
|
-
if value == true
|
257
|
-
options[option] = option.to_s.chars.first != '!'
|
258
|
-
else
|
259
|
-
options[option] = Thread.start do
|
260
|
-
$SAFE = 0;
|
261
|
-
case
|
262
|
-
when value =~ /^(?:true|T)$/i
|
263
|
-
true
|
264
|
-
when value =~ /^(?:false|F)$/i
|
265
|
-
false
|
266
|
-
when Symbol === value
|
267
|
-
value
|
268
|
-
when (String === value and value =~ /^\/(.*)\/$/)
|
269
|
-
Regexp.new /#{$1}/
|
270
|
-
else
|
271
|
-
begin
|
272
|
-
Kernel.const_get value
|
273
|
-
rescue
|
274
|
-
begin
|
275
|
-
raise if value =~ /[a-z]/ and defined? value
|
276
|
-
eval(value)
|
277
|
-
rescue Exception
|
278
|
-
value
|
279
|
-
end
|
280
|
-
end
|
281
|
-
end
|
282
|
-
end.value
|
283
|
-
end
|
284
|
-
end
|
285
|
-
|
286
|
-
options
|
287
245
|
end
|
288
246
|
|
289
247
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'rbbt/util/cmd'
|
2
|
+
module ProcPath
|
3
|
+
CMD.tool :procpath do
|
4
|
+
'pip install procpath'
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.record(pid, path, options = {})
|
8
|
+
IndiferentHash.setup(options)
|
9
|
+
options = Misc.add_defaults options, "interval" => 30
|
10
|
+
|
11
|
+
cmd_options = %w(interval recnum reevalnum).inject({}){|acc,k| acc[k] = options[k]; acc}
|
12
|
+
|
13
|
+
Log.debug "ProcPath recording #{pid} in #{path} (#{Misc.fingerprint options})"
|
14
|
+
procpath_thread = Thread.new do
|
15
|
+
begin
|
16
|
+
procpath_pid = CMD.cmd_pid(:procpath, "record --database-file '#{path}' '$..children[?(@.stat.pid == #{pid})]'", cmd_options.merge(:nofail => true, :add_option_dashes => true))
|
17
|
+
rescue Exception
|
18
|
+
Log.exceptions $!
|
19
|
+
Process.kill "INT", procpath_pid
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
procpath_thread.report_on_exception = false
|
24
|
+
|
25
|
+
Process.wait pid.to_i
|
26
|
+
procpath_thread.raise Interrupt
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.plot(path, output, options = {})
|
30
|
+
IndiferentHash.setup(options)
|
31
|
+
options = Misc.add_defaults options, "query-name" => 'rss', 'epsilon' => 0.5, "moving-average-window" => 10
|
32
|
+
|
33
|
+
cmd_options = %w(query-name epsilon monitor-average-window title logarithmic after before custom-query-file custom-value-expr).inject({}){|acc,k| acc[k] = options[k]; acc}
|
34
|
+
CMD.cmd_log(:procpath, "plot --database-file '#{path}' --plot-file '#{output}' ", cmd_options.merge(:nofail => true, :add_option_dashes => true))
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.monitor(pid, path)
|
38
|
+
database, options_str = path.split("#")
|
39
|
+
options = options_str.nil? ? {} : Misc.string2hash(options_str)
|
40
|
+
|
41
|
+
database = File.expand_path database
|
42
|
+
Log.low "ProcPath monitor #{pid} in #{database} (#{Misc.fingerprint options})"
|
43
|
+
|
44
|
+
ProcPath.record(pid, database + '.sqlite3', options)
|
45
|
+
ProcPath.plot(database + '.sqlite3', database + '.cpu.svg', options.merge("query-name" => 'cpu'))
|
46
|
+
ProcPath.plot(database + '.sqlite3', database + '.rss.svg', options.merge("query-name" => 'rss'))
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
@@ -505,8 +505,8 @@ class Step
|
|
505
505
|
|
506
506
|
def running?
|
507
507
|
return false if ! (started? || status == :ending)
|
508
|
-
|
509
|
-
|
508
|
+
return nil unless Open.exist?(self.pid_file)
|
509
|
+
pid = Open.read(self.pid_file).to_i
|
510
510
|
|
511
511
|
return false if done? or error? or aborted?
|
512
512
|
|
@@ -530,8 +530,7 @@ class Step
|
|
530
530
|
end
|
531
531
|
|
532
532
|
def nopid?
|
533
|
-
|
534
|
-
! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
|
533
|
+
! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
|
535
534
|
end
|
536
535
|
|
537
536
|
def aborted?
|
@@ -373,7 +373,6 @@ class Step
|
|
373
373
|
Log.exception $!
|
374
374
|
ensure
|
375
375
|
Step.purge_stream_cache
|
376
|
-
set_info :pid, nil
|
377
376
|
Open.rm pid_file if Open.exist?(pid_file)
|
378
377
|
end
|
379
378
|
end
|
@@ -388,7 +387,6 @@ class Step
|
|
388
387
|
_clean_finished
|
389
388
|
rescue
|
390
389
|
stop_dependencies
|
391
|
-
set_info :pid, nil
|
392
390
|
Open.rm pid_file if Open.exist?(pid_file)
|
393
391
|
end
|
394
392
|
end
|
@@ -449,7 +447,7 @@ class Step
|
|
449
447
|
ensure
|
450
448
|
no_load = false unless IO === result
|
451
449
|
Open.rm pid_file if Open.exist?(pid_file) unless no_load
|
452
|
-
set_info :pid, nil unless no_load
|
450
|
+
#set_info :pid, nil unless no_load
|
453
451
|
end
|
454
452
|
end
|
455
453
|
|
@@ -559,7 +557,7 @@ class Step
|
|
559
557
|
RbbtSemaphore.post_semaphore(semaphore) if semaphore
|
560
558
|
Kernel.exit! -1
|
561
559
|
end
|
562
|
-
set_info :pid, nil
|
560
|
+
#set_info :pid, nil
|
563
561
|
ensure
|
564
562
|
RbbtSemaphore.post_semaphore(semaphore) if semaphore
|
565
563
|
end
|
data/lib/rbbt/workflow/usage.rb
CHANGED
@@ -57,7 +57,7 @@ module Task
|
|
57
57
|
puts Log.color(:magenta, "Input select options")
|
58
58
|
puts
|
59
59
|
selects.collect{|p| p}.uniq.each do |input,options|
|
60
|
-
puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| o.to_s} * ", ") << "\n"
|
60
|
+
puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
|
61
61
|
puts unless Log.compact
|
62
62
|
end
|
63
63
|
puts
|
@@ -21,6 +21,8 @@ $ rbbt mnl [options]
|
|
21
21
|
-s--search* Regular expression
|
22
22
|
-t--tail* Show the last lines of the STDERR
|
23
23
|
-SBP--sbatch_parameters show sbatch parameters
|
24
|
+
-PERF--procpath_performance show Procpath performance summary
|
25
|
+
-sacct--sacct_peformance show sacct performance summary
|
24
26
|
EOF
|
25
27
|
|
26
28
|
if options[:help]
|
@@ -140,9 +142,62 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
140
142
|
|
141
143
|
if options[:sbatch_parameters]
|
142
144
|
puts Log.color(:magenta, "SBATCH parameters: ")
|
143
|
-
|
145
|
+
text = CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
|
146
|
+
lines = text.split("\n").collect{|line| header, _sep, value = line.partition(/\s+/); Log.color(:yellow, header + ": ") + value}
|
147
|
+
puts Log.color :yellow, lines * "\n"
|
144
148
|
end
|
145
149
|
|
150
|
+
fprocpath = File.join(dir, 'procpath.sqlite3')
|
151
|
+
if options[:procpath_performance] && Open.exists?(fprocpath)
|
152
|
+
puts Log.color(:magenta, "Procpath summary: ")
|
153
|
+
require 'rbbt/tsv/csv'
|
154
|
+
meta = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from meta;' "))
|
155
|
+
perf = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from record;' "))
|
156
|
+
|
157
|
+
page_size = meta["page_size"].first.to_f
|
158
|
+
clock_ticks = meta["clock_ticks"].first.to_f
|
159
|
+
|
160
|
+
cpu_average = {}
|
161
|
+
rss_average = {}
|
162
|
+
perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
|
163
|
+
time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
|
164
|
+
time = time.to_f
|
165
|
+
|
166
|
+
cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
|
167
|
+
cpu_average[stat_pid] ||= {}
|
168
|
+
cpu_average[stat_pid][time] ||= []
|
169
|
+
cpu_average[stat_pid][time] << cpu.to_f
|
170
|
+
rss_average[time] ||= []
|
171
|
+
rss_average[time] << rss.to_f * page_size
|
172
|
+
end
|
173
|
+
|
174
|
+
ticks = 0
|
175
|
+
cpu_average.each do |stat_pid, cpu_average_pid|
|
176
|
+
start = cpu_average_pid.keys.sort.first
|
177
|
+
eend = cpu_average_pid.keys.sort.last
|
178
|
+
ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
|
179
|
+
end
|
180
|
+
start = rss_average.keys.sort.first
|
181
|
+
eend = rss_average.keys.sort.last
|
182
|
+
time_elapsed = eend - start
|
183
|
+
puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
|
184
|
+
puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s
|
185
|
+
|
186
|
+
end
|
187
|
+
|
188
|
+
if options[:sacct_peformance]
|
189
|
+
begin
|
190
|
+
tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
|
191
|
+
values = tsv[tsv.keys.first]
|
192
|
+
if values.compact.any?
|
193
|
+
puts Log.color(:magenta, "SACCT performance: ")
|
194
|
+
puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ") + v.to_s } * "\n"
|
195
|
+
end
|
196
|
+
rescue
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
|
146
201
|
if tail && File.exists?(File.join(dir, 'std.err'))
|
147
202
|
if exit_status && exit_status != 0
|
148
203
|
puts Log.color(:magenta, "First error or exception found: ")
|
@@ -21,6 +21,7 @@ $slurm_options = SOPT.get <<EOF
|
|
21
21
|
-CS--contain_and_sync Contain and sync to default locations
|
22
22
|
-ci--copy_image When using a container directory, copy image there
|
23
23
|
-t--tail Tail the logs
|
24
|
+
-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
|
24
25
|
-q--queue* Queue
|
25
26
|
-t--task_cpus* Tasks
|
26
27
|
-W--workflows* Additional workflows
|
@@ -20,6 +20,7 @@ $slurm_options = SOPT.get <<EOF
|
|
20
20
|
-CS--contain_and_sync Contain and sync to default locations
|
21
21
|
-ci--copy_image When using a container directory, copy image there
|
22
22
|
-t--tail Tail the logs
|
23
|
+
-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
|
23
24
|
-q--queue* Queue
|
24
25
|
-t--task_cpus* Tasks
|
25
26
|
-W--workflows* Additional workflows
|
@@ -35,7 +35,7 @@ file = case file
|
|
35
35
|
fields = options[:fields]
|
36
36
|
raise ParameterException, "Please specify the fields to slice" if fields.nil?
|
37
37
|
|
38
|
-
options[:header_hash]
|
38
|
+
options[:header_hash] ||= options["header_hash"]
|
39
39
|
|
40
40
|
case
|
41
41
|
when options[:tokyocabinet]
|
@@ -45,8 +45,8 @@ when options[:tokyocabinet_bd]
|
|
45
45
|
tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
|
46
46
|
puts tsv.summary
|
47
47
|
else
|
48
|
-
stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :
|
49
|
-
|
48
|
+
stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :fields => fields.split(","), :unnamed => true) do |k,fields,names|
|
49
|
+
[k,fields].flatten * "\t"
|
50
50
|
end
|
51
51
|
puts stream.read
|
52
52
|
exit 0
|
@@ -86,7 +86,7 @@ messages = info[:messages]
|
|
86
86
|
backtrace = info[:backtrace]
|
87
87
|
pid = info[:pid]
|
88
88
|
exception = info[:exception]
|
89
|
-
rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :
|
89
|
+
rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :archived_info]
|
90
90
|
|
91
91
|
|
92
92
|
puts Log.color(:magenta, "File") << ": " << step.path
|
@@ -203,6 +203,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
|
|
203
203
|
-prec--prepare_cpus* Number of dependencies prepared in parallel
|
204
204
|
-rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
|
205
205
|
-od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
|
206
|
+
-PERF--procpath_performance* Measure performance using procpath
|
206
207
|
EOF
|
207
208
|
|
208
209
|
workflow = ARGV.shift
|
@@ -407,6 +408,13 @@ begin
|
|
407
408
|
exit 0
|
408
409
|
end
|
409
410
|
|
411
|
+
if options[:procpath_performance]
|
412
|
+
require 'rbbt/util/procpath'
|
413
|
+
job.fork
|
414
|
+
job.soft_grace
|
415
|
+
pid = job.info[:pid]
|
416
|
+
ProcPath.monitor(pid, options[:procpath_performance])
|
417
|
+
end
|
410
418
|
|
411
419
|
if do_fork
|
412
420
|
ENV["RBBT_NO_PROGRESS"] = "true"
|
@@ -423,7 +431,6 @@ begin
|
|
423
431
|
res = job
|
424
432
|
end
|
425
433
|
|
426
|
-
|
427
434
|
if options.delete(:printpath)
|
428
435
|
job.join
|
429
436
|
raise job.messages.last if (job.error? || job.aborted?) && job.messages
|
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rbbt/workflow'
|
4
|
+
|
5
|
+
require 'rbbt-util'
|
6
|
+
require 'rbbt-util'
|
7
|
+
require 'rbbt/util/simpleopt'
|
8
|
+
|
9
|
+
$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
|
10
|
+
|
11
|
+
options = SOPT.setup <<EOF
|
12
|
+
Examine the info of a job result
|
13
|
+
|
14
|
+
$ rbbt workflow info <job-result> <key> <value>
|
15
|
+
|
16
|
+
-h--help Help
|
17
|
+
-f--force Write info even if key is already present
|
18
|
+
-r--recursive Write info for all dependencies as well
|
19
|
+
-p--check_pid Check that recursive jobs where created by the same process
|
20
|
+
EOF
|
21
|
+
|
22
|
+
SOPT.usage if options[:help]
|
23
|
+
|
24
|
+
file, key, value = ARGV
|
25
|
+
|
26
|
+
force, recursive, check_pid = options.values_at :force, :recursive, :check_pid
|
27
|
+
|
28
|
+
def get_step(file)
|
29
|
+
file = file.sub(/\.(info|files)/,'')
|
30
|
+
step = Workflow.load_step file
|
31
|
+
step
|
32
|
+
end
|
33
|
+
|
34
|
+
raise ParameterException if key.nil? || value.nil?
|
35
|
+
|
36
|
+
if %w(DELETE nil).include? value
|
37
|
+
value = nil
|
38
|
+
force = true
|
39
|
+
end
|
40
|
+
|
41
|
+
step = get_step file
|
42
|
+
|
43
|
+
step.set_info key, value if force || ! step.info.include?(key)
|
44
|
+
|
45
|
+
pid = step.info[:pid]
|
46
|
+
host = step.info[:pid_hostname]
|
47
|
+
|
48
|
+
step.rec_dependencies.each do |dep|
|
49
|
+
dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
|
50
|
+
rescue
|
51
|
+
Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
|
52
|
+
end if recursive
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
|
2
|
+
require 'rbbt/util/procpath'
|
3
|
+
|
4
|
+
class TestProcPath < Test::Unit::TestCase
|
5
|
+
def test_record_and_plot
|
6
|
+
Log.with_severity 0 do
|
7
|
+
pid = Process.fork do
|
8
|
+
a = ""
|
9
|
+
(0..1000).each do
|
10
|
+
a << (0..rand(10000).to_i).to_a.collect{|i| "TEST #{i}" } * " "
|
11
|
+
sleep 0.1
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
TmpFile.with_file(nil, false) do |db|
|
16
|
+
|
17
|
+
ProcPath.record(pid, db, :interval => '1', "recnum" => 100)
|
18
|
+
ProcPath.plot(db, db + '.svg', "moving-average-window" => 1 )
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.30.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-01
|
11
|
+
date: 2021-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -300,6 +300,7 @@ files:
|
|
300
300
|
- lib/rbbt/util/misc/system.rb
|
301
301
|
- lib/rbbt/util/named_array.rb
|
302
302
|
- lib/rbbt/util/open.rb
|
303
|
+
- lib/rbbt/util/procpath.rb
|
303
304
|
- lib/rbbt/util/python.rb
|
304
305
|
- lib/rbbt/util/semaphore.rb
|
305
306
|
- lib/rbbt/util/simpleDSL.rb
|
@@ -438,6 +439,7 @@ files:
|
|
438
439
|
- share/rbbt_commands/workflow/server
|
439
440
|
- share/rbbt_commands/workflow/task
|
440
441
|
- share/rbbt_commands/workflow/trace
|
442
|
+
- share/rbbt_commands/workflow/write_info
|
441
443
|
- share/unicorn.rb
|
442
444
|
- share/workflow_config.ru
|
443
445
|
- test/rbbt/annotations/test_util.rb
|
@@ -516,6 +518,7 @@ files:
|
|
516
518
|
- test/rbbt/util/test_log.rb
|
517
519
|
- test/rbbt/util/test_misc.rb
|
518
520
|
- test/rbbt/util/test_open.rb
|
521
|
+
- test/rbbt/util/test_procpath.rb
|
519
522
|
- test/rbbt/util/test_python.rb
|
520
523
|
- test/rbbt/util/test_semaphore.rb
|
521
524
|
- test/rbbt/util/test_simpleDSL.rb
|
@@ -563,6 +566,7 @@ test_files:
|
|
563
566
|
- test/rbbt/workflow/test_task.rb
|
564
567
|
- test/rbbt/resource/test_path.rb
|
565
568
|
- test/rbbt/util/test_colorize.rb
|
569
|
+
- test/rbbt/util/test_procpath.rb
|
566
570
|
- test/rbbt/util/misc/test_omics.rb
|
567
571
|
- test/rbbt/util/misc/test_pipes.rb
|
568
572
|
- test/rbbt/util/misc/test_format.rb
|