rbbt-util 5.25.8 → 5.25.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc.rb +44 -25
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f1eb2767b3e3f8adcb67bfc0fbc05d79000130a9
|
4
|
+
data.tar.gz: d3314ccd8960e4780887978e9b8745a31276b9f1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 465aefada76bce2c950aa535098ef03652a2793fe7030c1532557f72d9cd0d856479511247339152ef8e8a2084abe0436662a9e8ba6fc9ba1bf8833fa50f3e50
|
7
|
+
data.tar.gz: 84827530eee0ed691bfadfd1e137f801921f96c071b7ef016a9542896190e3ba6ad0f24603188fd81eba4f8e070364ebde39f7f27f2b9aa20feb5742ef97c0c3
|
data/lib/rbbt/hpc.rb
CHANGED
@@ -16,6 +16,7 @@ module Marenostrum
|
|
16
16
|
copy_image = options.delete :copy_image
|
17
17
|
exclusive = options.delete :exclusive
|
18
18
|
highmem = options.delete :highmem
|
19
|
+
inputs_dir = options.delete :inputs_dir
|
19
20
|
|
20
21
|
if contain_and_sync
|
21
22
|
contain = "/scratch/tmp/rbbt" if contain.nil?
|
@@ -101,19 +102,35 @@ mkdir -p "$SINGULARITY_RUBY_INLINE"
|
|
101
102
|
|
102
103
|
if contain
|
103
104
|
env +=<<-EOF
|
105
|
+
|
106
|
+
# Prepare container dir
|
104
107
|
CONTAINER_DIR="#{contain}"
|
105
108
|
mkdir -p $CONTAINER_DIR/.rbbt/etc/
|
106
109
|
for tmpd in persist_locks produce_locks R_sockets sensiblewrite sensiblewrite_locks step_info_locks tsv_open_locks; do
|
107
110
|
mkdir -p $CONTAINER_DIR/.rbbt/tmp/$tmpd
|
108
111
|
done
|
112
|
+
|
113
|
+
# Copy environment
|
109
114
|
cp ~/.rbbt/etc/environment $CONTAINER_DIR/.rbbt/etc/
|
115
|
+
|
116
|
+
# Set search_paths
|
110
117
|
echo "rbbt_user: /home/rbbt/.rbbt/{TOPLEVEL}/{SUBPATH}" > $CONTAINER_DIR/.rbbt/etc/search_paths
|
111
118
|
echo "home: $CONTAINER_DIR/home/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
|
112
119
|
echo "group_projects: $CONTAINER_DIR/projects/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
|
113
120
|
echo "group_scratch: $CONTAINER_DIR/scratch/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
|
114
121
|
echo "user_projects: $CONTAINER_DIR/projects/#{ENV['USER']}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
|
115
122
|
echo "user_scratch: $CONTAINER_DIR/scratch/#{ENV['USER']}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
|
123
|
+
echo "/scratch/tmp/rbbt/projects/rbbt/workflows/" > $CONTAINER_DIR/.rbbt/etc/workflow_dir
|
116
124
|
EOF
|
125
|
+
|
126
|
+
if inputs_dir
|
127
|
+
env +=<<-EOF
|
128
|
+
|
129
|
+
# Copy inputs
|
130
|
+
cp -R '#{inputs_dir}' $CONTAINER_DIR/inputs
|
131
|
+
EOF
|
132
|
+
rbbt_cmd = rbbt_cmd.sub(inputs_dir, "#{contain}/inputs")
|
133
|
+
end
|
117
134
|
|
118
135
|
if copy_image
|
119
136
|
env +=<<EOF
|
@@ -162,7 +179,7 @@ EOF
|
|
162
179
|
#{cmd}
|
163
180
|
|
164
181
|
# Save exit status
|
165
|
-
|
182
|
+
exit_status=$?
|
166
183
|
|
167
184
|
# Clean job.id, since we are done
|
168
185
|
rm #{fjob}
|
@@ -195,6 +212,12 @@ unset sync_es
|
|
195
212
|
EOF
|
196
213
|
end
|
197
214
|
end
|
215
|
+
coda +=<<-EOF
|
216
|
+
|
217
|
+
# Write exit status to file
|
218
|
+
echo $exit_status > #{fexit}
|
219
|
+
unset exit_status
|
220
|
+
EOF
|
198
221
|
|
199
222
|
template = [header, env, run, coda] * "\n"
|
200
223
|
|
@@ -281,7 +304,7 @@ EOF
|
|
281
304
|
STDERR.puts Log.color(:magenta, "Output:")
|
282
305
|
begin
|
283
306
|
CMD.cmd("squeue --job #{job} > #{fstatus}")
|
284
|
-
out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists?
|
307
|
+
out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists?(fout) and not tail == :STDERR
|
285
308
|
err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exists? ferr
|
286
309
|
|
287
310
|
Misc.consume_stream(err, true, STDERR) if err
|
@@ -309,27 +332,8 @@ EOF
|
|
309
332
|
fjob = File.join(workdir, 'job.id')
|
310
333
|
job = Open.read(fjob) if Open.exists?(fjob)
|
311
334
|
|
312
|
-
status_txt = CMD.cmd("squeue --job #{job}").read
|
313
|
-
STDERR.puts Log.color(:magenta, "Status [#{job.to_i}]:")
|
314
|
-
STDERR.puts status_txt
|
315
|
-
lines = status_txt.split("\n").length
|
316
335
|
|
317
336
|
while ! Open.exists?(fexit)
|
318
|
-
STDERR.puts
|
319
|
-
Log.clear_line(STDERR)
|
320
|
-
STDERR.write Log.color(:magenta, "Waiting for end")
|
321
|
-
3.times do
|
322
|
-
STDERR.write Log.color(:magenta, ".")
|
323
|
-
sleep 1
|
324
|
-
end
|
325
|
-
status_txt = CMD.cmd("squeue --job #{job}").read
|
326
|
-
lines.times do
|
327
|
-
Log.clear_line(STDERR)
|
328
|
-
end
|
329
|
-
Log.clear_line(STDERR)
|
330
|
-
STDERR.puts Log.color(:magenta, "Status [#{job.to_i}]:")
|
331
|
-
STDERR.puts status_txt
|
332
|
-
lines = status_txt.split("\n").length
|
333
337
|
sleep time
|
334
338
|
end
|
335
339
|
end
|
@@ -338,18 +342,33 @@ EOF
|
|
338
342
|
workflow = job.workflow
|
339
343
|
task = job.task_name
|
340
344
|
name = job.clean_name
|
341
|
-
|
345
|
+
keep_workdir = options.delete :keep_SLURM_workdir
|
346
|
+
TmpFile.with_file(nil, !keep_workdir) do |tmp_directory|
|
342
347
|
workdir = options[:workdir] ||= File.join(tmp_directory, 'workdir')
|
343
348
|
inputs_dir = File.join(tmp_directory, 'inputs_dir')
|
344
349
|
Step.save_job_inputs(job, inputs_dir)
|
350
|
+
options[:inputs_dir] = inputs_dir
|
345
351
|
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '-jn', name, '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
|
352
|
+
|
353
|
+
%w(workflows requires remote_workflow_tasks override_deps).each do |key|
|
354
|
+
next unless options[key]
|
355
|
+
cmd += ["--#{key.to_s}", options[key]]
|
356
|
+
end
|
357
|
+
|
346
358
|
template = self.template(cmd, options)
|
347
359
|
self.issue_template(template, options)
|
360
|
+
t_monitor = Thread.new do
|
361
|
+
self.follow_job(workdir, :STDERR)
|
362
|
+
end
|
348
363
|
self.wait_for_job(workdir)
|
349
|
-
|
350
|
-
|
351
|
-
|
364
|
+
t_monitor.raise Aborted
|
365
|
+
return unless Open.read(File.join(workdir, 'exit.status')).strip == '0'
|
366
|
+
path = Open.read(File.join(workdir, 'std.out')).strip
|
367
|
+
if Open.exists?(path) && job.path != path
|
368
|
+
Log.info "Path of SLURM job #{path} is different from original job #{job.path}. Stablishing link."
|
352
369
|
Open.ln path, job.path
|
370
|
+
Open.ln path + '.info', job.path + '.info' if Open.exists?(path + '.info')
|
371
|
+
Open.ln path + '.files', job.path + '.files' if Open.exists?(path + '.files')
|
353
372
|
end
|
354
373
|
end
|
355
374
|
end
|