rbbt-util 5.30.11 → 5.31.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc.rb +3 -0
- data/lib/rbbt/hpc/batch.rb +621 -0
- data/lib/rbbt/hpc/lsf.rb +119 -0
- data/lib/rbbt/hpc/orchestrate.rb +12 -11
- data/lib/rbbt/hpc/slurm.rb +62 -567
- data/lib/rbbt/resource/path.rb +3 -1
- data/lib/rbbt/tsv/accessor.rb +5 -2
- data/lib/rbbt/tsv/dumper.rb +1 -0
- data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
- data/lib/rbbt/tsv/stream.rb +5 -6
- data/lib/rbbt/util/log.rb +22 -1
- data/lib/rbbt/util/misc/development.rb +2 -2
- data/lib/rbbt/util/misc/inspect.rb +17 -1
- data/lib/rbbt/util/misc/options.rb +5 -0
- data/lib/rbbt/workflow/integration/ansible.rb +7 -3
- data/lib/rbbt/workflow/step/accessor.rb +10 -6
- data/lib/rbbt/workflow/usage.rb +13 -13
- data/share/config.ru +3 -3
- data/share/rbbt_commands/{slurm → hpc}/clean +91 -18
- data/share/rbbt_commands/{slurm → hpc}/list +103 -33
- data/share/rbbt_commands/hpc/orchestrate +81 -0
- data/share/rbbt_commands/hpc/tail +81 -0
- data/share/rbbt_commands/hpc/task +80 -0
- data/test/rbbt/hpc/test_batch.rb +65 -0
- data/test/rbbt/hpc/test_slurm.rb +30 -0
- data/test/rbbt/util/misc/test_development.rb +11 -0
- data/test/test_helper.rb +3 -1
- metadata +16 -7
- data/share/rbbt_commands/slurm/orchestrate +0 -48
- data/share/rbbt_commands/slurm/task +0 -46
data/lib/rbbt/resource/path.rb
CHANGED
@@ -3,7 +3,7 @@ require 'rbbt/util/misc/indiferent_hash'
|
|
3
3
|
require 'yaml'
|
4
4
|
|
5
5
|
module Path
|
6
|
-
attr_accessor :resource, :pkgdir, :original, :search_paths, :search_order, :libdir
|
6
|
+
attr_accessor :resource, :pkgdir, :original, :search_paths, :search_order, :libdir, :where
|
7
7
|
|
8
8
|
def self.setup(string, pkgdir = nil, resource = nil, search_paths = nil, search_order = nil, libdir = nil)
|
9
9
|
return string if string.nil?
|
@@ -99,6 +99,7 @@ module Path
|
|
99
99
|
|
100
100
|
paths = paths.each do |p|
|
101
101
|
p.original = File.join(found.original, p.sub(/^#{found}/, ''))
|
102
|
+
p.where = where
|
102
103
|
end if found.original and pattern
|
103
104
|
|
104
105
|
location_paths[where] = paths
|
@@ -256,6 +257,7 @@ module Path
|
|
256
257
|
end
|
257
258
|
|
258
259
|
res.original = self.original || self
|
260
|
+
res.where = where
|
259
261
|
|
260
262
|
res
|
261
263
|
end
|
data/lib/rbbt/tsv/accessor.rb
CHANGED
@@ -713,8 +713,11 @@ module TSV
|
|
713
713
|
break
|
714
714
|
end
|
715
715
|
|
716
|
-
filename =
|
717
|
-
filename
|
716
|
+
filename = @filename
|
717
|
+
filename = "No filename" if filename.nil? || filename.empty?
|
718
|
+
filename.find if Path === filename
|
719
|
+
filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
|
720
|
+
|
718
721
|
with_unnamed do
|
719
722
|
<<-EOF
|
720
723
|
Filename = #{filename}
|
data/lib/rbbt/tsv/dumper.rb
CHANGED
@@ -610,7 +610,7 @@ module TSV
|
|
610
610
|
thread = Thread.new do
|
611
611
|
begin
|
612
612
|
traverse_run(obj, threads, cpus, options, &block)
|
613
|
-
into.close if into.respond_to?(:close) and not (into.respond_to?
|
613
|
+
into.close if into.respond_to?(:close) and not (into.respond_to?(:closed?) and into.closed?)
|
614
614
|
rescue Exception
|
615
615
|
abort_stream obj
|
616
616
|
abort_stream into
|
data/lib/rbbt/tsv/stream.rb
CHANGED
@@ -294,20 +294,19 @@ module TSV
|
|
294
294
|
end
|
295
295
|
|
296
296
|
|
297
|
-
def self.reorder_stream_tsv(stream, key_field, fields=nil, zipped = true)
|
297
|
+
def self.reorder_stream_tsv(stream, key_field, fields=nil, zipped = true, bar = nil)
|
298
298
|
parser = TSV::Parser.new TSV.get_stream(stream), :key_field => key_field, :fields => fields
|
299
299
|
dumper_options = parser.options
|
300
300
|
dumper = TSV::Dumper.new dumper_options
|
301
301
|
dumper.init
|
302
302
|
case parser.type
|
303
303
|
when :single
|
304
|
-
TSV.traverse parser, :into => dumper do |keys,values|
|
304
|
+
TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
|
305
305
|
key = keys.first
|
306
306
|
[key, [values]]
|
307
307
|
end
|
308
308
|
when :double
|
309
|
-
TSV.traverse parser, :into => dumper do |keys,values|
|
310
|
-
raise [keys, values].inspect if keys.include? 'gain'
|
309
|
+
TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
|
311
310
|
res = []
|
312
311
|
keys.each_with_index do |key,i|
|
313
312
|
vs = zipped ? values.collect{|l| l.length == 1 ? l : [l[i]] } : values
|
@@ -317,12 +316,12 @@ module TSV
|
|
317
316
|
res
|
318
317
|
end
|
319
318
|
when :list
|
320
|
-
TSV.traverse parser, :into => dumper do |keys,values|
|
319
|
+
TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
|
321
320
|
key = keys === Array ? keys.first : keys
|
322
321
|
[key, values]
|
323
322
|
end
|
324
323
|
when :flat
|
325
|
-
TSV.traverse parser, :into => dumper do |keys,values|
|
324
|
+
TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
|
326
325
|
key = keys === Array ? keys.first : keys
|
327
326
|
[key, values]
|
328
327
|
end
|
data/lib/rbbt/util/log.rb
CHANGED
@@ -341,10 +341,31 @@ module Log
|
|
341
341
|
end unless stack.nil?
|
342
342
|
end
|
343
343
|
|
344
|
-
def self.tsv(tsv)
|
344
|
+
def self.tsv(tsv, example = false)
|
345
345
|
STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
|
346
346
|
STDERR.puts Log.color(:blue, "=> "<< Misc.fingerprint(tsv), true)
|
347
347
|
STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
|
348
|
+
if example && ! tsv.empty?
|
349
|
+
key = case example
|
350
|
+
when TrueClass, :first, "first"
|
351
|
+
tsv.keys.first
|
352
|
+
when :random, "random"
|
353
|
+
tsv.keys.shuffle.first
|
354
|
+
else
|
355
|
+
example
|
356
|
+
end
|
357
|
+
|
358
|
+
values = tsv[key]
|
359
|
+
values = [values] if tsv.type == :flat || tsv.type == :single
|
360
|
+
if values.nil?
|
361
|
+
STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
|
362
|
+
else
|
363
|
+
STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
|
364
|
+
tsv.fields.zip(values).each do |field,value|
|
365
|
+
STDERR.puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
|
366
|
+
end
|
367
|
+
end
|
368
|
+
end
|
348
369
|
end
|
349
370
|
|
350
371
|
def self.stack(stack)
|
@@ -428,9 +428,9 @@ def self.add_libdir(dir=nil)
|
|
428
428
|
end
|
429
429
|
end
|
430
430
|
|
431
|
-
|
432
|
-
def self.ssh_run(server, script)
|
431
|
+
def self.ssh_run(server, script = nil, &block)
|
433
432
|
Log.debug "Run ssh script in #{server}:\n#{script}"
|
433
|
+
|
434
434
|
CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
|
435
435
|
end
|
436
436
|
|
@@ -271,6 +271,22 @@ module Misc
|
|
271
271
|
end
|
272
272
|
|
273
273
|
|
274
|
+
def self.step_file?(path)
|
275
|
+
return true if defined?(Step) && Step === path.resource
|
276
|
+
return false unless path.include?('.files/')
|
277
|
+
parts = path.split("/")
|
278
|
+
job = parts.select{|p| p =~ /\.files$/}.first
|
279
|
+
if job
|
280
|
+
i = parts.index job
|
281
|
+
begin
|
282
|
+
workflow, task = parts.values_at i - 2, i - 1
|
283
|
+
return Kernel.const_get(workflow).tasks.include? task.to_sym
|
284
|
+
rescue
|
285
|
+
end
|
286
|
+
end
|
287
|
+
false
|
288
|
+
end
|
289
|
+
|
274
290
|
def self.obj2str(obj)
|
275
291
|
_obj = obj
|
276
292
|
obj = Annotated.purge(obj) if Annotated === obj
|
@@ -289,7 +305,7 @@ module Misc
|
|
289
305
|
when (defined?(Path) and Path)
|
290
306
|
if defined?(Step) && Open.exists?(Step.info_file(obj))
|
291
307
|
obj2str(Workflow.load_step(obj))
|
292
|
-
elsif
|
308
|
+
elsif step_file?(obj)
|
293
309
|
"Step file: " + obj
|
294
310
|
else
|
295
311
|
if obj.exists?
|
@@ -135,9 +135,14 @@ module Misc
|
|
135
135
|
end
|
136
136
|
|
137
137
|
new_options
|
138
|
+
|
139
|
+
options.replace new_options
|
138
140
|
end
|
139
141
|
|
140
142
|
def self.process_options(hash, *keys)
|
143
|
+
defaults = keys.pop if Hash === keys.last
|
144
|
+
hahs = Misc.add_defaults hash, defaults if defaults
|
145
|
+
|
141
146
|
if keys.length == 1
|
142
147
|
hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s)
|
143
148
|
else
|
@@ -2,7 +2,7 @@ require_relative 'ansible/workflow'
|
|
2
2
|
require 'rbbt/workflow/usage'
|
3
3
|
|
4
4
|
module Ansible
|
5
|
-
def self.play(playbook, inventory = nil)
|
5
|
+
def self.play(playbook, inventory = nil, verbose = false)
|
6
6
|
inventory = Rbbt.etc.ansible_inventory.find
|
7
7
|
Log.with_severity 0 do
|
8
8
|
TmpFile.with_file do |tmp|
|
@@ -10,7 +10,11 @@ module Ansible
|
|
10
10
|
Open.write(tmp, [playbook].to_yaml)
|
11
11
|
playbook = tmp
|
12
12
|
end
|
13
|
-
|
13
|
+
if verbose
|
14
|
+
CMD.cmd_log("ansible-playbook -i #{inventory} -v #{playbook}")
|
15
|
+
else
|
16
|
+
CMD.cmd_log("ansible-playbook -i #{inventory} #{playbook}")
|
17
|
+
end
|
14
18
|
end
|
15
19
|
end
|
16
20
|
end
|
@@ -46,7 +50,7 @@ module Ansible
|
|
46
50
|
def self.playbook(file, task = nil, options = {})
|
47
51
|
task = 'default' if task.nil?
|
48
52
|
|
49
|
-
workflow = Workflow.require_workflow
|
53
|
+
workflow = Workflow === file ? file : Workflow.require_workflow(file)
|
50
54
|
task = workflow.tasks.keys.last if workflow.tasks[task].nil?
|
51
55
|
workflow2playbook workflow, task, options
|
52
56
|
end
|
@@ -99,6 +99,7 @@ class Step
|
|
99
99
|
if String === value && File.exists?(value)
|
100
100
|
Open.ln_s(value, path)
|
101
101
|
else
|
102
|
+
value = "#{value}" if Path === value
|
102
103
|
Open.write(path + '.yaml', value.to_yaml)
|
103
104
|
end
|
104
105
|
when Array === value
|
@@ -122,15 +123,18 @@ class Step
|
|
122
123
|
workflow = job.workflow
|
123
124
|
workflow = Kernel.const_get workflow if String === workflow
|
124
125
|
if workflow
|
125
|
-
task_info = workflow.task_info(task_name)
|
126
|
-
input_types = task_info[:input_types]
|
127
|
-
task_inputs = task_info[:inputs]
|
128
|
-
input_defaults = task_info[:input_defaults]
|
126
|
+
task_info = IndiferentHash.setup(workflow.task_info(task_name))
|
127
|
+
input_types = IndiferentHash.setup(task_info[:input_types])
|
128
|
+
task_inputs = IndiferentHash.setup(task_info[:inputs])
|
129
|
+
input_defaults = IndiferentHash.setup(task_info[:input_defaults])
|
129
130
|
else
|
130
|
-
task_info =
|
131
|
+
task_info = IndiferentHash.setup({})
|
132
|
+
input_types = IndiferentHash.setup({})
|
133
|
+
task_inputs = IndiferentHash.setup({})
|
134
|
+
input_defaults = IndiferentHash.setup({})
|
131
135
|
end
|
132
136
|
|
133
|
-
inputs = {}
|
137
|
+
inputs = IndiferentHash.setup({})
|
134
138
|
real_inputs = job.real_inputs || job.info[:real_inputs]
|
135
139
|
job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
|
136
140
|
next unless task_inputs.include? name.to_sym
|
data/lib/rbbt/workflow/usage.rb
CHANGED
@@ -6,19 +6,6 @@ module Task
|
|
6
6
|
puts "\n" << Misc.format_paragraph(description.strip) << "\n" if description and not description.empty?
|
7
7
|
puts
|
8
8
|
|
9
|
-
case
|
10
|
-
when (input_types.values & [:array]).any?
|
11
|
-
puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
|
12
|
-
also works in addition to the others. You may use the '--array_separator' option
|
13
|
-
the change this default. Whenever a file is specified it may also accept STDIN using
|
14
|
-
the '-' character."))
|
15
|
-
puts
|
16
|
-
|
17
|
-
when (input_types.values & [:text, :tsv]).any?
|
18
|
-
puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
|
19
|
-
puts
|
20
|
-
end
|
21
|
-
|
22
9
|
selects = []
|
23
10
|
if inputs.any?
|
24
11
|
inputs.zip(input_types.values_at(*inputs)).select{|i,t| t.to_sym == :select && input_options[i] && input_options[i][:select_options] }.each{|i,t| selects << [i, input_options[i][:select_options]] }
|
@@ -50,6 +37,19 @@ module Task
|
|
50
37
|
puts
|
51
38
|
end
|
52
39
|
|
40
|
+
case
|
41
|
+
when (input_types.values & [:array]).any?
|
42
|
+
puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
|
43
|
+
also works in addition to the others. You may use the '--array_separator' option
|
44
|
+
the change this default. Whenever a file is specified it may also accept STDIN using
|
45
|
+
the '-' character."))
|
46
|
+
puts
|
47
|
+
|
48
|
+
when (input_types.values & [:text, :tsv]).any?
|
49
|
+
puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
|
50
|
+
puts
|
51
|
+
end
|
52
|
+
|
53
53
|
puts Log.color(:magenta, "Returns: ") << Log.color(:blue, result_type.to_s) << "\n"
|
54
54
|
puts
|
55
55
|
|
data/share/config.ru
CHANGED
@@ -43,6 +43,9 @@ app_eval app, Rbbt.etc['app.d/remote_workflow_tasks.rb'].find_all
|
|
43
43
|
#{{{ BASE
|
44
44
|
app_eval app, Rbbt.etc['app.d/base.rb'].find
|
45
45
|
|
46
|
+
#{{{ SINATRA
|
47
|
+
load_file Rbbt.lib['sinatra.rb'].find_all
|
48
|
+
|
46
49
|
#{{{ RESOURCES
|
47
50
|
load_file Rbbt.etc['app.d/resources.rb'].find
|
48
51
|
|
@@ -70,9 +73,6 @@ load_file Rbbt.etc['app.d/preload.rb'].find_all
|
|
70
73
|
#{{{ PRELOAD
|
71
74
|
load_file Rbbt.etc['app.d/semaphores.rb'].find_all
|
72
75
|
|
73
|
-
#{{{ SINATRA
|
74
|
-
load_file Rbbt.lib['sinatra.rb'].find_all
|
75
|
-
|
76
76
|
Entity.entity_list_cache = Rbbt.var.sinatra.app[app_name].find.entity_lists
|
77
77
|
Entity.entity_map_cache = Rbbt.var.sinatra.app[app_name].find.entity_maps
|
78
78
|
Entity.entity_property_cache = Rbbt.var.sinatra.app[app_name].find.entity_properties
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'rbbt-util'
|
4
4
|
require 'rbbt/util/simpleopt'
|
5
|
+
require 'rbbt/hpc'
|
5
6
|
|
6
7
|
#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
|
7
8
|
|
@@ -9,16 +10,17 @@ options = SOPT.setup <<EOF
|
|
9
10
|
|
10
11
|
Clean error or aborted jobs
|
11
12
|
|
12
|
-
$ rbbt
|
13
|
+
$ rbbt slurm clean [options]
|
13
14
|
|
14
15
|
-h--help Print this help
|
15
16
|
-d--done Done jobs only
|
16
17
|
-e--error Error jobs only
|
17
18
|
-a--aborted SLURM aboted jobs
|
19
|
+
-q--queued Queued jobs only
|
18
20
|
-j--job* Job ids
|
19
21
|
-s--search* Regular expression
|
20
22
|
-t--tail* Show the last lines of the STDERR
|
21
|
-
-
|
23
|
+
-BP--batch_parameters show batch parameters
|
22
24
|
-dr--dry_run Do not erase anything
|
23
25
|
EOF
|
24
26
|
|
@@ -31,14 +33,47 @@ if options[:help]
|
|
31
33
|
exit 0
|
32
34
|
end
|
33
35
|
|
36
|
+
batch_system = options.delete :batch_system
|
37
|
+
batch_system ||= 'auto'
|
38
|
+
|
39
|
+
HPC::BATCH_MODULE = case batch_system.to_s.downcase
|
40
|
+
when 'slurm'
|
41
|
+
HPC::SLURM
|
42
|
+
when 'lsf'
|
43
|
+
HPC::LSF
|
44
|
+
when 'auto'
|
45
|
+
case $previous_commands.last
|
46
|
+
when 'slurm'
|
47
|
+
HPC::SLURM
|
48
|
+
when 'lsf'
|
49
|
+
HPC::LSF
|
50
|
+
else
|
51
|
+
case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
|
52
|
+
when 'slurm'
|
53
|
+
HPC::SLURM
|
54
|
+
when 'lsf'
|
55
|
+
HPC::LSF
|
56
|
+
else
|
57
|
+
case ENV["BATCH_SYSTEM"].to_s.downcase
|
58
|
+
when 'slurm'
|
59
|
+
HPC::SLURM
|
60
|
+
when 'lsf'
|
61
|
+
HPC::LSF
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
|
68
|
+
|
34
69
|
Log.severity = 4
|
35
|
-
done, error, aborted, jobid, search, tail,
|
70
|
+
done, error, aborted, queued, jobid, search, tail, batch_parameters, dry_run = options.values_at :done, :error, :aborted, :queued, :job, :search, :tail, :batch_parameters, :dry_run
|
36
71
|
|
37
|
-
workdir = File.expand_path('~/rbbt-
|
72
|
+
workdir = File.expand_path('~/rbbt-batch')
|
38
73
|
Path.setup(workdir)
|
39
74
|
|
40
75
|
running_jobs = begin
|
41
|
-
squeue_txt =
|
76
|
+
squeue_txt = HPC::BATCH_MODULE.job_status
|
42
77
|
squeue_txt.split("\n").collect{|l| l.to_i.to_s}
|
43
78
|
rescue
|
44
79
|
Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
|
@@ -58,27 +93,35 @@ else
|
|
58
93
|
end
|
59
94
|
|
60
95
|
count = 0
|
61
|
-
workdir.glob("**/command.
|
96
|
+
workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
62
97
|
dir = File.dirname(fcmd)
|
98
|
+
command_txt = Open.read(fcmd)
|
63
99
|
|
64
|
-
if m =
|
100
|
+
if m = command_txt.match(/#CMD: (.*)/)
|
65
101
|
cmd = m[1]
|
66
102
|
else
|
67
103
|
cmd = nil
|
68
104
|
end
|
69
105
|
|
70
|
-
if m =
|
106
|
+
if m = command_txt.match(/# Run command\n(.*?)\n/im)
|
71
107
|
exe = m[1]
|
72
108
|
else
|
73
109
|
exe = nil
|
74
110
|
end
|
75
111
|
|
76
|
-
if m =
|
112
|
+
if m = command_txt.match(/^CONTAINER_DIR=(.*)/)
|
77
113
|
container_home = m[1]
|
78
114
|
else
|
79
115
|
container_home = nil
|
80
116
|
end
|
81
117
|
|
118
|
+
if m = command_txt.match(/^BATCH_SYSTEM=(.*)/)
|
119
|
+
job_batch_system = m[1].downcase
|
120
|
+
else
|
121
|
+
job_batch_system = nil
|
122
|
+
end
|
123
|
+
|
124
|
+
different_system = job_batch_system != batch_system
|
82
125
|
|
83
126
|
if File.exists?(fid = File.join(dir, 'job.id'))
|
84
127
|
id = Open.read(fid).chomp
|
@@ -93,7 +136,16 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
93
136
|
end
|
94
137
|
|
95
138
|
if File.exists?(fstatus = File.join(dir, 'job.status'))
|
96
|
-
|
139
|
+
fstatus_txt = Open.read(fstatus)
|
140
|
+
begin
|
141
|
+
if job_batch_system == "lsf"
|
142
|
+
nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
|
143
|
+
else
|
144
|
+
nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
|
145
|
+
end
|
146
|
+
rescue
|
147
|
+
nodes = []
|
148
|
+
end
|
97
149
|
elsif job_nodes[id]
|
98
150
|
nodes = job_nodes[id]
|
99
151
|
else
|
@@ -113,20 +165,36 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
113
165
|
cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
|
114
166
|
|
115
167
|
aborted = error = true if aborted.nil? && error.nil?
|
116
|
-
if done || error || aborted || running || queued || jobid || search
|
168
|
+
#if done || error || aborted || running || queued || jobid || search
|
169
|
+
# select = false
|
170
|
+
# select = true if done && exit_status && exit_status.to_i == 0
|
171
|
+
# select = true if error && exit_status && exit_status.to_i != 0
|
172
|
+
# select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
|
173
|
+
# select = select && jobid.split(",").include?(id) if jobid
|
174
|
+
# select = select && cmd.match(/#{search}/) if search
|
175
|
+
# next unless select
|
176
|
+
#end
|
177
|
+
|
178
|
+
if done || error || aborted || queued || jobid
|
117
179
|
select = false
|
118
|
-
select = true if done && exit_status
|
119
|
-
select = true if error && exit_status && exit_status
|
180
|
+
select = true if done && exit_status == 0
|
181
|
+
select = true if error && exit_status && exit_status != 0
|
120
182
|
select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
|
121
|
-
|
183
|
+
is_running = exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
|
184
|
+
select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
|
185
|
+
select = true if jobid && jobid.split(",").include?(id)
|
122
186
|
select = select && cmd.match(/#{search}/) if search
|
123
187
|
next unless select
|
188
|
+
elsif search
|
189
|
+
select = false
|
190
|
+
select = true if search && cmd.match(/#{search}/)
|
191
|
+
next unless select
|
124
192
|
end
|
125
193
|
|
126
194
|
|
127
195
|
puts Log.color(:yellow, "**ERASING**")
|
128
196
|
puts Log.color :blue, dir
|
129
|
-
puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.
|
197
|
+
puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
|
130
198
|
puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
|
131
199
|
puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
|
132
200
|
puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
|
@@ -137,9 +205,14 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
137
205
|
puts Log.color(:magenta, "Nodes: ") << nodes * ", "
|
138
206
|
puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
|
139
207
|
|
140
|
-
if options[:
|
141
|
-
puts Log.color(:magenta, "
|
142
|
-
|
208
|
+
if options[:batch_parameters]
|
209
|
+
puts Log.color(:magenta, "BATCH parameters: ")
|
210
|
+
case job_batch_system
|
211
|
+
when 'slurm'
|
212
|
+
puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
|
213
|
+
when 'lsf'
|
214
|
+
puts Log.color :blue, CMD.cmd('grep "^#BSUB" |tail -n +6', :in => Open.read(fcmd)).read.strip
|
215
|
+
end
|
143
216
|
end
|
144
217
|
|
145
218
|
if tail && File.exists?(File.join(dir, 'std.err'))
|