autoflow 0.8.7 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/AutoFlow +5 -0
- data/lib/autoflow/batch.rb +68 -3
- data/lib/autoflow/queue_managers/bash_manager.rb +2 -2
- data/lib/autoflow/queue_managers/slurm_manager.rb +18 -4
- data/lib/autoflow/stack.rb +3 -2
- data/lib/autoflow/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 03a2fce5cddce218e8903d7b7cff49acd3c6962c
|
4
|
+
data.tar.gz: 1c2019a7d65d8710bc37baa649301c43a1b61585
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a52c86fde55c4854e587710c1969e62c282d7fbdd51f645c9a94ce1454882097cecf8f58566f1711cd6c6f4fb92fc02f6e4aef9a3fa46154453b6c09b8a01c6e
|
7
|
+
data.tar.gz: 1d8efc8bd847a2c1fea463497a0a02637582cd744d4654951551759a0e425dd74839622ec23f25a94b938c0a21e35edec129188a04409d9f84079c6171345486
|
data/bin/AutoFlow
CHANGED
@@ -83,6 +83,11 @@ optparse = OptionParser.new do |opts|
|
|
83
83
|
options[:add] = add
|
84
84
|
end
|
85
85
|
|
86
|
+
options[:additional_job_options] = nil
|
87
|
+
opts.on( '-A', '--additional_job_options STRING', 'Additional option in queue system jobs. Format: "parameter:value"' ) do |opt|
|
88
|
+
options[:additional_job_options] = opt.split(':')
|
89
|
+
end
|
90
|
+
|
86
91
|
options[:batch] = false
|
87
92
|
opts.on( '-b', '--batch', 'Workflow execution using batch' ) do
|
88
93
|
options[:batch] = true
|
data/lib/autoflow/batch.rb
CHANGED
@@ -4,14 +4,15 @@ class Batch
|
|
4
4
|
@@all_batch = {}
|
5
5
|
@@jobs_names = []
|
6
6
|
@@batch_iterator_relations = {}
|
7
|
-
@@
|
7
|
+
@@nested_iteration_relations = {}
|
8
8
|
@@general_computation_attrib = {
|
9
9
|
:cpu => nil,
|
10
10
|
:mem => nil,
|
11
11
|
:time => nil,
|
12
12
|
:node => nil,
|
13
13
|
:multinode => nil,
|
14
|
-
:ntask => nil
|
14
|
+
:ntask => nil,
|
15
|
+
:additional_job_options => nil
|
15
16
|
}
|
16
17
|
|
17
18
|
def self.set_general_attrib(attrib_hash)
|
@@ -20,6 +21,7 @@ class Batch
|
|
20
21
|
|
21
22
|
|
22
23
|
def initialize(tag, init, main_command, id, exec_folder)
|
24
|
+
@regex_deps = nil
|
23
25
|
replace_regexp(tag, init, main_command)
|
24
26
|
@name = nil
|
25
27
|
@id = id
|
@@ -63,6 +65,7 @@ class Batch
|
|
63
65
|
data = /!JobRegExp:([^ \n]+):([^ \n]+)!([^ \n]+)/.match(command) # *to1 with regexp
|
64
66
|
#data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern, data[3] => adyacent string to regexp as regexp/file_name
|
65
67
|
job_names = get_dependencies_by_regexp(data[1], data[2])
|
68
|
+
@regex_deps = 'command' if job_names.length > 0
|
66
69
|
new_string = job_names.map{|jn| jn + ')' + data[3] }.join(' ')
|
67
70
|
command.gsub!(data[0], new_string)
|
68
71
|
#puts command.inspect
|
@@ -72,6 +75,9 @@ class Batch
|
|
72
75
|
data = /JobRegExp:([^ \n]+):([^;\] \n]+)/.match(tag) # 1to1 with regexp
|
73
76
|
#data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern
|
74
77
|
job_names = get_dependencies_by_regexp(data[1], data[2])
|
78
|
+
if job_names.length > 0
|
79
|
+
@regex_deps = 'tag'
|
80
|
+
end
|
75
81
|
new_string = job_names.map{|jn| jn + ')'}.join(';')
|
76
82
|
tag.gsub!(data[0], new_string)
|
77
83
|
end
|
@@ -188,6 +194,8 @@ class Batch
|
|
188
194
|
@attrib[:time] = fields[index+1]
|
189
195
|
elsif field == '-u'
|
190
196
|
@attrib[:multinode] = fields[index+1].to_i
|
197
|
+
elsif field == '-A'
|
198
|
+
@attrib[:additional_job_options] = fields[index+1].split(':')
|
191
199
|
end
|
192
200
|
end
|
193
201
|
if fields.include?('-s')
|
@@ -253,6 +261,7 @@ class Batch
|
|
253
261
|
new_job = duplicate_job(tmp_j, iter)
|
254
262
|
check_dependencies(new_job, iter, temp_jobs)
|
255
263
|
parse_iter(iter, @name, new_job)
|
264
|
+
add_nested_iteration_relation(tmp_j, new_job)
|
256
265
|
@@jobs_names << new_job.name
|
257
266
|
jobs << new_job
|
258
267
|
@jobs << new_job
|
@@ -261,6 +270,7 @@ class Batch
|
|
261
270
|
end
|
262
271
|
temp_jobs = delete_jobs(jobs2delete, temp_jobs) #Remove temporal jobs
|
263
272
|
else
|
273
|
+
check_regex_dependencies
|
264
274
|
@iterator.each_with_index do |iter, num|
|
265
275
|
job_attrib = @attrib.dup
|
266
276
|
if !iter.nil?
|
@@ -283,6 +293,61 @@ class Batch
|
|
283
293
|
return jobs
|
284
294
|
end
|
285
295
|
|
296
|
+
def add_nested_iteration_relation(tmp_j, new_job)
|
297
|
+
query = @@nested_iteration_relations[tmp_j.name]
|
298
|
+
if query.nil?
|
299
|
+
@@nested_iteration_relations[tmp_j.name] = [new_job.name]
|
300
|
+
else
|
301
|
+
query << new_job.name
|
302
|
+
end
|
303
|
+
end
|
304
|
+
|
305
|
+
def check_regex_dependencies
|
306
|
+
if @regex_deps == 'tag'
|
307
|
+
new_job_names = []
|
308
|
+
@iterator.each do |iter|
|
309
|
+
new_names = find_job_names(iter.gsub(')', ''))
|
310
|
+
new_job_names.concat(new_names)
|
311
|
+
end
|
312
|
+
@iterator = new_job_names.map{|nj| nj + ')'} if !new_job_names.empty?
|
313
|
+
elsif @regex_deps == 'command'
|
314
|
+
[@initialization, @main_command].each do |command|
|
315
|
+
patterns = command.scan(/([^\s)]+)\)([^\s]*)/)
|
316
|
+
if !patterns.empty?
|
317
|
+
patterns.each do |putative_job, sufix|
|
318
|
+
job_names = find_job_names(putative_job)
|
319
|
+
if !job_names.empty?
|
320
|
+
new_string = job_names.map{|jn| "#{jn})#{sufix}"}.join(' ')
|
321
|
+
old_string = "#{putative_job})#{sufix}"
|
322
|
+
command.gsub!(old_string, new_string)
|
323
|
+
end
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
330
|
+
def find_job_names(name)
|
331
|
+
final_names = []
|
332
|
+
intermediary_names = @@nested_iteration_relations[name]
|
333
|
+
if !intermediary_names.nil?
|
334
|
+
while !intermediary_names.empty?
|
335
|
+
final_names = intermediary_names
|
336
|
+
i_names = []
|
337
|
+
intermediary_names.each do |i_n|
|
338
|
+
query = @@nested_iteration_relations[i_n]
|
339
|
+
i_names.concat(query) if !query.nil?
|
340
|
+
end
|
341
|
+
if !i_names.empty?
|
342
|
+
intermediary_names = i_names
|
343
|
+
else
|
344
|
+
break
|
345
|
+
end
|
346
|
+
end
|
347
|
+
end
|
348
|
+
return final_names
|
349
|
+
end
|
350
|
+
|
286
351
|
#tmp_j => job to set dependencies in iteration
|
287
352
|
#iter => sufix of current iteration
|
288
353
|
#jobs => array of jobs which has the job dependency
|
@@ -313,7 +378,7 @@ class Batch
|
|
313
378
|
def handle_dependencies(dinamic_variables)
|
314
379
|
[@initialization, @main_command].each do |instructions|
|
315
380
|
if instructions.class.to_s == 'String'
|
316
|
-
scan_dependencies(instructions)
|
381
|
+
#scan_dependencies(instructions) # NOT NECESSARY? REMOVED BY COLLISION CON REGEX SYSTEM. THE DINAMYC VARIABLES ARE NO USED
|
317
382
|
dinamic_variables.concat(collect_dinamic_variables(instructions))
|
318
383
|
@dependencies.concat(check_dependencies_with_DinVar(instructions, dinamic_variables))
|
319
384
|
end
|
@@ -1,20 +1,34 @@
|
|
1
1
|
require 'queue_manager'
|
2
2
|
class SlurmManager < QueueManager
|
3
|
+
def parse_additional_options(string, attribs)
|
4
|
+
expresions = %w[%C %T %M %N ]
|
5
|
+
values = [attribs[:cpu], attribs[:time], attribs[:mem], attribs[:node]]
|
6
|
+
new_string = string.dup
|
7
|
+
expresions.each_with_index do |exp, i|
|
8
|
+
new_string.gsub!(exp, "#{values[i]}")
|
9
|
+
end
|
10
|
+
return new_string
|
11
|
+
end
|
12
|
+
|
3
13
|
def write_header(id, job, sh_name)
|
4
14
|
if !job.attrib[:ntask]
|
5
15
|
write_file(sh_name, "#SBATCH --cpus=#{job.attrib[:cpu]}")
|
6
16
|
else
|
7
17
|
write_file(sh_name, "#SBATCH --ntasks=#{job.attrib[:cpu]}")
|
8
18
|
write_file(sh_name, "#SBATCH --nodes=#{job.attrib[:multinode]}") if job.attrib[:multinode] > 0
|
9
|
-
write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
|
10
19
|
end
|
11
20
|
write_file(sh_name, "#SBATCH --mem=#{job.attrib[:mem]}")
|
12
21
|
write_file(sh_name, "#SBATCH --time=#{job.attrib[:time]}")
|
13
22
|
write_file(sh_name, "#SBATCH --constraint=#{job.attrib[:node]}") if !job.attrib[:node].nil?
|
14
23
|
write_file(sh_name, '#SBATCH --error=job.%J.err')
|
15
24
|
write_file(sh_name, '#SBATCH --output=job.%J.out')
|
25
|
+
write_file(sh_name, "#SBATCH --#{job.attrib[:additional_job_options][0]}=#{parse_additional_options(job.attrib[:additional_job_options][1], job.attrib)}") if !job.attrib[:additional_job_options].nil?
|
26
|
+
if job.attrib[:ntask]
|
27
|
+
write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
|
28
|
+
end
|
16
29
|
end
|
17
30
|
|
31
|
+
|
18
32
|
def submit_job(job, ar_dependencies)
|
19
33
|
final_dep = get_all_deps(ar_dependencies)
|
20
34
|
dependencies = nil
|
@@ -34,13 +48,13 @@ class SlurmManager < QueueManager
|
|
34
48
|
end
|
35
49
|
|
36
50
|
def self.available?(options)
|
37
|
-
available =
|
51
|
+
available = true
|
38
52
|
shell_output = system_call("type 'sbatch'", nil, options[:remote], options[:ssh])
|
39
|
-
available =
|
53
|
+
available = false if shell_output.empty?
|
40
54
|
return available
|
41
55
|
end
|
42
56
|
|
43
57
|
def self.priority
|
44
58
|
return 100
|
45
59
|
end
|
46
|
-
end
|
60
|
+
end
|
data/lib/autoflow/stack.rb
CHANGED
@@ -16,7 +16,8 @@ class Stack
|
|
16
16
|
:time => options[:time],
|
17
17
|
:node => options[:node_type],
|
18
18
|
:multinode => options[:use_multinode],
|
19
|
-
:ntask => options[:use_ntasks]
|
19
|
+
:ntask => options[:use_ntasks],
|
20
|
+
:additional_job_options => options[:additional_job_options]
|
20
21
|
})
|
21
22
|
@@folder_name = :program_name
|
22
23
|
@@folder_name = :job_name if options[:key_name]
|
@@ -297,7 +298,7 @@ class Stack
|
|
297
298
|
end
|
298
299
|
file.puts '}'
|
299
300
|
file.close
|
300
|
-
system('dot -
|
301
|
+
system('dot -Tpdf '+name+representation_type+'.dot -o '+name+representation_type+'.pdf')
|
301
302
|
end
|
302
303
|
|
303
304
|
end
|
data/lib/autoflow/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: autoflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pedro Seoane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-10-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: net-ssh
|
@@ -161,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
161
161
|
version: '0'
|
162
162
|
requirements: []
|
163
163
|
rubyforge_project:
|
164
|
-
rubygems_version: 2.
|
164
|
+
rubygems_version: 2.6.14
|
165
165
|
signing_key:
|
166
166
|
specification_version: 4
|
167
167
|
summary: '"This gem take a pipeline and launch it on a queue system"'
|