autoflow 0.3.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/autoflow.gemspec +2 -0
- data/bin/AutoFlow +163 -34
- data/lib/autoflow/batch.rb +342 -0
- data/lib/autoflow/program.rb +19 -16
- data/lib/autoflow/queue_manager.rb +75 -108
- data/lib/autoflow/queue_managers/bash_manager.rb +4 -4
- data/lib/autoflow/queue_managers/slurm_manager.rb +12 -14
- data/lib/autoflow/stack.rb +210 -146
- data/lib/autoflow/version.rb +1 -1
- metadata +19 -2
data/autoflow.gemspec
CHANGED
@@ -19,6 +19,8 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.require_paths = ["lib"]
|
20
20
|
|
21
21
|
spec.add_runtime_dependency 'net-ssh', '>= 2.8.0'
|
22
|
+
spec.add_runtime_dependency 'win32console', '>= 1.3.2' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
|
23
|
+
spec.add_runtime_dependency 'colorize', '~> 0.7.3'
|
22
24
|
spec.add_development_dependency "bundler", "~> 1.3"
|
23
25
|
spec.add_development_dependency "rake"
|
24
26
|
end
|
data/bin/AutoFlow
CHANGED
@@ -9,6 +9,62 @@ require 'optparse'
|
|
9
9
|
require 'autoflow'
|
10
10
|
require 'io/console'
|
11
11
|
require 'net/ssh'
|
12
|
+
require 'queue_manager'
|
13
|
+
require 'fileutils'
|
14
|
+
|
15
|
+
#################################################################################################
|
16
|
+
# METHODS
|
17
|
+
#################################################################################################
|
18
|
+
def get_templates(string_template)
|
19
|
+
templates = string_template.split(',')
|
20
|
+
return templates
|
21
|
+
end
|
22
|
+
|
23
|
+
def get_repositories
|
24
|
+
directories = []
|
25
|
+
directories = ENV['WORKFLOW_REPOSITORY'].split(':') if !ENV['WORKFLOW_REPOSITORY'].nil?
|
26
|
+
return directories
|
27
|
+
end
|
28
|
+
|
29
|
+
def exit_exec(message)
|
30
|
+
puts message
|
31
|
+
Process.exit(-1)
|
32
|
+
end
|
33
|
+
|
34
|
+
def list_repository_templates
|
35
|
+
templates = []
|
36
|
+
if !ENV['WORKFLOW_REPOSITORY']. nil?
|
37
|
+
directories = get_repositories
|
38
|
+
directories.each do |dir|
|
39
|
+
templates.concat(Dir.entries(dir))
|
40
|
+
end
|
41
|
+
end
|
42
|
+
templates.delete('.')
|
43
|
+
templates.delete('..')
|
44
|
+
return templates
|
45
|
+
end
|
46
|
+
|
47
|
+
def get_template_path(template)
|
48
|
+
path = nil
|
49
|
+
directories = get_repositories
|
50
|
+
directories.each do |dir|
|
51
|
+
if Dir.entries(dir).include?(template)
|
52
|
+
path = File.join(dir, template)
|
53
|
+
break
|
54
|
+
end
|
55
|
+
end
|
56
|
+
return path
|
57
|
+
end
|
58
|
+
|
59
|
+
def get_description(path)
|
60
|
+
File.open(path).each do |line|
|
61
|
+
line.chomp!
|
62
|
+
if line =~ /^#=/
|
63
|
+
line.gsub!('#=','')
|
64
|
+
puts line
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
12
68
|
|
13
69
|
#################################################################################################
|
14
70
|
# INPUT PARSING
|
@@ -16,6 +72,11 @@ require 'net/ssh'
|
|
16
72
|
options = {}
|
17
73
|
template_file = ''
|
18
74
|
optparse = OptionParser.new do |opts|
|
75
|
+
options[:add] = nil
|
76
|
+
opts.on( '-a', '--add STRING', 'Puts a copy of a selected workflow template in repository' ) do |add|
|
77
|
+
options[:add] = add
|
78
|
+
end
|
79
|
+
|
19
80
|
options[:batch] = FALSE
|
20
81
|
opts.on( '-b', '--batch', 'Workflow execution using batch' ) do
|
21
82
|
options[:batch] = TRUE
|
@@ -31,21 +92,26 @@ optparse = OptionParser.new do |opts|
|
|
31
92
|
options[:external_dependencies] = external_dependencies.split(',')
|
32
93
|
end
|
33
94
|
|
34
|
-
options[:exp_cpu] = 0
|
35
|
-
opts.on( '-e', '--exp_cpu INTEGER', 'Exponent of cpu assigment series' ) do |exp_cpu|
|
36
|
-
options[:exp_cpu] = exp_cpu.to_i
|
37
|
-
end
|
38
|
-
|
39
95
|
options[:retry] = FALSE
|
40
96
|
opts.on( '-f', '--force', 'Execute all jobs, included the jobs commented with %' ) do
|
41
97
|
options[:retry] = TRUE
|
42
98
|
end
|
43
99
|
|
44
100
|
options[:graph] = nil
|
45
|
-
opts.on( '-g', '--graph STRING', 'Draw the template. t for use tag like names or f for use folders names instead' ) do |graph|
|
101
|
+
opts.on( '-g', '--graph STRING', 'Draw the template. t for use tag like names or f for use folders names instead. This option cancel the workflow launching' ) do |graph|
|
46
102
|
options[:graph] = graph
|
47
103
|
end
|
48
104
|
|
105
|
+
options[:identifier] = FALSE
|
106
|
+
opts.on( '-i', '--job_identifier STRING', 'Identifier tag for each launching script' ) do |identifier|
|
107
|
+
options[:identifier] = identifier
|
108
|
+
end
|
109
|
+
|
110
|
+
options[:list] = nil
|
111
|
+
opts.on( 'l', '--list_repository STRING', 'List template names in repository') do |name|
|
112
|
+
options[:list] = name
|
113
|
+
end
|
114
|
+
|
49
115
|
options[:memory] = '4gb'
|
50
116
|
opts.on( '-m', '--memory STRING', 'Max memory can be used in a task' ) do |mem|
|
51
117
|
options[:memory] = mem
|
@@ -61,6 +127,11 @@ optparse = OptionParser.new do |opts|
|
|
61
127
|
options[:output] = output
|
62
128
|
end
|
63
129
|
|
130
|
+
options[:pull] = nil
|
131
|
+
opts.on( '-p', '--pull STRING', 'Get a copy of a workflow template in repository.' ) do |pull|
|
132
|
+
options[:pull] = pull
|
133
|
+
end
|
134
|
+
|
64
135
|
options[:remote] = FALSE
|
65
136
|
opts.on( '-r', '--remote', 'Connect with remote machine and launch the workflow' ) do
|
66
137
|
options[:remote] = TRUE
|
@@ -74,8 +145,13 @@ optparse = OptionParser.new do |opts|
|
|
74
145
|
puts options[:ssh].exec!('hostname')
|
75
146
|
end
|
76
147
|
|
148
|
+
options[:use_ntasks] = FALSE
|
149
|
+
opts.on( '-s', '--use_ntasks', 'Use -ntasks flag with sh' ) do
|
150
|
+
options[:use_ntasks] = TRUE
|
151
|
+
end
|
152
|
+
|
77
153
|
options[:time] = '20:00:00'
|
78
|
-
opts.on( '-t', '--time STRING', 'Max time that can be needed in a task' ) do |time|
|
154
|
+
opts.on( '-t', '--time STRING', 'Max time that can be needed in a task. Format: dd-hh:mm:ss' ) do |time|
|
79
155
|
options[:time] = time
|
80
156
|
end
|
81
157
|
|
@@ -84,10 +160,6 @@ optparse = OptionParser.new do |opts|
|
|
84
160
|
options[:use_multinode] = use_multinode.to_i
|
85
161
|
end
|
86
162
|
|
87
|
-
options[:use_ntasks] = FALSE
|
88
|
-
opts.on( '-s', '--use_ntasks', 'Use -ntasks flag with sh' ) do
|
89
|
-
options[:use_ntasks] = TRUE
|
90
|
-
end
|
91
163
|
|
92
164
|
options[:verbose] = FALSE
|
93
165
|
opts.on( '-v', '--verbose', 'Show info without launch jobs' ) do
|
@@ -95,8 +167,8 @@ optparse = OptionParser.new do |opts|
|
|
95
167
|
end
|
96
168
|
|
97
169
|
options[:Variables] = nil
|
98
|
-
opts.on( '-V', '--Variables STRING', 'Variables to be parsed on template. Format: \'$variable_name1=value1
|
99
|
-
options[:Variables] = mem
|
170
|
+
opts.on( '-V', '--Variables STRING', 'Variables to be parsed on template. Format: \'$variable_name1=value1,$variable_name2=value2,...\'' ) do |mem|
|
171
|
+
options[:Variables] = [mem]
|
100
172
|
end
|
101
173
|
|
102
174
|
options[:workflow] = FALSE
|
@@ -105,10 +177,6 @@ optparse = OptionParser.new do |opts|
|
|
105
177
|
template_file = workflow
|
106
178
|
end
|
107
179
|
|
108
|
-
options[:identifier] = FALSE
|
109
|
-
opts.on( '-i', '--job_identifier STRING', 'Identifier tag for each launching script' ) do |identifier|
|
110
|
-
options[:identifier] = identifier
|
111
|
-
end
|
112
180
|
|
113
181
|
# Set a banner, displayed at the top of the help screen.
|
114
182
|
opts.banner = "Usage: AutoFlow.rb -w worflow_file -c n_cpus \n\n"
|
@@ -124,23 +192,85 @@ end # End opts
|
|
124
192
|
# parse options and remove from ARGV
|
125
193
|
optparse.parse!
|
126
194
|
|
127
|
-
if !options[:workflow] || !File.exists?(options[:workflow])
|
128
|
-
puts 'Workflow file not especified or not exists'
|
129
|
-
Process.exit(-1)
|
130
|
-
else
|
131
|
-
options[:identifier] = "#{options[:workflow]}_#{Time.new.to_i}" if !options[:identifier]
|
132
|
-
options[:workflow] = File.open(options[:workflow]).read
|
133
|
-
end
|
134
|
-
|
135
195
|
#################################################################################################
|
136
196
|
# MAIN
|
137
197
|
#################################################################################################
|
198
|
+
|
199
|
+
# List templates
|
200
|
+
templates_rep_names = list_repository_templates
|
201
|
+
if !options[:list].nil?
|
202
|
+
if options[:list] == 'all'
|
203
|
+
puts templates_rep_names
|
204
|
+
else
|
205
|
+
path = get_template_path(options[:list])
|
206
|
+
get_description(path) if !path.nil?
|
207
|
+
end
|
208
|
+
exit_exec('')
|
209
|
+
end
|
210
|
+
|
211
|
+
# Get a template
|
212
|
+
if !options[:pull].nil?
|
213
|
+
path = get_template_path(options[:pull])
|
214
|
+
if !path.nil?
|
215
|
+
FileUtils.cp(path, Dir.pwd)
|
216
|
+
else
|
217
|
+
exit_exec("Template not found: #{options[:pull]}")
|
218
|
+
end
|
219
|
+
exit_exec('')
|
220
|
+
end
|
221
|
+
|
222
|
+
# Upload a template
|
223
|
+
if !options[:add].nil? && File.exists?(options[:add])
|
224
|
+
directories = get_repositories
|
225
|
+
writable_dirs = []
|
226
|
+
directories.each_with_index do |dir, i|
|
227
|
+
if File.directory?(dir) && File.writable?(dir)
|
228
|
+
puts "#{i}\t#{dir}"
|
229
|
+
writable_dirs << dir
|
230
|
+
end
|
231
|
+
end
|
232
|
+
if !writable_dirs.empty?
|
233
|
+
msg = ''
|
234
|
+
puts "Select a directory"
|
235
|
+
select = gets.chomp.to_i
|
236
|
+
FileUtils.cp(options[:add], writable_dirs[select])
|
237
|
+
else
|
238
|
+
msg = "You don't have a writable directory"
|
239
|
+
end
|
240
|
+
exit_exec(msg)
|
241
|
+
end
|
242
|
+
|
243
|
+
# Merge templates
|
244
|
+
templates = get_templates(options[:workflow])
|
245
|
+
options[:workflow] = ''
|
246
|
+
options[:identifier] = "#{File.basename(templates.first)}_#{Time.new.to_i}" if !options[:identifier]
|
247
|
+
|
248
|
+
if !options[:workflow]
|
249
|
+
exit_exec('Workflow not especified')
|
250
|
+
else
|
251
|
+
templates.each do |template|
|
252
|
+
if !File.exists?(template) && !templates_rep_names.empty? && !templates_rep_names.include?(template)
|
253
|
+
exit_exec("Workflow not found: #{template}")
|
254
|
+
Process.exit(-1)
|
255
|
+
else
|
256
|
+
if File.exists?(template)
|
257
|
+
path = template
|
258
|
+
else
|
259
|
+
path = get_template_path(template)
|
260
|
+
end
|
261
|
+
options[:workflow] += File.open(path).read+"\n"
|
262
|
+
end
|
263
|
+
end
|
264
|
+
end
|
265
|
+
|
266
|
+
# Set local or remote execution
|
138
267
|
if options[:remote]
|
139
268
|
main_path = options[:ssh].exec!('pwd').chomp
|
140
269
|
else
|
141
270
|
main_path = Dir.pwd
|
142
271
|
end
|
143
272
|
|
273
|
+
# Set output directory
|
144
274
|
if options[:output] == 'exec'
|
145
275
|
exec_folder = File.join(main_path,'exec')
|
146
276
|
else
|
@@ -151,17 +281,16 @@ end
|
|
151
281
|
#--------------------------------------------------------------------------------
|
152
282
|
# Flow parse
|
153
283
|
#--------------------------------------------------------------------------------
|
154
|
-
stack=Stack.new(exec_folder, options)
|
155
|
-
|
156
|
-
|
284
|
+
stack = Stack.new(exec_folder, options)
|
285
|
+
manager = QueueManager.select_queue_manager(stack, options)
|
286
|
+
|
157
287
|
#--------------------------------------------------------------------------------
|
158
288
|
# Flow exec
|
159
289
|
#--------------------------------------------------------------------------------
|
160
|
-
if options[:
|
161
|
-
stack.
|
162
|
-
|
163
|
-
|
164
|
-
|
290
|
+
if !options[:graph].nil?
|
291
|
+
stack.draw(template_file, options[:graph])
|
292
|
+
else
|
293
|
+
stack.inspect if options[:verbose]
|
294
|
+
manager.exec
|
165
295
|
end
|
166
296
|
options[:ssh].close if options[:remote]
|
167
|
-
|
@@ -0,0 +1,342 @@
|
|
1
|
+
class Batch
|
2
|
+
attr_accessor :name, :iterator, :dependencies, :init, :main_command, :attrib, :id, :jobs, :parent
|
3
|
+
|
4
|
+
@@all_batch = {}
|
5
|
+
@@all_jobs_relations = {}
|
6
|
+
@@batch_iterator_relations = {}
|
7
|
+
@@state_iterations = {}
|
8
|
+
@@general_computation_attrib = {
|
9
|
+
:cpu => nil,
|
10
|
+
:mem => nil,
|
11
|
+
:time => nil,
|
12
|
+
:node => nil,
|
13
|
+
:multinode => nil,
|
14
|
+
:ntask => nil
|
15
|
+
}
|
16
|
+
|
17
|
+
def self.set_general_attrib(attrib_hash)
|
18
|
+
@@general_computation_attrib = attrib_hash
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.get_job_relations
|
22
|
+
return @@all_jobs_relations
|
23
|
+
end
|
24
|
+
|
25
|
+
def initialize(tag, init, main_command, id, exec_folder)
|
26
|
+
@name = nil
|
27
|
+
@id = id
|
28
|
+
@iterator = [nil]
|
29
|
+
@parent = nil
|
30
|
+
@dependencies = [] # [batch_name, dependency_type, keyword2replace]
|
31
|
+
# nil => There isn't dependecies,
|
32
|
+
# 'simple' => One job needs a previous job,
|
33
|
+
# '1to1' => A job in a batch needs another job in other batch,
|
34
|
+
# '*to1' => A job need a previous full batch of jobs
|
35
|
+
# 'local' => A simple job needs one job of a batch
|
36
|
+
@initialization = init
|
37
|
+
@main_command = main_command
|
38
|
+
@attrib = {
|
39
|
+
:done => FALSE,
|
40
|
+
:folder => TRUE,
|
41
|
+
:buffer => FALSE,
|
42
|
+
:exec_folder => exec_folder,
|
43
|
+
:cpu_asign => nil # number, list or mono
|
44
|
+
}.merge(@@general_computation_attrib)
|
45
|
+
get_name_and_iterators_and_modifiers(tag)
|
46
|
+
set_execution_attrib
|
47
|
+
set_cpu
|
48
|
+
@jobs = []
|
49
|
+
@@all_batch[@name] = self
|
50
|
+
end
|
51
|
+
|
52
|
+
def has_jobs?
|
53
|
+
res = !@jobs.empty?
|
54
|
+
return res
|
55
|
+
end
|
56
|
+
|
57
|
+
def get_name_and_iterators_and_modifiers(tag)
|
58
|
+
tag =~ /(^.+)\[([^\]]+)/
|
59
|
+
name = $1
|
60
|
+
if $1.nil?
|
61
|
+
tag =~ /(^.+)\)/
|
62
|
+
name = $1
|
63
|
+
end
|
64
|
+
@name , @attrib[:done], @attrib[:folder], @attrib[:buffer] = check_execution_modifiers(name)
|
65
|
+
if !$2.nil?
|
66
|
+
@iterator = []
|
67
|
+
$2.split(';').map{|interval|
|
68
|
+
if interval.include?('-')
|
69
|
+
limits = interval.split('-')
|
70
|
+
@iterator.concat((limits.first..limits.last).to_a.map{|n| n.to_s})
|
71
|
+
else
|
72
|
+
@iterator << interval
|
73
|
+
end
|
74
|
+
}
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def check_execution_modifiers(name)
|
79
|
+
done = FALSE
|
80
|
+
folder = TRUE
|
81
|
+
buffer = FALSE
|
82
|
+
done = TRUE if name.include?('%')
|
83
|
+
folder = FALSE if name.include?('!')
|
84
|
+
buffer = TRUE if name.include?('&')
|
85
|
+
name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
|
86
|
+
return name, done, folder, buffer
|
87
|
+
end
|
88
|
+
|
89
|
+
def set_execution_attrib
|
90
|
+
@initialization = scan_resources(@initialization) if !@initialization.nil?
|
91
|
+
@main_command = scan_resources(@main_command) if @main_command.class.to_s == 'String'
|
92
|
+
end
|
93
|
+
|
94
|
+
def scan_resources(command)
|
95
|
+
resources_line = nil
|
96
|
+
command.each_line do |line|
|
97
|
+
if line.include?('resources:')
|
98
|
+
resources_line = line
|
99
|
+
fields = line.split(' ')
|
100
|
+
fields.each_with_index do |field, index|
|
101
|
+
if field == '-c'
|
102
|
+
@attrib[:cpu] = fields[index+1].to_i
|
103
|
+
elsif field == '-m'
|
104
|
+
@attrib[:mem] = fields[index+1]
|
105
|
+
elsif field == '-n'
|
106
|
+
@attrib[:node] = fields[index+1]
|
107
|
+
elsif field == '-t'
|
108
|
+
@attrib[:time] = fields[index+1]
|
109
|
+
elsif field == '-u'
|
110
|
+
@attrib[:multinode] = fields[index+1].to_i
|
111
|
+
end
|
112
|
+
end
|
113
|
+
if fields.include?('-s')
|
114
|
+
@attrib[:ntask] = TRUE
|
115
|
+
else
|
116
|
+
@attrib[:ntask] = FALSE
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
command.gsub!(resources_line, '') if !resources_line.nil?
|
121
|
+
return command
|
122
|
+
end
|
123
|
+
|
124
|
+
def set_cpu
|
125
|
+
@initialization = scan_cpu(@initialization) if !@initialization.nil?
|
126
|
+
@main_command = scan_cpu(@main_command) if @main_command.class.to_s == 'String'
|
127
|
+
@attrib[:cpu] = 1 if @attrib[:cpu_asign] == 'mono'
|
128
|
+
end
|
129
|
+
|
130
|
+
def scan_cpu(command)
|
131
|
+
if command.include?('[cpu]')
|
132
|
+
command.gsub!('[cpu]', @attrib[:cpu].to_s)
|
133
|
+
@attrib[:cpu_asign] = 'number'
|
134
|
+
elsif command.include?('[lcpu]')
|
135
|
+
command.gsub!('[lcpu]', 'workers')
|
136
|
+
@attrib[:cpu_asign] = 'list'
|
137
|
+
elsif @attrib[:cpu_asign].nil?
|
138
|
+
@attrib[:cpu_asign] = 'mono'
|
139
|
+
end
|
140
|
+
return command
|
141
|
+
end
|
142
|
+
|
143
|
+
|
144
|
+
def asign_folder(local_command = nil)
|
145
|
+
if local_command.nil? #we check a simple job
|
146
|
+
command = @main_command
|
147
|
+
else #we check a cloned job
|
148
|
+
command = local_command
|
149
|
+
end
|
150
|
+
if command.class.to_s == 'Array'
|
151
|
+
folder = nil
|
152
|
+
elsif @attrib[:folder]
|
153
|
+
program = File.join(@attrib[:exec_folder], command.split(' ', 2).first)
|
154
|
+
count = 0
|
155
|
+
folder = program + "_#{"%04d" % count}"
|
156
|
+
while @@all_jobs_relations.values.include?(folder)
|
157
|
+
folder = program + "_#{"%04d" % count}"
|
158
|
+
count += 1
|
159
|
+
end
|
160
|
+
else
|
161
|
+
folder = @attrib[:exec_folder]
|
162
|
+
end
|
163
|
+
return folder
|
164
|
+
end
|
165
|
+
|
166
|
+
def duplicate_job(tmp_j, sufix_name = '')
|
167
|
+
new_job = tmp_j.clone
|
168
|
+
new_job.name = tmp_j.name+'_'+sufix_name
|
169
|
+
new_job.attrib = tmp_j.attrib.clone
|
170
|
+
new_job.dependencies = tmp_j.dependencies.clone
|
171
|
+
new_job.initialization = tmp_j.initialization.clone
|
172
|
+
new_job.parameters = tmp_j.parameters.clone
|
173
|
+
new_job.attrib[:exec_folder] = asign_folder(new_job.parameters)
|
174
|
+
return new_job
|
175
|
+
end
|
176
|
+
|
177
|
+
def delete_jobs(jobs2delete, job_array)
|
178
|
+
jobs2delete.uniq!
|
179
|
+
jobs2delete.sort{|s1, s2| s2 <=> s1}.each do |index|
|
180
|
+
job_array.delete_at(index)
|
181
|
+
end
|
182
|
+
return job_array
|
183
|
+
end
|
184
|
+
|
185
|
+
def get_jobs
|
186
|
+
jobs = []
|
187
|
+
@@batch_iterator_relations[@name] = @iterator
|
188
|
+
if @main_command.class.to_s == 'Array' # There are nested batchs
|
189
|
+
temp_jobs = []
|
190
|
+
@main_command.each do |batch|
|
191
|
+
temp_jobs.concat(batch.get_jobs)
|
192
|
+
end
|
193
|
+
jobs2delete = []
|
194
|
+
@iterator.each_with_index do |iter, i|
|
195
|
+
temp_jobs.each_with_index do |tmp_j, tmp_i|
|
196
|
+
new_job = duplicate_job(tmp_j, iter)
|
197
|
+
check_dependencies(new_job, iter, temp_jobs)
|
198
|
+
parse_iter(iter, @name, new_job)
|
199
|
+
jobs << new_job
|
200
|
+
@jobs << new_job
|
201
|
+
@@all_jobs_relations[new_job.name] = new_job.attrib[:exec_folder]
|
202
|
+
jobs2delete << tmp_i
|
203
|
+
end
|
204
|
+
end
|
205
|
+
temp_jobs = delete_jobs(jobs2delete, temp_jobs) #Remove temporal jobs
|
206
|
+
else
|
207
|
+
@iterator.each_with_index do |iter, num|
|
208
|
+
job_attrib = @attrib.dup
|
209
|
+
if !iter.nil?
|
210
|
+
iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter)
|
211
|
+
job_attrib[:done] = done if !@attrib[:done] # To keep attrib priority in batch on job
|
212
|
+
end
|
213
|
+
name = "#{@name}#{iter}"
|
214
|
+
job_attrib[:exec_folder] = asign_folder if @parent.nil? #Don't asign folder to nested batches (iterative batchs)
|
215
|
+
job_dependencies = []
|
216
|
+
initialization = replace_dependencies(@initialization, job_dependencies, iter, num)
|
217
|
+
parameters = replace_dependencies(@main_command, job_dependencies, iter, num)
|
218
|
+
job = Program.new(name, initialization, parameters, job_dependencies, job_attrib)
|
219
|
+
job.batch = @name
|
220
|
+
jobs << job
|
221
|
+
@jobs << job
|
222
|
+
@@all_jobs_relations[name] = job_attrib[:exec_folder]
|
223
|
+
end
|
224
|
+
end
|
225
|
+
return jobs
|
226
|
+
end
|
227
|
+
|
228
|
+
#tmp_j => job to set dependencies in iteration
|
229
|
+
#iter => sufix of current iteration
|
230
|
+
#jobs => array of jobs which has the job dependency
|
231
|
+
def check_dependencies(tmp_j, iter, jobs)
|
232
|
+
jobs_names = jobs.map{|job| job.name}
|
233
|
+
deps = {}
|
234
|
+
tmp_j.dependencies.each_with_index do |dep, i|
|
235
|
+
deps[dep] = i if jobs_names.include?(dep)
|
236
|
+
end
|
237
|
+
deps.each do |name, index|
|
238
|
+
dep = name+'_'+iter
|
239
|
+
tmp_j.initialization.gsub!(name+')', dep+')')
|
240
|
+
tmp_j.parameters.gsub!(name+')', dep+')')
|
241
|
+
tmp_j.dependencies[index] = dep
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
def parse_iter(iter, name, job)
|
246
|
+
job.parameters = set_iter(name, iter, job.parameters)
|
247
|
+
job.initialization = set_iter(name, iter, job.initialization)
|
248
|
+
end
|
249
|
+
|
250
|
+
def set_iter(name, iter, string)
|
251
|
+
string = string.gsub(name+'(+)', iter)
|
252
|
+
return string
|
253
|
+
end
|
254
|
+
|
255
|
+
def replace_dependencies(command, job_dependencies, iter, num)
|
256
|
+
if !command.nil?
|
257
|
+
@dependencies.each do |batch_name, dep_type, dep_keyword2replace, dep_info|
|
258
|
+
if dep_type == 'simple'
|
259
|
+
if @@all_batch[batch_name].parent.nil?
|
260
|
+
new_string = @@all_jobs_relations[batch_name]
|
261
|
+
end
|
262
|
+
job_dependencies << batch_name
|
263
|
+
elsif dep_type == '1to1'
|
264
|
+
if @@all_batch[batch_name].parent.nil? || !@parent.nil?
|
265
|
+
dep_name = "#{batch_name}#{@@batch_iterator_relations[batch_name][num]}"
|
266
|
+
else
|
267
|
+
root_batch = get_root(batch_name)
|
268
|
+
selected_jobs = root_batch.get_jobs_by_batch_name(batch_name)
|
269
|
+
dep_name = selected_jobs[num].name
|
270
|
+
end
|
271
|
+
job_dependencies << dep_name
|
272
|
+
if !@parent.nil? && !@@all_batch[batch_name].parent.nil?
|
273
|
+
new_string = dep_name + ')'
|
274
|
+
else
|
275
|
+
new_string = @@all_jobs_relations[dep_name]
|
276
|
+
end
|
277
|
+
elsif dep_type == '*to1'
|
278
|
+
if @@all_batch[batch_name].parent.nil?
|
279
|
+
new_string = @@batch_iterator_relations[batch_name].map{|iter|
|
280
|
+
dep_name = batch_name + iter
|
281
|
+
job_dependencies << dep_name
|
282
|
+
"#{@@all_jobs_relations[dep_name]}#{dep_info}"
|
283
|
+
}.join(' ')
|
284
|
+
dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
|
285
|
+
elsif !@parent.nil?
|
286
|
+
new_string = @@batch_iterator_relations[batch_name].map{|iter|
|
287
|
+
dep_name = batch_name + iter
|
288
|
+
job_dependencies << dep_name
|
289
|
+
"#{dep_name})#{dep_info}"
|
290
|
+
}.join(' ')
|
291
|
+
dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
|
292
|
+
else
|
293
|
+
root_batch = get_root(batch_name)
|
294
|
+
selected_jobs = root_batch.get_jobs_by_batch_name(batch_name)
|
295
|
+
new_string = selected_jobs.map{|j|
|
296
|
+
job_dependencies << j.name
|
297
|
+
"#{@@all_jobs_relations[j.name]}#{dep_info}"
|
298
|
+
}.join(' ')
|
299
|
+
dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
|
300
|
+
end
|
301
|
+
elsif dep_type == 'local'
|
302
|
+
if @@all_batch[batch_name].parent.nil? || !@parent.nil?
|
303
|
+
if @@batch_iterator_relations[batch_name].include?(dep_info) #This avoids cross dependencies by similar names
|
304
|
+
dep_name = batch_name + dep_info
|
305
|
+
job_dependencies << dep_name
|
306
|
+
if !@parent.nil? && !@@all_batch[batch_name].parent.nil?
|
307
|
+
new_string = dep_name + ')'
|
308
|
+
else
|
309
|
+
new_string = @@all_jobs_relations[dep_name]
|
310
|
+
end
|
311
|
+
end
|
312
|
+
else
|
313
|
+
dep_name = dep_keyword2replace.gsub(')','') #This avoids cross dependencies by similar names
|
314
|
+
if !@@all_jobs_relations[dep_name].nil?
|
315
|
+
job_dependencies << dep_name
|
316
|
+
new_string = @@all_jobs_relations[dep_name]
|
317
|
+
end
|
318
|
+
end
|
319
|
+
elsif dep_type == 'DinVar'
|
320
|
+
job_dependencies << batch_name if batch_name != @name # This condition avoids autodependencies
|
321
|
+
end
|
322
|
+
job_dependencies.uniq!
|
323
|
+
command = command.gsub(dep_keyword2replace, new_string) if dep_type != 'DinVar' && !dep_keyword2replace.nil? && !new_string.nil?
|
324
|
+
end
|
325
|
+
|
326
|
+
command = command.gsub('(*)', "#{iter}") if command.class.to_s == 'String'
|
327
|
+
end
|
328
|
+
return command
|
329
|
+
end
|
330
|
+
|
331
|
+
def get_root(batch_name)
|
332
|
+
root_batch = @@all_batch[batch_name]
|
333
|
+
root_batch = root_batch.get_root(root_batch.parent) if !root_batch.nil? && !root_batch.parent.nil?
|
334
|
+
return root_batch
|
335
|
+
end
|
336
|
+
|
337
|
+
def get_jobs_by_batch_name(batch_name)
|
338
|
+
jobs = @jobs.select{|j| j.batch == batch_name}
|
339
|
+
return jobs
|
340
|
+
end
|
341
|
+
|
342
|
+
end
|
data/lib/autoflow/program.rb
CHANGED
@@ -1,22 +1,25 @@
|
|
1
1
|
class Program
|
2
|
-
attr_accessor :name, :
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
@initialization=initialization
|
7
|
-
|
8
|
-
@
|
9
|
-
@
|
10
|
-
@
|
11
|
-
@
|
12
|
-
@monocpu=TRUE
|
13
|
-
@cloned_id = nil
|
14
|
-
if @parameters =~ /\[cpu|lcpu\]/ #||@parameters =~ /\[lcpu\]/
|
15
|
-
@monocpu=FALSE
|
16
|
-
end
|
2
|
+
attr_accessor :name, :initialization, :parameters, :dependencies, :attrib, :queue_id, :batch
|
3
|
+
|
4
|
+
def initialize(name, initialization, parameters, dependencies, job_attrib)
|
5
|
+
@name = name
|
6
|
+
@initialization = initialization
|
7
|
+
@parameters = parameters
|
8
|
+
@dependencies = dependencies
|
9
|
+
@attrib = job_attrib
|
10
|
+
@queue_id = nil
|
11
|
+
@batch = nil
|
17
12
|
end
|
18
13
|
|
19
14
|
def inspect
|
20
|
-
|
15
|
+
if @parameters.class.to_s == 'String'
|
16
|
+
program = @parameters.split(' ').first
|
17
|
+
command = @parameters.gsub("\n","\n\t")
|
18
|
+
else
|
19
|
+
program = 'iterative_job'
|
20
|
+
command = @parameters.map{|b| b}.join(' ')
|
21
|
+
end
|
22
|
+
string="\e[31m#{program}\n\e[0m\t\e[33m#{command}\e[0m\e[34m#{@attrib[:exec_folder]}\e[0m"
|
21
23
|
end
|
24
|
+
|
22
25
|
end
|