autoflow 0.5.1 → 0.5.3
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/AutoFlow +5 -0
- data/lib/autoflow/batch.rb +17 -8
- data/lib/autoflow/queue_manager.rb +2 -2
- data/lib/autoflow/stack.rb +8 -2
- data/lib/autoflow/version.rb +1 -1
- metadata +2 -2
data/bin/AutoFlow
CHANGED
@@ -107,6 +107,11 @@ optparse = OptionParser.new do |opts|
|
|
107
107
|
options[:identifier] = identifier
|
108
108
|
end
|
109
109
|
|
110
|
+
options[:key_name] = FALSE
|
111
|
+
opts.on( '-k', '--use_key_name', 'Use job names like folder name' ) do
|
112
|
+
options[:key_name] = TRUE
|
113
|
+
end
|
114
|
+
|
110
115
|
options[:list] = nil
|
111
116
|
opts.on( 'l', '--list_repository STRING', 'List template names in repository') do |name|
|
112
117
|
options[:list] = name
|
data/lib/autoflow/batch.rb
CHANGED
@@ -13,6 +13,11 @@ class Batch
|
|
13
13
|
:multinode => nil,
|
14
14
|
:ntask => nil
|
15
15
|
}
|
16
|
+
@@folder_name = :program_name # can :job_name
|
17
|
+
|
18
|
+
def self.set_job_folder_definition(definition)
|
19
|
+
@@folder_name = definition
|
20
|
+
end
|
16
21
|
|
17
22
|
def self.set_general_attrib(attrib_hash)
|
18
23
|
@@general_computation_attrib = attrib_hash
|
@@ -141,7 +146,7 @@ class Batch
|
|
141
146
|
end
|
142
147
|
|
143
148
|
|
144
|
-
def asign_folder(local_command = nil)
|
149
|
+
def asign_folder(name, local_command = nil)
|
145
150
|
if local_command.nil? #we check a simple job
|
146
151
|
command = @main_command
|
147
152
|
else #we check a cloned job
|
@@ -150,12 +155,16 @@ class Batch
|
|
150
155
|
if command.class.to_s == 'Array'
|
151
156
|
folder = nil
|
152
157
|
elsif @attrib[:folder]
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
while @@all_jobs_relations.values.include?(folder)
|
158
|
+
if @@folder_name == :program_name
|
159
|
+
program = File.join(@attrib[:exec_folder], command.split(' ', 2).first)
|
160
|
+
count = 0
|
157
161
|
folder = program + "_#{"%04d" % count}"
|
158
|
-
|
162
|
+
while @@all_jobs_relations.values.include?(folder)
|
163
|
+
folder = program + "_#{"%04d" % count}"
|
164
|
+
count += 1
|
165
|
+
end
|
166
|
+
elsif @@folder_name == :job_name
|
167
|
+
folder = File.join(@attrib[:exec_folder], name)
|
159
168
|
end
|
160
169
|
else
|
161
170
|
folder = @attrib[:exec_folder]
|
@@ -170,7 +179,7 @@ class Batch
|
|
170
179
|
new_job.dependencies = tmp_j.dependencies.clone
|
171
180
|
new_job.initialization = tmp_j.initialization.clone
|
172
181
|
new_job.parameters = tmp_j.parameters.clone
|
173
|
-
new_job.attrib[:exec_folder] = asign_folder(new_job.parameters)
|
182
|
+
new_job.attrib[:exec_folder] = asign_folder(new_job.name, new_job.parameters)
|
174
183
|
return new_job
|
175
184
|
end
|
176
185
|
|
@@ -211,7 +220,7 @@ class Batch
|
|
211
220
|
job_attrib[:done] = done if !@attrib[:done] # To keep attrib priority in batch on job
|
212
221
|
end
|
213
222
|
name = "#{@name}#{iter}"
|
214
|
-
job_attrib[:exec_folder] = asign_folder if @parent.nil? #Don't asign folder to nested batches (iterative batchs)
|
223
|
+
job_attrib[:exec_folder] = asign_folder(name) if @parent.nil? #Don't asign folder to nested batches (iterative batchs)
|
215
224
|
job_dependencies = []
|
216
225
|
initialization = replace_dependencies(@initialization, job_dependencies, iter, num)
|
217
226
|
parameters = replace_dependencies(@main_command, job_dependencies, iter, num)
|
@@ -70,10 +70,10 @@ class QueueManager
|
|
70
70
|
rm_done_dependencies(job)
|
71
71
|
end
|
72
72
|
buffered_jobs = launch_job_in_folder(job, name, buffered_jobs)
|
73
|
-
end
|
73
|
+
end
|
74
74
|
end
|
75
75
|
|
76
|
-
def sort_jobs_by_dependencies
|
76
|
+
def sort_jobs_by_dependencies # We need job ids from queue system so we ask for each job and we give the previous queue system ids as dependencies if necessary
|
77
77
|
ar_jobs = @commands.to_a
|
78
78
|
sorted_jobs = []
|
79
79
|
jobs_without_dep = ar_jobs.select{|job| job.last.dependencies.empty?}
|
data/lib/autoflow/stack.rb
CHANGED
@@ -17,12 +17,14 @@ class Stack
|
|
17
17
|
:node => options[:node_type],
|
18
18
|
:multinode => options[:use_multinode],
|
19
19
|
:ntask => options[:use_ntasks]
|
20
|
-
})
|
20
|
+
})
|
21
|
+
Batch.set_job_folder_definition(:job_name) if options[:key_name]
|
21
22
|
@commands = {}
|
22
23
|
@variables = {}
|
23
24
|
@persist_variables = {}
|
24
25
|
@exec_folder = exec_folder
|
25
26
|
@do_retry = options[:retry]
|
27
|
+
@options = options
|
26
28
|
parse(options[:workflow], options[:Variables])
|
27
29
|
@jobs = get_jobs_relations
|
28
30
|
end
|
@@ -142,8 +144,10 @@ class Stack
|
|
142
144
|
new_batch = Batch.new(tag, init, command, index, @exec_folder)
|
143
145
|
|
144
146
|
#Dependencies
|
147
|
+
#-- Check initialize
|
145
148
|
scan_dependencies(init, new_batch.dependencies, dinamic_variables)
|
146
149
|
dinamic_variables.concat(collect_dinamic_variables(init, tag))
|
150
|
+
#-- Check command
|
147
151
|
if command.class.to_s == 'String'
|
148
152
|
scan_dependencies(command, new_batch.dependencies, dinamic_variables)
|
149
153
|
dinamic_variables.concat(collect_dinamic_variables(command, tag))
|
@@ -151,6 +155,8 @@ class Stack
|
|
151
155
|
|
152
156
|
@commands[new_batch.name] = new_batch
|
153
157
|
end
|
158
|
+
|
159
|
+
# link each parent batch to a child batch
|
154
160
|
@commands.each do |name, batch|
|
155
161
|
batch.main_command = asign_child_batch(batch.main_command, batch.name) if batch.main_command.class.to_s == 'Array'
|
156
162
|
end
|
@@ -245,7 +251,7 @@ class Stack
|
|
245
251
|
@commands.each do |name, batch|
|
246
252
|
next if batch.has_jobs?
|
247
253
|
batch.get_jobs.each do |j|
|
248
|
-
set_path(j, all_jobs_relations)
|
254
|
+
set_path(j, all_jobs_relations)
|
249
255
|
jobs << [j.name, j]
|
250
256
|
end
|
251
257
|
end
|
data/lib/autoflow/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: autoflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2015-04-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: net-ssh
|