autoflow 0.3.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/autoflow.gemspec +2 -0
- data/bin/AutoFlow +163 -34
- data/lib/autoflow/batch.rb +342 -0
- data/lib/autoflow/program.rb +19 -16
- data/lib/autoflow/queue_manager.rb +75 -108
- data/lib/autoflow/queue_managers/bash_manager.rb +4 -4
- data/lib/autoflow/queue_managers/slurm_manager.rb +12 -14
- data/lib/autoflow/stack.rb +210 -146
- data/lib/autoflow/version.rb +1 -1
- metadata +19 -2
data/lib/autoflow/stack.rb
CHANGED
@@ -1,29 +1,30 @@
|
|
1
1
|
require 'program'
|
2
|
-
require '
|
3
|
-
|
2
|
+
require 'batch'
|
4
3
|
|
4
|
+
require 'win32console' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
|
5
|
+
require 'colorize'
|
5
6
|
class Stack
|
6
|
-
|
7
|
-
TAG = 0
|
8
|
-
INIT = 1
|
9
|
-
COMMAND = 2
|
10
|
-
PROG_NAME = 0
|
11
|
-
PROG_PARAM = 1
|
7
|
+
attr_accessor :jobs, :exec_folder, :persist_variables
|
12
8
|
|
13
9
|
##########################################################################################
|
14
10
|
## PARSE TEMPLATE
|
15
11
|
##########################################################################################
|
16
12
|
def initialize(exec_folder, options)
|
17
|
-
|
13
|
+
Batch.set_general_attrib({
|
14
|
+
:cpu => options[:cpus],
|
15
|
+
:mem => options[:memory],
|
16
|
+
:time => options[:time],
|
17
|
+
:node => options[:node_type],
|
18
|
+
:multinode => options[:use_multinode],
|
19
|
+
:ntask => options[:use_ntasks]
|
20
|
+
})
|
21
|
+
@commands = {}
|
18
22
|
@variables = {}
|
19
|
-
@one2one_dependencies = {}
|
20
23
|
@persist_variables = {}
|
21
|
-
@dirs = []
|
22
24
|
@exec_folder = exec_folder
|
23
|
-
@file_workflow = options[:workflow]
|
24
25
|
@do_retry = options[:retry]
|
25
26
|
parse(options[:workflow], options[:Variables])
|
26
|
-
@
|
27
|
+
@jobs = get_jobs_relations
|
27
28
|
end
|
28
29
|
|
29
30
|
def parse(workflow, external_variables)
|
@@ -31,14 +32,24 @@ class Stack
|
|
31
32
|
workflow.gsub!(/\#.+$/,'') #Delete comments
|
32
33
|
workflow.gsub!("\t",'') #Drop tabs
|
33
34
|
workflow.gsub!(/\n+/,"\n") #Drop empty lines
|
35
|
+
workflow.gsub!(/^\s*/,'')
|
34
36
|
|
35
37
|
#Parse template
|
36
38
|
variables_lines = []
|
37
39
|
persist_variables_lines = []
|
38
40
|
node_lines = []
|
39
|
-
|
41
|
+
|
42
|
+
node_beg = FALSE
|
40
43
|
workflow.each_line do |line|
|
41
|
-
if line
|
44
|
+
node_beg = TRUE if line.include?('{') # This check the context of a variable
|
45
|
+
if line.include?('}') # if a variable is within a node,
|
46
|
+
if node_beg # we consider tha is a bash variable not a static autoflow variable
|
47
|
+
node_beg = FALSE
|
48
|
+
else
|
49
|
+
node_beg = TRUE
|
50
|
+
end
|
51
|
+
end
|
52
|
+
if line =~ /^\$/ && !node_beg
|
42
53
|
variables_lines << line
|
43
54
|
elsif line =~ /^\@/
|
44
55
|
persist_variables_lines << line.gsub('@','')
|
@@ -57,7 +68,7 @@ class Stack
|
|
57
68
|
variables_lines.each do |line|
|
58
69
|
line.chomp!
|
59
70
|
line.gsub!(/\s/,'')
|
60
|
-
pairs = line.split('
|
71
|
+
pairs = line.split(',')
|
61
72
|
pairs.each do |pair|
|
62
73
|
pair =~ /(.+)=(.+)/
|
63
74
|
variable_type[$1] = $2
|
@@ -66,161 +77,187 @@ class Stack
|
|
66
77
|
end
|
67
78
|
end
|
68
79
|
|
69
|
-
def
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
80
|
+
def scan_nodes(execution_lines)
|
81
|
+
template_executions = execution_lines.join('')
|
82
|
+
replace_variables(template_executions)
|
83
|
+
# $1 => tag, $2 => initialize, $3 => main command
|
84
|
+
#executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?\s([^\}]{1,})\s{0,}\}/)
|
85
|
+
#=begin
|
86
|
+
executions = []
|
87
|
+
states = {} #name => [state, id(position)]
|
88
|
+
#t => tag, i => initialize , c => command
|
89
|
+
names = []
|
90
|
+
template_executions.each_line do |line|
|
91
|
+
line.strip! #Clean al whitespaces at beginning and the end of string
|
92
|
+
node = states[names.last] if !names.empty?
|
93
|
+
if line.empty?
|
94
|
+
next
|
95
|
+
elsif line =~ /(\S*\)){$/ #Check tag
|
96
|
+
name = $1
|
97
|
+
executions << [name, '', '']
|
98
|
+
states[name] = ['t', executions.length - 1]
|
99
|
+
names << name
|
100
|
+
elsif line == '?' #Check command
|
101
|
+
node[0] = 'c'
|
102
|
+
elsif states[names.last].first == 't' || states[names.last].first == 'i' #Check initialize
|
103
|
+
node[0] = 'i' if states[names.last].first == 't'
|
104
|
+
executions[node.last][1] << line +"\n"
|
105
|
+
elsif line == '}' #Close node
|
106
|
+
finished_node = names.pop
|
107
|
+
if !names.empty?
|
108
|
+
parent_node = states[names.last].last #position
|
109
|
+
child_node = states[finished_node].last
|
110
|
+
parent_execution = executions[parent_node]
|
111
|
+
if parent_execution[2].class.to_s == 'String'
|
112
|
+
parent_execution[2] = [child_node]
|
113
|
+
else
|
114
|
+
parent_execution[2] << child_node
|
115
|
+
end
|
116
|
+
end
|
117
|
+
elsif states[names.last].first == 'c' #Check command
|
118
|
+
executions[node.last][2] << line +"\n"
|
88
119
|
end
|
89
120
|
end
|
121
|
+
#=end
|
122
|
+
return executions
|
90
123
|
end
|
91
124
|
|
92
|
-
def
|
93
|
-
|
94
|
-
|
95
|
-
done = FALSE
|
96
|
-
|
97
|
-
if tag_root_name =~ /\!/ || (!dup_node_id.nil? && dup_node_id =~ /\!/)
|
98
|
-
folder = @exec_folder
|
99
|
-
else
|
100
|
-
folder = asign_folder(prog_parameters[PROG_NAME])
|
101
|
-
end
|
102
|
-
if tag_root_name =~ /&/ || (!dup_node_id.nil? && dup_node_id =~ /&/) #TODO comprobar como va esto
|
103
|
-
no_buffer_node = FALSE
|
104
|
-
folder = @exec_folder#This path is replaced later for the path of the main task that launch all buffered tasks
|
125
|
+
def replace_variables(string)
|
126
|
+
@variables.each do |name, value|
|
127
|
+
string.gsub!(name, value)
|
105
128
|
end
|
106
|
-
done = TRUE if tag_root_name =~ /\%/ && !@do_retry || (!dup_node_id.nil? && dup_node_id =~ /\%/ && !@do_retry)
|
107
|
-
|
108
|
-
tag_root_name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
|
109
|
-
dup_node_id.gsub!(/&|\!|\%|\)/,'') if !dup_node_id.nil?
|
110
|
-
return tag_root_name, dup_node_id, folder, no_buffer_node, done
|
111
129
|
end
|
112
130
|
|
113
|
-
def
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
131
|
+
def parse_nodes(execution_lines)
|
132
|
+
dinamic_variables = []
|
133
|
+
nodes = scan_nodes(execution_lines)
|
134
|
+
nodes = create_ids(nodes)
|
135
|
+
|
136
|
+
#nodes.each do |tag, init, command, index|
|
137
|
+
# puts "#{tag.colorize(:red)}\t#{index}\n#{init.chomp.colorize(:blue)}\n#{command.to_s.colorize(:green)}"
|
138
|
+
#end
|
139
|
+
|
140
|
+
nodes.each do |tag, init, command, index| #Takes the info of each node of workflow for create the job
|
141
|
+
# Set batch
|
142
|
+
new_batch = Batch.new(tag, init, command, index, @exec_folder)
|
143
|
+
|
144
|
+
#Dependencies
|
145
|
+
scan_dependencies(init, new_batch.dependencies, dinamic_variables)
|
146
|
+
dinamic_variables.concat(collect_dinamic_variables(init, tag))
|
147
|
+
if command.class.to_s == 'String'
|
148
|
+
scan_dependencies(command, new_batch.dependencies, dinamic_variables)
|
149
|
+
dinamic_variables.concat(collect_dinamic_variables(command, tag))
|
122
150
|
end
|
123
151
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
152
|
+
@commands[new_batch.name] = new_batch
|
153
|
+
end
|
154
|
+
@commands.each do |name, batch|
|
155
|
+
batch.main_command = asign_child_batch(batch.main_command, batch.name) if batch.main_command.class.to_s == 'Array'
|
156
|
+
end
|
157
|
+
end
|
129
158
|
|
130
|
-
|
159
|
+
def create_ids(nodes)
|
160
|
+
nodes.each_with_index do |node, i|
|
161
|
+
node << i
|
131
162
|
end
|
132
|
-
|
133
|
-
return command, dependencies
|
163
|
+
return nodes
|
134
164
|
end
|
135
165
|
|
136
|
-
def
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
end
|
143
|
-
command.gsub!(stage_id+')', folder) #Change tag to folder path
|
166
|
+
def asign_child_batch(batch_ids, name)
|
167
|
+
batches = []
|
168
|
+
batch_ids.each do |id|
|
169
|
+
batch = get_batch(id)
|
170
|
+
batch.parent = name
|
171
|
+
batches << batch
|
144
172
|
end
|
173
|
+
return batches
|
145
174
|
end
|
146
175
|
|
147
|
-
def
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
176
|
+
def get_batch(id)
|
177
|
+
selected_batch = nil
|
178
|
+
@commands.each do |name, batch|
|
179
|
+
if batch.id == id
|
180
|
+
selected_batch = batch
|
152
181
|
break
|
153
182
|
end
|
154
183
|
end
|
184
|
+
return selected_batch
|
155
185
|
end
|
156
186
|
|
157
|
-
def
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
187
|
+
def collect_dinamic_variables(command, node_name)
|
188
|
+
dinamic_variables = []
|
189
|
+
node_name = node_name.gsub(/&|\!|\%|\)/,'')# Delete function characters
|
190
|
+
if !command.nil? && command.include?('env_manager')
|
191
|
+
command =~ /env_manager "([^"]+)/
|
192
|
+
command =~ /env_manager '([^']+)/ if $1.nil?
|
193
|
+
if !$1.nil?
|
194
|
+
$1.split(';').each do |variable|
|
195
|
+
name, value = variable.split('=')
|
196
|
+
name.gsub!(' ', '') #Remove spaces
|
197
|
+
dinamic_variables << [name, node_name.gsub(')','')]
|
198
|
+
end
|
163
199
|
end
|
164
200
|
end
|
165
|
-
|
166
|
-
command.gsub!("!#{dependency[0]}!#{dependency[1]}", string)
|
201
|
+
return dinamic_variables
|
167
202
|
end
|
168
203
|
|
169
|
-
def
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
else
|
178
|
-
dup_nodes_ids << interval
|
204
|
+
def scan_dependencies(command, dependencies, dinamic_variables)
|
205
|
+
if !command.nil?# When command is the initialize, sometimes can be undefined
|
206
|
+
@commands.each do |name, batch|
|
207
|
+
if command.include?(name+')')
|
208
|
+
dependencies << [name, 'simple', name+')']
|
209
|
+
end
|
210
|
+
if command.include?("!#{name}*!")
|
211
|
+
dependencies << [name, '1to1', "!#{name}*!"]
|
179
212
|
end
|
180
|
-
|
213
|
+
if command.include?("!#{name}!")
|
214
|
+
command =~ /!#{name}!([^ \n]+)/
|
215
|
+
dependencies << [name, '*to1', "!#{name}!", $1]
|
216
|
+
end
|
217
|
+
local_dependencies = command.scan(/#{name}([^\( \n]+)\)/)
|
218
|
+
local_dependencies.each do |local_dependency|
|
219
|
+
dependencies << [name, 'local', "#{name}#{local_dependency.first}"+')', local_dependency.first]
|
220
|
+
end
|
221
|
+
end
|
222
|
+
dependencies.concat(check_dependencies_with_DinVar(command, dinamic_variables))
|
181
223
|
end
|
182
|
-
return dup_nodes_ids
|
183
|
-
end
|
184
|
-
|
185
|
-
def scan_nodes(execution_lines)
|
186
|
-
template_executions = execution_lines.join('')
|
187
|
-
executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?([^\}]{1,})\s{0,}\}/)
|
188
|
-
return executions
|
189
224
|
end
|
190
225
|
|
191
|
-
def
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
end
|
196
|
-
|
197
|
-
def replace_variables(command)
|
198
|
-
@variables.each do |name, value|
|
199
|
-
command.gsub!(name, value)
|
226
|
+
def check_dependencies_with_DinVar(command, dinamic_variables)
|
227
|
+
dep = []
|
228
|
+
dinamic_variables.each do |var, name|
|
229
|
+
dep << [name, 'DinVar'] if command.include?(var)
|
200
230
|
end
|
231
|
+
return dep
|
201
232
|
end
|
202
233
|
|
203
|
-
def
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
count+=1
|
234
|
+
def set_path(job, all_jobs_relations)
|
235
|
+
job.dependencies.each do |dep|
|
236
|
+
path = all_jobs_relations[dep]
|
237
|
+
job.initialization.gsub!(dep+')', path)
|
238
|
+
job.parameters.gsub!(dep+')', path)
|
209
239
|
end
|
210
|
-
@dirs << folder
|
211
|
-
return folder
|
212
240
|
end
|
213
241
|
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
242
|
+
def get_jobs
|
243
|
+
jobs =[]
|
244
|
+
all_jobs_relations = Batch.get_job_relations
|
245
|
+
@commands.each do |name, batch|
|
246
|
+
next if batch.has_jobs?
|
247
|
+
batch.get_jobs.each do |j|
|
248
|
+
set_path(j, all_jobs_relations)
|
249
|
+
jobs << [j.name, j]
|
250
|
+
end
|
251
|
+
end
|
252
|
+
return jobs
|
220
253
|
end
|
221
254
|
|
222
|
-
def
|
223
|
-
|
255
|
+
def get_jobs_relations
|
256
|
+
hash = {}
|
257
|
+
get_jobs.each do |name, job|
|
258
|
+
hash[name] = job
|
259
|
+
end
|
260
|
+
return hash
|
224
261
|
end
|
225
262
|
|
226
263
|
##########################################################################################
|
@@ -228,41 +265,68 @@ class Stack
|
|
228
265
|
##########################################################################################
|
229
266
|
|
230
267
|
def inspect
|
231
|
-
@
|
232
|
-
puts "#{id} > #{
|
268
|
+
@jobs.each do |id, job|
|
269
|
+
puts "#{id} > #{job.inspect}\t#{job.attrib[:done]}\n\t\e[32m#{job.dependencies.join("\n\t")}\e[0m"
|
233
270
|
end
|
234
271
|
end
|
235
272
|
|
236
273
|
def draw(name, name_type)
|
237
274
|
representation_type = '_structural'
|
238
275
|
representation_type = '_semantic' if name_type.include?('t')
|
276
|
+
if name_type.include?('b')
|
277
|
+
representation_type << '_simplified'
|
278
|
+
set = @commands
|
279
|
+
else
|
280
|
+
set = @jobs
|
281
|
+
end
|
239
282
|
name.gsub!(/\.\S+/,'')
|
240
283
|
file = File.open(name+representation_type+'.dot','w')
|
241
284
|
file.puts 'digraph G {', 'node[shape=box]'
|
242
285
|
all_dependencies = []
|
243
286
|
all_tag = []
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
287
|
+
set.each do |id, tag|
|
288
|
+
if name_type.include?('b')
|
289
|
+
tag_name = tag.main_command.split(' ').first+"_#{tag.id}"
|
290
|
+
else
|
291
|
+
tag_name = File.basename(tag.attrib[:exec_folder])
|
248
292
|
end
|
293
|
+
tag_name = id if name_type.include?('t')
|
294
|
+
tag_name = tag_name + '(*)' if name_type.include?('b') && tag.iterator.length > 1
|
295
|
+
|
249
296
|
all_tag << tag_name
|
250
297
|
if tag.dependencies.length > 0
|
251
|
-
tag.dependencies.each do |dependencie|
|
252
|
-
|
253
|
-
|
254
|
-
|
298
|
+
tag.dependencies.each do |dependencie, type, string|
|
299
|
+
if name_type.include?('b')
|
300
|
+
dependencie_name = set[dependencie].main_command.split(' ').first+"_#{set[dependencie].id}"
|
301
|
+
else
|
302
|
+
dependencie_name = File.basename(set[dependencie].attrib[:exec_folder])
|
255
303
|
end
|
304
|
+
dependencie_name = dependencie if name_type.include?('t')
|
305
|
+
|
306
|
+
dependencie_name = dependencie_name + '(*)' if name_type.include?('b') && set[dependencie].iterator.length > 1
|
256
307
|
all_dependencies << dependencie_name
|
308
|
+
|
257
309
|
file.puts "\"#{dependencie_name}\"-> \"#{tag_name}\""
|
258
310
|
end
|
259
311
|
else
|
260
|
-
file.puts "\"#{tag_name}\"[color=
|
312
|
+
file.puts "\"#{tag_name}\"[color=black, peripheries=2, style=filled, fillcolor=yellow]"
|
261
313
|
end
|
262
314
|
end
|
263
315
|
all_tag.keep_if{|tag| !all_dependencies.include?(tag)}
|
264
316
|
all_tag.each do |tag|
|
265
|
-
|
317
|
+
if name_type.include?('b')
|
318
|
+
if !name_type.include?('f')
|
319
|
+
tag = tag + '(*)' if !tag.include?('(*)') && set[tag].iterator.length > 1
|
320
|
+
else
|
321
|
+
id = tag.reverse.split('_',2).first.reverse.to_i
|
322
|
+
batch = nil
|
323
|
+
set.each do |id,tag|
|
324
|
+
batch = tag if tag.id = id
|
325
|
+
end
|
326
|
+
tag = tag + '(*)' if batch.iterator.length > 1
|
327
|
+
end
|
328
|
+
end
|
329
|
+
file.puts "\"#{tag}\"[fontcolor=white, color=black, style=filled]"
|
266
330
|
end
|
267
331
|
file.puts '}'
|
268
332
|
file.close
|
data/lib/autoflow/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: autoflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-
|
12
|
+
date: 2014-11-04 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: net-ssh
|
@@ -27,6 +27,22 @@ dependencies:
|
|
27
27
|
- - ! '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: 2.8.0
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: colorize
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 0.7.3
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 0.7.3
|
30
46
|
- !ruby/object:Gem::Dependency
|
31
47
|
name: bundler
|
32
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +96,7 @@ files:
|
|
80
96
|
- bin/env_manager
|
81
97
|
- bin/flow_time
|
82
98
|
- lib/autoflow.rb
|
99
|
+
- lib/autoflow/batch.rb
|
83
100
|
- lib/autoflow/program.rb
|
84
101
|
- lib/autoflow/queue_manager.rb
|
85
102
|
- lib/autoflow/queue_managers/bash_manager.rb
|