autoflow 0.3.5 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,29 +1,30 @@
1
1
  require 'program'
2
- require 'queue_manager'
3
-
2
+ require 'batch'
4
3
 
4
+ require 'win32console' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
5
+ require 'colorize'
5
6
  class Stack
6
-
7
- TAG = 0
8
- INIT = 1
9
- COMMAND = 2
10
- PROG_NAME = 0
11
- PROG_PARAM = 1
7
+ attr_accessor :jobs, :exec_folder, :persist_variables
12
8
 
13
9
  ##########################################################################################
14
10
  ## PARSE TEMPLATE
15
11
  ##########################################################################################
16
12
  def initialize(exec_folder, options)
17
- @commands = {}
13
+ Batch.set_general_attrib({
14
+ :cpu => options[:cpus],
15
+ :mem => options[:memory],
16
+ :time => options[:time],
17
+ :node => options[:node_type],
18
+ :multinode => options[:use_multinode],
19
+ :ntask => options[:use_ntasks]
20
+ })
21
+ @commands = {}
18
22
  @variables = {}
19
- @one2one_dependencies = {}
20
23
  @persist_variables = {}
21
- @dirs = []
22
24
  @exec_folder = exec_folder
23
- @file_workflow = options[:workflow]
24
25
  @do_retry = options[:retry]
25
26
  parse(options[:workflow], options[:Variables])
26
- @q_mgr = load_queue_manager(options)
27
+ @jobs = get_jobs_relations
27
28
  end
28
29
 
29
30
  def parse(workflow, external_variables)
@@ -31,14 +32,24 @@ class Stack
31
32
  workflow.gsub!(/\#.+$/,'') #Delete comments
32
33
  workflow.gsub!("\t",'') #Drop tabs
33
34
  workflow.gsub!(/\n+/,"\n") #Drop empty lines
35
+ workflow.gsub!(/^\s*/,'')
34
36
 
35
37
  #Parse template
36
38
  variables_lines = []
37
39
  persist_variables_lines = []
38
40
  node_lines = []
39
-
41
+
42
+ node_beg = FALSE
40
43
  workflow.each_line do |line|
41
- if line =~ /^\$/
44
+ node_beg = TRUE if line.include?('{') # This check the context of a variable
45
+ if line.include?('}') # if a variable is within a node,
46
+ if node_beg # we consider tha is a bash variable not a static autoflow variable
47
+ node_beg = FALSE
48
+ else
49
+ node_beg = TRUE
50
+ end
51
+ end
52
+ if line =~ /^\$/ && !node_beg
42
53
  variables_lines << line
43
54
  elsif line =~ /^\@/
44
55
  persist_variables_lines << line.gsub('@','')
@@ -57,7 +68,7 @@ class Stack
57
68
  variables_lines.each do |line|
58
69
  line.chomp!
59
70
  line.gsub!(/\s/,'')
60
- pairs = line.split(';')
71
+ pairs = line.split(',')
61
72
  pairs.each do |pair|
62
73
  pair =~ /(.+)=(.+)/
63
74
  variable_type[$1] = $2
@@ -66,161 +77,187 @@ class Stack
66
77
  end
67
78
  end
68
79
 
69
- def parse_nodes(execution_lines)
70
- scan_nodes(execution_lines).each do |job_node| #Takes the info of each node of workflow for create the job
71
- list_dup_nodes_ids(job_node[TAG]).each do |dup_node_id| #if dup_node_id is nil, the node isn't cloned.
72
- # Set node attributes
73
- prog_parameters = job_node[COMMAND].split(' ', 2)
74
- tag_root_name = job_node[TAG].gsub(/\[([^\]]+\])\)/, '')#We remove the clone node expression
75
- tag_root_name, dup_node_id, folder, no_buffer_node, done = set_node_attributes(tag_root_name, dup_node_id, prog_parameters)
76
-
77
- #Dependencies
78
- initialization, init_dependencies = parse_parameters(job_node[INIT].dup, dup_node_id)
79
- parameters, dependencies = parse_parameters(prog_parameters[PROG_PARAM].dup, dup_node_id)
80
- all_dependencies = dependencies + init_dependencies
81
- all_dependencies.uniq!
82
-
83
- #Create node_job
84
- command_line = prog_parameters[PROG_NAME]+' '+parameters
85
- node_name = "#{tag_root_name}#{dup_node_id}"
86
- node = add_program(node_name, prog_parameters[PROG_NAME], command_line, initialization, folder, all_dependencies, done, no_buffer_node)
87
- node.cloned_id = dup_node_id
80
+ def scan_nodes(execution_lines)
81
+ template_executions = execution_lines.join('')
82
+ replace_variables(template_executions)
83
+ # $1 => tag, $2 => initialize, $3 => main command
84
+ #executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?\s([^\}]{1,})\s{0,}\}/)
85
+ #=begin
86
+ executions = []
87
+ states = {} #name => [state, id(position)]
88
+ #t => tag, i => initialize , c => command
89
+ names = []
90
+ template_executions.each_line do |line|
91
+ line.strip! #Clean al whitespaces at beginning and the end of string
92
+ node = states[names.last] if !names.empty?
93
+ if line.empty?
94
+ next
95
+ elsif line =~ /(\S*\)){$/ #Check tag
96
+ name = $1
97
+ executions << [name, '', '']
98
+ states[name] = ['t', executions.length - 1]
99
+ names << name
100
+ elsif line == '?' #Check command
101
+ node[0] = 'c'
102
+ elsif states[names.last].first == 't' || states[names.last].first == 'i' #Check initialize
103
+ node[0] = 'i' if states[names.last].first == 't'
104
+ executions[node.last][1] << line +"\n"
105
+ elsif line == '}' #Close node
106
+ finished_node = names.pop
107
+ if !names.empty?
108
+ parent_node = states[names.last].last #position
109
+ child_node = states[finished_node].last
110
+ parent_execution = executions[parent_node]
111
+ if parent_execution[2].class.to_s == 'String'
112
+ parent_execution[2] = [child_node]
113
+ else
114
+ parent_execution[2] << child_node
115
+ end
116
+ end
117
+ elsif states[names.last].first == 'c' #Check command
118
+ executions[node.last][2] << line +"\n"
88
119
  end
89
120
  end
121
+ #=end
122
+ return executions
90
123
  end
91
124
 
92
- def set_node_attributes(tag_root_name, dup_node_id, prog_parameters)
93
- folder = ''
94
- no_buffer_node = TRUE
95
- done = FALSE
96
-
97
- if tag_root_name =~ /\!/ || (!dup_node_id.nil? && dup_node_id =~ /\!/)
98
- folder = @exec_folder
99
- else
100
- folder = asign_folder(prog_parameters[PROG_NAME])
101
- end
102
- if tag_root_name =~ /&/ || (!dup_node_id.nil? && dup_node_id =~ /&/) #TODO comprobar como va esto
103
- no_buffer_node = FALSE
104
- folder = @exec_folder#This path is replaced later for the path of the main task that launch all buffered tasks
125
+ def replace_variables(string)
126
+ @variables.each do |name, value|
127
+ string.gsub!(name, value)
105
128
  end
106
- done = TRUE if tag_root_name =~ /\%/ && !@do_retry || (!dup_node_id.nil? && dup_node_id =~ /\%/ && !@do_retry)
107
-
108
- tag_root_name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
109
- dup_node_id.gsub!(/&|\!|\%|\)/,'') if !dup_node_id.nil?
110
- return tag_root_name, dup_node_id, folder, no_buffer_node, done
111
129
  end
112
130
 
113
- def parse_parameters(command, dup_node_id)
114
- dependencies = []
115
- if !command.nil?# When command is the initialize, sometimes can be undefined
116
- command.gsub!(/\(\*\)/, dup_node_id) if !dup_node_id.nil? # Define local parameter for duplicated nodes
117
- dependencies_direct(command, dependencies)
118
- if !dup_node_id.nil?
119
- command.scan(/!([\S]+)\*!/).each do |dependency| # Current node depends on a only node of a batch of duplicated nodes
120
- dependencies_one2one(command, dependency[0], dependencies, dup_node_id)
121
- end
131
+ def parse_nodes(execution_lines)
132
+ dinamic_variables = []
133
+ nodes = scan_nodes(execution_lines)
134
+ nodes = create_ids(nodes)
135
+
136
+ #nodes.each do |tag, init, command, index|
137
+ # puts "#{tag.colorize(:red)}\t#{index}\n#{init.chomp.colorize(:blue)}\n#{command.to_s.colorize(:green)}"
138
+ #end
139
+
140
+ nodes.each do |tag, init, command, index| #Takes the info of each node of workflow for create the job
141
+ # Set batch
142
+ new_batch = Batch.new(tag, init, command, index, @exec_folder)
143
+
144
+ #Dependencies
145
+ scan_dependencies(init, new_batch.dependencies, dinamic_variables)
146
+ dinamic_variables.concat(collect_dinamic_variables(init, tag))
147
+ if command.class.to_s == 'String'
148
+ scan_dependencies(command, new_batch.dependencies, dinamic_variables)
149
+ dinamic_variables.concat(collect_dinamic_variables(command, tag))
122
150
  end
123
151
 
124
- one2more = command.scan(/!([\S]+)!([^ \n]+)/) # Current node depends on a full batch of duplicated nodes
125
- one2more.concat(command.scan(/!([\S]+)![ \n]/))
126
- one2more.each do |dependency|
127
- dependencies_one2more(command, dependency, dependencies)
128
- end
152
+ @commands[new_batch.name] = new_batch
153
+ end
154
+ @commands.each do |name, batch|
155
+ batch.main_command = asign_child_batch(batch.main_command, batch.name) if batch.main_command.class.to_s == 'Array'
156
+ end
157
+ end
129
158
 
130
- replace_variables(command)
159
+ def create_ids(nodes)
160
+ nodes.each_with_index do |node, i|
161
+ node << i
131
162
  end
132
- #raise 'Missed dependency on: ' + command if command.include?(')')
133
- return command, dependencies
163
+ return nodes
134
164
  end
135
165
 
136
- def dependencies_direct(command, dependencies)
137
- @commands.each do |stage_id, node|
138
- #Second conditional is triggered when is parsing a batch of tags. The first tag replaces stage_id by path and for second tag, this dependecy is lost
139
- folder = node.exec_folder_program
140
- if command.include?(stage_id) || command.include?(folder)
141
- dependencies << stage_id
142
- end
143
- command.gsub!(stage_id+')', folder) #Change tag to folder path
166
+ def asign_child_batch(batch_ids, name)
167
+ batches = []
168
+ batch_ids.each do |id|
169
+ batch = get_batch(id)
170
+ batch.parent = name
171
+ batches << batch
144
172
  end
173
+ return batches
145
174
  end
146
175
 
147
- def dependencies_one2one(command, dependency, dependencies, dup_node_id)
148
- @commands.keys.each do |st_id|
149
- if st_id.include?(dependency) && @commands[st_id].cloned_id == dup_node_id # if a tag id match with expresion, we take it like a dependecy
150
- dependencies << st_id
151
- command.gsub!("!#{dependency}*!", @commands[st_id].exec_folder_program)
176
+ def get_batch(id)
177
+ selected_batch = nil
178
+ @commands.each do |name, batch|
179
+ if batch.id == id
180
+ selected_batch = batch
152
181
  break
153
182
  end
154
183
  end
184
+ return selected_batch
155
185
  end
156
186
 
157
- def dependencies_one2more(command, dependency, dependencies)
158
- batch = []
159
- @commands.keys.each do |stage_id|
160
- if stage_id.include?(dependency[0]) # if a tag id match with expresion, we take it like a dependecy
161
- dependencies << stage_id
162
- batch << @commands[stage_id].exec_folder_program
187
+ def collect_dinamic_variables(command, node_name)
188
+ dinamic_variables = []
189
+ node_name = node_name.gsub(/&|\!|\%|\)/,'')# Delete function characters
190
+ if !command.nil? && command.include?('env_manager')
191
+ command =~ /env_manager "([^"]+)/
192
+ command =~ /env_manager '([^']+)/ if $1.nil?
193
+ if !$1.nil?
194
+ $1.split(';').each do |variable|
195
+ name, value = variable.split('=')
196
+ name.gsub!(' ', '') #Remove spaces
197
+ dinamic_variables << [name, node_name.gsub(')','')]
198
+ end
163
199
  end
164
200
  end
165
- string = batch.map{|tag_id| "#{tag_id}#{dependency[1]}"}.join(' ')
166
- command.gsub!("!#{dependency[0]}!#{dependency[1]}", string)
201
+ return dinamic_variables
167
202
  end
168
203
 
169
- def list_dup_nodes_ids(tag_node)
170
- dup_nodes_ids = [nil]
171
- if tag_node =~ /\[([^\]]+)/
172
- dup_nodes_ids = []
173
- $1.split(';').map{|interval|
174
- if interval.include?('-')
175
- limits = interval.split('-')
176
- dup_nodes_ids.concat((limits.first..limits.last).to_a.map{|n| n.to_s})
177
- else
178
- dup_nodes_ids << interval
204
+ def scan_dependencies(command, dependencies, dinamic_variables)
205
+ if !command.nil?# When command is the initialize, sometimes can be undefined
206
+ @commands.each do |name, batch|
207
+ if command.include?(name+')')
208
+ dependencies << [name, 'simple', name+')']
209
+ end
210
+ if command.include?("!#{name}*!")
211
+ dependencies << [name, '1to1', "!#{name}*!"]
179
212
  end
180
- }
213
+ if command.include?("!#{name}!")
214
+ command =~ /!#{name}!([^ \n]+)/
215
+ dependencies << [name, '*to1', "!#{name}!", $1]
216
+ end
217
+ local_dependencies = command.scan(/#{name}([^\( \n]+)\)/)
218
+ local_dependencies.each do |local_dependency|
219
+ dependencies << [name, 'local', "#{name}#{local_dependency.first}"+')', local_dependency.first]
220
+ end
221
+ end
222
+ dependencies.concat(check_dependencies_with_DinVar(command, dinamic_variables))
181
223
  end
182
- return dup_nodes_ids
183
- end
184
-
185
- def scan_nodes(execution_lines)
186
- template_executions = execution_lines.join('')
187
- executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?([^\}]{1,})\s{0,}\}/)
188
- return executions
189
224
  end
190
225
 
191
- def add_program(stage_id, name, parameters, initialization, exec_folder_program, dependencies, done, no_buffer_node)
192
- tag=Program.new(name, parameters, initialization, exec_folder_program, dependencies, done, no_buffer_node)
193
- @commands[stage_id]=tag
194
- return tag
195
- end
196
-
197
- def replace_variables(command)
198
- @variables.each do |name, value|
199
- command.gsub!(name, value)
226
+ def check_dependencies_with_DinVar(command, dinamic_variables)
227
+ dep = []
228
+ dinamic_variables.each do |var, name|
229
+ dep << [name, 'DinVar'] if command.include?(var)
200
230
  end
231
+ return dep
201
232
  end
202
233
 
203
- def asign_folder(program_name)
204
- folder=File.join(@exec_folder,"#{program_name}_#{"%04d" % 0}")
205
- count=0
206
- while @dirs.include?(folder)
207
- folder=File.join(@exec_folder,"#{program_name}_#{"%04d" % count}")
208
- count+=1
234
+ def set_path(job, all_jobs_relations)
235
+ job.dependencies.each do |dep|
236
+ path = all_jobs_relations[dep]
237
+ job.initialization.gsub!(dep+')', path)
238
+ job.parameters.gsub!(dep+')', path)
209
239
  end
210
- @dirs << folder
211
- return folder
212
240
  end
213
241
 
214
- ##########################################################################################
215
- ## LAUNCH WORKFLOW
216
- ##########################################################################################
217
-
218
- def load_queue_manager(options)
219
- return QueueManager.select_queue_manager(@exec_folder, options, @commands, @persist_variables)
242
+ def get_jobs
243
+ jobs =[]
244
+ all_jobs_relations = Batch.get_job_relations
245
+ @commands.each do |name, batch|
246
+ next if batch.has_jobs?
247
+ batch.get_jobs.each do |j|
248
+ set_path(j, all_jobs_relations)
249
+ jobs << [j.name, j]
250
+ end
251
+ end
252
+ return jobs
220
253
  end
221
254
 
222
- def send
223
- @q_mgr.exec
255
+ def get_jobs_relations
256
+ hash = {}
257
+ get_jobs.each do |name, job|
258
+ hash[name] = job
259
+ end
260
+ return hash
224
261
  end
225
262
 
226
263
  ##########################################################################################
@@ -228,41 +265,68 @@ class Stack
228
265
  ##########################################################################################
229
266
 
230
267
  def inspect
231
- @commands.each do |id, tag|
232
- puts "#{id} > #{tag.inspect}\t#{tag.done}\n\t\e[32m#{tag.dependencies.join("\n\t")}\e[0m"
268
+ @jobs.each do |id, job|
269
+ puts "#{id} > #{job.inspect}\t#{job.attrib[:done]}\n\t\e[32m#{job.dependencies.join("\n\t")}\e[0m"
233
270
  end
234
271
  end
235
272
 
236
273
  def draw(name, name_type)
237
274
  representation_type = '_structural'
238
275
  representation_type = '_semantic' if name_type.include?('t')
276
+ if name_type.include?('b')
277
+ representation_type << '_simplified'
278
+ set = @commands
279
+ else
280
+ set = @jobs
281
+ end
239
282
  name.gsub!(/\.\S+/,'')
240
283
  file = File.open(name+representation_type+'.dot','w')
241
284
  file.puts 'digraph G {', 'node[shape=box]'
242
285
  all_dependencies = []
243
286
  all_tag = []
244
- @commands.each do |id, tag|
245
- tag_name = File.basename(tag.exec_folder_program)
246
- if name_type.include?('t')
247
- tag_name = id
287
+ set.each do |id, tag|
288
+ if name_type.include?('b')
289
+ tag_name = tag.main_command.split(' ').first+"_#{tag.id}"
290
+ else
291
+ tag_name = File.basename(tag.attrib[:exec_folder])
248
292
  end
293
+ tag_name = id if name_type.include?('t')
294
+ tag_name = tag_name + '(*)' if name_type.include?('b') && tag.iterator.length > 1
295
+
249
296
  all_tag << tag_name
250
297
  if tag.dependencies.length > 0
251
- tag.dependencies.each do |dependencie|
252
- dependencie_name = File.basename(@commands[dependencie].exec_folder_program)
253
- if name_type.include?('t')
254
- dependencie_name = dependencie
298
+ tag.dependencies.each do |dependencie, type, string|
299
+ if name_type.include?('b')
300
+ dependencie_name = set[dependencie].main_command.split(' ').first+"_#{set[dependencie].id}"
301
+ else
302
+ dependencie_name = File.basename(set[dependencie].attrib[:exec_folder])
255
303
  end
304
+ dependencie_name = dependencie if name_type.include?('t')
305
+
306
+ dependencie_name = dependencie_name + '(*)' if name_type.include?('b') && set[dependencie].iterator.length > 1
256
307
  all_dependencies << dependencie_name
308
+
257
309
  file.puts "\"#{dependencie_name}\"-> \"#{tag_name}\""
258
310
  end
259
311
  else
260
- file.puts "\"#{tag_name}\"[color=mediumseagreen, style=filled]"
312
+ file.puts "\"#{tag_name}\"[color=black, peripheries=2, style=filled, fillcolor=yellow]"
261
313
  end
262
314
  end
263
315
  all_tag.keep_if{|tag| !all_dependencies.include?(tag)}
264
316
  all_tag.each do |tag|
265
- file.puts "\"#{tag}\"[color=palevioletred, style=filled]"
317
+ if name_type.include?('b')
318
+ if !name_type.include?('f')
319
+ tag = tag + '(*)' if !tag.include?('(*)') && set[tag].iterator.length > 1
320
+ else
321
+ id = tag.reverse.split('_',2).first.reverse.to_i
322
+ batch = nil
323
+ set.each do |id,tag|
324
+ batch = tag if tag.id = id
325
+ end
326
+ tag = tag + '(*)' if batch.iterator.length > 1
327
+ end
328
+ end
329
+ file.puts "\"#{tag}\"[fontcolor=white, color=black, style=filled]"
266
330
  end
267
331
  file.puts '}'
268
332
  file.close
@@ -1,3 +1,3 @@
1
1
  module Autoflow
2
- VERSION = "0.3.5"
2
+ VERSION = "0.5.0"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: autoflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.5
4
+ version: 0.5.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-07-22 00:00:00.000000000 Z
12
+ date: 2014-11-04 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: net-ssh
@@ -27,6 +27,22 @@ dependencies:
27
27
  - - ! '>='
28
28
  - !ruby/object:Gem::Version
29
29
  version: 2.8.0
30
+ - !ruby/object:Gem::Dependency
31
+ name: colorize
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ~>
36
+ - !ruby/object:Gem::Version
37
+ version: 0.7.3
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: 0.7.3
30
46
  - !ruby/object:Gem::Dependency
31
47
  name: bundler
32
48
  requirement: !ruby/object:Gem::Requirement
@@ -80,6 +96,7 @@ files:
80
96
  - bin/env_manager
81
97
  - bin/flow_time
82
98
  - lib/autoflow.rb
99
+ - lib/autoflow/batch.rb
83
100
  - lib/autoflow/program.rb
84
101
  - lib/autoflow/queue_manager.rb
85
102
  - lib/autoflow/queue_managers/bash_manager.rb