autoflow 0.5.3 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -1 +1,9 @@
1
1
  require "bundler/gem_tasks"
2
+
3
+ require 'rake/testtask'
4
+
5
+ Rake::TestTask.new do |t|
6
+ t.libs << 'test'
7
+ t.pattern = "test/*_test.rb"
8
+ end
9
+
data/bin/AutoFlow CHANGED
@@ -1,9 +1,9 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
3
  ROOT_PATH=File.dirname(__FILE__)
4
- $: << File.expand_path(File.join(ROOT_PATH, "../lib/"))
5
- $: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/"))
6
- $: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/queue_managers"))
4
+ $: << File.expand_path(File.join(ROOT_PATH, "..", "lib"))
5
+ $: << File.expand_path(File.join(ROOT_PATH, "..", "lib", "autoflow"))
6
+ $: << File.expand_path(File.join(ROOT_PATH, "..", "lib", "autoflow", "queue_managers"))
7
7
 
8
8
  require 'optparse'
9
9
  require 'autoflow'
@@ -21,8 +21,8 @@ def get_templates(string_template)
21
21
  end
22
22
 
23
23
  def get_repositories
24
- directories = []
25
- directories = ENV['WORKFLOW_REPOSITORY'].split(':') if !ENV['WORKFLOW_REPOSITORY'].nil?
24
+ directories = [File.join(ENV['HOME'], 'autoflow_templates')]
25
+ directories.concat(ENV['WORKFLOW_REPOSITORY'].split(':')) if !ENV['WORKFLOW_REPOSITORY'].nil?
26
26
  return directories
27
27
  end
28
28
 
@@ -33,11 +33,9 @@ end
33
33
 
34
34
  def list_repository_templates
35
35
  templates = []
36
- if !ENV['WORKFLOW_REPOSITORY']. nil?
37
- directories = get_repositories
38
- directories.each do |dir|
39
- templates.concat(Dir.entries(dir))
40
- end
36
+ directories = get_repositories
37
+ directories.each do |dir|
38
+ templates.concat(Dir.entries(dir))
41
39
  end
42
40
  templates.delete('.')
43
41
  templates.delete('..')
@@ -73,7 +71,7 @@ options = {}
73
71
  template_file = ''
74
72
  optparse = OptionParser.new do |opts|
75
73
  options[:add] = nil
76
- opts.on( '-a', '--add STRING', 'Puts a copy of a selected workflow template in repository' ) do |add|
74
+ opts.on( '-a', '--add STRING', 'Put a copy of any selected workflow template in repository' ) do |add|
77
75
  options[:add] = add
78
76
  end
79
77
 
@@ -83,22 +81,27 @@ optparse = OptionParser.new do |opts|
83
81
  end
84
82
 
85
83
  options[:cpus] = 16
86
- opts.on( '-c', '--cpus INTEGER', 'Max cpus can be used in all workflow' ) do |cpus|
84
+ opts.on( '-c', '--cpus INTEGER', 'Max number of CPUs that can be used in all workflow' ) do |cpus|
87
85
  options[:cpus] = cpus.to_i
88
86
  end
89
87
 
88
+ options[:comment] = FALSE
89
+ opts.on( '-C', '--comment_main_command', 'Comment first line of main command job' ) do
90
+ options[:comment] = TRUE
91
+ end
92
+
90
93
  options[:external_dependencies] = []
91
- opts.on( '-d', '--external_dependencies STRING', 'The workflow will start when indicated jobs finish on queue system. Format: \'id1,id2,id3..\'') do |external_dependencies|
94
+ opts.on( '-d', '--external_dependencies STRING', 'The workflow will start when indicated jobs are finished on queue system. Format: \'id1,id2,id3..\'') do |external_dependencies|
92
95
  options[:external_dependencies] = external_dependencies.split(',')
93
96
  end
94
97
 
95
98
  options[:retry] = FALSE
96
- opts.on( '-f', '--force', 'Execute all jobs, included the jobs commented with %' ) do
99
+ opts.on( '-f', '--force', 'Execute all jobs, including any job commented with %' ) do
97
100
  options[:retry] = TRUE
98
101
  end
99
102
 
100
103
  options[:graph] = nil
101
- opts.on( '-g', '--graph STRING', 'Draw the template. t for use tag like names or f for use folders names instead. This option cancel the workflow launching' ) do |graph|
104
+ opts.on( '-g', '--graph STRING', 'Draw a chart for the template. The workflow is not executed \'t\' use TIDs for box names \'f\' use folder names for boxes.' ) do |graph|
102
105
  options[:graph] = graph
103
106
  end
104
107
 
@@ -108,17 +111,17 @@ optparse = OptionParser.new do |opts|
108
111
  end
109
112
 
110
113
  options[:key_name] = FALSE
111
- opts.on( '-k', '--use_key_name', 'Use job names like folder name' ) do
114
+ opts.on( '-k', '--use_key_name', ' Use job names as folder names' ) do
112
115
  options[:key_name] = TRUE
113
116
  end
114
117
 
115
118
  options[:list] = nil
116
- opts.on( 'l', '--list_repository STRING', 'List template names in repository') do |name|
119
+ opts.on( '-l', '--list_repository STRING', 'List template names in repository') do |name|
117
120
  options[:list] = name
118
121
  end
119
122
 
120
123
  options[:memory] = '4gb'
121
- opts.on( '-m', '--memory STRING', 'Max memory can be used in a task' ) do |mem|
124
+ opts.on( '-m', '--memory STRING', 'Max memory that can be allocated in a task' ) do |mem|
122
125
  options[:memory] = mem
123
126
  end
124
127
 
@@ -128,7 +131,7 @@ optparse = OptionParser.new do |opts|
128
131
  end
129
132
 
130
133
  options[:output] = 'exec'
131
- opts.on( '-o', '--output STRING', 'Output folder of flow' ) do |output|
134
+ opts.on( '-o', '--output STRING', 'Define an output folder name' ) do |output|
132
135
  options[:output] = output
133
136
  end
134
137
 
@@ -151,7 +154,7 @@ optparse = OptionParser.new do |opts|
151
154
  end
152
155
 
153
156
  options[:use_ntasks] = FALSE
154
- opts.on( '-s', '--use_ntasks', 'Use -ntasks flag with sh' ) do
157
+ opts.on( '-s', '--use_ntasks', 'Use several nodes on execution' ) do
155
158
  options[:use_ntasks] = TRUE
156
159
  end
157
160
 
@@ -167,7 +170,7 @@ optparse = OptionParser.new do |opts|
167
170
 
168
171
 
169
172
  options[:verbose] = FALSE
170
- opts.on( '-v', '--verbose', 'Show info without launch jobs' ) do
173
+ opts.on( '-v', '--verbose', 'Show info without launching jobs' ) do
171
174
  options[:verbose] = TRUE
172
175
  end
173
176
 
@@ -200,6 +203,11 @@ optparse.parse!
200
203
  #################################################################################################
201
204
  # MAIN
202
205
  #################################################################################################
206
+ #Create repository
207
+ if !File.exists?(File.join(ENV['HOME'], 'autoflow_templates'))
208
+ Dir.mkdir(File.join(ENV['HOME'], 'autoflow_templates'))
209
+ end
210
+
203
211
  # List templates
204
212
  templates_rep_names = list_repository_templates
205
213
  if !options[:list].nil?
@@ -295,6 +303,7 @@ if !options[:graph].nil?
295
303
  stack.draw(template_file, options[:graph])
296
304
  else
297
305
  stack.inspect if options[:verbose]
306
+ stack.comment_main_command if options[:comment]
298
307
  manager.exec
299
308
  end
300
309
  options[:ssh].close if options[:remote]
@@ -2,7 +2,7 @@ class Batch
2
2
  attr_accessor :name, :iterator, :dependencies, :init, :main_command, :attrib, :id, :jobs, :parent
3
3
 
4
4
  @@all_batch = {}
5
- @@all_jobs_relations = {}
5
+ @@jobs_names = []
6
6
  @@batch_iterator_relations = {}
7
7
  @@state_iterations = {}
8
8
  @@general_computation_attrib = {
@@ -13,21 +13,14 @@ class Batch
13
13
  :multinode => nil,
14
14
  :ntask => nil
15
15
  }
16
- @@folder_name = :program_name # can :job_name
17
-
18
- def self.set_job_folder_definition(definition)
19
- @@folder_name = definition
20
- end
21
-
16
+
22
17
  def self.set_general_attrib(attrib_hash)
23
18
  @@general_computation_attrib = attrib_hash
24
19
  end
25
20
 
26
- def self.get_job_relations
27
- return @@all_jobs_relations
28
- end
29
21
 
30
22
  def initialize(tag, init, main_command, id, exec_folder)
23
+ replace_regexp(tag, init, main_command)
31
24
  @name = nil
32
25
  @id = id
33
26
  @iterator = [nil]
@@ -54,40 +47,121 @@ class Batch
54
47
  @@all_batch[@name] = self
55
48
  end
56
49
 
50
+ def replace_regexp(tag, init, main_command)
51
+ scan_JobRegExp_tag(tag) if tag.include?('JobRegExp:')
52
+ [init, main_command].each do |intructions|
53
+ if intructions.class.to_s == 'String'
54
+ while intructions.include?('!JobRegExp:')
55
+ scan_JobRegExp(intructions)
56
+ end
57
+ end
58
+ end
59
+ #puts main_command.inspect
60
+ end
61
+
62
+ def scan_JobRegExp(command)
63
+ data = /!JobRegExp:([^ \n]+):([^ \n]+)!([^ \n]+)/.match(command) # *to1 with regexp
64
+ #data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern, data[3] => adyacent string to regexp as regexp/file_name
65
+ job_names = get_dependencies_by_regexp(data[1], data[2])
66
+ new_string = job_names.map{|jn| jn + ')' + data[3] }.join(' ')
67
+ command.gsub!(data[0], new_string)
68
+ #puts command.inspect
69
+ end
70
+
71
+ def scan_JobRegExp_tag(tag)
72
+ data = /JobRegExp:([^ \n]+):([^;\] \n]+)/.match(tag) # 1to1 with regexp
73
+ #data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern
74
+ job_names = get_dependencies_by_regexp(data[1], data[2])
75
+ new_string = job_names.map{|jn| jn + ')'}.join(';')
76
+ tag.gsub!(data[0], new_string)
77
+ end
78
+
79
+ def get_dependencies_by_regexp(batch_pattern, iterator_pattern)
80
+ selected_batches = @@all_batch.keys.select{|cmd_name| cmd_name =~ /#{batch_pattern}/}
81
+ job_names = []
82
+ selected_batches.each do |batch_name|
83
+ iterators = @@all_batch[batch_name].iterator
84
+ iterators = iterators.map{|it| it.gsub(/&|\!|\%|\)/,'')} if !iterators.first.nil?
85
+ if iterator_pattern != '-'
86
+ next if iterators.first.nil?
87
+ iterators = iterators.select{|iter| iter =~ /#{iterator_pattern}/}
88
+ end
89
+ if !iterators.empty?
90
+ if iterators.first.nil?
91
+ job_names << batch_name
92
+ else
93
+ iterators.each do |iter|
94
+ job_names << batch_name+iter
95
+ end
96
+ end
97
+ end
98
+ end
99
+ return job_names
100
+ end
101
+
57
102
  def has_jobs?
58
103
  res = !@jobs.empty?
59
104
  return res
60
105
  end
61
106
 
107
+ def asign_child_batch
108
+ batches = []
109
+ if @main_command.class.to_s == 'Array'
110
+ @main_command.each do |id|
111
+ batch = get_batch(id)
112
+ batch.parent = @name
113
+ batches << batch
114
+ end
115
+ @main_command = batches
116
+ end
117
+ end
118
+
119
+ def get_batch(id)
120
+ selected_batch = nil
121
+ @@all_batch.each do |name, batch|
122
+ if batch.id == id
123
+ selected_batch = batch
124
+ break
125
+ end
126
+ end
127
+ return selected_batch
128
+ end
129
+
62
130
  def get_name_and_iterators_and_modifiers(tag)
63
- tag =~ /(^.+)\[([^\]]+)/
131
+ tag =~ /(^.+)\[([^\]]+)\]\)/ # iterative node
64
132
  name = $1
65
- if $1.nil?
133
+ if $1.nil? # Non iterative node (simple node)
66
134
  tag =~ /(^.+)\)/
67
135
  name = $1
68
136
  end
69
137
  @name , @attrib[:done], @attrib[:folder], @attrib[:buffer] = check_execution_modifiers(name)
70
138
  if !$2.nil?
71
139
  @iterator = []
72
- $2.split(';').map{|interval|
140
+ #$2.split(';').map{|iter| iter.gsub(')','')}.each do |interval|
141
+ $2.split(';').each do |interval|
73
142
  if interval.include?('-')
74
143
  limits = interval.split('-')
75
144
  @iterator.concat((limits.first..limits.last).to_a.map{|n| n.to_s})
76
145
  else
77
146
  @iterator << interval
78
147
  end
79
- }
148
+ end
80
149
  end
150
+ @@batch_iterator_relations[@name] = @iterator
81
151
  end
82
152
 
83
- def check_execution_modifiers(name)
153
+ def check_execution_modifiers(name, iter_type = FALSE) #The last paremeter iused to indicate tha name is a iterator not an orignal node name
84
154
  done = FALSE
85
155
  folder = TRUE
86
156
  buffer = FALSE
87
157
  done = TRUE if name.include?('%')
88
158
  folder = FALSE if name.include?('!')
89
159
  buffer = TRUE if name.include?('&')
90
- name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
160
+ if !iter_type
161
+ name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
162
+ else
163
+ name.gsub!(/&|\!|\%/,'')# Delete function characters
164
+ end
91
165
  return name, done, folder, buffer
92
166
  end
93
167
 
@@ -146,32 +220,6 @@ class Batch
146
220
  end
147
221
 
148
222
 
149
- def asign_folder(name, local_command = nil)
150
- if local_command.nil? #we check a simple job
151
- command = @main_command
152
- else #we check a cloned job
153
- command = local_command
154
- end
155
- if command.class.to_s == 'Array'
156
- folder = nil
157
- elsif @attrib[:folder]
158
- if @@folder_name == :program_name
159
- program = File.join(@attrib[:exec_folder], command.split(' ', 2).first)
160
- count = 0
161
- folder = program + "_#{"%04d" % count}"
162
- while @@all_jobs_relations.values.include?(folder)
163
- folder = program + "_#{"%04d" % count}"
164
- count += 1
165
- end
166
- elsif @@folder_name == :job_name
167
- folder = File.join(@attrib[:exec_folder], name)
168
- end
169
- else
170
- folder = @attrib[:exec_folder]
171
- end
172
- return folder
173
- end
174
-
175
223
  def duplicate_job(tmp_j, sufix_name = '')
176
224
  new_job = tmp_j.clone
177
225
  new_job.name = tmp_j.name+'_'+sufix_name
@@ -179,7 +227,6 @@ class Batch
179
227
  new_job.dependencies = tmp_j.dependencies.clone
180
228
  new_job.initialization = tmp_j.initialization.clone
181
229
  new_job.parameters = tmp_j.parameters.clone
182
- new_job.attrib[:exec_folder] = asign_folder(new_job.name, new_job.parameters)
183
230
  return new_job
184
231
  end
185
232
 
@@ -193,7 +240,7 @@ class Batch
193
240
 
194
241
  def get_jobs
195
242
  jobs = []
196
- @@batch_iterator_relations[@name] = @iterator
243
+ #@@batch_iterator_relations[@name] = @iterator #??????
197
244
  if @main_command.class.to_s == 'Array' # There are nested batchs
198
245
  temp_jobs = []
199
246
  @main_command.each do |batch|
@@ -205,9 +252,9 @@ class Batch
205
252
  new_job = duplicate_job(tmp_j, iter)
206
253
  check_dependencies(new_job, iter, temp_jobs)
207
254
  parse_iter(iter, @name, new_job)
255
+ @@jobs_names << new_job.name
208
256
  jobs << new_job
209
257
  @jobs << new_job
210
- @@all_jobs_relations[new_job.name] = new_job.attrib[:exec_folder]
211
258
  jobs2delete << tmp_i
212
259
  end
213
260
  end
@@ -216,19 +263,20 @@ class Batch
216
263
  @iterator.each_with_index do |iter, num|
217
264
  job_attrib = @attrib.dup
218
265
  if !iter.nil?
219
- iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter)
266
+ iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter, TRUE)
220
267
  job_attrib[:done] = done if !@attrib[:done] # To keep attrib priority in batch on job
221
268
  end
222
269
  name = "#{@name}#{iter}"
223
- job_attrib[:exec_folder] = asign_folder(name) if @parent.nil? #Don't asign folder to nested batches (iterative batchs)
224
270
  job_dependencies = []
271
+ batch_deps = @dependencies.length
225
272
  initialization = replace_dependencies(@initialization, job_dependencies, iter, num)
226
273
  parameters = replace_dependencies(@main_command, job_dependencies, iter, num)
274
+ @dependencies.pop(@dependencies.length - batch_deps) # Clean temporal dependencies by regexp
227
275
  job = Program.new(name, initialization, parameters, job_dependencies, job_attrib)
228
276
  job.batch = @name
277
+ @@jobs_names << job.name
229
278
  jobs << job
230
279
  @jobs << job
231
- @@all_jobs_relations[name] = job_attrib[:exec_folder]
232
280
  end
233
281
  end
234
282
  return jobs
@@ -261,12 +309,101 @@ class Batch
261
309
  return string
262
310
  end
263
311
 
312
+ def handle_dependencies(dinamic_variables)
313
+ [@initialization, @main_command].each do |instructions|
314
+ if instructions.class.to_s == 'String'
315
+ scan_dependencies(instructions)
316
+ dinamic_variables.concat(collect_dinamic_variables(instructions))
317
+ @dependencies.concat(check_dependencies_with_DinVar(instructions, dinamic_variables))
318
+ end
319
+ end
320
+ return dinamic_variables
321
+ end
322
+
323
+ def scan_dependencies(command)
324
+ if !command.nil?# When command is the initialize, sometimes can be undefined
325
+ matched_regions = []
326
+ batches = []
327
+ @@all_batch.each do |k , val| #sorting is used to match last jobs first, and avoid small matches of first nodes
328
+ batches << [k , val]
329
+ end
330
+ batches.reverse.each do |name, batch|
331
+ if command.include?(name+')') && !string_overlap(matched_regions, name+')', command)
332
+ @dependencies << [name, 'simple', name+')']
333
+ end
334
+ if command.include?("!#{name}*!") && !string_overlap(matched_regions, "!#{name}*!", command)
335
+ @dependencies << [name, '1to1', "!#{name}*!"]
336
+ end
337
+ if command.include?("!#{name}!") && !string_overlap(matched_regions, "!#{name}!", command)
338
+ command =~ /!#{name}!([^ \n]+)/
339
+ @dependencies << [name, '*to1', "!#{name}!", $1]
340
+ end
341
+ local_dependencies = command.scan(/#{name}([^\( \n]+)\)/)
342
+ local_dependencies.each do |local_dependency|
343
+ if !string_overlap(matched_regions, "#{name}#{local_dependency.first}"+')', command)
344
+ @dependencies << [name, 'local', "#{name}#{local_dependency.first}"+')', local_dependency.first]
345
+ end
346
+ end
347
+ end
348
+ end
349
+ end
350
+
351
+ def get_string_position(substr, string)
352
+ start = string.index(substr)
353
+ ending = start + substr.length - 1
354
+ range = [start, ending]
355
+ return range
356
+ end
357
+
358
+ def string_overlap(matched_regions, substr, string)
359
+ match = FALSE
360
+ range = get_string_position(substr, string)
361
+ if !range.empty?
362
+ matched_regions.each do |start, ending|
363
+ if (range.first >= start && range.first <= ending) ||
364
+ (range.last >= start && range.last <= ending) ||
365
+ (range.first <= start && range.last >= ending)
366
+ match = TRUE
367
+ break
368
+ end
369
+ end
370
+ matched_regions << range
371
+ end
372
+ return match
373
+ end
374
+
375
+ def collect_dinamic_variables(command)
376
+ dinamic_variables = []
377
+ if !command.nil? && command.include?('env_manager')
378
+ command =~ /env_manager "([^"]+)/
379
+ command =~ /env_manager '([^']+)/ if $1.nil?
380
+ if !$1.nil?
381
+ $1.split(';').each do |variable|
382
+ name, value = variable.split('=')
383
+ name.gsub!(' ', '') #Remove spaces
384
+ dinamic_variables << [name, @name]
385
+ end
386
+ end
387
+ end
388
+ return dinamic_variables
389
+ end
390
+
391
+ def check_dependencies_with_DinVar(command, dinamic_variables)
392
+ dep = []
393
+ dinamic_variables.each do |var, name|
394
+ dep << [name, 'DinVar'] if command.include?(var)
395
+ end
396
+ return dep
397
+ end
398
+
264
399
  def replace_dependencies(command, job_dependencies, iter, num)
265
400
  if !command.nil?
401
+ command = command.gsub('(*)', "#{iter}") if command.class.to_s == 'String'
402
+ scan_dependencies(command)
266
403
  @dependencies.each do |batch_name, dep_type, dep_keyword2replace, dep_info|
267
404
  if dep_type == 'simple'
268
405
  if @@all_batch[batch_name].parent.nil?
269
- new_string = @@all_jobs_relations[batch_name]
406
+ new_string = batch_name + ')'
270
407
  end
271
408
  job_dependencies << batch_name
272
409
  elsif dep_type == '1to1'
@@ -278,52 +415,39 @@ class Batch
278
415
  dep_name = selected_jobs[num].name
279
416
  end
280
417
  job_dependencies << dep_name
281
- if !@parent.nil? && !@@all_batch[batch_name].parent.nil?
282
- new_string = dep_name + ')'
283
- else
284
- new_string = @@all_jobs_relations[dep_name]
285
- end
418
+ new_string = dep_name + ')'
286
419
  elsif dep_type == '*to1'
287
- if @@all_batch[batch_name].parent.nil?
288
- new_string = @@batch_iterator_relations[batch_name].map{|iter|
289
- dep_name = batch_name + iter
290
- job_dependencies << dep_name
291
- "#{@@all_jobs_relations[dep_name]}#{dep_info}"
292
- }.join(' ')
293
- dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
294
- elsif !@parent.nil?
420
+ if @@all_batch[batch_name].parent.nil? || !@parent.nil?
295
421
  new_string = @@batch_iterator_relations[batch_name].map{|iter|
296
422
  dep_name = batch_name + iter
297
423
  job_dependencies << dep_name
298
424
  "#{dep_name})#{dep_info}"
299
425
  }.join(' ')
300
- dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
301
426
  else
302
427
  root_batch = get_root(batch_name)
303
428
  selected_jobs = root_batch.get_jobs_by_batch_name(batch_name)
304
429
  new_string = selected_jobs.map{|j|
305
430
  job_dependencies << j.name
306
- "#{@@all_jobs_relations[j.name]}#{dep_info}"
431
+ "#{j.name + ')'}#{dep_info}"
307
432
  }.join(' ')
308
- dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
309
433
  end
434
+ dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
310
435
  elsif dep_type == 'local'
311
436
  if @@all_batch[batch_name].parent.nil? || !@parent.nil?
312
- if @@batch_iterator_relations[batch_name].include?(dep_info) #This avoids cross dependencies by similar names
437
+ #@@batch_iterator_relations.each do |key,val|
438
+ # puts "#{key}\t#{val.inspect}"
439
+ #end
440
+ if @@batch_iterator_relations[batch_name].map{|iter| iter.gsub(')','')}.include?(dep_info) #This avoids cross dependencies by similar names, map used for regexp deps
313
441
  dep_name = batch_name + dep_info
314
442
  job_dependencies << dep_name
315
- if !@parent.nil? && !@@all_batch[batch_name].parent.nil?
316
- new_string = dep_name + ')'
317
- else
318
- new_string = @@all_jobs_relations[dep_name]
319
- end
443
+ new_string = dep_name + ')'
320
444
  end
321
445
  else
322
446
  dep_name = dep_keyword2replace.gsub(')','') #This avoids cross dependencies by similar names
323
- if !@@all_jobs_relations[dep_name].nil?
324
- job_dependencies << dep_name
325
- new_string = @@all_jobs_relations[dep_name]
326
- end
447
+ #if !@@jobs_names.include?(dep_name)
448
+ job_dependencies << dep_name
449
+ new_string = dep_name + ')'
450
+ #end
327
451
  end
328
452
  elsif dep_type == 'DinVar'
329
453
  job_dependencies << batch_name if batch_name != @name # This condition avoids autodependencies
@@ -331,9 +455,7 @@ class Batch
331
455
  job_dependencies.uniq!
332
456
  command = command.gsub(dep_keyword2replace, new_string) if dep_type != 'DinVar' && !dep_keyword2replace.nil? && !new_string.nil?
333
457
  end
334
-
335
- command = command.gsub('(*)', "#{iter}") if command.class.to_s == 'String'
336
- end
458
+ end
337
459
  return command
338
460
  end
339
461
 
@@ -139,6 +139,8 @@ class QueueManager
139
139
  write_file(sh_name, "source #{File.join(@exec_folder, 'env_file')}") if !@persist_variables.empty?
140
140
  write_job(job, sh_name)
141
141
  write_file(sh_name, "echo -e \"FINISHED #{id} #{job.parameters.split.first}:\\t`date`\" >> #{log_folder}")
142
+ write_file(sh_name, "echo 'General time'")
143
+ write_file(sh_name, "times")
142
144
  close_file(sh_name, 0755)
143
145
 
144
146
  #Submit node
@@ -18,11 +18,13 @@ class Stack
18
18
  :multinode => options[:use_multinode],
19
19
  :ntask => options[:use_ntasks]
20
20
  })
21
- Batch.set_job_folder_definition(:job_name) if options[:key_name]
21
+ @@folder_name = :program_name
22
+ @@folder_name = :job_name if options[:key_name]
22
23
  @commands = {}
23
24
  @variables = {}
24
25
  @persist_variables = {}
25
- @exec_folder = exec_folder
26
+ @@all_jobs_relations = {}
27
+ @exec_folder = exec_folder #TODO move this to queue_manager
26
28
  @do_retry = options[:retry]
27
29
  @options = options
28
30
  parse(options[:workflow], options[:Variables])
@@ -85,38 +87,41 @@ class Stack
85
87
  # $1 => tag, $2 => initialize, $3 => main command
86
88
  #executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?\s([^\}]{1,})\s{0,}\}/)
87
89
  #=begin
88
- executions = []
90
+ executions = [] # tag, initialize, main_command
89
91
  states = {} #name => [state, id(position)]
90
92
  #t => tag, i => initialize , c => command
91
- names = []
93
+ open_nodes = []
92
94
  template_executions.each_line do |line|
93
95
  line.strip! #Clean al whitespaces at beginning and the end of string
94
- node = states[names.last] if !names.empty?
96
+ node = states[open_nodes.last] if !open_nodes.empty?
95
97
  if line.empty?
96
98
  next
97
- elsif line =~ /(\S*\)){$/ #Check tag
99
+ # Create nodes and asign nodes states
100
+ #----------------------------------------
101
+ elsif line =~ /(\S*\)){$/ #Check tag and create node
98
102
  name = $1
99
- executions << [name, '', '']
100
- states[name] = ['t', executions.length - 1]
101
- names << name
103
+ executions << [name, '', ''] # create node
104
+ states[name] = [:i, executions.length - 1]
105
+ open_nodes << name
102
106
  elsif line == '?' #Check command
103
- node[0] = 'c'
104
- elsif states[names.last].first == 't' || states[names.last].first == 'i' #Check initialize
105
- node[0] = 'i' if states[names.last].first == 't'
106
- executions[node.last][1] << line +"\n"
107
+ node[0] = :c
107
108
  elsif line == '}' #Close node
108
- finished_node = names.pop
109
- if !names.empty?
110
- parent_node = states[names.last].last #position
109
+ finished_node = open_nodes.pop
110
+ if !open_nodes.empty?
111
+ parent_node = states[open_nodes.last].last #position
111
112
  child_node = states[finished_node].last
112
113
  parent_execution = executions[parent_node]
113
- if parent_execution[2].class.to_s == 'String'
114
+ if parent_execution[2].class == String
114
115
  parent_execution[2] = [child_node]
115
116
  else
116
117
  parent_execution[2] << child_node
117
118
  end
118
119
  end
119
- elsif states[names.last].first == 'c' #Check command
120
+ # Add lines to nodes
121
+ #----------------------
122
+ elsif states[open_nodes.last].first == :i #Add initialize line
123
+ executions[node.last][1] << line +"\n"
124
+ elsif states[open_nodes.last].first == :c #Add command line
120
125
  executions[node.last][2] << line +"\n"
121
126
  end
122
127
  end
@@ -136,29 +141,19 @@ class Stack
136
141
  nodes = create_ids(nodes)
137
142
 
138
143
  #nodes.each do |tag, init, command, index|
139
- # puts "#{tag.colorize(:red)}\t#{index}\n#{init.chomp.colorize(:blue)}\n#{command.to_s.colorize(:green)}"
144
+ # puts "#{tag.colorize(:red)}\t#{index}\n#{('-'*tag.length).colorize(:red)}\n#{init.chomp.colorize(:blue)}\n#{command.to_s.colorize(:green)}"
140
145
  #end
141
146
 
142
- nodes.each do |tag, init, command, index| #Takes the info of each node of workflow for create the job
143
- # Set batch
144
- new_batch = Batch.new(tag, init, command, index, @exec_folder)
145
-
146
- #Dependencies
147
- #-- Check initialize
148
- scan_dependencies(init, new_batch.dependencies, dinamic_variables)
149
- dinamic_variables.concat(collect_dinamic_variables(init, tag))
150
- #-- Check command
151
- if command.class.to_s == 'String'
152
- scan_dependencies(command, new_batch.dependencies, dinamic_variables)
153
- dinamic_variables.concat(collect_dinamic_variables(command, tag))
154
- end
155
-
147
+ nodes.each do |tag, init, command, index| #Takes the info of each node of workflow to create the job
148
+ # Set batch
149
+ new_batch = Batch.new(tag, init, command, index, @exec_folder)
150
+ dinamic_variables.concat(new_batch.handle_dependencies(dinamic_variables))
156
151
  @commands[new_batch.name] = new_batch
157
152
  end
158
153
 
159
154
  # link each parent batch to a child batch
160
155
  @commands.each do |name, batch|
161
- batch.main_command = asign_child_batch(batch.main_command, batch.name) if batch.main_command.class.to_s == 'Array'
156
+ batch.asign_child_batch
162
157
  end
163
158
  end
164
159
 
@@ -169,95 +164,54 @@ class Stack
169
164
  return nodes
170
165
  end
171
166
 
172
- def asign_child_batch(batch_ids, name)
173
- batches = []
174
- batch_ids.each do |id|
175
- batch = get_batch(id)
176
- batch.parent = name
177
- batches << batch
178
- end
179
- return batches
180
- end
181
-
182
- def get_batch(id)
183
- selected_batch = nil
184
- @commands.each do |name, batch|
185
- if batch.id == id
186
- selected_batch = batch
187
- break
188
- end
167
+ def set_dependencies_path(job) #TODO move this to queue_manager
168
+ job.dependencies.sort{|d1, d2| d2.length <=> d1.length}.each do |dep|
169
+ path = @@all_jobs_relations[dep]
170
+ job.initialization.gsub!(dep+')', path) if !job.initialization.nil?
171
+ job.parameters.gsub!(dep+')', path)
189
172
  end
190
- return selected_batch
191
173
  end
192
174
 
193
- def collect_dinamic_variables(command, node_name)
194
- dinamic_variables = []
195
- node_name = node_name.gsub(/&|\!|\%|\)/,'')# Delete function characters
196
- if !command.nil? && command.include?('env_manager')
197
- command =~ /env_manager "([^"]+)/
198
- command =~ /env_manager '([^']+)/ if $1.nil?
199
- if !$1.nil?
200
- $1.split(';').each do |variable|
201
- name, value = variable.split('=')
202
- name.gsub!(' ', '') #Remove spaces
203
- dinamic_variables << [name, node_name.gsub(')','')]
175
+ def asign_folder(job) #TODO move this to queue_manager
176
+ folder = nil
177
+ if job.attrib[:folder]
178
+ if @@folder_name == :program_name
179
+ program = File.join(job.attrib[:exec_folder], job.parameters.split(' ', 2).first)
180
+ count = 0
181
+ folder = program + "_#{"%04d" % count}"
182
+ while @@all_jobs_relations.values.include?(folder)
183
+ folder = program + "_#{"%04d" % count}"
184
+ count += 1
204
185
  end
186
+ elsif @@folder_name == :job_name
187
+ folder = File.join(job.attrib[:exec_folder], job.name)
205
188
  end
189
+ else
190
+ folder = job.attrib[:exec_folder]
206
191
  end
207
- return dinamic_variables
208
- end
209
-
210
- def scan_dependencies(command, dependencies, dinamic_variables)
211
- if !command.nil?# When command is the initialize, sometimes can be undefined
212
- @commands.each do |name, batch|
213
- if command.include?(name+')')
214
- dependencies << [name, 'simple', name+')']
215
- end
216
- if command.include?("!#{name}*!")
217
- dependencies << [name, '1to1', "!#{name}*!"]
218
- end
219
- if command.include?("!#{name}!")
220
- command =~ /!#{name}!([^ \n]+)/
221
- dependencies << [name, '*to1', "!#{name}!", $1]
222
- end
223
- local_dependencies = command.scan(/#{name}([^\( \n]+)\)/)
224
- local_dependencies.each do |local_dependency|
225
- dependencies << [name, 'local', "#{name}#{local_dependency.first}"+')', local_dependency.first]
226
- end
227
- end
228
- dependencies.concat(check_dependencies_with_DinVar(command, dinamic_variables))
229
- end
230
- end
231
-
232
- def check_dependencies_with_DinVar(command, dinamic_variables)
233
- dep = []
234
- dinamic_variables.each do |var, name|
235
- dep << [name, 'DinVar'] if command.include?(var)
236
- end
237
- return dep
238
- end
239
-
240
- def set_path(job, all_jobs_relations)
241
- job.dependencies.each do |dep|
242
- path = all_jobs_relations[dep]
243
- job.initialization.gsub!(dep+')', path)
244
- job.parameters.gsub!(dep+')', path)
245
- end
192
+ @@all_jobs_relations[job.name.gsub(')','')] = folder
193
+ job.attrib[:exec_folder] = folder
246
194
  end
247
-
195
+
248
196
  def get_jobs
249
197
  jobs =[]
250
- all_jobs_relations = Batch.get_job_relations
251
198
  @commands.each do |name, batch|
252
- next if batch.has_jobs?
199
+ next if batch.has_jobs? #parent batch (intermediates)
253
200
  batch.get_jobs.each do |j|
254
- set_path(j, all_jobs_relations)
201
+ folder = asign_folder(j) #TODO move this to queue_manager
255
202
  jobs << [j.name, j]
256
203
  end
204
+
205
+ end
206
+ jobs.each do |j_name, job| #TODO move this to queue_manager
207
+ set_dependencies_path(job)
208
+ j_name.gsub!(')','') #Clean function characters on name
209
+ job.name.gsub!(')','')
257
210
  end
258
211
  return jobs
259
212
  end
260
213
 
214
+
261
215
  def get_jobs_relations
262
216
  hash = {}
263
217
  get_jobs.each do |name, job|
@@ -266,6 +220,11 @@ class Stack
266
220
  return hash
267
221
  end
268
222
 
223
+ def comment_main_command
224
+ @jobs.each do |name, job|
225
+ job.parameters = "##{job.parameters}"
226
+ end
227
+ end
269
228
  ##########################################################################################
270
229
  ## WORKFLOW REPRESENTATION
271
230
  ##########################################################################################
@@ -1,3 +1,3 @@
1
1
  module Autoflow
2
- VERSION = "0.5.3"
2
+ VERSION = "0.6.4"
3
3
  end
@@ -0,0 +1,148 @@
1
+ require File.dirname(__FILE__) + '/test_helper.rb'
2
+
3
+ class StackTest < Test::Unit::TestCase
4
+
5
+ def setup
6
+
7
+ end
8
+
9
+ def test_scan_nodes
10
+ stack = Stack.new('exec', {
11
+ :cpu => 16,
12
+ :mem => '4gb',
13
+ :time => '20:00:00',
14
+ :node => nil,
15
+ :multinode => 0,
16
+ :ntask => FALSE,
17
+ :key_name => FALSE,
18
+ :retry => FALSE,
19
+ :Variables => nil,
20
+ :workflow => ''
21
+ })
22
+
23
+ # test simple node
24
+ #---------------------------------------------
25
+
26
+ node_lines = [
27
+ #Single node
28
+ "result){\n",
29
+ "null\n",
30
+ "?\n",
31
+ "touch algo\n",
32
+ "}\n"
33
+ ]
34
+
35
+ result = [
36
+ ["result)", "null\n", "touch algo\n"]
37
+ ]
38
+
39
+ test = stack.scan_nodes(node_lines)
40
+ assert_equal(result, test)
41
+
42
+ # test double node
43
+ #---------------------------------------------
44
+
45
+ node_lines = [
46
+ #Single node
47
+ "algo){\n",
48
+ "null\n",
49
+ "?\n",
50
+ "echo 'OK'\n",
51
+ "}\n",
52
+ #Single node
53
+ "result){\n",
54
+ "null\n",
55
+ "?\n",
56
+ "touch algo\n",
57
+ "}\n"
58
+ ]
59
+
60
+ result = [
61
+ ["algo)", "null\n", "echo 'OK'\n"],
62
+ ["result)", "null\n", "touch algo\n"]
63
+ ]
64
+
65
+ test = stack.scan_nodes(node_lines)
66
+ assert_equal(result, test)
67
+
68
+ # test nested iterative node
69
+ #---------------------------------------------
70
+
71
+ node_lines = [
72
+ #Nested iterative nodes
73
+ "itera_[11;22]){\n",
74
+ "algo_[aa;bb]){\n",
75
+ "null\n",
76
+ "?\n",
77
+ "echo 'OK'itera_(+)(*)\n",
78
+ "}\n",
79
+ "}\n"
80
+ ]
81
+
82
+ result = [
83
+ ["itera_[11;22])", "", [1]],
84
+ ["algo_[aa;bb])", "null\n", "echo 'OK'itera_(+)(*)\n"]
85
+ ]
86
+
87
+ test = stack.scan_nodes(node_lines)
88
+ assert_equal(result, test)
89
+
90
+ # test nested 3 iterative node
91
+ #---------------------------------------------
92
+
93
+ node_lines = [
94
+ #Nested iterative nodes
95
+ "itera_[11;22]){\n",
96
+ "?\n",
97
+ "algo_[aa;bb]){\n",
98
+ "?\n",
99
+ "mas_[ZZ;YY]){\n",
100
+ "null\n",
101
+ "?\n",
102
+ "echo 'OK'itera_(+)(*)mas_(+)\n",
103
+ "}\n",
104
+ "}\n",
105
+ "}\n"
106
+ ]
107
+
108
+ result = [
109
+ ["itera_[11;22])", "", [1]],
110
+ ["algo_[aa;bb])", "", [2]],
111
+ ["mas_[ZZ;YY])", "null\n", "echo 'OK'itera_(+)(*)mas_(+)\n"]
112
+ ]
113
+
114
+ test = stack.scan_nodes(node_lines)
115
+ assert_equal(result, test)
116
+
117
+ #test nested iterative nodes with a final simgle node (bug)
118
+ #--------------------------------------------------------------
119
+
120
+ node_lines = [
121
+ #Nested iterative nodes
122
+ "itera_[11;22]){\n",
123
+ "algo_[aa;bb]){\n",
124
+ "null\n",
125
+ "?\n",
126
+ "echo 'OK'itera_(+)(*)\n",
127
+ "}\n",
128
+ "}\n",
129
+ #Single node
130
+ "result){\n",
131
+ "null\n",
132
+ "?\n",
133
+ "touch algo\n",
134
+ "}\n"
135
+ ]
136
+
137
+ result = [
138
+ ["itera_[11;22])", "", [1]],
139
+ ["algo_[aa;bb])", "null\n", "echo 'OK'itera_(+)(*)\n"],
140
+ ["result)", "null\n", "touch algo\n"]
141
+ ]
142
+
143
+ test = stack.scan_nodes(node_lines)
144
+ assert_equal(result, test)
145
+ end
146
+
147
+
148
+ end
@@ -0,0 +1,11 @@
1
+ require 'stringio'
2
+ require 'test/unit'
3
+
4
+ ROOT_PATH=File.dirname(__FILE__)
5
+ $: << File.expand_path(File.join(ROOT_PATH, "../lib/"))
6
+ $: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/"))
7
+ $: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/queue_managers"))
8
+
9
+ require File.dirname(__FILE__) + '/../lib/autoflow/program'
10
+ require File.dirname(__FILE__) + '/../lib/autoflow/batch'
11
+ require File.dirname(__FILE__) + '/../lib/autoflow/stack'
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: autoflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.3
4
+ version: 0.6.4
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2015-04-16 00:00:00.000000000 Z
12
+ date: 2016-03-10 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: net-ssh
@@ -103,6 +103,8 @@ files:
103
103
  - lib/autoflow/queue_managers/slurm_manager.rb
104
104
  - lib/autoflow/stack.rb
105
105
  - lib/autoflow/version.rb
106
+ - test/stack_test.rb
107
+ - test/test_helper.rb
106
108
  homepage: ''
107
109
  licenses:
108
110
  - MIT
@@ -128,4 +130,6 @@ rubygems_version: 1.8.23
128
130
  signing_key:
129
131
  specification_version: 3
130
132
  summary: ! '"This gem take a pipeline and launch it on a queue system"'
131
- test_files: []
133
+ test_files:
134
+ - test/stack_test.rb
135
+ - test/test_helper.rb