autoflow 0.5.3 → 0.6.4

data/Rakefile

@@ -1 +1,9 @@
 require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = "test/*_test.rb"
+end
+

data/bin/AutoFlow

@@ -1,9 +1,9 @@
 #!/usr/bin/env ruby
 
 ROOT_PATH=File.dirname(__FILE__)
-$: << File.expand_path(File.join(ROOT_PATH, "../lib/"))
-$: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/"))
-$: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/queue_managers"))
+$: << File.expand_path(File.join(ROOT_PATH, "..", "lib"))
+$: << File.expand_path(File.join(ROOT_PATH, "..", "lib", "autoflow"))
+$: << File.expand_path(File.join(ROOT_PATH, "..", "lib", "autoflow", "queue_managers"))
 
 require 'optparse'
 require 'autoflow'
@@ -21,8 +21,8 @@ def get_templates(string_template)
 end
 
 def get_repositories
-    directories = []
-    directories = ENV['WORKFLOW_REPOSITORY'].split(':') if !ENV['WORKFLOW_REPOSITORY'].nil?
+    directories = [File.join(ENV['HOME'], 'autoflow_templates')]
+    directories.concat(ENV['WORKFLOW_REPOSITORY'].split(':')) if !ENV['WORKFLOW_REPOSITORY'].nil?
     return directories
 end
 
@@ -33,11 +33,9 @@ end
 
 def list_repository_templates
         templates = []
-        if !ENV['WORKFLOW_REPOSITORY']. nil?
-                directories = get_repositories
-                directories.each do |dir|
-                        templates.concat(Dir.entries(dir))
-                end
+        directories = get_repositories
+        directories.each do |dir|
+                templates.concat(Dir.entries(dir))
         end
         templates.delete('.')
         templates.delete('..')
@@ -73,7 +71,7 @@ options = {}
 template_file = ''
 optparse = OptionParser.new do |opts|
         options[:add] = nil
-        opts.on( '-a', '--add STRING', 'Puts a copy of a selected workflow template in repository' ) do |add|
+        opts.on( '-a', '--add STRING', 'Put a copy of any selected workflow template in repository' ) do |add|
                 options[:add] = add
         end
 
@@ -83,22 +81,27 @@ optparse = OptionParser.new do |opts|
         end
 
         options[:cpus] = 16
-        opts.on( '-c', '--cpus INTEGER', 'Max cpus can be used in all workflow' ) do |cpus|
+        opts.on( '-c', '--cpus INTEGER', 'Max number of CPUs that can be used in all workflow' ) do |cpus|
                 options[:cpus] = cpus.to_i
         end
 
+        options[:comment] = FALSE
+        opts.on( '-C', '--comment_main_command', 'Comment first line of main command job' ) do 
+                options[:comment] = TRUE
+        end
+
         options[:external_dependencies] = []
-        opts.on( '-d', '--external_dependencies STRING', 'The workflow will start when indicated jobs finish on queue system. Format: \'id1,id2,id3..\'') do |external_dependencies|
+        opts.on( '-d', '--external_dependencies STRING', 'The workflow will start when indicated jobs are finished on queue system. Format: \'id1,id2,id3..\'') do |external_dependencies|
                 options[:external_dependencies] = external_dependencies.split(',')
         end
 
         options[:retry] = FALSE
-        opts.on( '-f', '--force', 'Execute all jobs, included the jobs commented with %' ) do 
+        opts.on( '-f', '--force', 'Execute all jobs, including any job commented with %' ) do 
                 options[:retry] = TRUE
         end
 
         options[:graph] = nil
-        opts.on( '-g', '--graph STRING', 'Draw the template. t for use tag like names or f for use folders names instead. This option cancel the workflow launching' ) do |graph| 
+        opts.on( '-g', '--graph STRING', 'Draw a chart for the template. The workflow is not executed \'t\' use TIDs for box names \'f\' use folder names for boxes.' ) do |graph| 
                 options[:graph] = graph
         end
 
@@ -108,17 +111,17 @@ optparse = OptionParser.new do |opts|
         end
 
         options[:key_name] = FALSE
-        opts.on( '-k', '--use_key_name', 'Use job names like folder name' ) do
+        opts.on( '-k', '--use_key_name', ' Use job names as folder names' ) do
                 options[:key_name] = TRUE
         end
 
         options[:list] = nil
-        opts.on( 'l', '--list_repository STRING', 'List template names in repository') do |name|
+        opts.on( '-l', '--list_repository STRING', 'List template names in repository') do |name|
                 options[:list] = name
         end
 
         options[:memory] = '4gb'
-        opts.on( '-m', '--memory STRING', 'Max memory can be used in a task' ) do |mem|
+        opts.on( '-m', '--memory STRING', 'Max memory that can be allocated in a task' ) do |mem|
                 options[:memory] = mem
         end
 
@@ -128,7 +131,7 @@ optparse = OptionParser.new do |opts|
         end
 
         options[:output] = 'exec'
-        opts.on( '-o', '--output STRING', 'Output folder of flow' ) do |output|
+        opts.on( '-o', '--output STRING', 'Define an output folder name' ) do |output|
                 options[:output] = output
         end
 
@@ -151,7 +154,7 @@ optparse = OptionParser.new do |opts|
         end
 
         options[:use_ntasks] = FALSE
-        opts.on( '-s', '--use_ntasks', 'Use -ntasks flag with sh' ) do 
+        opts.on( '-s', '--use_ntasks', 'Use several nodes on execution' ) do 
                 options[:use_ntasks] = TRUE
         end
 
@@ -167,7 +170,7 @@ optparse = OptionParser.new do |opts|
 
 
         options[:verbose] = FALSE
-        opts.on( '-v', '--verbose', 'Show info without launch jobs' ) do 
+        opts.on( '-v', '--verbose', 'Show info without launching jobs' ) do 
                 options[:verbose] = TRUE
         end
 
@@ -200,6 +203,11 @@ optparse.parse!
 #################################################################################################
 # MAIN
 #################################################################################################
+#Create repository
+if !File.exists?(File.join(ENV['HOME'], 'autoflow_templates'))
+    Dir.mkdir(File.join(ENV['HOME'], 'autoflow_templates'))
+end
+
 # List templates
 templates_rep_names = list_repository_templates
 if !options[:list].nil?
@@ -295,6 +303,7 @@ if !options[:graph].nil?
         stack.draw(template_file, options[:graph])
 else
         stack.inspect if options[:verbose]
+        stack.comment_main_command if options[:comment]
         manager.exec
 end
 options[:ssh].close if options[:remote]

data/lib/autoflow/batch.rb

@@ -2,7 +2,7 @@ class Batch
 	attr_accessor :name, :iterator, :dependencies, :init, :main_command, :attrib, :id, :jobs, :parent
 
 	@@all_batch = {}
-	@@all_jobs_relations = {}
+	@@jobs_names = []
 	@@batch_iterator_relations = {}
 	@@state_iterations = {}
 	@@general_computation_attrib = {
@@ -13,21 +13,14 @@ class Batch
 		:multinode => nil,
 		:ntask => nil
 	}
-	@@folder_name = :program_name # can  :job_name
-
-	def self.set_job_folder_definition(definition)
-		@@folder_name = definition
-	end
-
+ 
 	def self.set_general_attrib(attrib_hash)
 		@@general_computation_attrib = attrib_hash
 	end
 
-	def self.get_job_relations
-		return @@all_jobs_relations
-	end
 
 	def initialize(tag, init, main_command, id, exec_folder)
+		replace_regexp(tag, init, main_command)
 		@name = nil
 		@id = id
 		@iterator = [nil] 
@@ -54,40 +47,121 @@ class Batch
 		@@all_batch[@name] = self
 	end
 
+	def replace_regexp(tag, init, main_command)
+		scan_JobRegExp_tag(tag) if tag.include?('JobRegExp:') 
+		[init, main_command].each do |intructions|
+			if intructions.class.to_s == 'String'
+				while intructions.include?('!JobRegExp:')
+					scan_JobRegExp(intructions)
+				end
+			end
+		end
+		#puts main_command.inspect
+	end
+
+	def scan_JobRegExp(command)
+		data = /!JobRegExp:([^ \n]+):([^ \n]+)!([^ \n]+)/.match(command) # *to1 with regexp
+		#data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern, data[3] => adyacent string to regexp as regexp/file_name
+		job_names = get_dependencies_by_regexp(data[1], data[2])
+		new_string = job_names.map{|jn| jn + ')' + data[3] }.join(' ')
+		command.gsub!(data[0], new_string)
+		#puts command.inspect
+	end
+
+	def scan_JobRegExp_tag(tag)
+		data = /JobRegExp:([^ \n]+):([^;\] \n]+)/.match(tag) # 1to1 with regexp
+		#data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern
+		job_names = get_dependencies_by_regexp(data[1], data[2])
+		new_string = job_names.map{|jn| jn + ')'}.join(';')
+		tag.gsub!(data[0], new_string)
+	end
+
+	def get_dependencies_by_regexp(batch_pattern, iterator_pattern)
+		selected_batches = @@all_batch.keys.select{|cmd_name| cmd_name =~ /#{batch_pattern}/}
+		job_names = []
+		selected_batches.each do |batch_name|
+			iterators = @@all_batch[batch_name].iterator
+			iterators = iterators.map{|it| it.gsub(/&|\!|\%|\)/,'')} if !iterators.first.nil?
+			if iterator_pattern != '-'
+				next if iterators.first.nil?
+				iterators = iterators.select{|iter| iter =~ /#{iterator_pattern}/} 
+			end
+			if !iterators.empty?
+				if iterators.first.nil? 
+					job_names << batch_name
+				else
+					iterators.each do |iter|
+						job_names << batch_name+iter
+					end
+				end
+			end
+		end
+		return job_names
+	end
+
 	def has_jobs?
 		res = !@jobs.empty?
 		return res
 	end
 
+	def asign_child_batch
+		batches = []
+		if @main_command.class.to_s == 'Array'
+			@main_command.each do |id|
+				batch = get_batch(id)
+				batch.parent = @name
+				batches << batch
+			end
+			@main_command = batches
+		end
+	end
+
+	def get_batch(id)
+		selected_batch = nil
+		@@all_batch.each do |name, batch|
+			if batch.id == id
+				selected_batch = batch  
+				break
+			end
+		end
+		return selected_batch
+	end
+
 	def get_name_and_iterators_and_modifiers(tag)
-		tag =~ /(^.+)\[([^\]]+)/
+		tag =~ /(^.+)\[([^\]]+)\]\)/ # iterative node
 		name = $1
-		if $1.nil?
+		if $1.nil? # Non iterative node (simple node)
 			tag =~ /(^.+)\)/
 			name = $1
 		end
 		@name , @attrib[:done], @attrib[:folder], @attrib[:buffer] = check_execution_modifiers(name)
 		if !$2.nil?
 			@iterator = []
-			$2.split(';').map{|interval|
+			#$2.split(';').map{|iter| iter.gsub(')','')}.each do |interval|
+			$2.split(';').each do |interval|
 				if interval.include?('-')
 					limits = interval.split('-')
 					@iterator.concat((limits.first..limits.last).to_a.map{|n| n.to_s})
 				else
 					@iterator << interval
 				end
-			}
+			end
 		end
+		@@batch_iterator_relations[@name] = @iterator
 	end
 
-	def check_execution_modifiers(name)
+	def check_execution_modifiers(name, iter_type = FALSE) #The last paremeter iused to indicate tha name is a iterator not an orignal node name
 		done = FALSE
 		folder = TRUE
 		buffer = FALSE
 		done = TRUE if name.include?('%')
 		folder = FALSE if name.include?('!')
 		buffer = TRUE if name.include?('&')
-		name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
+		if !iter_type
+			name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
+		else
+			name.gsub!(/&|\!|\%/,'')# Delete function characters
+		end
 		return name, done, folder, buffer
 	end
 
@@ -146,32 +220,6 @@ class Batch
 	end
 
 
-	def asign_folder(name, local_command = nil)
-		if local_command.nil? #we check a simple job
-			command = @main_command
-		else #we check a cloned job
-			command = local_command 
-		end 
-		if command.class.to_s == 'Array'
-			folder = nil
-		elsif @attrib[:folder]
-			if @@folder_name == :program_name
-				program = File.join(@attrib[:exec_folder], command.split(' ', 2).first) 			
-				count = 0
-				folder = program + "_#{"%04d" % count}"
-				while @@all_jobs_relations.values.include?(folder)
-					folder = program + "_#{"%04d" % count}"
-					count += 1
-				end
-			elsif @@folder_name == :job_name
-				folder = File.join(@attrib[:exec_folder], name)
-			end
-		else
-			folder = @attrib[:exec_folder]
-		end
-		return folder
-	end
-
 	def duplicate_job(tmp_j, sufix_name = '')
 		new_job = tmp_j.clone
 		new_job.name = tmp_j.name+'_'+sufix_name
@@ -179,7 +227,6 @@ class Batch
 		new_job.dependencies = tmp_j.dependencies.clone
 		new_job.initialization = tmp_j.initialization.clone
 		new_job.parameters = tmp_j.parameters.clone
-		new_job.attrib[:exec_folder] = asign_folder(new_job.name, new_job.parameters)
 		return new_job
 	end
 
@@ -193,7 +240,7 @@ class Batch
 
 	def get_jobs
 		jobs = []
-		@@batch_iterator_relations[@name] = @iterator
+		#@@batch_iterator_relations[@name] = @iterator #??????
 		if @main_command.class.to_s == 'Array' # There are nested batchs
 			temp_jobs = []
 			@main_command.each do |batch|
@@ -205,9 +252,9 @@ class Batch
 					new_job = duplicate_job(tmp_j, iter)
 					check_dependencies(new_job, iter, temp_jobs)
 					parse_iter(iter, @name, new_job)
+					@@jobs_names << new_job.name
 					jobs << new_job
 					@jobs << new_job
-					@@all_jobs_relations[new_job.name] = new_job.attrib[:exec_folder]
 					jobs2delete << tmp_i
 				end
 			end
@@ -216,19 +263,20 @@ class Batch
 			@iterator.each_with_index do |iter, num|
 				job_attrib = @attrib.dup
 				if !iter.nil?
-					iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter)
+					iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter, TRUE)
 					job_attrib[:done] = done if !@attrib[:done] # To keep attrib priority in batch on job
 				end
 				name = "#{@name}#{iter}"
-				job_attrib[:exec_folder] = asign_folder(name) if @parent.nil? #Don't asign folder to nested batches (iterative batchs)
 				job_dependencies = []
+				batch_deps = @dependencies.length
 				initialization = replace_dependencies(@initialization, job_dependencies, iter, num)
 				parameters = replace_dependencies(@main_command, job_dependencies, iter, num)
+				@dependencies.pop(@dependencies.length - batch_deps) # Clean temporal dependencies by regexp
 				job = Program.new(name, initialization, parameters, job_dependencies, job_attrib)
 				job.batch = @name
+				@@jobs_names << job.name
 				jobs << job
 				@jobs << job 
-				@@all_jobs_relations[name] = job_attrib[:exec_folder]
 			end
 		end
 		return jobs
@@ -261,12 +309,101 @@ class Batch
 		return string
 	end
 
+	def handle_dependencies(dinamic_variables)
+		[@initialization, @main_command].each do |instructions|
+			if instructions.class.to_s == 'String'
+				scan_dependencies(instructions)
+				dinamic_variables.concat(collect_dinamic_variables(instructions))
+				@dependencies.concat(check_dependencies_with_DinVar(instructions, dinamic_variables))
+			end
+		end
+		return dinamic_variables
+	end
+
+	def scan_dependencies(command)
+		if !command.nil?# When command is the initialize, sometimes can be undefined
+			matched_regions = []
+			batches = []
+			@@all_batch.each do |k , val| #sorting is used to match last jobs first, and avoid small matches of first nodes
+				batches << [k , val]
+			end
+			batches.reverse.each do |name, batch| 
+				if command.include?(name+')') && !string_overlap(matched_regions, name+')', command)
+					@dependencies << [name, 'simple', name+')']
+				end
+				if command.include?("!#{name}*!") && !string_overlap(matched_regions, "!#{name}*!", command)
+					@dependencies << [name, '1to1', "!#{name}*!"]
+				end
+				if command.include?("!#{name}!") && !string_overlap(matched_regions, "!#{name}!", command)
+					command =~ /!#{name}!([^ \n]+)/
+					@dependencies << [name, '*to1', "!#{name}!", $1]
+				end
+				local_dependencies = command.scan(/#{name}([^\( \n]+)\)/)
+				local_dependencies.each do |local_dependency|
+					if !string_overlap(matched_regions, "#{name}#{local_dependency.first}"+')', command)
+						@dependencies << [name, 'local', "#{name}#{local_dependency.first}"+')', local_dependency.first] 
+					end
+				end
+			end
+		end
+	end
+
+	def get_string_position(substr, string)
+		start = string.index(substr)
+		ending = start + substr.length - 1
+		range = [start, ending]
+		return range
+	end
+
+	def string_overlap(matched_regions, substr, string)
+		match = FALSE
+		range = get_string_position(substr, string)
+		if !range.empty?
+			matched_regions.each do |start, ending|
+				if (range.first >= start && range.first <= ending) || 
+					(range.last >= start && range.last <= ending) ||
+					(range.first <= start && range.last >= ending)
+					match = TRUE
+					break
+				end
+			end
+			matched_regions << range 
+		end
+		return match
+	end
+
+	def collect_dinamic_variables(command)
+		dinamic_variables = []
+		if !command.nil? && command.include?('env_manager')
+			command =~ /env_manager "([^"]+)/
+			command =~ /env_manager '([^']+)/ if $1.nil?
+			if !$1.nil?
+				$1.split(';').each do |variable|
+					name, value = variable.split('=')
+					name.gsub!(' ', '') #Remove spaces
+					dinamic_variables << [name, @name]
+				end
+			end
+		end
+		return dinamic_variables
+	end
+
+	def check_dependencies_with_DinVar(command, dinamic_variables)
+		dep = []
+		dinamic_variables.each do |var, name|
+			dep << [name, 'DinVar'] if command.include?(var)
+		end
+		return dep
+	end
+
 	def replace_dependencies(command, job_dependencies, iter, num)
 		if !command.nil?
+			command = command.gsub('(*)', "#{iter}") if command.class.to_s == 'String'
+			scan_dependencies(command)
 			@dependencies.each do |batch_name, dep_type, dep_keyword2replace, dep_info|
 				if dep_type == 'simple'
 					if @@all_batch[batch_name].parent.nil?
-						new_string = @@all_jobs_relations[batch_name]
+						new_string = batch_name + ')'
 					end
 					job_dependencies << batch_name					
 				elsif dep_type == '1to1'
@@ -278,52 +415,39 @@ class Batch
 						dep_name = selected_jobs[num].name
 					end
 					job_dependencies << dep_name
-					if !@parent.nil? && !@@all_batch[batch_name].parent.nil?
-						new_string = dep_name + ')' 
-					else
-						new_string = @@all_jobs_relations[dep_name]
-					end
+					new_string = dep_name + ')' 
 				elsif dep_type == '*to1'
-					if @@all_batch[batch_name].parent.nil?
-						new_string = @@batch_iterator_relations[batch_name].map{|iter|
-							dep_name =  batch_name + iter
-							job_dependencies << dep_name
-							"#{@@all_jobs_relations[dep_name]}#{dep_info}"
-						}.join(' ')
-						dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
-					elsif !@parent.nil? 
+					if @@all_batch[batch_name].parent.nil? || !@parent.nil?
 						new_string = @@batch_iterator_relations[batch_name].map{|iter|
 							dep_name =  batch_name + iter
 							job_dependencies << dep_name
 							"#{dep_name})#{dep_info}"
 						}.join(' ')
-						dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"						
 					else
 						root_batch = get_root(batch_name)
 						selected_jobs = root_batch.get_jobs_by_batch_name(batch_name)
 						new_string = selected_jobs.map{|j|
 							job_dependencies << j.name
-							"#{@@all_jobs_relations[j.name]}#{dep_info}"
+							"#{j.name + ')'}#{dep_info}"
 						}.join(' ')
-						dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
 					end
+					dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
 				elsif dep_type == 'local'
 					if @@all_batch[batch_name].parent.nil? || !@parent.nil?
-						if @@batch_iterator_relations[batch_name].include?(dep_info) #This avoids cross dependencies by similar names
+						#@@batch_iterator_relations.each do |key,val|
+						#	puts "#{key}\t#{val.inspect}"
+						#end
+						if @@batch_iterator_relations[batch_name].map{|iter| iter.gsub(')','')}.include?(dep_info) #This avoids cross dependencies by similar names, map used for regexp deps
 							dep_name = batch_name + dep_info
 							job_dependencies << dep_name
-							if !@parent.nil? && !@@all_batch[batch_name].parent.nil?
-								new_string = dep_name + ')'
-							else
-								new_string = @@all_jobs_relations[dep_name]
-							end
+							new_string = dep_name + ')'
 						end
 					else
 						dep_name = dep_keyword2replace.gsub(')','') #This avoids cross dependencies by similar names
-						if !@@all_jobs_relations[dep_name].nil?
-							job_dependencies << dep_name
-							new_string = @@all_jobs_relations[dep_name]
-						end
+						#if !@@jobs_names.include?(dep_name)
+						job_dependencies << dep_name
+						new_string = dep_name + ')'
+						#end
 					end
 				elsif dep_type == 'DinVar'
 					job_dependencies << batch_name if batch_name != @name # This condition avoids autodependencies
@@ -331,9 +455,7 @@ class Batch
 				job_dependencies.uniq!
 				command = command.gsub(dep_keyword2replace, new_string) if dep_type != 'DinVar' && !dep_keyword2replace.nil? && !new_string.nil?
 			end
-
-			command = command.gsub('(*)', "#{iter}") if command.class.to_s == 'String'
-		end 
+		end
 		return command
 	end
 

data/lib/autoflow/queue_manager.rb

@@ -139,6 +139,8 @@ class QueueManager
 		write_file(sh_name, "source #{File.join(@exec_folder, 'env_file')}") if !@persist_variables.empty?
 		write_job(job, sh_name)
 		write_file(sh_name, "echo -e \"FINISHED  #{id}  #{job.parameters.split.first}:\\t`date`\" >> #{log_folder}")
+		write_file(sh_name, "echo 'General time'")
+		write_file(sh_name, "times")
 		close_file(sh_name, 0755)
 
 		#Submit node

data/lib/autoflow/stack.rb

@@ -18,11 +18,13 @@ class Stack
 			:multinode => options[:use_multinode],
 			:ntask => options[:use_ntasks]
 		})
-		Batch.set_job_folder_definition(:job_name) if options[:key_name]
+		@@folder_name = :program_name
+		@@folder_name = :job_name if options[:key_name]
 		@commands = {}	
 		@variables = {}
 		@persist_variables = {}
-		@exec_folder = exec_folder
+		@@all_jobs_relations = {}
+		@exec_folder = exec_folder #TODO move this to queue_manager
 		@do_retry = options[:retry]
 		@options = options
 		parse(options[:workflow], options[:Variables])
@@ -85,38 +87,41 @@ class Stack
 		# $1 => tag, $2 => initialize, $3 => main command
 		#executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?\s([^\}]{1,})\s{0,}\}/)
 #=begin
-		executions = [] 
+		executions = [] # tag, initialize, main_command
 		states = {} #name => [state, id(position)]
 					#t => tag, i => initialize , c => command
-		names = []
+		open_nodes = []
 		template_executions.each_line do |line|
 			line.strip! #Clean al whitespaces at beginning and the end of string
-			node = states[names.last] if !names.empty?
+			node = states[open_nodes.last] if !open_nodes.empty?
 			if line.empty?
 				next
-			elsif line =~ /(\S*\)){$/ #Check tag
+			# Create nodes and asign nodes states
+			#----------------------------------------
+			elsif line =~ /(\S*\)){$/ #Check tag and create node
 				name = $1
-				executions << [name, '', '']
-				states[name] = ['t', executions.length - 1]
-				names << name
+				executions << [name, '', ''] # create node
+				states[name] = [:i, executions.length - 1]
+				open_nodes << name
 			elsif line == '?' #Check command
-				node[0] = 'c'
-			elsif states[names.last].first == 't' || states[names.last].first == 'i' #Check initialize
-				node[0] = 'i' if states[names.last].first == 't'
-				executions[node.last][1] << line +"\n"
+				node[0] = :c
 			elsif line == '}' #Close node
-				finished_node = names.pop
-				if !names.empty?
-					parent_node = states[names.last].last #position
+				finished_node = open_nodes.pop
+				if !open_nodes.empty?
+					parent_node = states[open_nodes.last].last #position
 					child_node = states[finished_node].last
 					parent_execution = executions[parent_node]
-					if parent_execution[2].class.to_s == 'String'   
+					if parent_execution[2].class == String   
 						parent_execution[2] = [child_node]
 					else
 						parent_execution[2] << child_node
 					end
 				end
-			elsif states[names.last].first == 'c' #Check command
+			# Add lines to nodes
+			#----------------------
+			elsif states[open_nodes.last].first == :i #Add initialize line
+				executions[node.last][1] << line +"\n"
+			elsif states[open_nodes.last].first == :c #Add command line
 				executions[node.last][2] << line +"\n"			
 			end
 		end
@@ -136,29 +141,19 @@ class Stack
 		nodes = create_ids(nodes)
 		
 		#nodes.each do |tag, init, command, index|
-		#	puts "#{tag.colorize(:red)}\t#{index}\n#{init.chomp.colorize(:blue)}\n#{command.to_s.colorize(:green)}"
+		#	puts "#{tag.colorize(:red)}\t#{index}\n#{('-'*tag.length).colorize(:red)}\n#{init.chomp.colorize(:blue)}\n#{command.to_s.colorize(:green)}"
 		#end
 
-		nodes.each do |tag, init, command, index| #Takes the info of each node of workflow for create the job
-			# Set batch
-			new_batch = Batch.new(tag, init, command, index, @exec_folder)			
-
-			#Dependencies
-			#-- Check initialize
-			scan_dependencies(init, new_batch.dependencies, dinamic_variables)
-			dinamic_variables.concat(collect_dinamic_variables(init, tag))
-			#-- Check command
-			if command.class.to_s == 'String'
-				scan_dependencies(command, new_batch.dependencies, dinamic_variables)
-				dinamic_variables.concat(collect_dinamic_variables(command, tag))
-			end
-
+		nodes.each do |tag, init, command, index| #Takes the info of each node of workflow to create the job
+			# Set batch	
+			new_batch = Batch.new(tag, init, command, index, @exec_folder)
+			dinamic_variables.concat(new_batch.handle_dependencies(dinamic_variables))
 			@commands[new_batch.name] = new_batch
 		end
 
 		# link each parent batch to a child batch
 		@commands.each do |name, batch|
-			batch.main_command = asign_child_batch(batch.main_command, batch.name) if batch.main_command.class.to_s == 'Array'
+			batch.asign_child_batch
 		end
 	end
 
@@ -169,95 +164,54 @@ class Stack
 		return nodes
 	end
 
-	def asign_child_batch(batch_ids, name)
-		batches = []
-		batch_ids.each do |id|
-			batch = get_batch(id)
-			batch.parent = name
-			batches << batch
-		end
-		return batches
-	end
-
-	def get_batch(id)
-		selected_batch = nil
-		@commands.each do |name, batch|
-			if batch.id == id
-				selected_batch = batch  
-				break
-			end
+	def set_dependencies_path(job) #TODO move this to queue_manager
+		job.dependencies.sort{|d1, d2| d2.length <=> d1.length}.each do |dep|
+			path =  @@all_jobs_relations[dep]
+			job.initialization.gsub!(dep+')', path) if !job.initialization.nil?
+			job.parameters.gsub!(dep+')', path)
 		end
-		return selected_batch
 	end
 
-	def collect_dinamic_variables(command, node_name)
-		dinamic_variables = []
-		node_name = node_name.gsub(/&|\!|\%|\)/,'')# Delete function characters
-		if !command.nil? && command.include?('env_manager')
-			command =~ /env_manager "([^"]+)/
-			command =~ /env_manager '([^']+)/ if $1.nil?
-			if !$1.nil?
-				$1.split(';').each do |variable|
-					name, value = variable.split('=')
-					name.gsub!(' ', '') #Remove spaces
-					dinamic_variables << [name, node_name.gsub(')','')]
+	def asign_folder(job) #TODO move this to queue_manager
+		folder = nil
+		if job.attrib[:folder]
+			if @@folder_name == :program_name
+				program = File.join(job.attrib[:exec_folder], job.parameters.split(' ', 2).first) 			
+				count = 0
+				folder = program + "_#{"%04d" % count}"
+				while @@all_jobs_relations.values.include?(folder)
+					folder = program + "_#{"%04d" % count}"
+					count += 1
 				end
+			elsif @@folder_name == :job_name
+				folder = File.join(job.attrib[:exec_folder], job.name)
 			end
+		else
+			folder = job.attrib[:exec_folder]
 		end
-		return dinamic_variables
-	end
-
-	def scan_dependencies(command, dependencies, dinamic_variables)
-		if !command.nil?# When command is the initialize, sometimes can be undefined
-			@commands.each do |name, batch|
-				if command.include?(name+')')
-					dependencies << [name, 'simple', name+')']
-				end
-				if command.include?("!#{name}*!")
-					dependencies << [name, '1to1', "!#{name}*!"]
-				end
-				if command.include?("!#{name}!")
-					command =~ /!#{name}!([^ \n]+)/
-					dependencies << [name, '*to1', "!#{name}!", $1]
-				end
-				local_dependencies = command.scan(/#{name}([^\( \n]+)\)/)
-				local_dependencies.each do |local_dependency|
-					dependencies << [name, 'local', "#{name}#{local_dependency.first}"+')', local_dependency.first] 
-				end
-			end
-			dependencies.concat(check_dependencies_with_DinVar(command, dinamic_variables))
-		end
-	end
-
-	def check_dependencies_with_DinVar(command, dinamic_variables)
-		dep = []
-		dinamic_variables.each do |var, name|
-			dep << [name, 'DinVar'] if command.include?(var)
-		end
-		return dep
-	end
-
-	def set_path(job, all_jobs_relations)
-		job.dependencies.each do |dep|
-			path = all_jobs_relations[dep]
-			job.initialization.gsub!(dep+')', path)
-			job.parameters.gsub!(dep+')', path)
-		end
+		@@all_jobs_relations[job.name.gsub(')','')] = folder
+		job.attrib[:exec_folder] = folder 
 	end
-
+	
 	def get_jobs
 		jobs =[]
-		all_jobs_relations = Batch.get_job_relations
 		@commands.each do |name, batch|
-			next if batch.has_jobs?
+			next if batch.has_jobs? #parent batch (intermediates)
 			batch.get_jobs.each do |j|
-				set_path(j, all_jobs_relations)
+				folder = asign_folder(j) #TODO move this to queue_manager
 				jobs << [j.name, j]
 			end
+
+		end
+		jobs.each do |j_name, job| #TODO move this to queue_manager
+			set_dependencies_path(job)
+			j_name.gsub!(')','') #Clean function characters on name
+			job.name.gsub!(')','')
 		end
 		return jobs
 	end
 
+
 	def get_jobs_relations
 		hash = {}
 		get_jobs.each do |name, job|
@@ -266,6 +220,11 @@ class Stack
 		return hash
 	end
 
+	def comment_main_command
+		@jobs.each do |name, job|
+			job.parameters = "##{job.parameters}"
+		end
+	end
 ##########################################################################################
 ## WORKFLOW REPRESENTATION
 ##########################################################################################

data/lib/autoflow/version.rb

@@ -1,3 +1,3 @@
 module Autoflow
-  VERSION = "0.5.3"
+  VERSION = "0.6.4"
 end

data/test/stack_test.rb

@@ -0,0 +1,148 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class StackTest < Test::Unit::TestCase
+
+  def setup
+
+  end
+
+  def test_scan_nodes
+    stack = Stack.new('exec', {
+      :cpu => 16, 
+      :mem => '4gb', 
+      :time => '20:00:00',
+      :node =>  nil,
+      :multinode => 0,
+      :ntask => FALSE,
+      :key_name => FALSE,
+      :retry => FALSE,
+      :Variables => nil,
+      :workflow => ''
+      })
+
+    # test simple node
+    #---------------------------------------------
+
+    node_lines = [
+      #Single node
+      "result){\n",
+        "null\n", 
+        "?\n", 
+        "touch algo\n", 
+      "}\n"
+    ]
+    
+    result = [
+      ["result)", "null\n", "touch algo\n"]
+    ]
+    
+    test = stack.scan_nodes(node_lines)
+    assert_equal(result, test)
+
+    # test double node
+    #---------------------------------------------
+
+    node_lines = [
+      #Single node
+      "algo){\n", 
+        "null\n", 
+        "?\n", 
+        "echo 'OK'\n", 
+      "}\n",
+      #Single node 
+      "result){\n", 
+        "null\n", 
+        "?\n", 
+        "touch algo\n", 
+      "}\n"
+    ]
+    
+    result = [
+      ["algo)", "null\n", "echo 'OK'\n"], 
+      ["result)", "null\n", "touch algo\n"]
+    ]
+    
+    test = stack.scan_nodes(node_lines)
+    assert_equal(result, test)
+
+    # test nested iterative node
+    #---------------------------------------------
+
+    node_lines = [
+      #Nested iterative nodes
+      "itera_[11;22]){\n", 
+        "algo_[aa;bb]){\n", 
+          "null\n", 
+          "?\n", 
+          "echo 'OK'itera_(+)(*)\n", 
+        "}\n", 
+      "}\n"
+    ]
+    
+    result = [
+      ["itera_[11;22])", "", [1]], 
+      ["algo_[aa;bb])", "null\n", "echo 'OK'itera_(+)(*)\n"]
+    ]
+    
+    test = stack.scan_nodes(node_lines)
+    assert_equal(result, test)
+
+    # test nested 3 iterative node
+    #---------------------------------------------
+
+    node_lines = [
+      #Nested iterative nodes
+      "itera_[11;22]){\n", 
+        "?\n", 
+        "algo_[aa;bb]){\n", 
+         "?\n", 
+          "mas_[ZZ;YY]){\n", 
+            "null\n", 
+            "?\n", 
+            "echo 'OK'itera_(+)(*)mas_(+)\n", 
+          "}\n", 
+        "}\n", 
+      "}\n"
+    ]
+    
+    result = [
+      ["itera_[11;22])", "", [1]], 
+      ["algo_[aa;bb])", "", [2]], 
+      ["mas_[ZZ;YY])", "null\n", "echo 'OK'itera_(+)(*)mas_(+)\n"]
+    ]
+    
+    test = stack.scan_nodes(node_lines)
+    assert_equal(result, test)
+
+    #test nested iterative nodes with a final simgle node (bug)
+    #--------------------------------------------------------------
+
+    node_lines = [
+      #Nested iterative nodes
+      "itera_[11;22]){\n", 
+        "algo_[aa;bb]){\n", 
+          "null\n", 
+          "?\n", 
+          "echo 'OK'itera_(+)(*)\n", 
+        "}\n", 
+      "}\n", 
+      #Single node
+      "result){\n", 
+        "null\n", 
+        "?\n", 
+        "touch algo\n", 
+      "}\n"
+    ]
+    
+    result = [
+      ["itera_[11;22])", "", [1]], 
+      ["algo_[aa;bb])", "null\n", "echo 'OK'itera_(+)(*)\n"], 
+      ["result)", "null\n", "touch algo\n"]
+    ] 
+    
+    test = stack.scan_nodes(node_lines)
+    assert_equal(result, test)
+  end
+  
+    
+end

data/test/test_helper.rb

@@ -0,0 +1,11 @@
+require 'stringio'
+require 'test/unit'
+
+ROOT_PATH=File.dirname(__FILE__)
+$: << File.expand_path(File.join(ROOT_PATH, "../lib/"))
+$: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/"))
+$: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/queue_managers"))
+
+require File.dirname(__FILE__) + '/../lib/autoflow/program'
+require File.dirname(__FILE__) + '/../lib/autoflow/batch'
+require File.dirname(__FILE__) + '/../lib/autoflow/stack'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: autoflow
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.6.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-16 00:00:00.000000000 Z
+date: 2016-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -103,6 +103,8 @@ files:
 - lib/autoflow/queue_managers/slurm_manager.rb
 - lib/autoflow/stack.rb
 - lib/autoflow/version.rb
+- test/stack_test.rb
+- test/test_helper.rb
 homepage: ''
 licenses:
 - MIT
@@ -128,4 +130,6 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: ! '"This gem take a pipeline and launch it on a queue system"'
-test_files: []
+test_files:
+- test/stack_test.rb
+- test/test_helper.rb