autoflow 0.3.5 → 0.5.0

data/lib/autoflow/stack.rb

@@ -1,29 +1,30 @@
 require 'program'
-require 'queue_manager'
-
+require 'batch'
 
+require 'win32console' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
+require 'colorize'
 class Stack
-
-	TAG = 0
-	INIT = 1
-	COMMAND = 2
-	PROG_NAME = 0
-	PROG_PARAM = 1
+	attr_accessor :jobs, :exec_folder, :persist_variables
 
 ##########################################################################################
 ## PARSE TEMPLATE
 ##########################################################################################
 	def initialize(exec_folder, options)
-		@commands = {}
+		Batch.set_general_attrib({
+			:cpu => options[:cpus], 
+			:mem => options[:memory], 
+			:time => options[:time],
+			:node => options[:node_type],
+			:multinode => options[:use_multinode],
+			:ntask => options[:use_ntasks]
+		}) 
+		@commands = {}	
 		@variables = {}
-		@one2one_dependencies = {}
 		@persist_variables = {}
-		@dirs = []
 		@exec_folder = exec_folder
-		@file_workflow = options[:workflow]
 		@do_retry = options[:retry]
 		parse(options[:workflow], options[:Variables])
-		@q_mgr = load_queue_manager(options)
+		@jobs = get_jobs_relations
 	end
 	
 	def parse(workflow, external_variables)
@@ -31,14 +32,24 @@ class Stack
 		workflow.gsub!(/\#.+$/,'')	#Delete comments
 		workflow.gsub!("\t",'')		#Drop tabs
 		workflow.gsub!(/\n+/,"\n")	#Drop empty lines
+		workflow.gsub!(/^\s*/,'')
 
 		#Parse template
 		variables_lines = []
 		persist_variables_lines = []
 		node_lines = []
-		
+
+		node_beg = FALSE		
 		workflow.each_line do |line|
-			if line =~ /^\$/
+			node_beg = TRUE if line.include?('{') 	# This check the context of a variable
+			if line.include?('}')					# if a variable is within a node,
+				if node_beg							# we consider tha is a bash variable not a static autoflow variable
+					node_beg = FALSE
+				else
+					node_beg = TRUE
+				end
+			end
+			if line =~ /^\$/ && !node_beg
 				variables_lines << line
 			elsif line =~ /^\@/
 				persist_variables_lines << line.gsub('@','')
@@ -57,7 +68,7 @@ class Stack
 			variables_lines.each do |line|
 				line.chomp!
 				line.gsub!(/\s/,'')
-				pairs = line.split(';')
+				pairs = line.split(',')
 				pairs.each do |pair|
 					pair =~ /(.+)=(.+)/
 					variable_type[$1] = $2
@@ -66,161 +77,187 @@ class Stack
 		end
 	end
 
-	def parse_nodes(execution_lines)
-		scan_nodes(execution_lines).each do |job_node| #Takes the info of each node of workflow for create the job
-			list_dup_nodes_ids(job_node[TAG]).each do |dup_node_id| #if dup_node_id is nil, the node isn't cloned.
-				# Set node attributes
-				prog_parameters = job_node[COMMAND].split(' ', 2)
-				tag_root_name = job_node[TAG].gsub(/\[([^\]]+\])\)/, '')#We remove the clone node expression
-				tag_root_name, dup_node_id, folder, no_buffer_node, done = set_node_attributes(tag_root_name, dup_node_id, prog_parameters)
-
-				#Dependencies
-				initialization, init_dependencies = parse_parameters(job_node[INIT].dup, dup_node_id)
-				parameters, dependencies = parse_parameters(prog_parameters[PROG_PARAM].dup, dup_node_id)
-				all_dependencies = dependencies + init_dependencies
-				all_dependencies.uniq!
-
-				#Create node_job
-				command_line = prog_parameters[PROG_NAME]+' '+parameters
-				node_name = "#{tag_root_name}#{dup_node_id}"
-				node = add_program(node_name, prog_parameters[PROG_NAME], command_line, initialization, folder, all_dependencies, done, no_buffer_node)
-				node.cloned_id = dup_node_id
+	def scan_nodes(execution_lines)
+		template_executions = execution_lines.join('')
+		replace_variables(template_executions)
+		# $1 => tag, $2 => initialize, $3 => main command
+		#executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?\s([^\}]{1,})\s{0,}\}/)
+#=begin
+		executions = [] 
+		states = {} #name => [state, id(position)]
+					#t => tag, i => initialize , c => command
+		names = []
+		template_executions.each_line do |line|
+			line.strip! #Clean al whitespaces at beginning and the end of string
+			node = states[names.last] if !names.empty?
+			if line.empty?
+				next
+			elsif line =~ /(\S*\)){$/ #Check tag
+				name = $1
+				executions << [name, '', '']
+				states[name] = ['t', executions.length - 1]
+				names << name
+			elsif line == '?' #Check command
+				node[0] = 'c'
+			elsif states[names.last].first == 't' || states[names.last].first == 'i' #Check initialize
+				node[0] = 'i' if states[names.last].first == 't'
+				executions[node.last][1] << line +"\n"
+			elsif line == '}' #Close node
+				finished_node = names.pop
+				if !names.empty?
+					parent_node = states[names.last].last #position
+					child_node = states[finished_node].last
+					parent_execution = executions[parent_node]
+					if parent_execution[2].class.to_s == 'String'   
+						parent_execution[2] = [child_node]
+					else
+						parent_execution[2] << child_node
+					end
+				end
+			elsif states[names.last].first == 'c' #Check command
+				executions[node.last][2] << line +"\n"			
 			end
 		end
+#=end
+		return executions
 	end
 
-	def set_node_attributes(tag_root_name, dup_node_id, prog_parameters)
-		folder = ''
-		no_buffer_node = TRUE			
-		done = FALSE
-
-		if tag_root_name =~ /\!/ || (!dup_node_id.nil? && dup_node_id =~ /\!/)
-			folder = @exec_folder
-		else
-			folder = asign_folder(prog_parameters[PROG_NAME])
-		end
-		if tag_root_name =~ /&/ || (!dup_node_id.nil? && dup_node_id =~ /&/) #TODO comprobar como va esto
-			no_buffer_node = FALSE
-			folder = @exec_folder#This path is replaced later for the path of the main task that launch all buffered tasks
+	def replace_variables(string)
+		@variables.each do |name, value|
+			string.gsub!(name, value)
 		end
-		done = TRUE if tag_root_name =~ /\%/ && !@do_retry || (!dup_node_id.nil? && dup_node_id =~ /\%/ && !@do_retry)
-
-		tag_root_name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
-		dup_node_id.gsub!(/&|\!|\%|\)/,'') if !dup_node_id.nil?
-		return tag_root_name, dup_node_id, folder, no_buffer_node, done
 	end
 
-	def parse_parameters(command, dup_node_id)
-		dependencies = []
-		if !command.nil?# When command is the initialize, sometimes can be undefined			
-			command.gsub!(/\(\*\)/, dup_node_id) if !dup_node_id.nil? # Define local parameter for duplicated nodes
-			dependencies_direct(command, dependencies)
-			if !dup_node_id.nil?
-				command.scan(/!([\S]+)\*!/).each do |dependency| # Current node depends on a only node of a batch of duplicated nodes
-					dependencies_one2one(command, dependency[0], dependencies, dup_node_id)
-				end
+	def parse_nodes(execution_lines)
+		dinamic_variables = []
+		nodes = scan_nodes(execution_lines)
+		nodes = create_ids(nodes)
+		
+		#nodes.each do |tag, init, command, index|
+		#	puts "#{tag.colorize(:red)}\t#{index}\n#{init.chomp.colorize(:blue)}\n#{command.to_s.colorize(:green)}"
+		#end
+
+		nodes.each do |tag, init, command, index| #Takes the info of each node of workflow for create the job
+			# Set batch
+			new_batch = Batch.new(tag, init, command, index, @exec_folder)			
+
+			#Dependencies
+			scan_dependencies(init, new_batch.dependencies, dinamic_variables)
+			dinamic_variables.concat(collect_dinamic_variables(init, tag))
+			if command.class.to_s == 'String'
+				scan_dependencies(command, new_batch.dependencies, dinamic_variables)
+				dinamic_variables.concat(collect_dinamic_variables(command, tag))
 			end
 
-			one2more = command.scan(/!([\S]+)!([^ \n]+)/) # Current node depends on a full batch of duplicated nodes
-			one2more.concat(command.scan(/!([\S]+)![ \n]/))
-			one2more.each do |dependency|
-				dependencies_one2more(command, dependency, dependencies)
-			end		
+			@commands[new_batch.name] = new_batch
+		end
+		@commands.each do |name, batch|
+			batch.main_command = asign_child_batch(batch.main_command, batch.name) if batch.main_command.class.to_s == 'Array'
+		end
+	end
 
-			replace_variables(command)
+	def create_ids(nodes)
+		nodes.each_with_index do |node, i|
+			node << i
 		end
-		#raise 'Missed dependency on: ' + command if command.include?(')')
-		return command, dependencies
+		return nodes
 	end
 
-	def	dependencies_direct(command, dependencies)
-		@commands.each do |stage_id, node|
-			#Second conditional is triggered when is parsing a batch of tags. The first tag replaces stage_id by path and for second tag, this dependecy is lost
-			folder = node.exec_folder_program
-			if command.include?(stage_id) || command.include?(folder)  
-				dependencies << stage_id
-			end
-			command.gsub!(stage_id+')', folder) #Change tag to folder path
+	def asign_child_batch(batch_ids, name)
+		batches = []
+		batch_ids.each do |id|
+			batch = get_batch(id)
+			batch.parent = name
+			batches << batch
 		end
+		return batches
 	end
 
-	def dependencies_one2one(command, dependency, dependencies, dup_node_id)
-		@commands.keys.each do |st_id|
-			if st_id.include?(dependency) && @commands[st_id].cloned_id == dup_node_id # if a tag id match with expresion, we take it like a dependecy
-				dependencies << st_id
-				command.gsub!("!#{dependency}*!", @commands[st_id].exec_folder_program)
+	def get_batch(id)
+		selected_batch = nil
+		@commands.each do |name, batch|
+			if batch.id == id
+				selected_batch = batch  
 				break
 			end
 		end
+		return selected_batch
 	end
 
-	def dependencies_one2more(command, dependency, dependencies)
-		batch = []
-		@commands.keys.each do |stage_id|
-			if stage_id.include?(dependency[0]) # if a tag id match with expresion, we take it like a dependecy
-				dependencies << stage_id
-				batch << @commands[stage_id].exec_folder_program
+	def collect_dinamic_variables(command, node_name)
+		dinamic_variables = []
+		node_name = node_name.gsub(/&|\!|\%|\)/,'')# Delete function characters
+		if !command.nil? && command.include?('env_manager')
+			command =~ /env_manager "([^"]+)/
+			command =~ /env_manager '([^']+)/ if $1.nil?
+			if !$1.nil?
+				$1.split(';').each do |variable|
+					name, value = variable.split('=')
+					name.gsub!(' ', '') #Remove spaces
+					dinamic_variables << [name, node_name.gsub(')','')]
+				end
 			end
 		end
-		string = batch.map{|tag_id| "#{tag_id}#{dependency[1]}"}.join(' ')
-		command.gsub!("!#{dependency[0]}!#{dependency[1]}", string)
+		return dinamic_variables
 	end
 
-	def list_dup_nodes_ids(tag_node)
-		dup_nodes_ids = [nil]
-		if tag_node =~ /\[([^\]]+)/
-			dup_nodes_ids = []
-			$1.split(';').map{|interval|
-				if interval.include?('-')
-					limits = interval.split('-')
-					dup_nodes_ids.concat((limits.first..limits.last).to_a.map{|n| n.to_s})
-				else
-					dup_nodes_ids << interval
+	def scan_dependencies(command, dependencies, dinamic_variables)
+		if !command.nil?# When command is the initialize, sometimes can be undefined
+			@commands.each do |name, batch|
+				if command.include?(name+')')
+					dependencies << [name, 'simple', name+')']
+				end
+				if command.include?("!#{name}*!")
+					dependencies << [name, '1to1', "!#{name}*!"]
 				end
-			}
+				if command.include?("!#{name}!")
+					command =~ /!#{name}!([^ \n]+)/
+					dependencies << [name, '*to1', "!#{name}!", $1]
+				end
+				local_dependencies = command.scan(/#{name}([^\( \n]+)\)/)
+				local_dependencies.each do |local_dependency|
+					dependencies << [name, 'local', "#{name}#{local_dependency.first}"+')', local_dependency.first] 
+				end
+			end
+			dependencies.concat(check_dependencies_with_DinVar(command, dinamic_variables))
 		end
-		return dup_nodes_ids
-	end
- 
-	def scan_nodes(execution_lines)
-		template_executions = execution_lines.join('')
-		executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?([^\}]{1,})\s{0,}\}/)
-		return executions
 	end
 
-	def add_program(stage_id, name, parameters, initialization, exec_folder_program, dependencies, done, no_buffer_node)
-		tag=Program.new(name, parameters, initialization, exec_folder_program, dependencies, done, no_buffer_node)
-		@commands[stage_id]=tag
-		return tag
-	end
-
-	def replace_variables(command)
-		@variables.each do |name, value|
-			command.gsub!(name, value)
+	def check_dependencies_with_DinVar(command, dinamic_variables)
+		dep = []
+		dinamic_variables.each do |var, name|
+			dep << [name, 'DinVar'] if command.include?(var)
 		end
+		return dep
 	end
 
-	def asign_folder(program_name)			
-		folder=File.join(@exec_folder,"#{program_name}_#{"%04d" % 0}")
-		count=0
-		while @dirs.include?(folder)
-			folder=File.join(@exec_folder,"#{program_name}_#{"%04d" % count}")
-			count+=1
+	def set_path(job, all_jobs_relations)
+		job.dependencies.each do |dep|
+			path = all_jobs_relations[dep]
+			job.initialization.gsub!(dep+')', path)
+			job.parameters.gsub!(dep+')', path)
 		end
-		@dirs << folder
-		return folder
 	end
 
-##########################################################################################
-## LAUNCH WORKFLOW
-##########################################################################################
-
-	def load_queue_manager(options)
-		return QueueManager.select_queue_manager(@exec_folder, options, @commands, @persist_variables)
+	def get_jobs
+		jobs =[]
+		all_jobs_relations = Batch.get_job_relations
+		@commands.each do |name, batch|
+			next if batch.has_jobs?
+			batch.get_jobs.each do |j|
+				set_path(j, all_jobs_relations) 
+				jobs << [j.name, j]
+			end
+		end
+		return jobs
 	end
 
-	def send
-		@q_mgr.exec
+	def get_jobs_relations
+		hash = {}
+		get_jobs.each do |name, job|
+			hash[name] = job
+		end
+		return hash
 	end
 
 ##########################################################################################
@@ -228,41 +265,68 @@ class Stack
 ##########################################################################################
 
 	def inspect
-		@commands.each do |id, tag|
-			puts "#{id} > #{tag.inspect}\t#{tag.done}\n\t\e[32m#{tag.dependencies.join("\n\t")}\e[0m"
+		@jobs.each do |id, job|
+			puts "#{id} > #{job.inspect}\t#{job.attrib[:done]}\n\t\e[32m#{job.dependencies.join("\n\t")}\e[0m"
 		end
 	end
 
 	def draw(name, name_type)
 		representation_type = '_structural'
 		representation_type = '_semantic' if name_type.include?('t')
+		if name_type.include?('b')
+			representation_type << '_simplified'
+			set = @commands
+		else
+			set = @jobs
+		end
 		name.gsub!(/\.\S+/,'')
 		file = File.open(name+representation_type+'.dot','w')
 		file.puts 'digraph G {', 'node[shape=box]'
 		all_dependencies = []
 		all_tag = []
-		@commands.each do |id, tag|
-			tag_name = File.basename(tag.exec_folder_program)
-			if name_type.include?('t')
-				tag_name = id
+		set.each do |id, tag|
+			if name_type.include?('b')
+				tag_name = tag.main_command.split(' ').first+"_#{tag.id}"
+			else 
+				tag_name = File.basename(tag.attrib[:exec_folder])
 			end
+			tag_name = id if name_type.include?('t')
+			tag_name = tag_name + '(*)' if name_type.include?('b') && tag.iterator.length > 1
+			
 			all_tag << tag_name
 			if tag.dependencies.length > 0
-				tag.dependencies.each do |dependencie|
-					dependencie_name = File.basename(@commands[dependencie].exec_folder_program)
-					if name_type.include?('t')
-						dependencie_name = dependencie
+				tag.dependencies.each do |dependencie, type, string|
+					if name_type.include?('b')
+						dependencie_name = set[dependencie].main_command.split(' ').first+"_#{set[dependencie].id}"
+					else 
+						dependencie_name = File.basename(set[dependencie].attrib[:exec_folder])				
 					end
+					dependencie_name = dependencie if name_type.include?('t')
+
+					dependencie_name = dependencie_name + '(*)' if name_type.include?('b') && set[dependencie].iterator.length > 1
 					all_dependencies << dependencie_name
+
 					file.puts "\"#{dependencie_name}\"-> \"#{tag_name}\""
 				end
 			else
-				file.puts "\"#{tag_name}\"[color=mediumseagreen, style=filled]"
+				file.puts "\"#{tag_name}\"[color=black, peripheries=2, style=filled, fillcolor=yellow]"
 			end
 		end
 		all_tag.keep_if{|tag| !all_dependencies.include?(tag)}
 		all_tag.each do |tag|
-			file.puts "\"#{tag}\"[color=palevioletred, style=filled]"
+			if name_type.include?('b')
+				if !name_type.include?('f')
+					tag = tag + '(*)'  if !tag.include?('(*)') && set[tag].iterator.length > 1
+				else
+					id = tag.reverse.split('_',2).first.reverse.to_i
+					batch = nil
+					set.each do |id,tag|
+						batch = tag if tag.id = id 
+					end
+					tag = tag + '(*)'  if batch.iterator.length > 1
+				end
+			end
+			file.puts "\"#{tag}\"[fontcolor=white, color=black, style=filled]"
 		end
 		file.puts '}'
 		file.close

data/lib/autoflow/version.rb

@@ -1,3 +1,3 @@
 module Autoflow
-  VERSION = "0.3.5"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: autoflow
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-22 00:00:00.000000000 Z
+date: 2014-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.8.0
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +96,7 @@ files:
 - bin/env_manager
 - bin/flow_time
 - lib/autoflow.rb
+- lib/autoflow/batch.rb
 - lib/autoflow/program.rb
 - lib/autoflow/queue_manager.rb
 - lib/autoflow/queue_managers/bash_manager.rb