autoflow 0.3.5 → 0.5.0

data/autoflow.gemspec

@@ -19,6 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency 'net-ssh', '>= 2.8.0'
+  spec.add_runtime_dependency 'win32console', '>= 1.3.2' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
+  spec.add_runtime_dependency 'colorize', '~> 0.7.3'
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
 end

data/bin/AutoFlow

@@ -9,6 +9,62 @@ require 'optparse'
 require 'autoflow'
 require 'io/console'
 require 'net/ssh'
+require 'queue_manager'
+require 'fileutils'
+
+#################################################################################################
+# METHODS
+#################################################################################################
+def get_templates(string_template)
+        templates = string_template.split(',') 
+        return templates
+end
+
+def get_repositories
+    directories = []  
+    directories = ENV['WORKFLOW_REPOSITORY'].split(':') if !ENV['WORKFLOW_REPOSITORY'].nil?
+    return directories
+end
+
+def exit_exec(message)
+        puts message
+        Process.exit(-1)
+end
+
+def list_repository_templates
+        templates = []
+        if !ENV['WORKFLOW_REPOSITORY']. nil?
+                directories = get_repositories
+                directories.each do |dir|
+                        templates.concat(Dir.entries(dir))
+                end
+        end
+        templates.delete('.')
+        templates.delete('..')
+        return templates
+end
+
+def get_template_path(template)
+        path = nil
+        directories = get_repositories
+        directories.each do |dir|
+                if Dir.entries(dir).include?(template)
+                        path = File.join(dir, template) 
+                        break
+                end
+        end       
+        return path
+end
+
+def get_description(path)
+        File.open(path).each do |line|
+                line.chomp!
+                if line =~ /^#=/
+                        line.gsub!('#=','')
+                        puts line
+                end
+        end
+end
 
 #################################################################################################
 # INPUT PARSING
@@ -16,6 +72,11 @@ require 'net/ssh'
 options = {}
 template_file = ''
 optparse = OptionParser.new do |opts|
+        options[:add] = nil
+        opts.on( '-a', '--add STRING', 'Puts a copy of a selected workflow template in repository' ) do |add|
+                options[:add] = add
+        end
+
         options[:batch] = FALSE
         opts.on( '-b', '--batch', 'Workflow execution using batch' ) do 
                 options[:batch] = TRUE
@@ -31,21 +92,26 @@ optparse = OptionParser.new do |opts|
                 options[:external_dependencies] = external_dependencies.split(',')
         end
 
-        options[:exp_cpu] = 0
-        opts.on( '-e', '--exp_cpu INTEGER', 'Exponent of cpu assigment series' ) do |exp_cpu|
-                options[:exp_cpu] = exp_cpu.to_i
-        end
-
         options[:retry] = FALSE
         opts.on( '-f', '--force', 'Execute all jobs, included the jobs commented with %' ) do 
                 options[:retry] = TRUE
         end
 
         options[:graph] = nil
-        opts.on( '-g', '--graph STRING', 'Draw the template. t for use tag like names or f for use folders names instead' ) do |graph| 
+        opts.on( '-g', '--graph STRING', 'Draw the template. t for use tag like names or f for use folders names instead. This option cancel the workflow launching' ) do |graph| 
                 options[:graph] = graph
         end
 
+        options[:identifier] = FALSE
+        opts.on( '-i', '--job_identifier STRING', 'Identifier tag for each launching script' ) do |identifier|
+                options[:identifier] = identifier
+        end
+
+        options[:list] = nil
+        opts.on( 'l', '--list_repository STRING', 'List template names in repository') do |name|
+                options[:list] = name
+        end
+
         options[:memory] = '4gb'
         opts.on( '-m', '--memory STRING', 'Max memory can be used in a task' ) do |mem|
                 options[:memory] = mem
@@ -61,6 +127,11 @@ optparse = OptionParser.new do |opts|
                 options[:output] = output
         end
 
+        options[:pull] = nil
+        opts.on( '-p', '--pull STRING', 'Get a copy of a workflow template in repository.' ) do |pull|
+                options[:pull] = pull
+        end    
+
         options[:remote] = FALSE
         opts.on( '-r', '--remote', 'Connect with remote machine and launch the workflow' ) do 
                 options[:remote] = TRUE
@@ -74,8 +145,13 @@ optparse = OptionParser.new do |opts|
                 puts options[:ssh].exec!('hostname')
         end
 
+        options[:use_ntasks] = FALSE
+        opts.on( '-s', '--use_ntasks', 'Use -ntasks flag with sh' ) do 
+                options[:use_ntasks] = TRUE
+        end
+
         options[:time] = '20:00:00'
-        opts.on( '-t', '--time STRING', 'Max time that can be needed in a task' ) do |time|
+        opts.on( '-t', '--time STRING', 'Max time that can be needed in a task. Format: dd-hh:mm:ss' ) do |time|
                 options[:time] = time
         end
 
@@ -84,10 +160,6 @@ optparse = OptionParser.new do |opts|
                 options[:use_multinode] = use_multinode.to_i
         end     
 
-        options[:use_ntasks] = FALSE
-        opts.on( '-s', '--use_ntasks', 'Use -ntasks flag with sh' ) do 
-                options[:use_ntasks] = TRUE
-        end
 
         options[:verbose] = FALSE
         opts.on( '-v', '--verbose', 'Show info without launch jobs' ) do 
@@ -95,8 +167,8 @@ optparse = OptionParser.new do |opts|
         end
 
         options[:Variables] = nil
-        opts.on( '-V', '--Variables STRING', 'Variables to be parsed on template. Format: \'$variable_name1=value1;$variable_name2=value2;...\'' ) do |mem|
-                options[:Variables] = mem.split(';')
+        opts.on( '-V', '--Variables STRING', 'Variables to be parsed on template. Format: \'$variable_name1=value1,$variable_name2=value2,...\'' ) do |mem|
+                options[:Variables] = [mem]
         end
 
         options[:workflow] = FALSE
@@ -105,10 +177,6 @@ optparse = OptionParser.new do |opts|
                 template_file = workflow
         end
 
-        options[:identifier] = FALSE
-        opts.on( '-i', '--job_identifier STRING', 'Identifier tag for each launching script' ) do |identifier|
-                options[:identifier] = identifier
-        end
 
         # Set a banner, displayed at the top of the help screen.
         opts.banner = "Usage: AutoFlow.rb -w worflow_file -c n_cpus \n\n"
@@ -124,23 +192,85 @@ end # End opts
 # parse options and remove from ARGV
 optparse.parse!
 
-if !options[:workflow] || !File.exists?(options[:workflow]) 
-	puts 'Workflow file not especified or not exists'
-	Process.exit(-1)
-else
-        options[:identifier] = "#{options[:workflow]}_#{Time.new.to_i}" if !options[:identifier]
-        options[:workflow] = File.open(options[:workflow]).read
-end
-
 #################################################################################################
 # MAIN
 #################################################################################################
+
+# List templates
+templates_rep_names = list_repository_templates
+if !options[:list].nil?
+        if options[:list] == 'all'
+                puts templates_rep_names
+        else
+                path = get_template_path(options[:list])
+                get_description(path) if !path.nil?
+        end
+        exit_exec('')
+end
+
+# Get a template
+if !options[:pull].nil?
+        path = get_template_path(options[:pull])
+        if !path.nil?
+                FileUtils.cp(path, Dir.pwd)
+        else
+                exit_exec("Template not found: #{options[:pull]}")
+        end
+        exit_exec('')
+end
+
+# Upload a template
+if !options[:add].nil? && File.exists?(options[:add])
+        directories = get_repositories
+        writable_dirs = []
+        directories.each_with_index do |dir, i|
+                if File.directory?(dir) && File.writable?(dir)
+                       puts "#{i}\t#{dir}"
+                       writable_dirs << dir 
+                end
+        end
+        if !writable_dirs.empty?
+                msg = ''
+                puts "Select a directory"
+                select = gets.chomp.to_i
+                FileUtils.cp(options[:add], writable_dirs[select])
+        else
+                msg = "You don't have a writable directory"
+        end
+        exit_exec(msg)
+end
+
+# Merge templates
+templates = get_templates(options[:workflow])
+options[:workflow] = ''
+options[:identifier] = "#{File.basename(templates.first)}_#{Time.new.to_i}" if !options[:identifier]
+
+if !options[:workflow]
+        exit_exec('Workflow not especified')
+else
+        templates.each do |template|
+                if  !File.exists?(template) && !templates_rep_names.empty? && !templates_rep_names.include?(template)
+                        exit_exec("Workflow not found: #{template}")
+                        Process.exit(-1)
+                else
+                        if File.exists?(template)
+                                path = template
+                        else
+                                path = get_template_path(template)
+                        end
+                        options[:workflow] += File.open(path).read+"\n"
+                end
+        end
+end
+
+# Set local or remote execution
 if options[:remote]
         main_path = options[:ssh].exec!('pwd').chomp
 else
         main_path = Dir.pwd
 end
 
+# Set output directory
 if options[:output] == 'exec'
         exec_folder = File.join(main_path,'exec')
 else
@@ -151,17 +281,16 @@ end
 #--------------------------------------------------------------------------------
 # Flow parse
 #--------------------------------------------------------------------------------
-stack=Stack.new(exec_folder, options)
-stack.draw(template_file, options[:graph]) if !options[:graph].nil?
-
+stack = Stack.new(exec_folder, options)
+manager = QueueManager.select_queue_manager(stack, options)
+ 
 #--------------------------------------------------------------------------------
 # Flow exec
 #--------------------------------------------------------------------------------
-if options[:verbose]
-        stack.inspect
-        stack.send
-elsif options[:graph].nil?
-        stack.send
+if !options[:graph].nil?
+        stack.draw(template_file, options[:graph])
+else
+        stack.inspect if options[:verbose]
+        manager.exec
 end
 options[:ssh].close if options[:remote]
-

data/lib/autoflow/batch.rb

@@ -0,0 +1,342 @@
+class Batch
+	attr_accessor :name, :iterator, :dependencies, :init, :main_command, :attrib, :id, :jobs, :parent
+
+	@@all_batch = {}
+	@@all_jobs_relations = {}
+	@@batch_iterator_relations = {}
+	@@state_iterations = {}
+	@@general_computation_attrib = {
+		:cpu => nil, 
+		:mem => nil, 
+		:time => nil,
+		:node => nil,
+		:multinode => nil,
+		:ntask => nil
+	}
+
+	def self.set_general_attrib(attrib_hash)
+		@@general_computation_attrib = attrib_hash
+	end
+
+	def self.get_job_relations
+		return @@all_jobs_relations
+	end
+
+	def initialize(tag, init, main_command, id, exec_folder)
+		@name = nil
+		@id = id
+		@iterator = [nil] 
+		@parent = nil
+		@dependencies = [] # [batch_name, dependency_type, keyword2replace]
+			# nil => There isn't dependecies, 
+			# 'simple' => One job needs a previous job, 
+			# '1to1' => A job in a batch needs another job in other batch, 
+			# '*to1' => A job need a previous full batch of jobs
+			# 'local' => A simple job needs one job of a batch
+		@initialization = init
+		@main_command = main_command
+		@attrib = {
+			:done => FALSE,
+			:folder => TRUE,
+			:buffer => FALSE,
+			:exec_folder => exec_folder,
+			:cpu_asign => nil # number, list or mono
+		}.merge(@@general_computation_attrib)
+		get_name_and_iterators_and_modifiers(tag)
+		set_execution_attrib
+		set_cpu
+		@jobs = []
+		@@all_batch[@name] = self
+	end
+
+	def has_jobs?
+		res = !@jobs.empty?
+		return res
+	end
+
+	def get_name_and_iterators_and_modifiers(tag)
+		tag =~ /(^.+)\[([^\]]+)/
+		name = $1
+		if $1.nil?
+			tag =~ /(^.+)\)/
+			name = $1
+		end
+		@name , @attrib[:done], @attrib[:folder], @attrib[:buffer] = check_execution_modifiers(name)
+		if !$2.nil?
+			@iterator = []
+			$2.split(';').map{|interval|
+				if interval.include?('-')
+					limits = interval.split('-')
+					@iterator.concat((limits.first..limits.last).to_a.map{|n| n.to_s})
+				else
+					@iterator << interval
+				end
+			}
+		end
+	end
+
+	def check_execution_modifiers(name)
+		done = FALSE
+		folder = TRUE
+		buffer = FALSE
+		done = TRUE if name.include?('%')
+		folder = FALSE if name.include?('!')
+		buffer = TRUE if name.include?('&')
+		name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
+		return name, done, folder, buffer
+	end
+
+	def set_execution_attrib
+		@initialization = scan_resources(@initialization) if !@initialization.nil?
+		@main_command = scan_resources(@main_command) if @main_command.class.to_s == 'String'
+	end
+
+	def scan_resources(command)
+		resources_line = nil
+		command.each_line do |line|
+			if line.include?('resources:')
+				resources_line = line
+				fields = line.split(' ')
+				fields.each_with_index do |field, index|
+					if field == '-c'
+						@attrib[:cpu] = fields[index+1].to_i
+					elsif field == '-m'
+						@attrib[:mem] = fields[index+1]
+					elsif field == '-n'
+						@attrib[:node] = fields[index+1]
+					elsif field == '-t'
+						@attrib[:time] = fields[index+1]
+					elsif field == '-u'
+						@attrib[:multinode] = fields[index+1].to_i
+					end	
+				end
+				if fields.include?('-s')
+					@attrib[:ntask] = TRUE
+				else
+					@attrib[:ntask] = FALSE
+				end
+			end
+		end
+		command.gsub!(resources_line, '') if !resources_line.nil?
+		return command
+	end
+
+	def set_cpu
+		@initialization = scan_cpu(@initialization) if !@initialization.nil?
+		@main_command = scan_cpu(@main_command) if @main_command.class.to_s == 'String'
+		@attrib[:cpu] = 1 if @attrib[:cpu_asign] == 'mono'
+	end
+
+	def scan_cpu(command)
+		if command.include?('[cpu]')
+			command.gsub!('[cpu]', @attrib[:cpu].to_s)
+			@attrib[:cpu_asign] = 'number'
+		elsif command.include?('[lcpu]')
+			command.gsub!('[lcpu]', 'workers')
+			@attrib[:cpu_asign] = 'list'
+		elsif @attrib[:cpu_asign].nil?
+			@attrib[:cpu_asign] = 'mono'
+		end				
+		return command
+	end
+
+
+	def asign_folder(local_command = nil)
+		if local_command.nil? #we check a simple job
+			command = @main_command
+		else #we check a cloned job
+			command = local_command 
+		end 
+		if command.class.to_s == 'Array'
+			folder = nil
+		elsif @attrib[:folder]
+			program = File.join(@attrib[:exec_folder], command.split(' ', 2).first) 			
+			count = 0
+			folder = program + "_#{"%04d" % count}"
+			while @@all_jobs_relations.values.include?(folder)
+				folder = program + "_#{"%04d" % count}"
+				count += 1
+			end
+		else
+			folder = @attrib[:exec_folder]
+		end
+		return folder
+	end
+
+	def duplicate_job(tmp_j, sufix_name = '')
+		new_job = tmp_j.clone
+		new_job.name = tmp_j.name+'_'+sufix_name
+		new_job.attrib = tmp_j.attrib.clone
+		new_job.dependencies = tmp_j.dependencies.clone
+		new_job.initialization = tmp_j.initialization.clone
+		new_job.parameters = tmp_j.parameters.clone
+		new_job.attrib[:exec_folder] = asign_folder(new_job.parameters)
+		return new_job
+	end
+
+	def delete_jobs(jobs2delete, job_array)
+		jobs2delete.uniq!
+		jobs2delete.sort{|s1, s2| s2 <=> s1}.each do |index|
+			job_array.delete_at(index)
+		end
+		return job_array
+	end
+
+	def get_jobs
+		jobs = []
+		@@batch_iterator_relations[@name] = @iterator
+		if @main_command.class.to_s == 'Array' # There are nested batchs
+			temp_jobs = []
+			@main_command.each do |batch|
+				temp_jobs.concat(batch.get_jobs)
+			end
+			jobs2delete = []
+			@iterator.each_with_index do |iter, i|
+				temp_jobs.each_with_index do |tmp_j, tmp_i|
+					new_job = duplicate_job(tmp_j, iter)
+					check_dependencies(new_job, iter, temp_jobs)
+					parse_iter(iter, @name, new_job)
+					jobs << new_job
+					@jobs << new_job
+					@@all_jobs_relations[new_job.name] = new_job.attrib[:exec_folder]
+					jobs2delete << tmp_i
+				end
+			end
+			temp_jobs = delete_jobs(jobs2delete, temp_jobs) #Remove temporal jobs
+		else
+			@iterator.each_with_index do |iter, num|
+				job_attrib = @attrib.dup
+				if !iter.nil?
+					iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter)
+					job_attrib[:done] = done if !@attrib[:done] # To keep attrib priority in batch on job
+				end
+				name = "#{@name}#{iter}"
+				job_attrib[:exec_folder] = asign_folder if @parent.nil? #Don't asign folder to nested batches (iterative batchs)
+				job_dependencies = []
+				initialization = replace_dependencies(@initialization, job_dependencies, iter, num)
+				parameters = replace_dependencies(@main_command, job_dependencies, iter, num)
+				job = Program.new(name, initialization, parameters, job_dependencies, job_attrib)
+				job.batch = @name
+				jobs << job
+				@jobs << job 
+				@@all_jobs_relations[name] = job_attrib[:exec_folder]
+			end
+		end
+		return jobs
+	end
+
+	#tmp_j => job to set dependencies in iteration
+	#iter => sufix of current iteration
+	#jobs => array of jobs which has the job dependency	
+	def check_dependencies(tmp_j, iter, jobs)
+		jobs_names = jobs.map{|job| job.name}
+		deps = {}
+		tmp_j.dependencies.each_with_index do |dep, i|		
+			deps[dep] = i if jobs_names.include?(dep)
+		end
+		deps.each do |name, index|
+			dep = name+'_'+iter
+			tmp_j.initialization.gsub!(name+')', dep+')')
+			tmp_j.parameters.gsub!(name+')', dep+')')
+			tmp_j.dependencies[index] = dep 
+		end
+	end
+
+	def parse_iter(iter, name, job)
+		job.parameters = set_iter(name, iter, job.parameters)
+		job.initialization = set_iter(name, iter, job.initialization)
+	end
+
+	def set_iter(name, iter, string)
+		string = string.gsub(name+'(+)', iter)
+		return string
+	end
+
+	def replace_dependencies(command, job_dependencies, iter, num)
+		if !command.nil?
+			@dependencies.each do |batch_name, dep_type, dep_keyword2replace, dep_info|
+				if dep_type == 'simple'
+					if @@all_batch[batch_name].parent.nil?
+						new_string = @@all_jobs_relations[batch_name]
+					end
+					job_dependencies << batch_name					
+				elsif dep_type == '1to1'
+					if @@all_batch[batch_name].parent.nil? || !@parent.nil?
+						dep_name = "#{batch_name}#{@@batch_iterator_relations[batch_name][num]}"
+					else
+						root_batch = get_root(batch_name)
+						selected_jobs = root_batch.get_jobs_by_batch_name(batch_name)
+						dep_name = selected_jobs[num].name
+					end
+					job_dependencies << dep_name
+					if !@parent.nil? && !@@all_batch[batch_name].parent.nil?
+						new_string = dep_name + ')' 
+					else
+						new_string = @@all_jobs_relations[dep_name]
+					end
+				elsif dep_type == '*to1'
+					if @@all_batch[batch_name].parent.nil?
+						new_string = @@batch_iterator_relations[batch_name].map{|iter|
+							dep_name =  batch_name + iter
+							job_dependencies << dep_name
+							"#{@@all_jobs_relations[dep_name]}#{dep_info}"
+						}.join(' ')
+						dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
+					elsif !@parent.nil? 
+						new_string = @@batch_iterator_relations[batch_name].map{|iter|
+							dep_name =  batch_name + iter
+							job_dependencies << dep_name
+							"#{dep_name})#{dep_info}"
+						}.join(' ')
+						dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"						
+					else
+						root_batch = get_root(batch_name)
+						selected_jobs = root_batch.get_jobs_by_batch_name(batch_name)
+						new_string = selected_jobs.map{|j|
+							job_dependencies << j.name
+							"#{@@all_jobs_relations[j.name]}#{dep_info}"
+						}.join(' ')
+						dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}"
+					end
+				elsif dep_type == 'local'
+					if @@all_batch[batch_name].parent.nil? || !@parent.nil?
+						if @@batch_iterator_relations[batch_name].include?(dep_info) #This avoids cross dependencies by similar names
+							dep_name = batch_name + dep_info
+							job_dependencies << dep_name
+							if !@parent.nil? && !@@all_batch[batch_name].parent.nil?
+								new_string = dep_name + ')'
+							else
+								new_string = @@all_jobs_relations[dep_name]
+							end
+						end
+					else
+						dep_name = dep_keyword2replace.gsub(')','') #This avoids cross dependencies by similar names
+						if !@@all_jobs_relations[dep_name].nil?
+							job_dependencies << dep_name
+							new_string = @@all_jobs_relations[dep_name]
+						end
+					end
+				elsif dep_type == 'DinVar'
+					job_dependencies << batch_name if batch_name != @name # This condition avoids autodependencies
+				end
+				job_dependencies.uniq!
+				command = command.gsub(dep_keyword2replace, new_string) if dep_type != 'DinVar' && !dep_keyword2replace.nil? && !new_string.nil?
+			end
+
+			command = command.gsub('(*)', "#{iter}") if command.class.to_s == 'String'
+		end 
+		return command
+	end
+
+	def get_root(batch_name)
+		root_batch = @@all_batch[batch_name]
+		root_batch = root_batch.get_root(root_batch.parent) if !root_batch.nil? && !root_batch.parent.nil?
+		return root_batch
+	end
+
+	def get_jobs_by_batch_name(batch_name)
+		jobs = @jobs.select{|j| j.batch == batch_name}
+		return jobs
+	end
+
+end

data/lib/autoflow/program.rb

@@ -1,22 +1,25 @@
 class Program
-	attr_accessor :name, :exec_folder_program, :parameters, :initialization, :queue_id, :done, :dependencies, :monocpu, :no_buffer_node,:cloned_id
-	def initialize(name, parameters, initialization, exec_folder_program, dependencies, done, no_buffer_node)
-		@name=name
-    	@parameters=parameters
-		@initialization=initialization
-		@exec_folder_program=exec_folder_program
-		@queue_id=nil
-		@done=done
-		@dependencies=dependencies
-		@no_buffer_node=no_buffer_node
-		@monocpu=TRUE
-		@cloned_id = nil
-		if @parameters =~ /\[cpu|lcpu\]/ #||@parameters =~ /\[lcpu\]/ 
-			@monocpu=FALSE
-		end
+	attr_accessor :name, :initialization, :parameters, :dependencies, :attrib, :queue_id, :batch
+
+	def initialize(name, initialization, parameters, dependencies, job_attrib)
+		@name = name
+		@initialization = initialization
+    	@parameters = parameters
+		@dependencies = dependencies
+		@attrib = job_attrib
+		@queue_id = nil
+		@batch = nil
 	end
 
 	def inspect
-		string="\e[31m#{@name}\e[0m\n\t\e[33m#{@parameters.gsub("\n","\n\t")}\e[0m\t\e[34m#{@exec_folder_program}\e[0m"
+		if @parameters.class.to_s == 'String'
+			program = @parameters.split(' ').first
+			command = @parameters.gsub("\n","\n\t")
+		else
+			program = 'iterative_job'
+			command = @parameters.map{|b| b}.join(' ')
+		end
+		string="\e[31m#{program}\n\e[0m\t\e[33m#{command}\e[0m\e[34m#{@attrib[:exec_folder]}\e[0m"
 	end
+
 end