autoflow 0.3.5 → 0.5.0

data/lib/autoflow/queue_manager.rb

@@ -1,21 +1,10 @@
 class QueueManager
 
-	########################################################################################
-	## SELECT AND PREPARE MANAGER
-	########################################################################################
 	def initialize(exec_folder, options, commands, persist_variables)
 		@exec_folder = exec_folder
 		@commands = commands
 		@persist_variables = persist_variables
 		@verbose = options[:verbose]
-		@cpus = options[:cpus]
-		@exp_cpu = options[:exp_cpu]
-		@count_cpu = 0
-		@time = options[:time]
-		@memory = options[:memory]
-		@node_type = options[:node_type]
-		@use_multinode = options[:use_multinode]
-		@use_ntasks = options[:use_ntasks]
 		@job_identifier = options[:identifier]
 		@files = {}
 		@remote = options[:remote]
@@ -23,11 +12,15 @@ class QueueManager
 		@external_dependencies = options[:external_dependencies]
 	end
 
+	########################################################################################
+	## SELECT AND PREPARE MANAGER
+	########################################################################################
+
 	def self.descendants
 		ObjectSpace.each_object(Class).select { |klass| klass < self }
 	end
 
-	def self.select_queue_manager(exec_folder, options, commands, persist_variables)
+	def self.select_queue_manager(stack, options)
 		path_managers = File.join(File.dirname(__FILE__),'queue_managers')
 		Dir.glob(path_managers+'/*').each do |manager|
 			require manager
@@ -37,7 +30,7 @@ class QueueManager
 		else
 			queue_manager = select_manager(options)
 		end
-		return queue_manager.new(exec_folder, options, commands, persist_variables)
+		return queue_manager.new(stack.exec_folder, options, stack.jobs, stack.persist_variables)
 	end
 
 	def self.select_manager(options)
@@ -59,71 +52,101 @@ class QueueManager
 	def exec
 		create_folder(@exec_folder)
 		make_environment_file if !@persist_variables.empty?
+		create_file('versions', @exec_folder)
+		write_file('versions',"autoflow\t#{Autoflow::VERSION}")
+		close_file('versions')
 		create_file('index_execution', @exec_folder)
-		launch_all_tasks
+		launch_all_jobs
 		close_file('index_execution')
 	end
 
-	def launch_all_tasks
-		buffered_nodes = []
-		sort_commands_by_dependencies.each do |id, node|
-			write_file('index_execution', "#{id}\t#{node.exec_folder_program}") 
-			if node.done
+	def launch_all_jobs
+		buffered_jobs = []
+		sort_jobs_by_dependencies.each do |name, job|
+			write_file('index_execution', "#{name}\t#{job.attrib[:exec_folder]}") 
+			if job.attrib[:done]
 				next
 			else
-				rm_done_dependencies(node)
+				rm_done_dependencies(job)
 			end	
-			buffered_nodes = launch_task_in_folder(node, id, buffered_nodes)
+			buffered_jobs = launch_job_in_folder(job, name, buffered_jobs)
+		end
+	end
+
+	def sort_jobs_by_dependencies
+		ar_jobs = @commands.to_a
+		sorted_jobs = []
+		jobs_without_dep = ar_jobs.select{|job| job.last.dependencies.empty?}
+		sorted_jobs.concat(jobs_without_dep)
+		while ar_jobs.length != sorted_jobs.length
+			ids = sorted_jobs.map{|job| job.first}
+			ar_jobs.each do |job|
+				if !sorted_jobs.include?(job) 
+					deps = job.last.dependencies - ids
+					sorted_jobs << job if deps.empty?
+				end
+			end
+		end
+		return sorted_jobs
+	end
+
+	def rm_done_dependencies(job)
+		remove=[]
+		job.dependencies.each do |dependency|			
+			remove << dependency if @commands[dependency].attrib[:done]
+		end
+		remove.each do |rm|
+			job.dependencies.delete(rm)
 		end
 	end
 
-	def launch_task_in_folder(node, id, buffered_nodes)
-		create_folder(node.exec_folder_program)
-		if node.no_buffer_node  # Launch with queue_system nodes and all buffered
-			launch2queue_system(id, node, buffered_nodes)
-			buffered_nodes = []#Clean buffer
-		else # Buffer node
-			buffered_nodes << [id, node]
+	def launch_job_in_folder(job, id, buffered_jobs)
+		create_folder(job.attrib[:exec_folder])
+		if !job.attrib[:buffer]  # Launch with queue_system the job and all buffered jobs
+			launch2queue_system(job, id, buffered_jobs)
+			buffered_jobs = []#Clean buffer
+		else # Buffer job
+			buffered_jobs << [id, job]
 		end
-		return buffered_nodes	
+		return buffered_jobs	
 	end
 
 
-	def launch2queue_system(id, node, buffered_nodes)
+	def launch2queue_system(job, id, buffered_jobs)
 		# Write sh file
 		#--------------------------------
 		log_folder = File.join(@exec_folder, 'log')
-		sh_name = node.name+'.sh'
-		create_file(sh_name, node.exec_folder_program)
+		sh_name = job.name+'.sh'
+		create_file(sh_name, job.attrib[:exec_folder])
 		write_file(sh_name, '#!/usr/bin/env bash')
 		write_file(sh_name, '##JOB_GROUP_ID='+@job_identifier)
-		write_header(id, node, sh_name)
+		write_header(id, job, sh_name)
 
 		#Get dependencies
 		#------------------------------------
-		ar_dependencies = get_dependencies(node, id)
-		buffered_nodes.each do |id_buff_node,buff_node|
-			write_node(buff_node, sh_name)
+		ar_dependencies = get_dependencies(job, id)
+		buffered_jobs.each do |id_buff_job, buff_job|
+			write_job(buff_job, sh_name)
 			ar_dependencies += get_dependencies(buff_node, id_buff_node)
-			buff_node.exec_folder_program = node.exec_folder_program
+			buff_job.attrib[:exec_folder] = job.attrib[:exec_folder]
 		end
 		ar_dependencies.uniq!
 
 		#Write sh body
 		#--------------------------------
 		write_file(sh_name, 'hostname')
-		write_file(sh_name, "echo -e \"STARTED   #{id.gsub(')','')}  #{node.name}:\\t`date`\" >> #{log_folder}")
+		write_file(sh_name, "echo -e \"STARTED   #{id}  #{job.parameters.split.first}:\\t`date`\" >> #{log_folder}")
 		write_file(sh_name, "source #{File.join(@exec_folder, 'env_file')}") if !@persist_variables.empty?
-		write_node(node, sh_name)
-		write_file(sh_name, "echo -e \"FINISHED  #{id.gsub(')','')}  #{node.name}:\\t`date`\" >> #{log_folder}")
+		write_job(job, sh_name)
+		write_file(sh_name, "echo -e \"FINISHED  #{id}  #{job.parameters.split.first}:\\t`date`\" >> #{log_folder}")
 		close_file(sh_name, 0755)
 
 		#Submit node
 		#-----------------------------------
 		if !@verbose
-			queue_id = submit_node(node, ar_dependencies)
-			node.queue_id = queue_id # Returns id of running tag on queue system 
-			asign_queue_id(buffered_nodes, queue_id)
+			queue_id = submit_job(job, ar_dependencies)
+			job.queue_id = queue_id # Returns id of running tag on queue system 
+			asign_queue_id(buffered_jobs, queue_id)
 		end
 	end
 
@@ -185,77 +208,21 @@ class QueueManager
 		return call
 	end
 
-	def rm_done_dependencies(node)
-		remove=[]
-		node.dependencies.each do |dependency|
-			if @commands[dependency].done
-				remove << dependency
-			end
-		end
-		remove.each do |rm|
-			node.dependencies.delete(rm)
-		end
-	end
-
-	def sort_commands_by_dependencies
-		ar_commands = @commands.to_a
-		sorted_commands = []
-		task_without_dep = ar_commands.select{|node| node.last.dependencies.empty?}
-		sorted_commands.concat(task_without_dep)
-		while ar_commands.length != sorted_commands.length
-			ids = sorted_commands.map{|command| command.first}
-			ar_commands.each do |com|
-				if !sorted_commands.include?(com) 
-					deps = com.last.dependencies - ids
-					sorted_commands << com if deps.empty?
-				end
-			end
-		end
-		return sorted_commands
-	end
-
-	def write_node(tag, sh_name)	
-		write_file(sh_name, tag.initialization) if !tag.initialization.nil?
-		cmd = 'time '
-		if !tag.monocpu
-			if tag.parameters.include?('[lcpu]')
-				string = '[lcpu]'
-				used_cpu = 'workers'
-			elsif tag.parameters.include?('[cpu]')
-				string = '[cpu]'
-				used_cpu = @cpus.to_s
-			end
-			tag.parameters.gsub!(string, used_cpu) #Use asigned cpus
-		end
-		cmd << tag.parameters
-		write_file(sh_name, cmd)
-	end
-
-	def asign_cpu(node)
-		used_cpu = 1
-		if !node.monocpu
-			if @exp_cpu == 0 
-				used_cpu = @cpus
-			else #asign exponential cpus
-				if @exp_cpu**(@count_cpu) < @cpus
-					@count_cpu +=1
-				end
-				used_cpu = @exp_cpu**@count_cpu
-			end 
-		end
-		return used_cpu
+	def write_job(job, sh_name)	
+		write_file(sh_name, job.initialization) if !job.initialization.nil?
+		write_file(sh_name, 'time ' + job.parameters)
 	end
 
-	def get_dependencies(node, id = nil)
+	def get_dependencies(job, id = nil)
 		ar_dependencies = []
-		ar_dependencies += node.dependencies
+		ar_dependencies += job.dependencies
 		ar_dependencies.delete(id) if !id.nil? #Delete autodependency
 		return ar_dependencies
 	end
 
-	def asign_queue_id(ar_tags,id)
-		ar_tags.each do |id_ar_tag, ar_tag|
-			ar_tag.queue_id=id
+	def asign_queue_id(ar_jobs, id)
+		ar_jobs.each do |id_job, job|
+			job.queue_id=id
 		end
 	end
 	
@@ -281,7 +248,7 @@ class QueueManager
 
 	end
 
-	def submit_node(node, ar_dependencies)
+	def submit_job(job, ar_dependencies)
 
 	end
 

data/lib/autoflow/queue_managers/bash_manager.rb

@@ -10,7 +10,7 @@ class BashManager < QueueManager
 		write_file('execution.sh', '#! /usr/bin/env bash')
 	end
 
-	def launch_all_tasks
+	def launch_all_jobs
 		super
 		close_file('execution.sh', 0755)
 		system_call("#{@path2execution_script} > #{File.join(File.dirname(@path2execution_script),'output')} & ", @exec_folder)
@@ -20,7 +20,7 @@ class BashManager < QueueManager
 		@queued << id # For dependencies purposes
 	end
 
-	def submit_node(node, ar_dependencies)
+	def submit_job(job, ar_dependencies)
 		write_file('execution.sh','')
 		if !ar_dependencies.empty? 
 			deps = ar_dependencies - @last_deps
@@ -31,8 +31,8 @@ class BashManager < QueueManager
 		end
 		@last_deps.concat(ar_dependencies)
 		@last_deps.uniq!
-		write_file('execution.sh', "cd #{node.exec_folder_program}")
-		write_file('execution.sh', "./#{node.name}.sh &")
+		write_file('execution.sh', "cd #{job.attrib[:exec_folder]}")
+		write_file('execution.sh', "./#{job.name}.sh &")
 		return nil
 	end
 

data/lib/autoflow/queue_managers/slurm_manager.rb

@@ -1,28 +1,26 @@
 require 'queue_manager'
 class SlurmManager < QueueManager
-	def write_header(id, node, sh_name)
-		used_cpu = asign_cpu(node)
-		if !@use_ntasks
-			write_file(sh_name, "#SBATCH --cpus=#{used_cpu}")
+	def write_header(id, job, sh_name)
+		if !job.attrib[:ntask]
+			write_file(sh_name, "#SBATCH --cpus=#{job.attrib[:cpu]}")
 		else
-			write_file(sh_name, "#SBATCH --ntasks=#{used_cpu}")
-			write_file(sh_name, "#SBATCH --nodes=#{@use_multinode}") if @use_multinode > 0
-			write_file(sh_name, 'srun hostname -s > workers') if node.parameters.include?('[lcpu]')
+			write_file(sh_name, "#SBATCH --ntasks=#{job.attrib[:cpu]}")
+			write_file(sh_name, "#SBATCH --nodes=#{job.attrib[:multinode]}") if job.attrib[:multinode] > 0
+			write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
 		end
-		constraint = '#SBATCH --constraint='+@node_type if !@node_type.nil?
-		write_file(sh_name,	"#SBATCH --mem=#{@memory}")
-		write_file(sh_name, "#SBATCH --time=#{@time}")
-		write_file(sh_name,	"#{constraint}")
+		write_file(sh_name,	"#SBATCH --mem=#{job.attrib[:mem]}")
+		write_file(sh_name, "#SBATCH --time=#{job.attrib[:time]}")
+		write_file(sh_name,	"#SBATCH --constraint=#{job.attrib[:node]}") if !job.attrib[:node].nil?
 		write_file(sh_name, '#SBATCH --error=job.%J.err')
 		write_file(sh_name, '#SBATCH --output=job.%J.out')
 	end
 
-	def submit_node(node, ar_dependencies)
+	def submit_job(job, ar_dependencies)
 		final_dep = get_all_deps(ar_dependencies)
 		dependencies = nil
 		dependencies='--dependency=afterok:'+final_dep.join(':') if !final_dep.empty?  
-		cmd = "sbatch #{dependencies} #{node.name}.sh"
-		queue_id = get_queue_system_id(system_call(cmd, node.exec_folder_program))
+		cmd = "sbatch #{dependencies} #{job.name}.sh"
+		queue_id = get_queue_system_id(system_call(cmd, job.attrib[:exec_folder]))
 		return queue_id
 	end