autoflow 0.8.7 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5db665d92bc22dfdf35e0160f45af1510c89563f
-  data.tar.gz: b60d8c6b6f1e7891c5925e2d1abb8e02ce15724c
+  metadata.gz: 03a2fce5cddce218e8903d7b7cff49acd3c6962c
+  data.tar.gz: 1c2019a7d65d8710bc37baa649301c43a1b61585
 SHA512:
-  metadata.gz: 56247ffdc4228a3256fd259f9899af2c61921a6ef410ef09480591601c44f7b16daa3e0e89249f3e5d7e97285ce5a18bbff2036164345058ec7689aa00e355a8
-  data.tar.gz: 16e2f4b8255a2ec76213b001a50257a489d14eed148712a31d75b2c170debe82ef9a0f4138fe46e01eaf6c6983f3d745bbe167e4536dc681d8226e64846d8c37
+  metadata.gz: a52c86fde55c4854e587710c1969e62c282d7fbdd51f645c9a94ce1454882097cecf8f58566f1711cd6c6f4fb92fc02f6e4aef9a3fa46154453b6c09b8a01c6e
+  data.tar.gz: 1d8efc8bd847a2c1fea463497a0a02637582cd744d4654951551759a0e425dd74839622ec23f25a94b938c0a21e35edec129188a04409d9f84079c6171345486

data/bin/AutoFlow

@@ -83,6 +83,11 @@ optparse = OptionParser.new do |opts|
                 options[:add] = add
         end
 
+        options[:additional_job_options] = nil
+        opts.on( '-A', '--additional_job_options STRING', 'Additional option in queue system jobs. Format: "parameter:value"' ) do |opt|
+                options[:additional_job_options] = opt.split(':')
+        end
+
         options[:batch] = false
         opts.on( '-b', '--batch', 'Workflow execution using batch' ) do 
                 options[:batch] = true

data/lib/autoflow/batch.rb

@@ -4,14 +4,15 @@ class Batch
 	@@all_batch = {}
 	@@jobs_names = []
 	@@batch_iterator_relations = {}
-	@@state_iterations = {}
+	@@nested_iteration_relations = {}
 	@@general_computation_attrib = {
 		:cpu => nil, 
 		:mem => nil, 
 		:time => nil,
 		:node => nil,
 		:multinode => nil,
-		:ntask => nil
+		:ntask => nil,
+		:additional_job_options => nil
 	}
  
 	def self.set_general_attrib(attrib_hash)
@@ -20,6 +21,7 @@ class Batch
 
 
 	def initialize(tag, init, main_command, id, exec_folder)
+		@regex_deps = nil
 		replace_regexp(tag, init, main_command)
 		@name = nil
 		@id = id
@@ -63,6 +65,7 @@ class Batch
 		data = /!JobRegExp:([^ \n]+):([^ \n]+)!([^ \n]+)/.match(command) # *to1 with regexp
 		#data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern, data[3] => adyacent string to regexp as regexp/file_name
 		job_names = get_dependencies_by_regexp(data[1], data[2])
+		@regex_deps = 'command' if job_names.length > 0
 		new_string = job_names.map{|jn| jn + ')' + data[3] }.join(' ')
 		command.gsub!(data[0], new_string)
 		#puts command.inspect
@@ -72,6 +75,9 @@ class Batch
 		data = /JobRegExp:([^ \n]+):([^;\] \n]+)/.match(tag) # 1to1 with regexp
 		#data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern
 		job_names = get_dependencies_by_regexp(data[1], data[2])
+		if job_names.length > 0
+			@regex_deps = 'tag' 
+		end
 		new_string = job_names.map{|jn| jn + ')'}.join(';')
 		tag.gsub!(data[0], new_string)
 	end
@@ -188,6 +194,8 @@ class Batch
 						@attrib[:time] = fields[index+1]
 					elsif field == '-u'
 						@attrib[:multinode] = fields[index+1].to_i
+					elsif field == '-A'
+						@attrib[:additional_job_options] = fields[index+1].split(':')
 					end	
 				end
 				if fields.include?('-s')
@@ -253,6 +261,7 @@ class Batch
 					new_job = duplicate_job(tmp_j, iter)
 					check_dependencies(new_job, iter, temp_jobs)
 					parse_iter(iter, @name, new_job)
+					add_nested_iteration_relation(tmp_j, new_job)
 					@@jobs_names << new_job.name
 					jobs << new_job
 					@jobs << new_job
@@ -261,6 +270,7 @@ class Batch
 			end
 			temp_jobs = delete_jobs(jobs2delete, temp_jobs) #Remove temporal jobs
 		else
+			check_regex_dependencies
 			@iterator.each_with_index do |iter, num|
 				job_attrib = @attrib.dup
 				if !iter.nil?
@@ -283,6 +293,61 @@ class Batch
 		return jobs
 	end
 
+	def add_nested_iteration_relation(tmp_j, new_job)
+		query = @@nested_iteration_relations[tmp_j.name]
+		if query.nil?
+			@@nested_iteration_relations[tmp_j.name] = [new_job.name]
+		else
+			query << new_job.name
+		end
+	end
+
+	def check_regex_dependencies
+		if @regex_deps == 'tag'
+			new_job_names = []
+			@iterator.each do |iter|
+				new_names = find_job_names(iter.gsub(')', ''))
+				new_job_names.concat(new_names) 
+			end
+			@iterator = new_job_names.map{|nj| nj + ')'} if !new_job_names.empty?
+		elsif @regex_deps == 'command'
+			[@initialization, @main_command].each do |command|
+				patterns = command.scan(/([^\s)]+)\)([^\s]*)/)
+				if !patterns.empty?
+					patterns.each do |putative_job, sufix|
+						job_names = find_job_names(putative_job)
+						if !job_names.empty?
+							new_string = job_names.map{|jn| "#{jn})#{sufix}"}.join(' ')
+							old_string = "#{putative_job})#{sufix}"
+							command.gsub!(old_string, new_string)
+						end
+					end
+				end
+			end
+		end
+	end
+
+	def find_job_names(name)
+		final_names = []
+		intermediary_names = @@nested_iteration_relations[name]
+		if !intermediary_names.nil?
+			while !intermediary_names.empty?
+				final_names = intermediary_names
+				i_names = []
+				intermediary_names.each do |i_n|
+					query = @@nested_iteration_relations[i_n]
+					i_names.concat(query) if !query.nil?
+				end
+				if !i_names.empty?
+					intermediary_names = i_names
+				else 
+					break
+				end
+			end
+		end
+		return final_names
+	end
+
 	#tmp_j => job to set dependencies in iteration
 	#iter => sufix of current iteration
 	#jobs => array of jobs which has the job dependency	
@@ -313,7 +378,7 @@ class Batch
 	def handle_dependencies(dinamic_variables)
 		[@initialization, @main_command].each do |instructions|
 			if instructions.class.to_s == 'String'
-				scan_dependencies(instructions)
+				#scan_dependencies(instructions) # NOT NECESSARY? REMOVED BY COLLISION CON REGEX SYSTEM. THE DINAMYC VARIABLES ARE NO USED
 				dinamic_variables.concat(collect_dinamic_variables(instructions))
 				@dependencies.concat(check_dependencies_with_DinVar(instructions, dinamic_variables))
 			end

data/lib/autoflow/queue_managers/bash_manager.rb

@@ -42,10 +42,10 @@ class BashManager < QueueManager
 	end
 
 	def self.available?(options)
-		return TRUE
+		return true
 	end
 
 	def self.priority
 		return 0 
 	end
-end
+end

data/lib/autoflow/queue_managers/slurm_manager.rb

@@ -1,20 +1,34 @@
 require 'queue_manager'
 class SlurmManager < QueueManager
+	def parse_additional_options(string, attribs)
+		expresions = %w[%C %T %M %N ]
+		values = [attribs[:cpu], attribs[:time], attribs[:mem], attribs[:node]]
+		new_string = string.dup
+		expresions.each_with_index do |exp, i|
+			new_string.gsub!(exp, "#{values[i]}")
+		end
+		return new_string
+	end
+
 	def write_header(id, job, sh_name)
 		if !job.attrib[:ntask]
 			write_file(sh_name, "#SBATCH --cpus=#{job.attrib[:cpu]}")
 		else
 			write_file(sh_name, "#SBATCH --ntasks=#{job.attrib[:cpu]}")
 			write_file(sh_name, "#SBATCH --nodes=#{job.attrib[:multinode]}") if job.attrib[:multinode] > 0
-			write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
 		end
 		write_file(sh_name,	"#SBATCH --mem=#{job.attrib[:mem]}")
 		write_file(sh_name, "#SBATCH --time=#{job.attrib[:time]}")
 		write_file(sh_name,	"#SBATCH --constraint=#{job.attrib[:node]}") if !job.attrib[:node].nil?
 		write_file(sh_name, '#SBATCH --error=job.%J.err')
 		write_file(sh_name, '#SBATCH --output=job.%J.out')
+		write_file(sh_name, "#SBATCH --#{job.attrib[:additional_job_options][0]}=#{parse_additional_options(job.attrib[:additional_job_options][1], job.attrib)}") if !job.attrib[:additional_job_options].nil?
+		if job.attrib[:ntask]
+			write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
+		end
 	end
 
+
 	def submit_job(job, ar_dependencies)
 		final_dep = get_all_deps(ar_dependencies)
 		dependencies = nil
@@ -34,13 +48,13 @@ class SlurmManager < QueueManager
 	end
 
 	def self.available?(options)
-		available = TRUE
+		available = true
 		shell_output = system_call("type 'sbatch'", nil, options[:remote], options[:ssh])
-		available = FALSE if shell_output.empty?
+		available = false if shell_output.empty?
 		return available
 	end
 
 	def self.priority
 		return 100 
 	end
-end
+end

data/lib/autoflow/stack.rb

@@ -16,7 +16,8 @@ class Stack
 			:time => options[:time],
 			:node => options[:node_type],
 			:multinode => options[:use_multinode],
-			:ntask => options[:use_ntasks]
+			:ntask => options[:use_ntasks],
+			:additional_job_options => options[:additional_job_options]
 		})
 		@@folder_name = :program_name
 		@@folder_name = :job_name if options[:key_name]
@@ -297,7 +298,7 @@ class Stack
 		end
 		file.puts '}'
 		file.close
-		system('dot -Tpng '+name+representation_type+'.dot -o '+name+representation_type+'.png')
+		system('dot -Tpdf '+name+representation_type+'.dot -o '+name+representation_type+'.pdf')
 	end
 	
 end

data/lib/autoflow/version.rb

@@ -1,3 +1,3 @@
 module Autoflow
-  VERSION = "0.8.7"
+  VERSION = "0.9.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autoflow
 version: !ruby/object:Gem::Version
-  version: 0.8.7
+  version: 0.9.2
 platform: ruby
 authors:
 - Pedro Seoane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-04 00:00:00.000000000 Z
+date: 2019-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -161,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: '"This gem take a pipeline and launch it on a queue system"'