autoflow 0.8.5 → 0.9.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f312d125afdd65f9895264cda8383258bd155fb3
-  data.tar.gz: 005f0219a73e5f38811df45b30d5fdaba7f243ae
+SHA256:
+  metadata.gz: 1e2701342ff440d0d59eb570fe27b29207d53086ba8136a7a36cf4d6645282c1
+  data.tar.gz: 617a0049af3be6e649947f73eabe68ac38016fd0e687b42ef1c3081faebe3db7
 SHA512:
-  metadata.gz: 2106ec11e38495b7338bef9cf94a02f2df9a9ddaea8a464167872d4a5919ace690dfd9624ea68f60d3fde74fda5d4aab606f4905c3241eb434697b14049628f2
-  data.tar.gz: 2f046987dd54bee1c4c256e5475c7a544ecbaf5735b4b01677f841188457fe5eaf644dcbc2d0089a0f6ea30cc0cbe4c310f333d73b6a2b673a3be8bb11755033
+  metadata.gz: 768d88e9b31cce325d703a27659e63485ae801701bdc4a668218df71888d4517a62be5f95c681f29828a7ef93d00a3cacf51af66e354cd4c56b0d5b415511722
+  data.tar.gz: e5495d58e69440871e743613716866eeb1a93968214cc50fc5da2298beb5cb3d5dadcf67bdcf0ce0ce208696740866805ba6cbfc12cce1d056e2d0ead6f44ce7

data/autoflow.gemspec

@@ -19,10 +19,11 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency 'net-ssh', '>= 2.8.0'
-  spec.add_runtime_dependency 'git', '>= 1.3.0'
+  spec.add_runtime_dependency 'git', '>= 0.8.1'
   spec.add_runtime_dependency 'win32console', '>= 1.3.2' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
-  spec.add_runtime_dependency 'colorize', '~> 0.7.3'
-  spec.add_runtime_dependency 'terminal-table', '~> 1.6.0'
-  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_runtime_dependency 'colorize', '>= 0.7.3'
+  spec.add_runtime_dependency 'terminal-table', '>= 2.0.0'
+  spec.add_runtime_dependency 'openssl', '>= 2.2.0'
+  spec.add_development_dependency "bundler", ">= 2.2.7"
   spec.add_development_dependency "rake"
 end

data/bin/AutoFlow

@@ -83,6 +83,11 @@ optparse = OptionParser.new do |opts|
                 options[:add] = add
         end
 
+        options[:additional_job_options] = nil
+        opts.on( '-A', '--additional_job_options STRING', 'Additional option in queue system jobs. Format: "parameter:value"' ) do |opt|
+                options[:additional_job_options] = opt.split(':')
+        end
+
         options[:batch] = false
         opts.on( '-b', '--batch', 'Workflow execution using batch' ) do 
                 options[:batch] = true
@@ -103,6 +108,11 @@ optparse = OptionParser.new do |opts|
                 options[:external_dependencies] = external_dependencies.split(',')
         end
 
+        options[:extended_logging] = false
+        opts.on( '-e', '--extended_logging', 'If set the command /usr/bin/time will be used instead of shell built-in version. Data will be saved in process_data file of task folder' ) do 
+                options[:extended_logging] = true
+        end
+
         options[:retry] = false
         opts.on( '-f', '--force', 'Execute all jobs, including any job commented with %' ) do 
                 options[:retry] = true

data/bin/flow_logger

@@ -296,6 +296,15 @@ OptionParser.new do |opts|
   opts.on("-H", "--html", "Make a workflow execution full report in html format") do |opt|
     options[:html] = true
   end  
+
+	 # Set a banner, displayed at the top of the help screen.
+	opts.banner = "Usage: flow_logger [options] \n\n"
+
+	# This displays the help screen
+	opts.on( '-h', '--help', 'Display this screen' ) do
+	        puts opts
+	        exit
+	end
 end.parse!
 
 #################################################################################################

data/lib/autoflow/batch.rb

@@ -4,14 +4,15 @@ class Batch
 	@@all_batch = {}
 	@@jobs_names = []
 	@@batch_iterator_relations = {}
-	@@state_iterations = {}
+	@@nested_iteration_relations = {}
 	@@general_computation_attrib = {
 		:cpu => nil, 
 		:mem => nil, 
 		:time => nil,
 		:node => nil,
 		:multinode => nil,
-		:ntask => nil
+		:ntask => nil,
+		:additional_job_options => nil
 	}
  
 	def self.set_general_attrib(attrib_hash)
@@ -20,6 +21,7 @@ class Batch
 
 
 	def initialize(tag, init, main_command, id, exec_folder)
+		@regex_deps = nil
 		replace_regexp(tag, init, main_command)
 		@name = nil
 		@id = id
@@ -63,6 +65,7 @@ class Batch
 		data = /!JobRegExp:([^ \n]+):([^ \n]+)!([^ \n]+)/.match(command) # *to1 with regexp
 		#data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern, data[3] => adyacent string to regexp as regexp/file_name
 		job_names = get_dependencies_by_regexp(data[1], data[2])
+		@regex_deps = 'command' if job_names.length > 0
 		new_string = job_names.map{|jn| jn + ')' + data[3] }.join(' ')
 		command.gsub!(data[0], new_string)
 		#puts command.inspect
@@ -72,6 +75,9 @@ class Batch
 		data = /JobRegExp:([^ \n]+):([^;\] \n]+)/.match(tag) # 1to1 with regexp
 		#data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern
 		job_names = get_dependencies_by_regexp(data[1], data[2])
+		if job_names.length > 0
+			@regex_deps = 'tag' 
+		end
 		new_string = job_names.map{|jn| jn + ')'}.join(';')
 		tag.gsub!(data[0], new_string)
 	end
@@ -188,6 +194,8 @@ class Batch
 						@attrib[:time] = fields[index+1]
 					elsif field == '-u'
 						@attrib[:multinode] = fields[index+1].to_i
+					elsif field == '-A'
+						@attrib[:additional_job_options] = fields[index+1].split(':')
 					end	
 				end
 				if fields.include?('-s')
@@ -253,6 +261,7 @@ class Batch
 					new_job = duplicate_job(tmp_j, iter)
 					check_dependencies(new_job, iter, temp_jobs)
 					parse_iter(iter, @name, new_job)
+					add_nested_iteration_relation(tmp_j, new_job)
 					@@jobs_names << new_job.name
 					jobs << new_job
 					@jobs << new_job
@@ -261,6 +270,7 @@ class Batch
 			end
 			temp_jobs = delete_jobs(jobs2delete, temp_jobs) #Remove temporal jobs
 		else
+			check_regex_dependencies
 			@iterator.each_with_index do |iter, num|
 				job_attrib = @attrib.dup
 				if !iter.nil?
@@ -283,6 +293,61 @@ class Batch
 		return jobs
 	end
 
+	def add_nested_iteration_relation(tmp_j, new_job)
+		query = @@nested_iteration_relations[tmp_j.name]
+		if query.nil?
+			@@nested_iteration_relations[tmp_j.name] = [new_job.name]
+		else
+			query << new_job.name
+		end
+	end
+
+	def check_regex_dependencies
+		if @regex_deps == 'tag'
+			new_job_names = []
+			@iterator.each do |iter|
+				new_names = find_job_names(iter.gsub(')', ''))
+				new_job_names.concat(new_names) 
+			end
+			@iterator = new_job_names.map{|nj| nj + ')'} if !new_job_names.empty?
+		elsif @regex_deps == 'command'
+			[@initialization, @main_command].each do |command|
+				patterns = command.scan(/([^\s)]+)\)([^\s]*)/)
+				if !patterns.empty?
+					patterns.each do |putative_job, sufix|
+						job_names = find_job_names(putative_job)
+						if !job_names.empty?
+							new_string = job_names.map{|jn| "#{jn})#{sufix}"}.join(' ')
+							old_string = "#{putative_job})#{sufix}"
+							command.gsub!(old_string, new_string)
+						end
+					end
+				end
+			end
+		end
+	end
+
+	def find_job_names(name)
+		final_names = []
+		intermediary_names = @@nested_iteration_relations[name]
+		if !intermediary_names.nil?
+			while !intermediary_names.empty?
+				final_names = intermediary_names
+				i_names = []
+				intermediary_names.each do |i_n|
+					query = @@nested_iteration_relations[i_n]
+					i_names.concat(query) if !query.nil?
+				end
+				if !i_names.empty?
+					intermediary_names = i_names
+				else 
+					break
+				end
+			end
+		end
+		return final_names
+	end
+
 	#tmp_j => job to set dependencies in iteration
 	#iter => sufix of current iteration
 	#jobs => array of jobs which has the job dependency	
@@ -313,7 +378,7 @@ class Batch
 	def handle_dependencies(dinamic_variables)
 		[@initialization, @main_command].each do |instructions|
 			if instructions.class.to_s == 'String'
-				scan_dependencies(instructions)
+				#scan_dependencies(instructions) # NOT NECESSARY? REMOVED BY COLLISION CON REGEX SYSTEM. THE DINAMYC VARIABLES ARE NO USED
 				dinamic_variables.concat(collect_dinamic_variables(instructions))
 				@dependencies.concat(check_dependencies_with_DinVar(instructions, dinamic_variables))
 			end
@@ -336,8 +401,10 @@ class Batch
 					@dependencies << [name, '1to1', "!#{name}*!"]
 				end
 				if command.include?("!#{name}!") && !string_overlap(matched_regions, "!#{name}!", command)
-					command =~ /!#{name}!([^ \n]+)/
-					@dependencies << [name, '*to1', "!#{name}!", $1]
+					#command =~ /!#{name}!([^ \n]+)/
+					command.scan(/!#{name}!([^ \n]+)/).each do |string_match|
+						@dependencies << [name, '*to1', "!#{name}!", string_match.first]
+					end
 				end
 				local_dependencies = command.scan(/#{name}([^\( \n]+)\)/)
 				local_dependencies.each do |local_dependency|

data/lib/autoflow/queue_manager.rb

@@ -15,6 +15,7 @@ class QueueManager
 		@write_sh = options[:write_sh]
 		@external_dependencies = options[:external_dependencies]
 		@active_jobs = []
+		@extended_logging = options[:extended_logging]
 	end
 
 	########################################################################################
@@ -267,7 +268,12 @@ class QueueManager
 
 	def write_job(job, sh_name)	
 		write_file(sh_name, job.initialization) if !job.initialization.nil?
-		write_file(sh_name, 'time ' + job.parameters)
+		if @extended_logging
+			log_command = '/usr/bin/time -o process_data -v '
+		else
+			log_command = 'time '
+		end
+		write_file(sh_name, log_command + job.parameters)
 	end
 
 	def get_dependencies(job, id = nil)

data/lib/autoflow/queue_managers/bash_manager.rb

@@ -42,10 +42,10 @@ class BashManager < QueueManager
 	end
 
 	def self.available?(options)
-		return TRUE
+		return true
 	end
 
 	def self.priority
 		return 0 
 	end
-end
+end

data/lib/autoflow/queue_managers/slurm_manager.rb

@@ -1,20 +1,34 @@
 require 'queue_manager'
 class SlurmManager < QueueManager
+	def parse_additional_options(string, attribs)
+		expresions = %w[%C %T %M %N ]
+		values = [attribs[:cpu], attribs[:time], attribs[:mem], attribs[:node]]
+		new_string = string.dup
+		expresions.each_with_index do |exp, i|
+			new_string.gsub!(exp, "#{values[i]}")
+		end
+		return new_string
+	end
+
 	def write_header(id, job, sh_name)
 		if !job.attrib[:ntask]
 			write_file(sh_name, "#SBATCH --cpus=#{job.attrib[:cpu]}")
 		else
 			write_file(sh_name, "#SBATCH --ntasks=#{job.attrib[:cpu]}")
 			write_file(sh_name, "#SBATCH --nodes=#{job.attrib[:multinode]}") if job.attrib[:multinode] > 0
-			write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
 		end
 		write_file(sh_name,	"#SBATCH --mem=#{job.attrib[:mem]}")
 		write_file(sh_name, "#SBATCH --time=#{job.attrib[:time]}")
 		write_file(sh_name,	"#SBATCH --constraint=#{job.attrib[:node]}") if !job.attrib[:node].nil?
 		write_file(sh_name, '#SBATCH --error=job.%J.err')
 		write_file(sh_name, '#SBATCH --output=job.%J.out')
+		write_file(sh_name, "#SBATCH --#{job.attrib[:additional_job_options][0]}=#{parse_additional_options(job.attrib[:additional_job_options][1], job.attrib)}") if !job.attrib[:additional_job_options].nil?
+		if job.attrib[:ntask]
+			write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
+		end
 	end
 
+
 	def submit_job(job, ar_dependencies)
 		final_dep = get_all_deps(ar_dependencies)
 		dependencies = nil
@@ -34,13 +48,13 @@ class SlurmManager < QueueManager
 	end
 
 	def self.available?(options)
-		available = TRUE
+		available = true
 		shell_output = system_call("type 'sbatch'", nil, options[:remote], options[:ssh])
-		available = FALSE if shell_output.empty?
+		available = false if shell_output.empty?
 		return available
 	end
 
 	def self.priority
 		return 100 
 	end
-end
+end

data/lib/autoflow/stack.rb

@@ -16,7 +16,8 @@ class Stack
 			:time => options[:time],
 			:node => options[:node_type],
 			:multinode => options[:use_multinode],
-			:ntask => options[:use_ntasks]
+			:ntask => options[:use_ntasks],
+			:additional_job_options => options[:additional_job_options]
 		})
 		@@folder_name = :program_name
 		@@folder_name = :job_name if options[:key_name]
@@ -76,8 +77,10 @@ class Stack
 				line.gsub!(/\s/,'')
 				pairs = line.split(',')
 				pairs.each do |pair|
-					pair =~ /(.+)=(.+)/
-					variable_type[$1] = $2
+					#pair =~ /(.+)=(.+)/
+					#variable_type[$1] = $2
+					var, value = pair.split('=', 2)
+					variable_type[var] = value
 				end
 			end
 		end
@@ -297,7 +300,7 @@ class Stack
 		end
 		file.puts '}'
 		file.close
-		system('dot -Tpng '+name+representation_type+'.dot -o '+name+representation_type+'.png')
+		system('dot -Tpdf '+name+representation_type+'.dot -o '+name+representation_type+'.pdf')
 	end
 	
 end

data/lib/autoflow/version.rb

@@ -1,3 +1,3 @@
 module Autoflow
-  VERSION = "0.8.5"
+  VERSION = "0.9.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autoflow
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.9.6
 platform: ruby
 authors:
 - Pedro Seoane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-30 00:00:00.000000000 Z
+date: 2021-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -30,56 +30,70 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 0.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.3
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: openssl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 2.2.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 2.2.7
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -131,7 +145,7 @@ homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -146,9 +160,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.8
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: '"This gem take a pipeline and launch it on a queue system"'
 test_files: