autoflow 0.9.6 → 0.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e2701342ff440d0d59eb570fe27b29207d53086ba8136a7a36cf4d6645282c1
-  data.tar.gz: 617a0049af3be6e649947f73eabe68ac38016fd0e687b42ef1c3081faebe3db7
+  metadata.gz: 03720c11e2c6d967b2907823e0de305e34f8e141a792f6f4291ce6db41b3083b
+  data.tar.gz: cf00a1b7ef0f26e6a82c4044792d68b8e66783efe652bff50956c6c706bfa785
 SHA512:
-  metadata.gz: 768d88e9b31cce325d703a27659e63485ae801701bdc4a668218df71888d4517a62be5f95c681f29828a7ef93d00a3cacf51af66e354cd4c56b0d5b415511722
-  data.tar.gz: e5495d58e69440871e743613716866eeb1a93968214cc50fc5da2298beb5cb3d5dadcf67bdcf0ce0ce208696740866805ba6cbfc12cce1d056e2d0ead6f44ce7
+  metadata.gz: 6feeef52e7c4e92f630c2ad9e01d3d08b056a588c562fb87cae587e58dcf213fda8b86e7e61d72df6b507827ec6cb48360c25a3ce8e56fb2c4ed9be899057d52
+  data.tar.gz: 1b27d3309c7e1a30a2e31f01dde55b7b35d9be32bf4ef13c683b002941df60f61bb6385bb57bc6e226325c863f6eceae21170e5d8797f5e2623608da65bf48ce

data/bin/AutoFlow

@@ -9,7 +9,7 @@ require 'optparse'
 require 'autoflow'
 require 'io/console'
 require 'net/ssh'
-require 'queue_manager'
+#require 'queue_manager'
 require 'fileutils'
 require 'git'
 
@@ -347,7 +347,6 @@ if !options[:graph].nil?
         stack.draw(template_file, options[:graph])
 else
         stack.inspect if options[:verbose]
-        stack.comment_main_command if options[:comment]
         options[:write_sh] = true # Set in flow logger to FALSE, it is used for relaunch failed jobs 
         manager = QueueManager.select_queue_manager(stack.exec_folder, options, stack.jobs, stack.persist_variables)
         manager.exec

data/bin/flow_logger

@@ -7,11 +7,8 @@ $: << File.expand_path(File.join(ROOT_PATH, "..", "lib", "autoflow", "queue_mana
 require 'autoflow'
 require 'optparse'
 require 'colorize'
-require 'logging'
 require 'json'
 require 'terminal-table'
-require 'queue_manager'
-require 'program'
 require 'erb'
 
 #################################################################################################

data/lib/autoflow/queue_manager.rb

@@ -1,4 +1,3 @@
-require 'logging'
 require 'json'
 class QueueManager
 
@@ -16,6 +15,7 @@ class QueueManager
 		@external_dependencies = options[:external_dependencies]
 		@active_jobs = []
 		@extended_logging = options[:extended_logging]
+		@comment = options[:comment]
 	end
 
 	########################################################################################
@@ -36,6 +36,7 @@ class QueueManager
 		else
 			queue_manager = select_manager(options)
 		end
+		warn("Selected queue manager: #{queue_manager}")
 		return queue_manager.new(exec_folder, options, jobs, persist_variables)
 	end
 
@@ -268,12 +269,17 @@ class QueueManager
 
 	def write_job(job, sh_name)	
 		write_file(sh_name, job.initialization) if !job.initialization.nil?
-		if @extended_logging
-			log_command = '/usr/bin/time -o process_data -v '
+		if @comment
+			cmd = '#' + job.parameters
 		else
-			log_command = 'time '
+			if @extended_logging
+				log_command = '/usr/bin/time -o process_data -v '
+			else
+				log_command = 'time '
+			end
+			cmd = log_command + job.parameters
 		end
-		write_file(sh_name, log_command + job.parameters)
+		write_file(sh_name, cmd)
 	end
 
 	def get_dependencies(job, id = nil)

data/lib/autoflow/queue_managers/slurm2_manager.rb

@@ -0,0 +1,66 @@
+require 'queue_manager'
+class SlurmManager2 < QueueManager
+	# SLURM 20 or greater
+	def parse_additional_options(string, attribs)
+		expresions = %w[%C %T %M %N ]
+		values = [attribs[:cpu], attribs[:time], attribs[:mem], attribs[:node]]
+		new_string = string.dup
+		expresions.each_with_index do |exp, i|
+			new_string.gsub!(exp, "#{values[i]}")
+		end
+		return new_string
+	end
+
+	def write_header(id, job, sh_name)
+		if !job.attrib[:ntask]
+			write_file(sh_name, "#SBATCH --cpus-per-task=#{job.attrib[:cpu]}")
+		else
+			write_file(sh_name, "#SBATCH --ntasks=#{job.attrib[:cpu]}")
+			write_file(sh_name, "#SBATCH --nodes=#{job.attrib[:multinode]}") if job.attrib[:multinode] > 0
+		end
+		write_file(sh_name,	"#SBATCH --mem=#{job.attrib[:mem]}")
+		write_file(sh_name, "#SBATCH --time=#{job.attrib[:time]}")
+		write_file(sh_name,	"#SBATCH --constraint=#{job.attrib[:node]}") if !job.attrib[:node].nil?
+		write_file(sh_name, '#SBATCH --error=job.%J.err')
+		write_file(sh_name, '#SBATCH --output=job.%J.out')
+		write_file(sh_name, "#SBATCH --#{job.attrib[:additional_job_options][0]}=#{parse_additional_options(job.attrib[:additional_job_options][1], job.attrib)}") if !job.attrib[:additional_job_options].nil?
+		if job.attrib[:ntask]
+			write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
+		end
+	end
+
+
+	def submit_job(job, ar_dependencies)
+		final_dep = get_all_deps(ar_dependencies)
+		dependencies = nil
+		dependencies='--dependency=afterok:'+final_dep.join(':') if !final_dep.empty?  
+		cmd = "sbatch #{dependencies} #{job.name}.sh"
+		STDOUT.puts cmd if @show_submit
+		queue_id = get_queue_system_id(system_call(cmd, job.attrib[:exec_folder]))
+		return queue_id
+	end
+
+	def get_queue_system_id(shell_output)
+		queue_id = nil
+		shell_output.chomp!
+		shell_output =~ /Submitted batch job (\d+)/
+		queue_id = $1
+		raise("A queue id cannot be obtained. The queue manager has given this message:#{shell_output}") if queue_id.nil?
+		return queue_id
+	end
+
+	def self.available?(options)
+		available = false
+		shell_output = system_call("type 'sbatch'", nil, options[:remote], options[:ssh])
+		if !shell_output.empty?
+			shell_output = system_call("sbatch --version", nil, options[:remote], options[:ssh])
+			slurm_version = shell_output.split(' ').last.split('.').first.to_i # "slurm 17.11.4"
+			available = true if slurm_version >= 20
+		end
+		return available
+	end
+
+	def self.priority
+		return 100 
+	end
+end

data/lib/autoflow/queue_managers/slurm_manager.rb

@@ -42,19 +42,24 @@ class SlurmManager < QueueManager
 	def get_queue_system_id(shell_output)
 		queue_id = nil
 		shell_output.chomp!
-		fields = shell_output.split(' ')
-		queue_id = fields[3]
+		shell_output =~ /Submitted batch job (\d+)/
+		queue_id = $1
+		raise("A queue id cannot be obtained. The queue manager has given this message:#{shell_output}") if queue_id.nil?
 		return queue_id
 	end
 
 	def self.available?(options)
-		available = true
+		available = false
 		shell_output = system_call("type 'sbatch'", nil, options[:remote], options[:ssh])
-		available = false if shell_output.empty?
+		if !shell_output.empty?
+			shell_output = system_call("sbatch --version", nil, options[:remote], options[:ssh])
+			slurm_version = shell_output.split(' ').last.split('.').first.to_i # "slurm 17.11.4"
+			available = true if slurm_version < 20
+		end
 		return available
 	end
 
 	def self.priority
-		return 100 
+		return 120 
 	end
 end

data/lib/autoflow/stack.rb

@@ -1,5 +1,5 @@
-require 'program'
-require 'batch'
+#require 'program'
+#require 'batch'
 
 require 'win32console' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
 require 'colorize'
@@ -225,11 +225,6 @@ class Stack
 		return hash
 	end
 
-	def comment_main_command
-		@jobs.each do |name, job|
-			job.parameters = "##{job.parameters}"
-		end
-	end
 ##########################################################################################
 ## WORKFLOW REPRESENTATION
 ##########################################################################################

data/lib/autoflow/version.rb

@@ -1,3 +1,3 @@
 module Autoflow
-  VERSION = "0.9.6"
+  VERSION = "0.9.9"
 end

data/lib/autoflow.rb

@@ -1,6 +1,10 @@
 
 require "autoflow/version"
 require "autoflow/stack"
+require "autoflow/queue_manager"
+require "autoflow/program"
+require "autoflow/batch"
+require "autoflow/logging"
 
 module Autoflow
   # Your code goes here...

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autoflow
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 0.9.9
 platform: ruby
 authors:
 - Pedro Seoane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-22 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -136,6 +136,7 @@ files:
 - lib/autoflow/program.rb
 - lib/autoflow/queue_manager.rb
 - lib/autoflow/queue_managers/bash_manager.rb
+- lib/autoflow/queue_managers/slurm2_manager.rb
 - lib/autoflow/queue_managers/slurm_manager.rb
 - lib/autoflow/stack.rb
 - lib/autoflow/version.rb
@@ -160,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.15
 signing_key:
 specification_version: 4
 summary: '"This gem take a pipeline and launch it on a queue system"'