autoflow 0.9.2 → 0.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 03a2fce5cddce218e8903d7b7cff49acd3c6962c
-  data.tar.gz: 1c2019a7d65d8710bc37baa649301c43a1b61585
+SHA256:
+  metadata.gz: 03720c11e2c6d967b2907823e0de305e34f8e141a792f6f4291ce6db41b3083b
+  data.tar.gz: cf00a1b7ef0f26e6a82c4044792d68b8e66783efe652bff50956c6c706bfa785
 SHA512:
-  metadata.gz: a52c86fde55c4854e587710c1969e62c282d7fbdd51f645c9a94ce1454882097cecf8f58566f1711cd6c6f4fb92fc02f6e4aef9a3fa46154453b6c09b8a01c6e
-  data.tar.gz: 1d8efc8bd847a2c1fea463497a0a02637582cd744d4654951551759a0e425dd74839622ec23f25a94b938c0a21e35edec129188a04409d9f84079c6171345486
+  metadata.gz: 6feeef52e7c4e92f630c2ad9e01d3d08b056a588c562fb87cae587e58dcf213fda8b86e7e61d72df6b507827ec6cb48360c25a3ce8e56fb2c4ed9be899057d52
+  data.tar.gz: 1b27d3309c7e1a30a2e31f01dde55b7b35d9be32bf4ef13c683b002941df60f61bb6385bb57bc6e226325c863f6eceae21170e5d8797f5e2623608da65bf48ce

data/autoflow.gemspec

@@ -19,11 +19,11 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency 'net-ssh', '>= 2.8.0'
-  spec.add_runtime_dependency 'git', '>= 1.3.0'
+  spec.add_runtime_dependency 'git', '>= 0.8.1'
   spec.add_runtime_dependency 'win32console', '>= 1.3.2' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
-  spec.add_runtime_dependency 'colorize', '~> 0.7.3'
-  spec.add_runtime_dependency 'terminal-table', '~> 1.6.0'
-  spec.add_runtime_dependency 'openssl', '>= 2.0.6'
-  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_runtime_dependency 'colorize', '>= 0.7.3'
+  spec.add_runtime_dependency 'terminal-table', '>= 2.0.0'
+  spec.add_runtime_dependency 'openssl', '>= 2.2.0'
+  spec.add_development_dependency "bundler", ">= 2.2.7"
   spec.add_development_dependency "rake"
 end

data/bin/AutoFlow

@@ -9,7 +9,7 @@ require 'optparse'
 require 'autoflow'
 require 'io/console'
 require 'net/ssh'
-require 'queue_manager'
+#require 'queue_manager'
 require 'fileutils'
 require 'git'
 
@@ -108,6 +108,11 @@ optparse = OptionParser.new do |opts|
                 options[:external_dependencies] = external_dependencies.split(',')
         end
 
+        options[:extended_logging] = false
+        opts.on( '-e', '--extended_logging', 'If set the command /usr/bin/time will be used instead of shell built-in version. Data will be saved in process_data file of task folder' ) do 
+                options[:extended_logging] = true
+        end
+
         options[:retry] = false
         opts.on( '-f', '--force', 'Execute all jobs, including any job commented with %' ) do 
                 options[:retry] = true
@@ -342,7 +347,6 @@ if !options[:graph].nil?
         stack.draw(template_file, options[:graph])
 else
         stack.inspect if options[:verbose]
-        stack.comment_main_command if options[:comment]
         options[:write_sh] = true # Set in flow logger to FALSE, it is used for relaunch failed jobs 
         manager = QueueManager.select_queue_manager(stack.exec_folder, options, stack.jobs, stack.persist_variables)
         manager.exec

data/bin/flow_logger

@@ -7,11 +7,8 @@ $: << File.expand_path(File.join(ROOT_PATH, "..", "lib", "autoflow", "queue_mana
 require 'autoflow'
 require 'optparse'
 require 'colorize'
-require 'logging'
 require 'json'
 require 'terminal-table'
-require 'queue_manager'
-require 'program'
 require 'erb'
 
 #################################################################################################
@@ -296,6 +293,15 @@ OptionParser.new do |opts|
   opts.on("-H", "--html", "Make a workflow execution full report in html format") do |opt|
     options[:html] = true
   end  
+
+	 # Set a banner, displayed at the top of the help screen.
+	opts.banner = "Usage: flow_logger [options] \n\n"
+
+	# This displays the help screen
+	opts.on( '-h', '--help', 'Display this screen' ) do
+	        puts opts
+	        exit
+	end
 end.parse!
 
 #################################################################################################

data/lib/autoflow/queue_manager.rb

@@ -1,4 +1,3 @@
-require 'logging'
 require 'json'
 class QueueManager
 
@@ -15,6 +14,8 @@ class QueueManager
 		@write_sh = options[:write_sh]
 		@external_dependencies = options[:external_dependencies]
 		@active_jobs = []
+		@extended_logging = options[:extended_logging]
+		@comment = options[:comment]
 	end
 
 	########################################################################################
@@ -35,6 +36,7 @@ class QueueManager
 		else
 			queue_manager = select_manager(options)
 		end
+		warn("Selected queue manager: #{queue_manager}")
 		return queue_manager.new(exec_folder, options, jobs, persist_variables)
 	end
 
@@ -267,7 +269,17 @@ class QueueManager
 
 	def write_job(job, sh_name)	
 		write_file(sh_name, job.initialization) if !job.initialization.nil?
-		write_file(sh_name, 'time ' + job.parameters)
+		if @comment
+			cmd = '#' + job.parameters
+		else
+			if @extended_logging
+				log_command = '/usr/bin/time -o process_data -v '
+			else
+				log_command = 'time '
+			end
+			cmd = log_command + job.parameters
+		end
+		write_file(sh_name, cmd)
 	end
 
 	def get_dependencies(job, id = nil)

data/lib/autoflow/queue_managers/slurm2_manager.rb

@@ -0,0 +1,66 @@
+require 'queue_manager'
+class SlurmManager2 < QueueManager
+	# SLURM 20 or greater
+	def parse_additional_options(string, attribs)
+		expresions = %w[%C %T %M %N ]
+		values = [attribs[:cpu], attribs[:time], attribs[:mem], attribs[:node]]
+		new_string = string.dup
+		expresions.each_with_index do |exp, i|
+			new_string.gsub!(exp, "#{values[i]}")
+		end
+		return new_string
+	end
+
+	def write_header(id, job, sh_name)
+		if !job.attrib[:ntask]
+			write_file(sh_name, "#SBATCH --cpus-per-task=#{job.attrib[:cpu]}")
+		else
+			write_file(sh_name, "#SBATCH --ntasks=#{job.attrib[:cpu]}")
+			write_file(sh_name, "#SBATCH --nodes=#{job.attrib[:multinode]}") if job.attrib[:multinode] > 0
+		end
+		write_file(sh_name,	"#SBATCH --mem=#{job.attrib[:mem]}")
+		write_file(sh_name, "#SBATCH --time=#{job.attrib[:time]}")
+		write_file(sh_name,	"#SBATCH --constraint=#{job.attrib[:node]}") if !job.attrib[:node].nil?
+		write_file(sh_name, '#SBATCH --error=job.%J.err')
+		write_file(sh_name, '#SBATCH --output=job.%J.out')
+		write_file(sh_name, "#SBATCH --#{job.attrib[:additional_job_options][0]}=#{parse_additional_options(job.attrib[:additional_job_options][1], job.attrib)}") if !job.attrib[:additional_job_options].nil?
+		if job.attrib[:ntask]
+			write_file(sh_name, 'srun hostname -s > workers') if job.attrib[:cpu_asign] == 'list'
+		end
+	end
+
+
+	def submit_job(job, ar_dependencies)
+		final_dep = get_all_deps(ar_dependencies)
+		dependencies = nil
+		dependencies='--dependency=afterok:'+final_dep.join(':') if !final_dep.empty?  
+		cmd = "sbatch #{dependencies} #{job.name}.sh"
+		STDOUT.puts cmd if @show_submit
+		queue_id = get_queue_system_id(system_call(cmd, job.attrib[:exec_folder]))
+		return queue_id
+	end
+
+	def get_queue_system_id(shell_output)
+		queue_id = nil
+		shell_output.chomp!
+		shell_output =~ /Submitted batch job (\d+)/
+		queue_id = $1
+		raise("A queue id cannot be obtained. The queue manager has given this message:#{shell_output}") if queue_id.nil?
+		return queue_id
+	end
+
+	def self.available?(options)
+		available = false
+		shell_output = system_call("type 'sbatch'", nil, options[:remote], options[:ssh])
+		if !shell_output.empty?
+			shell_output = system_call("sbatch --version", nil, options[:remote], options[:ssh])
+			slurm_version = shell_output.split(' ').last.split('.').first.to_i # "slurm 17.11.4"
+			available = true if slurm_version >= 20
+		end
+		return available
+	end
+
+	def self.priority
+		return 100 
+	end
+end

data/lib/autoflow/queue_managers/slurm_manager.rb

@@ -42,19 +42,24 @@ class SlurmManager < QueueManager
 	def get_queue_system_id(shell_output)
 		queue_id = nil
 		shell_output.chomp!
-		fields = shell_output.split(' ')
-		queue_id = fields[3]
+		shell_output =~ /Submitted batch job (\d+)/
+		queue_id = $1
+		raise("A queue id cannot be obtained. The queue manager has given this message:#{shell_output}") if queue_id.nil?
 		return queue_id
 	end
 
 	def self.available?(options)
-		available = true
+		available = false
 		shell_output = system_call("type 'sbatch'", nil, options[:remote], options[:ssh])
-		available = false if shell_output.empty?
+		if !shell_output.empty?
+			shell_output = system_call("sbatch --version", nil, options[:remote], options[:ssh])
+			slurm_version = shell_output.split(' ').last.split('.').first.to_i # "slurm 17.11.4"
+			available = true if slurm_version < 20
+		end
 		return available
 	end
 
 	def self.priority
-		return 100 
+		return 120 
 	end
 end

data/lib/autoflow/stack.rb

@@ -1,5 +1,5 @@
-require 'program'
-require 'batch'
+#require 'program'
+#require 'batch'
 
 require 'win32console' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
 require 'colorize'
@@ -77,8 +77,10 @@ class Stack
 				line.gsub!(/\s/,'')
 				pairs = line.split(',')
 				pairs.each do |pair|
-					pair =~ /(.+)=(.+)/
-					variable_type[$1] = $2
+					#pair =~ /(.+)=(.+)/
+					#variable_type[$1] = $2
+					var, value = pair.split('=', 2)
+					variable_type[var] = value
 				end
 			end
 		end
@@ -223,11 +225,6 @@ class Stack
 		return hash
 	end
 
-	def comment_main_command
-		@jobs.each do |name, job|
-			job.parameters = "##{job.parameters}"
-		end
-	end
 ##########################################################################################
 ## WORKFLOW REPRESENTATION
 ##########################################################################################

data/lib/autoflow/version.rb

@@ -1,3 +1,3 @@
 module Autoflow
-  VERSION = "0.9.2"
+  VERSION = "0.9.9"
 end

data/lib/autoflow.rb

@@ -1,6 +1,10 @@
 
 require "autoflow/version"
 require "autoflow/stack"
+require "autoflow/queue_manager"
+require "autoflow/program"
+require "autoflow/batch"
+require "autoflow/logging"
 
 module Autoflow
   # Your code goes here...

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autoflow
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.9
 platform: ruby
 authors:
 - Pedro Seoane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-24 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -30,70 +30,70 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 0.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.3
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: openssl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 2.2.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 2.2.7
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +136,7 @@ files:
 - lib/autoflow/program.rb
 - lib/autoflow/queue_manager.rb
 - lib/autoflow/queue_managers/bash_manager.rb
+- lib/autoflow/queue_managers/slurm2_manager.rb
 - lib/autoflow/queue_managers/slurm_manager.rb
 - lib/autoflow/stack.rb
 - lib/autoflow/version.rb
@@ -145,7 +146,7 @@ homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -160,9 +161,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: '"This gem take a pipeline and launch it on a queue system"'
 test_files: