autoflow 0.3.5

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in autoflow.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Autoflow
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'autoflow'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install autoflow
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/autoflow.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'autoflow/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "autoflow"
+  spec.version       = Autoflow::VERSION
+  spec.authors       = ["Pedro Seoane"]
+  spec.email         = ["seoanezonjic@hotmail.com"]
+  spec.description   = %q{"Autoflow makes easy to launch big pipelines on a queue system. Only works with SLURM & PBS"}
+  spec.summary       = %q{"This gem take a pipeline and launch it on a queue system"}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'net-ssh', '>= 2.8.0'
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/bin/AutoFlow

@@ -0,0 +1,167 @@
+#!/usr/bin/env ruby
+
+ROOT_PATH=File.dirname(__FILE__)
+$: << File.expand_path(File.join(ROOT_PATH, "../lib/"))
+$: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/"))
+$: << File.expand_path(File.join(ROOT_PATH, "../lib/autoflow/queue_managers"))
+
+require 'optparse'
+require 'autoflow'
+require 'io/console'
+require 'net/ssh'
+
+#################################################################################################
+# INPUT PARSING
+#################################################################################################
+options = {}
+template_file = ''
+optparse = OptionParser.new do |opts|
+        options[:batch] = FALSE
+        opts.on( '-b', '--batch', 'Workflow execution using batch' ) do 
+                options[:batch] = TRUE
+        end
+
+        options[:cpus] = 16
+        opts.on( '-c', '--cpus INTEGER', 'Max cpus can be used in all workflow' ) do |cpus|
+                options[:cpus] = cpus.to_i
+        end
+
+        options[:external_dependencies] = []
+        opts.on( '-d', '--external_dependencies STRING', 'The workflow will start when indicated jobs finish on queue system. Format: \'id1,id2,id3..\'') do |external_dependencies|
+                options[:external_dependencies] = external_dependencies.split(',')
+        end
+
+        options[:exp_cpu] = 0
+        opts.on( '-e', '--exp_cpu INTEGER', 'Exponent of cpu assigment series' ) do |exp_cpu|
+                options[:exp_cpu] = exp_cpu.to_i
+        end
+
+        options[:retry] = FALSE
+        opts.on( '-f', '--force', 'Execute all jobs, included the jobs commented with %' ) do 
+                options[:retry] = TRUE
+        end
+
+        options[:graph] = nil
+        opts.on( '-g', '--graph STRING', 'Draw the template. t for use tag like names or f for use folders names instead' ) do |graph| 
+                options[:graph] = graph
+        end
+
+        options[:memory] = '4gb'
+        opts.on( '-m', '--memory STRING', 'Max memory can be used in a task' ) do |mem|
+                options[:memory] = mem
+        end
+
+        options[:node_type] = nil
+        opts.on( '-n', '--node_type STRING', 'Apply constraint attribute to tasks' ) do |node_type|
+                options[:node_type] = node_type
+        end
+
+        options[:output] = 'exec'
+        opts.on( '-o', '--output STRING', 'Output folder of flow' ) do |output|
+                options[:output] = output
+        end
+
+        options[:remote] = FALSE
+        opts.on( '-r', '--remote', 'Connect with remote machine and launch the workflow' ) do 
+                options[:remote] = TRUE
+                puts 'Host to connect and launch workflow:'
+                host = gets.chomp
+                puts 'User:'
+                user = gets.chomp
+                puts 'Password (hidden)'
+                password = STDIN.noecho(&:gets).chomp
+                options[:ssh] = Net::SSH.start(host, user, :password => password, :auth_methods => ['keyboard-interactive'])
+                puts options[:ssh].exec!('hostname')
+        end
+
+        options[:time] = '20:00:00'
+        opts.on( '-t', '--time STRING', 'Max time that can be needed in a task' ) do |time|
+                options[:time] = time
+        end
+
+        options[:use_multinode] = 0 
+        opts.on( '-u', '--use_multinode INTEGER', 'For use several nodes on execution' ) do |use_multinode|
+                options[:use_multinode] = use_multinode.to_i
+        end     
+
+        options[:use_ntasks] = FALSE
+        opts.on( '-s', '--use_ntasks', 'Use -ntasks flag with sh' ) do 
+                options[:use_ntasks] = TRUE
+        end
+
+        options[:verbose] = FALSE
+        opts.on( '-v', '--verbose', 'Show info without launch jobs' ) do 
+                options[:verbose] = TRUE
+        end
+
+        options[:Variables] = nil
+        opts.on( '-V', '--Variables STRING', 'Variables to be parsed on template. Format: \'$variable_name1=value1;$variable_name2=value2;...\'' ) do |mem|
+                options[:Variables] = mem.split(';')
+        end
+
+        options[:workflow] = FALSE
+        opts.on( '-w', '--workflow FILE', 'Input workflow file' ) do |workflow|
+                options[:workflow] = workflow
+                template_file = workflow
+        end
+
+        options[:identifier] = FALSE
+        opts.on( '-i', '--job_identifier STRING', 'Identifier tag for each launching script' ) do |identifier|
+                options[:identifier] = identifier
+        end
+
+        # Set a banner, displayed at the top of the help screen.
+        opts.banner = "Usage: AutoFlow.rb -w worflow_file -c n_cpus \n\n"
+
+        # This displays the help screen
+        opts.on( '-h', '--help', 'Display this screen' ) do
+                puts opts
+                exit
+        end
+
+end # End opts
+
+# parse options and remove from ARGV
+optparse.parse!
+
+if !options[:workflow] || !File.exists?(options[:workflow]) 
+	puts 'Workflow file not especified or not exists'
+	Process.exit(-1)
+else
+        options[:identifier] = "#{options[:workflow]}_#{Time.new.to_i}" if !options[:identifier]
+        options[:workflow] = File.open(options[:workflow]).read
+end
+
+#################################################################################################
+# MAIN
+#################################################################################################
+if options[:remote]
+        main_path = options[:ssh].exec!('pwd').chomp
+else
+        main_path = Dir.pwd
+end
+
+if options[:output] == 'exec'
+        exec_folder = File.join(main_path,'exec')
+else
+        exec_folder = options[:output]       
+        exec_folder = File.join(main_path, options[:output]) if exec_folder[0] != '/' && exec_folder[0] != '~'
+end 
+
+#--------------------------------------------------------------------------------
+# Flow parse
+#--------------------------------------------------------------------------------
+stack=Stack.new(exec_folder, options)
+stack.draw(template_file, options[:graph]) if !options[:graph].nil?
+
+#--------------------------------------------------------------------------------
+# Flow exec
+#--------------------------------------------------------------------------------
+if options[:verbose]
+        stack.inspect
+        stack.send
+elsif options[:graph].nil?
+        stack.send
+end
+options[:ssh].close if options[:remote]
+

data/bin/env_manager

@@ -0,0 +1,23 @@
+#! /usr/bin/env ruby
+
+# Parse enviroment variables file
+new_values = {}
+env_path = '../env_file' # A level up
+if !File.exists?(env_path)
+	env_path ='env_path' # Local
+	if !File.exists?(env_path)
+		raise 'Enviroment file not found'
+	end
+end
+pairs = File.open(env_path).read.scan(/export ([^=]+)=([\S]+)/)
+pairs.map{|pair| new_values[pair.first] = pair.last}
+
+# Change enviroment variables (Commandline must be 'var=value;var1=value2...varN=valueN;')
+ARGV[0].scan(/([^=]+)=([^;]+);/).map{|new_pair| new_values[new_pair.first]=new_pair.last}
+
+# Save modified enviroment variables
+new_enviroment = File.open(env_path,'w')
+new_values.each do |env_var, value|
+	new_enviroment.puts "export #{env_var}=#{value}"
+end
+new_enviroment.close

data/bin/flow_time

@@ -0,0 +1,52 @@
+#! /usr/bin/env ruby
+require 'scbi_plot'
+
+times = {}
+task = nil
+status = nil
+beg_time = Time.now
+end_time = Time.new(0)
+File.open(ARGV[0]).read.each_line do |line|
+	line.chomp!
+	description, date = line.split("\t")
+	status, task, program = description.split(' ')
+	day, month, n_day, time, mode, year = date.split(' ')
+	hours, minutes, seconds = time.split(':') 
+	task_time = Time.local(year.to_i, month, n_day.to_i, hours.to_i, minutes.to_i, seconds.to_i)
+	if task_time > end_time
+		end_time = task_time
+	end
+	if task_time < beg_time
+		beg_time = task_time
+	end
+	if status == 'STARTED'
+		times[task] = [task_time]
+	else
+		times[task] << task_time
+	end
+end
+
+tag_task = []
+plot_times = [] 
+total_time = (end_time - beg_time).to_i/3600.0
+tag_task << 'Total_time'
+plot_times << total_time
+puts "Total time: #{total_time} hours"
+times.each do |task, interval|
+	if interval.length == 2
+		task_time = (interval.last - interval.first).to_i/3600.0
+		tag_task << task
+		plot_times << task_time
+		puts "#{task}\t#{task_time} hours"
+	else
+		puts "#{task}\tStarted at #{interval.first}. Currently running"
+	end
+end
+
+# create Histogram
+tag_task.map!{|tag| tag.gsub('e','\\e')}
+tag_task.map!{|tag| tag.gsub('E','\\E')}
+p=ScbiPlot::Histogram.new('time_graph.png','Flow stats (hours)')
+p.add_x(tag_task) 
+p.add_y(plot_times)
+p.do_graph

data/lib/autoflow/program.rb

@@ -0,0 +1,22 @@
+class Program
+	attr_accessor :name, :exec_folder_program, :parameters, :initialization, :queue_id, :done, :dependencies, :monocpu, :no_buffer_node,:cloned_id
+	def initialize(name, parameters, initialization, exec_folder_program, dependencies, done, no_buffer_node)
+		@name=name
+    	@parameters=parameters
+		@initialization=initialization
+		@exec_folder_program=exec_folder_program
+		@queue_id=nil
+		@done=done
+		@dependencies=dependencies
+		@no_buffer_node=no_buffer_node
+		@monocpu=TRUE
+		@cloned_id = nil
+		if @parameters =~ /\[cpu|lcpu\]/ #||@parameters =~ /\[lcpu\]/ 
+			@monocpu=FALSE
+		end
+	end
+
+	def inspect
+		string="\e[31m#{@name}\e[0m\n\t\e[33m#{@parameters.gsub("\n","\n\t")}\e[0m\t\e[34m#{@exec_folder_program}\e[0m"
+	end
+end

data/lib/autoflow/queue_manager.rb

@@ -0,0 +1,299 @@
+class QueueManager
+
+	########################################################################################
+	## SELECT AND PREPARE MANAGER
+	########################################################################################
+	def initialize(exec_folder, options, commands, persist_variables)
+		@exec_folder = exec_folder
+		@commands = commands
+		@persist_variables = persist_variables
+		@verbose = options[:verbose]
+		@cpus = options[:cpus]
+		@exp_cpu = options[:exp_cpu]
+		@count_cpu = 0
+		@time = options[:time]
+		@memory = options[:memory]
+		@node_type = options[:node_type]
+		@use_multinode = options[:use_multinode]
+		@use_ntasks = options[:use_ntasks]
+		@job_identifier = options[:identifier]
+		@files = {}
+		@remote = options[:remote]
+		@ssh = options[:ssh]
+		@external_dependencies = options[:external_dependencies]
+	end
+
+	def self.descendants
+		ObjectSpace.each_object(Class).select { |klass| klass < self }
+	end
+
+	def self.select_queue_manager(exec_folder, options, commands, persist_variables)
+		path_managers = File.join(File.dirname(__FILE__),'queue_managers')
+		Dir.glob(path_managers+'/*').each do |manager|
+			require manager
+		end
+		if options[:batch]
+			queue_manager = BashManager
+		else
+			queue_manager = select_manager(options)
+		end
+		return queue_manager.new(exec_folder, options, commands, persist_variables)
+	end
+
+	def self.select_manager(options)
+		queue_manager = nil
+		priority = 0
+		descendants.each do |descendant|
+			if descendant.available?(options) && priority <= descendant.priority
+				queue_manager = descendant
+				priority = descendant.priority
+			end
+		end
+		return queue_manager
+	end
+
+	########################################################################################
+	## EXECUTING WORKFLOW WITH MANAGER
+	########################################################################################
+
+	def exec
+		create_folder(@exec_folder)
+		make_environment_file if !@persist_variables.empty?
+		create_file('index_execution', @exec_folder)
+		launch_all_tasks
+		close_file('index_execution')
+	end
+
+	def launch_all_tasks
+		buffered_nodes = []
+		sort_commands_by_dependencies.each do |id, node|
+			write_file('index_execution', "#{id}\t#{node.exec_folder_program}") 
+			if node.done
+				next
+			else
+				rm_done_dependencies(node)
+			end	
+			buffered_nodes = launch_task_in_folder(node, id, buffered_nodes)
+		end
+	end
+
+	def launch_task_in_folder(node, id, buffered_nodes)
+		create_folder(node.exec_folder_program)
+		if node.no_buffer_node  # Launch with queue_system nodes and all buffered
+			launch2queue_system(id, node, buffered_nodes)
+			buffered_nodes = []#Clean buffer
+		else # Buffer node
+			buffered_nodes << [id, node]
+		end
+		return buffered_nodes	
+	end
+
+
+	def launch2queue_system(id, node, buffered_nodes)
+		# Write sh file
+		#--------------------------------
+		log_folder = File.join(@exec_folder, 'log')
+		sh_name = node.name+'.sh'
+		create_file(sh_name, node.exec_folder_program)
+		write_file(sh_name, '#!/usr/bin/env bash')
+		write_file(sh_name, '##JOB_GROUP_ID='+@job_identifier)
+		write_header(id, node, sh_name)
+
+		#Get dependencies
+		#------------------------------------
+		ar_dependencies = get_dependencies(node, id)
+		buffered_nodes.each do |id_buff_node,buff_node|
+			write_node(buff_node, sh_name)
+			ar_dependencies += get_dependencies(buff_node, id_buff_node)
+			buff_node.exec_folder_program = node.exec_folder_program
+		end
+		ar_dependencies.uniq!
+
+		#Write sh body
+		#--------------------------------
+		write_file(sh_name, 'hostname')
+		write_file(sh_name, "echo -e \"STARTED   #{id.gsub(')','')}  #{node.name}:\\t`date`\" >> #{log_folder}")
+		write_file(sh_name, "source #{File.join(@exec_folder, 'env_file')}") if !@persist_variables.empty?
+		write_node(node, sh_name)
+		write_file(sh_name, "echo -e \"FINISHED  #{id.gsub(')','')}  #{node.name}:\\t`date`\" >> #{log_folder}")
+		close_file(sh_name, 0755)
+
+		#Submit node
+		#-----------------------------------
+		if !@verbose
+			queue_id = submit_node(node, ar_dependencies)
+			node.queue_id = queue_id # Returns id of running tag on queue system 
+			asign_queue_id(buffered_nodes, queue_id)
+		end
+	end
+
+	def make_environment_file
+		create_file('env_file', @exec_folder)
+		@persist_variables.each do |var, value|
+			write_file('env_file', "export #{var}=#{value}")
+		end
+		close_file('env_file')
+	end
+
+	def create_folder(folder_name)
+		if @remote
+			@ssh.exec!("if ! [ -d  #{folder_name} ]; then mkdir -p #{folder_name}; fi")
+		else
+			Dir.mkdir(folder_name) if !File.exists?(folder_name)
+		end
+	end
+
+	def create_file(file_name, path) 
+		@files[file_name] = [path, '']
+	end
+
+	def write_file(file_name, content)
+		@files[file_name].last << content+"\n"
+	end
+
+	def close_file(file_name, permissions = nil) #SSH
+		path, content = @files.delete(file_name)
+		file_path = File.join(path, file_name)
+		if @remote
+			@ssh.exec!("echo '#{content}' > #{file_path}")
+			@ssh.exec!("chmod #{permissions} #{file_path}") if !permissions.nil?
+		else
+			local_file = File.open(file_path,'w')
+			local_file.chmod(permissions) if !permissions.nil?
+			local_file.print content
+			local_file.close
+		end
+	end
+
+	def system_call(cmd, path = nil)
+		cmd = "cd #{path}; " + cmd if !path.nil?
+		if @remote
+			call = @ssh.exec!(cmd)
+		else
+			call = %x[#{cmd}] 
+		end
+		return call
+	end
+
+	def self.system_call(cmd, path = nil, remote = FALSE, ssh = nil)
+		cmd = "cd #{path}; " + cmd if !path.nil?
+		if remote
+			call = ssh.exec!(cmd)
+		else
+			call = %x[#{cmd}] 
+		end
+		return call
+	end
+
+	def rm_done_dependencies(node)
+		remove=[]
+		node.dependencies.each do |dependency|
+			if @commands[dependency].done
+				remove << dependency
+			end
+		end
+		remove.each do |rm|
+			node.dependencies.delete(rm)
+		end
+	end
+
+	def sort_commands_by_dependencies
+		ar_commands = @commands.to_a
+		sorted_commands = []
+		task_without_dep = ar_commands.select{|node| node.last.dependencies.empty?}
+		sorted_commands.concat(task_without_dep)
+		while ar_commands.length != sorted_commands.length
+			ids = sorted_commands.map{|command| command.first}
+			ar_commands.each do |com|
+				if !sorted_commands.include?(com) 
+					deps = com.last.dependencies - ids
+					sorted_commands << com if deps.empty?
+				end
+			end
+		end
+		return sorted_commands
+	end
+
+	def write_node(tag, sh_name)	
+		write_file(sh_name, tag.initialization) if !tag.initialization.nil?
+		cmd = 'time '
+		if !tag.monocpu
+			if tag.parameters.include?('[lcpu]')
+				string = '[lcpu]'
+				used_cpu = 'workers'
+			elsif tag.parameters.include?('[cpu]')
+				string = '[cpu]'
+				used_cpu = @cpus.to_s
+			end
+			tag.parameters.gsub!(string, used_cpu) #Use asigned cpus
+		end
+		cmd << tag.parameters
+		write_file(sh_name, cmd)
+	end
+
+	def asign_cpu(node)
+		used_cpu = 1
+		if !node.monocpu
+			if @exp_cpu == 0 
+				used_cpu = @cpus
+			else #asign exponential cpus
+				if @exp_cpu**(@count_cpu) < @cpus
+					@count_cpu +=1
+				end
+				used_cpu = @exp_cpu**@count_cpu
+			end 
+		end
+		return used_cpu
+	end
+
+	def get_dependencies(node, id = nil)
+		ar_dependencies = []
+		ar_dependencies += node.dependencies
+		ar_dependencies.delete(id) if !id.nil? #Delete autodependency
+		return ar_dependencies
+	end
+
+	def asign_queue_id(ar_tags,id)
+		ar_tags.each do |id_ar_tag, ar_tag|
+			ar_tag.queue_id=id
+		end
+	end
+	
+	def get_queue_system_dependencies(ar_dependencies)
+		queue_system_ids=[]
+		ar_dependencies.each do |dependency|
+			queue_system_ids << @commands[dependency].queue_id
+		end
+		return queue_system_ids
+	end
+
+	def get_all_deps(ar_dependencies)
+		final_dep = []
+		final_dep.concat(get_queue_system_dependencies(ar_dependencies)) if !ar_dependencies.empty?
+		final_dep.concat(@external_dependencies)
+		return final_dep
+	end
+
+	########################################################################################
+	## QUEUE DEPENDANT METHODS
+	########################################################################################
+	def write_header(id, node, sh)
+
+	end
+
+	def submit_node(node, ar_dependencies)
+
+	end
+
+	def get_queue_system_id(shell_output)
+
+	end
+
+	def self.available?
+		return FALSE
+	end
+
+	def self.priority
+		return -1
+	end
+end

data/lib/autoflow/queue_managers/bash_manager.rb

@@ -0,0 +1,50 @@
+require 'queue_manager'
+class BashManager < QueueManager
+
+	def initialize(exec_folder, options, commands, persist_variables)
+		super
+		@queued = []
+		@last_deps = []
+		@path2execution_script = File.join(@exec_folder, 'execution.sh') 
+		create_file('execution.sh', @exec_folder)
+		write_file('execution.sh', '#! /usr/bin/env bash')
+	end
+
+	def launch_all_tasks
+		super
+		close_file('execution.sh', 0755)
+		system_call("#{@path2execution_script} > #{File.join(File.dirname(@path2execution_script),'output')} & ", @exec_folder)
+	end
+
+	def write_header(id, node, sh)
+		@queued << id # For dependencies purposes
+	end
+
+	def submit_node(node, ar_dependencies)
+		write_file('execution.sh','')
+		if !ar_dependencies.empty? 
+			deps = ar_dependencies - @last_deps
+			if !deps.empty?
+				write_file('execution.sh', 'wait') 
+				@last_deps.concat(@queued)
+			end
+		end
+		@last_deps.concat(ar_dependencies)
+		@last_deps.uniq!
+		write_file('execution.sh', "cd #{node.exec_folder_program}")
+		write_file('execution.sh', "./#{node.name}.sh &")
+		return nil
+	end
+
+	def get_queue_system_id(shell_output)
+		return nil
+	end
+
+	def self.available?(options)
+		return TRUE
+	end
+
+	def self.priority
+		return 0 
+	end
+end

data/lib/autoflow/queue_managers/slurm_manager.rb

@@ -0,0 +1,47 @@
+require 'queue_manager'
+class SlurmManager < QueueManager
+	def write_header(id, node, sh_name)
+		used_cpu = asign_cpu(node)
+		if !@use_ntasks
+			write_file(sh_name, "#SBATCH --cpus=#{used_cpu}")
+		else
+			write_file(sh_name, "#SBATCH --ntasks=#{used_cpu}")
+			write_file(sh_name, "#SBATCH --nodes=#{@use_multinode}") if @use_multinode > 0
+			write_file(sh_name, 'srun hostname -s > workers') if node.parameters.include?('[lcpu]')
+		end
+		constraint = '#SBATCH --constraint='+@node_type if !@node_type.nil?
+		write_file(sh_name,	"#SBATCH --mem=#{@memory}")
+		write_file(sh_name, "#SBATCH --time=#{@time}")
+		write_file(sh_name,	"#{constraint}")
+		write_file(sh_name, '#SBATCH --error=job.%J.err')
+		write_file(sh_name, '#SBATCH --output=job.%J.out')
+	end
+
+	def submit_node(node, ar_dependencies)
+		final_dep = get_all_deps(ar_dependencies)
+		dependencies = nil
+		dependencies='--dependency=afterok:'+final_dep.join(':') if !final_dep.empty?  
+		cmd = "sbatch #{dependencies} #{node.name}.sh"
+		queue_id = get_queue_system_id(system_call(cmd, node.exec_folder_program))
+		return queue_id
+	end
+
+	def get_queue_system_id(shell_output)
+		queue_id = nil
+		shell_output.chomp!
+		fields = shell_output.split(' ')
+		queue_id = fields[3]
+		return queue_id
+	end
+
+	def self.available?(options)
+		available = TRUE
+		shell_output = system_call("type 'sbatch'", nil, options[:remote], options[:ssh])
+		available = FALSE if shell_output.empty?
+		return available
+	end
+
+	def self.priority
+		return 100 
+	end
+end

data/lib/autoflow/stack.rb

@@ -0,0 +1,272 @@
+require 'program'
+require 'queue_manager'
+
+
+class Stack
+
+	TAG = 0
+	INIT = 1
+	COMMAND = 2
+	PROG_NAME = 0
+	PROG_PARAM = 1
+
+##########################################################################################
+## PARSE TEMPLATE
+##########################################################################################
+	def initialize(exec_folder, options)
+		@commands = {}
+		@variables = {}
+		@one2one_dependencies = {}
+		@persist_variables = {}
+		@dirs = []
+		@exec_folder = exec_folder
+		@file_workflow = options[:workflow]
+		@do_retry = options[:retry]
+		parse(options[:workflow], options[:Variables])
+		@q_mgr = load_queue_manager(options)
+	end
+	
+	def parse(workflow, external_variables)
+		#Clean template
+		workflow.gsub!(/\#.+$/,'')	#Delete comments
+		workflow.gsub!("\t",'')		#Drop tabs
+		workflow.gsub!(/\n+/,"\n")	#Drop empty lines
+
+		#Parse template
+		variables_lines = []
+		persist_variables_lines = []
+		node_lines = []
+		
+		workflow.each_line do |line|
+			if line =~ /^\$/
+				variables_lines << line
+			elsif line =~ /^\@/
+				persist_variables_lines << line.gsub('@','')
+			else
+				node_lines << line
+			end
+		end
+		load_variables(variables_lines, @variables)
+		load_variables(external_variables, @variables)
+		load_variables(persist_variables_lines, @persist_variables)
+		parse_nodes(node_lines)
+	end
+
+	def load_variables(variables_lines, variable_type)
+		if !variables_lines.nil?
+			variables_lines.each do |line|
+				line.chomp!
+				line.gsub!(/\s/,'')
+				pairs = line.split(';')
+				pairs.each do |pair|
+					pair =~ /(.+)=(.+)/
+					variable_type[$1] = $2
+				end
+			end
+		end
+	end
+
+	def parse_nodes(execution_lines)
+		scan_nodes(execution_lines).each do |job_node| #Takes the info of each node of workflow for create the job
+			list_dup_nodes_ids(job_node[TAG]).each do |dup_node_id| #if dup_node_id is nil, the node isn't cloned.
+				# Set node attributes
+				prog_parameters = job_node[COMMAND].split(' ', 2)
+				tag_root_name = job_node[TAG].gsub(/\[([^\]]+\])\)/, '')#We remove the clone node expression
+				tag_root_name, dup_node_id, folder, no_buffer_node, done = set_node_attributes(tag_root_name, dup_node_id, prog_parameters)
+
+				#Dependencies
+				initialization, init_dependencies = parse_parameters(job_node[INIT].dup, dup_node_id)
+				parameters, dependencies = parse_parameters(prog_parameters[PROG_PARAM].dup, dup_node_id)
+				all_dependencies = dependencies + init_dependencies
+				all_dependencies.uniq!
+
+				#Create node_job
+				command_line = prog_parameters[PROG_NAME]+' '+parameters
+				node_name = "#{tag_root_name}#{dup_node_id}"
+				node = add_program(node_name, prog_parameters[PROG_NAME], command_line, initialization, folder, all_dependencies, done, no_buffer_node)
+				node.cloned_id = dup_node_id
+			end
+		end
+	end
+
+	def set_node_attributes(tag_root_name, dup_node_id, prog_parameters)
+		folder = ''
+		no_buffer_node = TRUE			
+		done = FALSE
+
+		if tag_root_name =~ /\!/ || (!dup_node_id.nil? && dup_node_id =~ /\!/)
+			folder = @exec_folder
+		else
+			folder = asign_folder(prog_parameters[PROG_NAME])
+		end
+		if tag_root_name =~ /&/ || (!dup_node_id.nil? && dup_node_id =~ /&/) #TODO comprobar como va esto
+			no_buffer_node = FALSE
+			folder = @exec_folder#This path is replaced later for the path of the main task that launch all buffered tasks
+		end
+		done = TRUE if tag_root_name =~ /\%/ && !@do_retry || (!dup_node_id.nil? && dup_node_id =~ /\%/ && !@do_retry)
+
+		tag_root_name.gsub!(/&|\!|\%|\)/,'')# Delete function characters
+		dup_node_id.gsub!(/&|\!|\%|\)/,'') if !dup_node_id.nil?
+		return tag_root_name, dup_node_id, folder, no_buffer_node, done
+	end
+
+	def parse_parameters(command, dup_node_id)
+		dependencies = []
+		if !command.nil?# When command is the initialize, sometimes can be undefined			
+			command.gsub!(/\(\*\)/, dup_node_id) if !dup_node_id.nil? # Define local parameter for duplicated nodes
+			dependencies_direct(command, dependencies)
+			if !dup_node_id.nil?
+				command.scan(/!([\S]+)\*!/).each do |dependency| # Current node depends on a only node of a batch of duplicated nodes
+					dependencies_one2one(command, dependency[0], dependencies, dup_node_id)
+				end
+			end
+
+			one2more = command.scan(/!([\S]+)!([^ \n]+)/) # Current node depends on a full batch of duplicated nodes
+			one2more.concat(command.scan(/!([\S]+)![ \n]/))
+			one2more.each do |dependency|
+				dependencies_one2more(command, dependency, dependencies)
+			end		
+
+			replace_variables(command)
+		end
+		#raise 'Missed dependency on: ' + command if command.include?(')')
+		return command, dependencies
+	end
+
+	def	dependencies_direct(command, dependencies)
+		@commands.each do |stage_id, node|
+			#Second conditional is triggered when is parsing a batch of tags. The first tag replaces stage_id by path and for second tag, this dependecy is lost
+			folder = node.exec_folder_program
+			if command.include?(stage_id) || command.include?(folder)  
+				dependencies << stage_id
+			end
+			command.gsub!(stage_id+')', folder) #Change tag to folder path
+		end
+	end
+
+	def dependencies_one2one(command, dependency, dependencies, dup_node_id)
+		@commands.keys.each do |st_id|
+			if st_id.include?(dependency) && @commands[st_id].cloned_id == dup_node_id # if a tag id match with expresion, we take it like a dependecy
+				dependencies << st_id
+				command.gsub!("!#{dependency}*!", @commands[st_id].exec_folder_program)
+				break
+			end
+		end
+	end
+
+	def dependencies_one2more(command, dependency, dependencies)
+		batch = []
+		@commands.keys.each do |stage_id|
+			if stage_id.include?(dependency[0]) # if a tag id match with expresion, we take it like a dependecy
+				dependencies << stage_id
+				batch << @commands[stage_id].exec_folder_program
+			end
+		end
+		string = batch.map{|tag_id| "#{tag_id}#{dependency[1]}"}.join(' ')
+		command.gsub!("!#{dependency[0]}!#{dependency[1]}", string)
+	end
+
+	def list_dup_nodes_ids(tag_node)
+		dup_nodes_ids = [nil]
+		if tag_node =~ /\[([^\]]+)/
+			dup_nodes_ids = []
+			$1.split(';').map{|interval|
+				if interval.include?('-')
+					limits = interval.split('-')
+					dup_nodes_ids.concat((limits.first..limits.last).to_a.map{|n| n.to_s})
+				else
+					dup_nodes_ids << interval
+				end
+			}
+		end
+		return dup_nodes_ids
+	end
+ 
+	def scan_nodes(execution_lines)
+		template_executions = execution_lines.join('')
+		executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?([^\}]{1,})\s{0,}\}/)
+		return executions
+	end
+
+	def add_program(stage_id, name, parameters, initialization, exec_folder_program, dependencies, done, no_buffer_node)
+		tag=Program.new(name, parameters, initialization, exec_folder_program, dependencies, done, no_buffer_node)
+		@commands[stage_id]=tag
+		return tag
+	end
+
+	def replace_variables(command)
+		@variables.each do |name, value|
+			command.gsub!(name, value)
+		end
+	end
+
+	def asign_folder(program_name)			
+		folder=File.join(@exec_folder,"#{program_name}_#{"%04d" % 0}")
+		count=0
+		while @dirs.include?(folder)
+			folder=File.join(@exec_folder,"#{program_name}_#{"%04d" % count}")
+			count+=1
+		end
+		@dirs << folder
+		return folder
+	end
+
+##########################################################################################
+## LAUNCH WORKFLOW
+##########################################################################################
+
+	def load_queue_manager(options)
+		return QueueManager.select_queue_manager(@exec_folder, options, @commands, @persist_variables)
+	end
+
+	def send
+		@q_mgr.exec
+	end
+
+##########################################################################################
+## WORKFLOW REPRESENTATION
+##########################################################################################
+
+	def inspect
+		@commands.each do |id, tag|
+			puts "#{id} > #{tag.inspect}\t#{tag.done}\n\t\e[32m#{tag.dependencies.join("\n\t")}\e[0m"
+		end
+	end
+
+	def draw(name, name_type)
+		representation_type = '_structural'
+		representation_type = '_semantic' if name_type.include?('t')
+		name.gsub!(/\.\S+/,'')
+		file = File.open(name+representation_type+'.dot','w')
+		file.puts 'digraph G {', 'node[shape=box]'
+		all_dependencies = []
+		all_tag = []
+		@commands.each do |id, tag|
+			tag_name = File.basename(tag.exec_folder_program)
+			if name_type.include?('t')
+				tag_name = id
+			end
+			all_tag << tag_name
+			if tag.dependencies.length > 0
+				tag.dependencies.each do |dependencie|
+					dependencie_name = File.basename(@commands[dependencie].exec_folder_program)
+					if name_type.include?('t')
+						dependencie_name = dependencie
+					end
+					all_dependencies << dependencie_name
+					file.puts "\"#{dependencie_name}\"-> \"#{tag_name}\""
+				end
+			else
+				file.puts "\"#{tag_name}\"[color=mediumseagreen, style=filled]"
+			end
+		end
+		all_tag.keep_if{|tag| !all_dependencies.include?(tag)}
+		all_tag.each do |tag|
+			file.puts "\"#{tag}\"[color=palevioletred, style=filled]"
+		end
+		file.puts '}'
+		file.close
+		system('dot -Tpng '+name+representation_type+'.dot -o '+name+representation_type+'.png')
+	end
+	
+end

data/lib/autoflow/version.rb

@@ -0,0 +1,3 @@
+module Autoflow
+  VERSION = "0.3.5"
+end

data/lib/autoflow.rb

@@ -0,0 +1,7 @@
+
+require "autoflow/version"
+require "autoflow/stack"
+
+module Autoflow
+  # Your code goes here...
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: autoflow
+version: !ruby/object:Gem::Version
+  version: 0.3.5
+  prerelease: 
+platform: ruby
+authors:
+- Pedro Seoane
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! '"Autoflow makes easy to launch big pipelines on a queue system. Only
+  works with SLURM & PBS"'
+email:
+- seoanezonjic@hotmail.com
+executables:
+- AutoFlow
+- env_manager
+- flow_time
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- autoflow.gemspec
+- bin/AutoFlow
+- bin/env_manager
+- bin/flow_time
+- lib/autoflow.rb
+- lib/autoflow/program.rb
+- lib/autoflow/queue_manager.rb
+- lib/autoflow/queue_managers/bash_manager.rb
+- lib/autoflow/queue_managers/slurm_manager.rb
+- lib/autoflow/stack.rb
+- lib/autoflow/version.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: ! '"This gem take a pipeline and launch it on a queue system"'
+test_files: []