bio-pipengine 0.6.0

data/VERSION

@@ -0,0 +1 @@
+0.6.0

data/bin/pipengine

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+
+$:<< File.expand_path(File.join(File.dirname(File.dirname __FILE__),"lib"))
+require 'bio-pipengine'
+
+banner_text = "\nLauncher for Complex Biological Pipelines . Copyright(C) 2013 Francesco Strozzi\n\n" 
+version_text = File.read File.expand_path(File.join(File.dirname(File.dirname __FILE__),"VERSION"))
+SUB_COMMANDS = %w(run jobs)
+
+
+Bio::Pipengine.check_config
+
+options = {}
+cmd = ARGV.first # get the subcommand
+opts = case cmd
+when "run" 	
+	options[:run] = true
+	ARGV.shift
+	Trollop::options do
+		opt :pipeline, "YAML file with pipeline and sample details", :short => "p", :type => :string, :default => "pipeline.yml"
+		opt :samples_file, "YAML file with samples name and directory paths", :short => "f", :type => :string, :default => "samples.yml"
+		opt :samples, "List of sample names to run the pipeline", :type => :strings, :short => "l"
+		opt :steps, "List of steps to be executed", :type => :strings, :short => "s"
+		opt :dry,"Dry run. Just create the job script without submitting it to the batch system", :short => "d"
+		opt :spooler,"Destination spooler PBS, plain shell script", :short =>"x", :type => :string, :default => "pbs"
+		opt :tmp, "Temporary output folder", :type => :string, :short => "t"
+		opt :create_samples, "Create samples.yml file from a Sample directory (only for CASAVA projects)", :short => "c", :type => :strings
+		opt :multi, "List of samples to be processed by a given step (the order matters)", :short => "m", :type => :strings
+		opt :group, "Specify the group of samples to run the pipeline steps on (do not specify --multi)", :short => "g", :type => :string
+		opt :name, "Analysis name", :short => "n", :type => :string
+		opt :output_dir, "Output directory (override standard output directory names)", :short => "o", :type => :string
+		opt :pbs_opts, "PBS options", :type => :strings, :short => "b"
+		opt :pbs_queue, "PBS queue", :type => :string, :short => "q"
+		opt :inspect_pipeline, "Show steps", :short => "i", :type => :string
+		opt :mail_exit, "Send an Email when the job terminates", :type => :string
+		opt :mail_start, "Send an Email when the job starts", :type => :string
+		opt :log, "Log script activities, by default stdin. Options are fluentd", :type => :string, :default => "stdin"
+		opt :log_adapter, "(stdin|syslog|fluentd) In case of fluentd use http://destination.hostname:port/yourtag", :type => :string
+	end
+when "jobs"
+	ARGV.shift
+	options[:jobs] = true
+	Trollop::options do
+		opt :job_id, "Search submitted jobs by Job ID", :type => :strings, :short => "i"
+		opt :job_name, "Search submitted jobs by Job Name", :type => :strings, :short => "n"
+		opt :delete, "Delete submitted jobs ('all' to erase everything or type one or more job IDs)", :short => "d", :type => :strings
+	end
+when "-h"
+	puts banner_text
+	puts "List of available commands:\n\trun\tSubmit pipelines to the job scheduler\n\tjobs\tShow statistics and interact with running jobs\n"
+	exit
+else
+	global_opts = Trollop::options do
+		banner banner_text
+		version "PipEngine v#{version_text}"
+	end
+end
+
+
+
+options = options.merge opts
+Trollop::die :multi, "Specifing both --group and --multi is not allowed" if options[:multi] and options[:group]
+
+if options[:create_samples]
+	Bio::Pipengine.create_samples options[:create_samples]
+elsif options[:jobs]
+	if options[:job_id] 
+		Bio::Pipengine.show_stats(options[:job_id])
+	elsif options[:job_name]
+		warn "Not yet implemented"
+		exit
+	elsif options[:delete]
+		if options[:delete].empty?
+			warn "Provide one or more Job IDs or write 'all' to delete all your running jobs".red
+			exit
+		end
+		puts "Warning: this will delete the following running jobs: ".light_blue + "#{options[:delete].join(",")}".green
+		print "Are you sure? (y|n):"
+		answer = gets.chomp
+		if answer == "y"
+			Bio::Pipengine.delete_jobs(options[:delete])
+		else
+			puts "Aborting..."
+			exit
+		end
+	else
+		Bio::Pipengine.show_stats(["all"])
+	end
+elsif options[:pipeline] && options[:samples_file]
+	if options[:inspect_pipeline]
+		Bio::Pipengine.inspect_steps(options[:inspect_pipeline])
+		exit
+	else
+		abort("File not found: #{options[:pipeline]}".red) unless File.exists? options[:pipeline]
+		abort("File not found: #{options[:samples_file]}".red) unless File.exists? options[:samples_file]
+		abort("Please provide a valid step name with the --step parameter".red) unless options[:steps]
+		Bio::Pipengine.run(options)
+	end
+end
+
+

data/lib/bio/pipengine/job.rb

@@ -0,0 +1,271 @@
+module Bio
+	module Pipengine
+		
+		class Job
+			
+			# a Job object holds information on a job to be submitted
+			# samples_groups and samples_obj are used to store information in case of steps that require to combine info
+			# from multiple samples
+			attr_accessor :name, :cpus, :nodes, :mem, :resources, :command_line, :local,
+			              :multi_samples, :samples_obj, :custom_output, :custom_name,
+			              :log, :log_adapter
+			def initialize(name)
+				@name = generate_uuid + "-" + name
+				@shortname = name
+				@command_line = []
+				@resources = {}
+				@cpus = 1
+				@nodes = "1"
+				@log = "stdin"
+				@log_adapter = nil
+			end
+
+			def add_resources(resources)
+				self.resources.merge! resources
+			end
+
+			def output
+				self.resources["output"]
+			end
+
+			# add all the command lines for a given step
+			def add_step(step,sample)	
+
+				# setting job working directory
+				working_dir = ""	
+				if self.local 
+					working_dir = self.local+"/"+self.name
+				else
+					working_dir = self.output
+
+					if step.is_multi?	
+						folder = (self.custom_output) ? self.custom_output : @shortname 
+						working_dir += "/#{folder}"
+					else
+						folder =
+						if self.custom_output 
+							self.custom_output
+						elsif self.custom_name
+							self.custom_name
+						else
+							step.name
+						end
+						working_dir += "/#{sample.name}/#{folder}"
+					end
+
+				end
+
+				# set job cpus number to the higher step cpus (this in case of multiple steps)
+				self.cpus = step.cpus if step.cpus > self.cpus
+				
+				# set number of nodes for job
+				self.nodes = (step.nodes) ? step.nodes : @nodes
+
+				# set the memory used
+				self.mem = step.mem
+
+				# adding job working directory
+				unless step.name.start_with? "_"
+					self.command_line << "if [ ! -f #{working_dir}/checkpoint ]"
+					self.command_line << "then"
+					self.command_line << logger(step, "start")
+					self.command_line << "\nmkdir -p #{working_dir}"
+					self.command_line << "cd #{working_dir}"
+				end
+
+				# generate command lines for this step
+				if step.run.kind_of? Array
+					step.run.each_with_index do |cmd, i|
+						command = generate_cmd_line(cmd,sample,step)
+						# TODO verify that logger works in this case
+						# self.command_line << "#{command} || { echo \"FAILED `date`: #{step.name}:#{i}\" ; exit 1; }"
+						self.command_line << "#{command} || { #{logger(step, "FAILED #{i}" )}; exit 1; }"
+					end
+				else
+					command = generate_cmd_line(step.run,sample,step)
+					# TODO verify that logger works in this case
+					# self.command_line << "#{command} || { echo \"FAILED `date`: #{step.name} \" ; exit 1; }"
+					self.command_line << "#{command} || { #{logger(step, "FAILED" )}; exit 1; }"
+				end
+				self.command_line << logger(step, "finished")
+                self.command_line << "touch #{working_dir}/checkpoint"
+				self.command_line << "else"
+				self.command_line << logger(step, "already executed, skipping this step")
+				self.command_line << "fi"
+			
+				# check if a temporary (i.e. different from 'output') directory is set
+				if self.local
+					final_output = ""
+
+					if step.is_multi?	
+						folder = (self.custom_output) ? self.custom_output : @shortname 
+						final_output = self.output+"/#{folder}"	 
+					else
+						folder = (self.custom_output) ? self.custom_output : step.name
+						final_output = self.output+"/#{sample.name}/#{folder}"
+					end
+
+					self.command_line << "mkdir -p #{final_output}"
+					self.command_line << "cp -r #{working_dir}/* #{final_output}"
+					self.command_line << "rm -fr #{working_dir}"
+				end
+
+			end
+
+			# convert the job object into a TORQUE::Qsub object
+			def to_pbs(options)
+				TORQUE::Qsub.new(options) do |torque_job|
+					torque_job.name = self.name
+					torque_job.working_directory = self.output # where pbs scripts and stdout / stderr files will be saved
+					if options[:pbs_opts] 
+						torque_job.l = options[:pbs_opts]
+					else
+						l_string = []
+						l_string << "nodes=#{self.nodes}:ppn=#{self.cpus}"
+						l_string << "mem=#{self.mem}" if self.mem
+						torque_job.l = l_string
+						if options[:mail_exit]
+							torque_job.m = "e"
+							torque_job.M = options[:mail_exit]
+						end
+						if options[:mail_start]
+							torque_job.m = "b"
+							torque_job.M = options[:mail_start]
+						end
+					end	
+					torque_job.q = options[:pbs_queue] if options[:pbs_queue]
+					torque_job.script = self.command_line.join("\n")+"\n"
+        end
+			end
+
+			def to_script(options)
+			  File.open(self.name+'.sh','w') do |file|
+		          file.puts "#!/usr/bin/env bash -l"
+			      file.puts self.command_line.join("\n")
+			  end
+			end
+
+		private
+			
+			# create a unique ID for each job
+			def generate_uuid
+				SecureRandom.hex(5)
+			end
+			
+			# this method call other methods to perform the right substitutions into the command lines
+			def generate_cmd_line(cmd,sample,step)
+				if step.is_multi? # if is a multi samples step call a different method
+					set_multi_cmd(step,self.multi_samples)
+					cmd = sub_multi(cmd,step)
+				else
+					cmd = sub_placeholders(cmd,sample,step) # normal step, perform usual substitutions
+				end
+				return cmd
+			end
+		
+			# perform substitutions on all the placeholders
+			def sub_placeholders(cmd,sample,step=nil)	
+				tmp_cmd = cmd.gsub(/<sample>/,sample.name)
+				if tmp_cmd =~/<sample_path>/
+					sample_path_glob = (tmp_cmd.scan(/<sample_path>(\S+)/).map {|e| e.first})
+					if sample_path_glob.empty?
+						tmp_cmd.gsub!(/<sample_path>/,sample.path.join("\s"))
+					else
+						sample_path_glob.each do |append|
+							tmp_cmd.gsub!(/<sample_path>#{Regexp.quote(append)}/,(sample.path.map {|s| s+append}).join("\s"))
+						end
+					end
+				end
+				# for resourcers and cpus
+				tmp_cmd = sub_resources_and_cpu(tmp_cmd,step)
+				
+				# for placeholders like <mapping/sample>
+				tmp_cmd.scan(/<(\S+)\/sample>/).map {|e| e.first}.each do |input_folder|
+					warn "Directory #{self.output+"/"+sample.name+"/"+input_folder} not found".magenta unless Dir.exists? self.output+"/"+sample.name+"/"+input_folder
+					tmp_cmd = tmp_cmd.gsub(/<#{input_folder}\/sample>/,self.output+"/"+sample.name+"/"+input_folder+"/"+sample.name)
+				end
+				
+				# for placeholders like <mapping/>
+				tmp_cmd.scan(/<(\S+)\/>/).map {|e| e.first}.each do |input_folder|
+					warn "Directory #{self.output+"/"+sample.name+"/"+input_folder} not found".magenta unless Dir.exists? self.output+"/"+sample.name+"/"+input_folder
+					tmp_cmd = tmp_cmd.gsub(/<#{input_folder}\/>/,self.output+"/"+sample.name+"/"+input_folder+"/")
+				end
+				return tmp_cmd
+			end
+
+			def sub_resources_and_cpu(cmd,step)	
+				# for all resources tags like <gtf> <index> <genome> <bwa> etc.
+				self.resources.each_key do |r|
+					cmd.gsub!(/<#{r}>/,self.resources[r])
+				end
+				# set number of cpus for this command line
+				cmd.gsub!(/<cpu>/,step.cpus.to_s) unless step.nil?
+				return cmd
+			end
+	
+
+			# creates actual multi-samples command lines to be substituted where <multi> placeholders are found
+			def set_multi_cmd(step,multi_samples)
+				if step.multi_def.kind_of? Array # in case of many multi-samples command lines
+					step.multi_cmd = []
+					step.multi_def.each do |m_def|
+						step.multi_cmd << generate_multi_cmd(m_def,multi_samples)
+					end
+				else
+					step.multi_cmd = generate_multi_cmd(step.multi_def,multi_samples)
+				end
+			end
+
+			# take the multi_cmd and perform the subsitutions into the step command lines
+			def sub_multi(cmd,step)
+				cmd = sub_resources_and_cpu(cmd,step)
+				if step.multi_cmd.kind_of? Array
+					step.multi_cmd.each_with_index do |m,index|
+						cmd.gsub!(/<multi#{index+1}>/,m)
+					end
+				else
+					cmd.gsub!(/<multi>/,step.multi_cmd)
+				end
+				return cmd
+			end
+
+			# this sub method handle different multi-samples definitions (like comma separated list, space separated etc.)
+			def generate_multi_cmd(multi_def,multi_samples)
+				multi_cmd = []	
+				multi_samples.each do |sample_name|
+					if sample_name.include? ","
+						multi_cmd << split_and_sub(",",multi_def,sample_name)
+					elsif sample_name.include? ";"
+						multi_cmd << split_and_sub(";",multi_def,sample_name)
+					else
+						multi_cmd << sub_placeholders(multi_def,self.samples_obj[sample_name])
+					end
+				end
+				return multi_cmd.join("\s")
+			end
+
+			# take a non-space separated list of samples and perform the substitution with the group defitions
+			def split_and_sub(sep,multi_def,multi)	
+				cmd_line = []
+				multi.split(sep).each do |sample_name|
+					cmd_line << sub_placeholders(multi_def,self.samples_obj[sample_name])
+				end
+				cmd_line.join(sep)
+			end
+
+			# log a step according to the selected adapter
+			def logger(step, message)  
+				case self.log
+					when "stdin"
+					   "echo \"#{step.name} #{name} #{message} `whoami` `hostname` `pwd` `date`.\""
+					when "syslog"
+						 "logger -t PIPENGINE \"#{step.name} #{name} #{message} `whoami` `hostname` `pwd`\""
+					when "fluentd"
+						 "curl -X POST -d 'json={\"source\":\"PIPENGINE\", \"step\":\"#{step.name}\", \"message\":\"#{message}\", \"job_id\":\"#{name}\", \"user\":\"\'\"`whoami`\"\'\", \"host\":\"\'\"`hostname`\"\'\", \"pwd\":\"\'\"`pwd`\"\'\"}' #{self.log_adapter}"
+					end
+			end #logger
+
+		end
+	end
+end
+

data/lib/bio/pipengine/sample.rb

@@ -0,0 +1,13 @@
+module Bio
+	module Pipengine
+		class Sample
+			# Sample holds all the information on a sample and its original input path (or multiple paths)	
+			attr_accessor :path, :name
+				def initialize(name,path_string)
+					@path = path_string.split(",")
+					@name = name 
+				end
+		end
+	end
+end
+

data/lib/bio/pipengine/step.rb

@@ -0,0 +1,39 @@
+module Bio
+	module Pipengine
+		
+		# Step holds information for a pipeline step
+		# groups_def is used to store information on groups definition (i.e. generic cmd lines with placeholders)
+		# groups_cmd is used to store the actual command lines for all the samples to be combined in a "groups" step
+		# this are generated by combining groups_def information with sample groups information and will be placed
+		# where <groups> placeholder is found into the step command lines.
+		class Step
+			attr_accessor :name, :run, :cpus, :mem, :nodes, :multi_def, :multi_cmd, :pre
+			def initialize(name,step_instructions)
+				@name = name
+				parse_yaml(step_instructions)	
+			end
+
+			def is_multi?
+				return (self.multi_def.nil?) ? false : true
+			end
+
+			def has_prerequisite?
+				return (self.pre.nil?) ? false : true
+			end
+
+			private
+
+			def parse_yaml(step_instructions)
+				self.cpus = step_instructions["cpu"].to_i
+				self.nodes = step_instructions["nodes"]
+				self.mem = step_instructions["mem"]
+				self.run = step_instructions["run"]
+				self.multi_def = step_instructions["multi"]
+				self.pre = step_instructions["pre"]
+			end
+
+		end
+
+	end
+end
+

data/lib/bio/pipengine.rb

@@ -0,0 +1,234 @@
+module Bio
+	module Pipengine
+	
+		def self.run(options)
+
+			# reading the yaml files
+			pipeline = YAML.load_file options[:pipeline]
+			samples_file = YAML.load_file options[:samples_file]
+			samples_file["samples"].each do |k,v|
+				if v.kind_of? Hash
+					samples_file["samples"][k] = Hash[samples_file["samples"][k].map{ |key, value| [key.to_s, value.to_s] }] 
+				else
+					samples_file["samples"][k] = v.to_s
+				end
+			end
+			# make sure everything in Samples and Resources is converted to string
+			#samples_file["samples"] = Hash[samples_file["samples"].map{ |key, value| [key.to_s, value.to_s] }] 
+			samples_file["resources"] = Hash[samples_file["resources"].map {|k,v| [k.to_s, v.to_s]}]	
+			
+			# pre-running checks	
+			check_steps(options[:steps],pipeline)	
+			check_samples(options[:samples],samples_file) if options[:samples]
+
+			# list of samples the jobs will work on
+			samples_list = nil
+			# check if a group is specified
+			if options[:group]
+				samples_list = options[:samples] ? samples_file["samples"][options[:group]].select {|k,v| options[:samples].include? k} : samples_file["samples"][options[:group]]
+				options[:multi] = samples_list.keys 
+				samples_file["resources"]["output"] << "/#{options[:group]}"	
+			else # if not, proceed normalizing the sample list to remove groups and get a list of all samples
+				full_list_samples = {}
+				samples_file["samples"].each_key do |k| 
+					if samples_file["samples"][k].kind_of? Hash
+						full_list_samples.merge! samples_file["samples"][k]
+					else
+						full_list_samples[k] = samples_file["samples"][k]
+					end
+				end
+				samples_list = options[:samples] ? full_list_samples.select {|k,v| options[:samples].include? k} : full_list_samples
+			end
+				
+			########### START ###########
+
+			# create output directory (jobs scripts will be saved there)
+			FileUtils.mkdir_p samples_file["resources"]["output"] unless options[:dry] #&& options[:spooler]!="pbs"
+
+			# check if the requested steps are multi-samples
+			run_multi = check_and_run_multi(samples_file,pipeline,samples_list,options)
+			
+			unless run_multi # there are no multi-samples steps, so iterate on samples and create one job per sample
+				samples_list.each_key do |sample_name|
+					sample = Bio::Pipengine::Sample.new(sample_name,samples_list[sample_name])
+					create_job(samples_file,pipeline,samples_list,options,sample)
+				end
+			end
+		end
+
+		# handle steps that run on multiple samples (i.e. sample groups job)
+		def self.check_and_run_multi(samples_file,pipeline,samples_list,options)
+			step_multi = options[:steps].map {|s| Bio::Pipengine::Step.new(s,pipeline["steps"][s]).is_multi?}
+			
+			if step_multi.include? false
+				if step_multi.uniq.size > 1
+					puts "\nAbort! You are trying to run both multi-samples and single sample steps in the same job".red
+					exit
+				else
+					return false
+				end
+			else
+				samples_obj = {}
+				samples_list.each_key {|sample_name| samples_obj[sample_name] = Bio::Pipengine::Sample.new(sample_name,samples_list[sample_name])}
+				create_job(samples_file,pipeline,samples_list,options,samples_obj)
+				return true
+			end
+		end
+
+		def self.create_job(samples_file,pipeline,samples_list,options,sample)
+			# getting the sample name (only if this is not a multi samples job)
+			sample_name = (sample.kind_of? Hash) ? nil : sample.name+"-"
+			# setting the job name
+			job_name = nil
+			if options[:name] 
+				job_name = options[:name]
+			elsif options[:steps].size > 1
+				job_name = "#{sample_name}#{options[:steps].join("-")}"
+			else
+				job_name = "#{sample_name}#{options[:steps].first}"
+			end	
+			# creating the Job object
+			job = Bio::Pipengine::Job.new(job_name)
+			job.local = options[:tmp]
+			job.custom_output = options[:output_dir]
+			job.custom_name = (options[:name]) ? options[:name] : nil
+			job.add_resources pipeline["resources"]
+			job.add_resources samples_file["resources"]
+			#setting the logging system
+			job.log = options[:log]
+			job.log_adapter = options[:log_adapter]
+			# setting sample groups either by cli option (if present) or by taking all available samples
+			job.multi_samples = (options[:multi]) ? options[:multi] : samples_list.keys
+			job.samples_obj = sample if sample.kind_of? Hash
+			# cycling through steps and add command lines to the job
+			options[:steps].each do |step_name| 
+				# TODO WARNING this can add multiple times the same step if the are multi dependencies
+				self.add_job(job, pipeline, step_name, sample)
+			end
+
+			if options[:dry] #&& options[:spooler] == "script"
+				job.to_script(options)
+			else
+			  script = job.to_pbs(options) # converting the Job into a TORQUE::Qsub PBS compatible object
+			  job_id = script.submit(options)
+			  puts "#{job_id}".green unless options[:dry]
+      end
+		end
+
+		# check if sample exists
+		def self.check_samples(passed_samples,samples)
+			passed_samples.each do |sample|
+				samples_names = []
+				samples["samples"].each_key do |k|
+					if samples["samples"][k].kind_of? Hash
+						samples["samples"][k].each_key {|s| samples_names << s}
+					else
+						samples_names << k
+					end
+				end
+				unless samples_names.include? sample
+					puts "Sample \"#{sample}\" does not exist in sample file!".red
+					exit
+				end
+			end
+		end
+
+		# check if step exists
+		def self.check_steps(passed_steps,pipeline)
+			passed_steps.each do |step|
+				unless pipeline["steps"].keys.include? step
+					puts "Step \"#{step}\" does not exist in pipeline file!".red
+					exit
+				end
+			end
+		end
+
+		# load the pipeline file and show a list of available steps
+		def self.inspect_steps(pipeline_file)
+			pipeline = YAML.load_file pipeline_file
+			print "\nPipeline: ".blue 
+			print "#{pipeline["pipeline"]}\n\n".green
+			puts "List of available steps:".light_blue
+			pipeline["steps"].each_key do |s|
+				print "\s\s#{s}:\s\s".blue 
+				print "#{pipeline["steps"][s]["desc"]}\n".green
+			end
+			puts "\n"
+		end
+		
+		# create the samples.yml file (CASAVA ONLY!)
+		def self.create_samples(dir)
+				File.open("samples.yml","w") do |file|
+					file.write "resources:\n\soutput: #{FileUtils.pwd}\n\nsamples:\n"
+					samples = Hash.new {|hash,key| hash[key] = []}
+					dir.each do |path|
+						projects = Dir.glob(path+"/*").sort.select {|folders| folders.split("/")[-1] =~/Project_/}
+						unless projects.empty?
+							projects.each do |project_folder|
+								Dir.glob(project_folder+"/*").sort.each {|s| samples[s.split("/")[-1]] << s}
+							end
+						else
+							Dir.glob(path+"/*").sort.each {|s| samples[s.split("/")[-1]] << s if Dir.exists? s}
+						end
+					end
+					samples.each_key do |sample|
+						file.write "\s"+sample+":\s"+samples[sample].join(",")+"\n"	
+					end
+				end
+		end
+
+		# show running jobs information
+		def self.show_stats(job_ids)
+			stats = TORQUE::Qstat.new
+			if job_ids.first == "all"
+				stats.display
+			else
+				stats.display(:job_ids => job_ids)
+			end
+		end
+
+		# delete running jobs from the scheduler
+		def self.delete_jobs(job_ids)
+			include TORQUE
+			if job_ids == ["all"]
+				Qdel.rm_all
+			else 
+				job_ids.each {|job_id| Qdel.rm job_id}
+			end
+		end #delete_jobs
+
+		# check if required configuration exists
+		def self.check_config
+			unless File.exists?("#{Dir.home}/.torque_rm.yaml")
+				ARGV.clear
+				current_user = Etc.getlogin
+				puts "\nIt seems you are running PipEngine for the first time. Please fill in the following information:"
+				print "\nHostname or IP address of authorized server from where jobs will be submitted: ".light_blue
+				server = gets.chomp
+				print "\n"
+				print "Specify the username you will be using to connect and submit jobs [#{current_user}]: ".light_blue
+				username = gets.chomp
+				username = (username == "") ? current_user : username
+				puts "Attempting connection to the server...".green
+				path = `ssh #{username}@#{server} -t "which qsub"`.split("/qsub").first
+				unless path=~/\/\S+\/\S+/
+					warn "Connection problems detected! Please check that you are able to connect to '#{server}' as '#{username}' via ssh.".red
+				else	
+					file = File.open("#{Dir.home}/.torque_rm.yaml","w")
+					file.write({:hostname => server, :path => path, :user => username}.to_yaml)
+					file.close
+					puts "First time configuration completed!".green
+					puts "It is strongly recommended to setup a password-less SSH connection to use PipEngine.".green
+					exit
+				end
+			end
+		end #check_config
+
+		def self.add_job(job, pipeline, step_name, sample)
+			step = Bio::Pipengine::Step.new(step_name,pipeline["steps"][step_name]) # parsing step instructions
+			self.add_job(job, pipeline, step.pre, sample) if step.has_prerequisite?
+			job.add_step(step,sample) # adding step command lines to the job	
+		end #add_job
+		
+	end
+end