autoflow 0.6.4 → 0.8.0

data/autoflow.gemspec

@@ -19,8 +19,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency 'net-ssh', '>= 2.8.0'
+  spec.add_runtime_dependency 'git', '>= 1.3.0'
   spec.add_runtime_dependency 'win32console', '>= 1.3.2' if !ENV['OS'].nil? && ENV['OS'].downcase.include?('windows')
   spec.add_runtime_dependency 'colorize', '~> 0.7.3'
+  spec.add_runtime_dependency 'terminal-table', '~> 1.6.0'
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
 end

data/bin/AutoFlow

@@ -11,6 +11,7 @@ require 'io/console'
 require 'net/ssh'
 require 'queue_manager'
 require 'fileutils'
+require 'git'
 
 #################################################################################################
 # METHODS
@@ -21,8 +22,13 @@ def get_templates(string_template)
 end
 
 def get_repositories
-    directories = [File.join(ENV['HOME'], 'autoflow_templates')]
+    main_repository = File.join(ENV['HOME'], 'autoflow_templates')
+    local_rep = File.join(main_repository, 'local')
+    remote_rep = File.join(main_repository, 'remote')
+    all_remote = Dir.glob(File.join(remote_rep, '*'))
+    directories = [local_rep]
     directories.concat(ENV['WORKFLOW_REPOSITORY'].split(':')) if !ENV['WORKFLOW_REPOSITORY'].nil?
+    directories.concat(all_remote)
     return directories
 end
 
@@ -39,6 +45,8 @@ def list_repository_templates
         end
         templates.delete('.')
         templates.delete('..')
+        templates.delete('README.md') # From git remote repos
+        templates.delete('.git') # From git remote repos
         return templates
 end
 
@@ -105,6 +113,11 @@ optparse = OptionParser.new do |opts|
                 options[:graph] = graph
         end
 
+        options[:get_template_repository] = nil
+        opts.on( '-G', '--get_template_repository GIT_URL', 'Git url to get template from a remote repository' ) do |url| 
+                options[:get_template_repository] = url
+        end
+
         options[:identifier] = FALSE
         opts.on( '-i', '--job_identifier STRING', 'Identifier tag for each launching script' ) do |identifier|
                 options[:identifier] = identifier
@@ -204,8 +217,24 @@ optparse.parse!
 # MAIN
 #################################################################################################
 #Create repository
-if !File.exists?(File.join(ENV['HOME'], 'autoflow_templates'))
-    Dir.mkdir(File.join(ENV['HOME'], 'autoflow_templates'))
+main_repository = File.join(ENV['HOME'], 'autoflow_templates')
+local_rep = File.join(main_repository, 'local')
+remote_rep = File.join(main_repository, 'remote')
+Dir.mkdir(main_repository) if !File.exists?(main_repository)
+Dir.mkdir(local_rep) if !File.exists?(local_rep)
+Dir.mkdir(remote_rep) if !File.exists?(remote_rep)
+
+#Move templates from legacy tree folder to current tree folder
+Dir.glob(File.join(main_repository, '*')) { |file| FileUtils.mv(file, local_rep) if file != local_rep && file != remote_rep }
+
+# Get git remote repos and include it in user templates
+if !options[:get_template_repository].nil?
+    fields = options[:get_template_repository].split(/\/+/)
+    repo_name = fields.last.gsub('.git','')
+    repo_author = fields[-2]
+    local_name = "REM_#{repo_author}_#{repo_name}"
+    Git.clone(options[:get_template_repository], local_name, :path => remote_rep)
+    exit_exec('Remote template repository added as ' + local_name)
 end
 
 # List templates
@@ -294,7 +323,7 @@ end
 # Flow parse
 #--------------------------------------------------------------------------------
 stack = Stack.new(exec_folder, options)
-manager = QueueManager.select_queue_manager(stack, options)
+stack.parse!
  
 #--------------------------------------------------------------------------------
 # Flow exec
@@ -304,6 +333,8 @@ if !options[:graph].nil?
 else
         stack.inspect if options[:verbose]
         stack.comment_main_command if options[:comment]
+        options[:write_sh] = TRUE # Set in flow logger to FALSE, it is used for relaunch failed jobs 
+        manager = QueueManager.select_queue_manager(stack.exec_folder, options, stack.jobs, stack.persist_variables)
         manager.exec
 end
 options[:ssh].close if options[:remote]

data/bin/flow_logger

@@ -0,0 +1,315 @@
+#! /usr/bin/env ruby
+
+ROOT_PATH=File.dirname(__FILE__)
+$: << File.expand_path(File.join(ROOT_PATH, "..", "lib", "autoflow"))
+$: << File.expand_path(File.join(ROOT_PATH, "..", "lib", "autoflow", "queue_managers"))
+
+require 'autoflow'
+require 'optparse'
+require 'colorize'
+require 'logging'
+require 'json'
+require 'terminal-table'
+require 'queue_manager'
+require 'program'
+require 'erb'
+
+#################################################################################################
+### METHODS
+#################################################################################################
+
+def report_log(log, initial_flow_attribs, mode, workflow_status, no_size)
+	set_task_state(log, workflow_status)
+	set_time(log)
+	if mode.nil? || mode.upcase == 'ALL'
+		tasks = log
+	else
+		tasks = log.select{|name, attribs| attribs['state'] == mode.upcase}
+	end
+	rows = []
+	tasks.each do |task_name, attribs|
+		job_path = initial_flow_attribs[task_name].first
+		size = nil
+		size = `du -sh #{job_path}`.split.first if !no_size
+		rows << [attribs['state_msg'], File.basename(job_path), attribs['time'], size, task_name]
+	end
+	puts Terminal::Table.new :headings => ['Status', 'Folder', 'Time', 'Size', 'Job Name'], :rows => rows
+end
+
+def launch_failed_jobs(log, initial_flow_attribs, exec_folder, batch)
+	options = {
+		:verbose => FALSE,
+		:identifier => nil,
+		:remote => FALSE,
+		:ssh => nil,
+		:external_dependencies => [],
+		:batch => batch,
+		:write_sh => FALSE
+	}
+	failed_jobs = get_failed_jobs(log)
+	jobs = {}
+	create_jobs(jobs, failed_jobs, initial_flow_attribs)
+	get_all_dependencies(jobs, failed_jobs, initial_flow_attribs)
+	manager = QueueManager.select_queue_manager(exec_folder, options, jobs, {})
+	manager.exec
+end
+
+def create_jobs(jobs, failed_jobs, initial_flow_attribs) 
+	failed_jobs.each do |job|
+		folder, dependencies = initial_flow_attribs[job]
+		job_attrib = {
+			:done => FALSE,
+			:folder => TRUE,
+			:buffer => FALSE,
+			:exec_folder => folder,
+			:cpu_asign => nil
+		}
+		verified_dependencies = []
+		dependencies.each do |dep|
+			verified_dependencies << dep if !jobs[dep].nil?
+		end
+		jobs[job] = Program.new(job, '', '', verified_dependencies, job_attrib)
+	end
+end
+
+def get_all_dependencies(jobs, failed_jobs, initial_flow_attribs)
+	failed_dependecies = []
+	failed_jobs.each do |fj|
+		initial_flow_attribs.each do |job, attribs|
+			folder, dependencies = attribs
+			failed_dependecies << job if dependencies.include?(fj) && !failed_dependecies.include?(job)
+		end
+	end
+	if !failed_dependecies.empty?
+		create_jobs(jobs, failed_dependecies, initial_flow_attribs)
+		get_all_dependencies(jobs, failed_dependecies, initial_flow_attribs)
+	end
+end
+
+def get_failed_jobs(log)
+	failed_jobs = [] 
+	position = 0
+	fails = []
+	log.each do |task, attribs|
+		abort = find_failed(attribs['start'], attribs['end'])
+		if !abort.nil?
+			position = abort if abort > position
+			fails << [task, abort]
+		end
+	end
+	if !fails.empty?
+		fails.each do |task, index| 
+			if position == index
+				failed_jobs << task
+			end
+		end
+	end
+	return failed_jobs
+end
+
+def find_failed(ar_start, ar_end)
+	position = nil
+	ar_start.reverse.each_with_index do |st, i|
+		reverse_pos = ar_start.length - i - 1
+		stop = ar_end[reverse_pos]
+		if st > 0 && stop == 0
+			next_executions = ar_end[reverse_pos..ar_end.length - 1]
+			if next_executions.nil? || next_executions.count(0) == next_executions.length
+				position = reverse_pos
+				break 
+			end
+		end
+	end
+	return position
+end
+
+def set_time(log)
+	log.each do |task, attribs|
+		start = attribs['start'].last
+		stop = attribs['end'].last
+		status = attribs['state']
+		time = 0
+		if status == 'SUCC'
+			time = stop - start
+		elsif status == 'RUN'
+			time = Time.now.to_i - start
+		end
+		attribs['seconds'] = time
+		magnitude = 's'
+		if time >= 60
+			magnitude = 'm'
+			time = time /60.0 # To minutes
+		end
+		if time >= 60 && magnitude == 'm'
+			magnitude = 'h'
+			time = time /60.0 # To hours
+		end
+		if time >= 24 && magnitude == 'h'
+			magnitude = 'd'
+			time = time /24.0 # To days
+		end
+		if time == 0
+			time_string = '-'
+		else
+			time_string = "#{time} #{magnitude}"
+		end
+		attribs['time'] = time_string
+	end
+end
+
+def set_task_state(log, workflow_status, position = -1)
+	log.each do | task, attribs|
+		start_position = attribs['start'].length - position
+		start = attribs['start'][position]
+		stop_position = attribs['end'].length - position
+		stop = attribs['end'][position]
+		if workflow_status # Workflow has finished
+			if start == 0 && stop == 0
+				status = 'NOT'
+				status_msg = 'NOT'.colorize(:blue)
+			elsif start > 0 && stop > 0
+				status = 'SUCC'
+				status_msg = 'SUCC'.colorize(:green)
+			elsif start > 0 && stop ==0
+				status = 'ABORT'
+				status_msg = 'ABORT'.colorize(:red)
+			end				
+		else # Workflow is still running
+			if start == 0 && stop == 0
+				status = 'PEND'
+				status_msg = 'PEND'.colorize(:blue)
+			elsif start > 0 && stop > 0
+				status = 'SUCC'
+				status_msg = 'SUCC'.colorize(:green)
+			elsif start > 0 && stop ==0
+				status = 'RUN'
+				status_msg = 'RUN'.colorize(:magenta)
+			end				
+		end
+		attribs['state'] = status
+		attribs['state_msg'] = status_msg
+	end
+end
+
+def add_timestamp(log_file, attrib, task_name)
+	File.open(log_file, 'a'){|f| f.puts "#{task_name}\t#{attrib}\t#{Time.now.to_i}"}
+end
+
+def report_html(log, initial_flow_attribs)
+	set_task_state(log, TRUE) 
+	set_time(log)
+	report ="
+	<table>
+		<% log.each do |task, attribs| %>
+			<tr>
+				<td><%= task %></td>
+				<td><%= attribs['seconds'] %></td>
+			<tr>
+		<% end %>
+	</table>
+	"
+	data_structure = {
+		'y' =>{ 
+			'vars' => ['Time'],
+			'smps' => log.keys,
+			'data' => [log.values.map{|attribs| attribs['seconds']}],
+			'desc' => ['seconds']
+		},
+		'a' => {
+			"xAxis" => ["Time"]
+		}
+	}
+	#puts log.inspect 
+
+	puts data_structure.inspect
+	#renderer = ERB.new(report)
+	demo = File.open('lines.html').read
+	demo.gsub!('data_structure', data_structure.to_json)
+	renderer = ERB.new(report + "\n" + demo)
+	File.open('report.html', 'w'){|f| f.puts renderer.result()}
+end
+#################################################################################################
+### PARSE OPTIONS
+#################################################################################################
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: __FILE__ [options]"
+
+  options[:workflow_execution] = Dir.pwd
+  opts.on("-e", "--workflow_execution PATH", "Path to workflow directory") do |opt|
+    options[:workflow_execution] = File.expand_path(opt)
+  end
+
+  options[:start] = nil
+  opts.on("-s", "--start TASK_NAME", "Write start timestamp of TASK_NAME to log") do |opt|
+    options[:start] = opt
+  end
+
+  options[:finish] = nil
+  opts.on("-f", "--finish TASK_NAME", "Write finish timestamp of TASK_NAME to log") do |opt|
+    options[:finish] = opt
+  end
+
+  options[:report] = nil
+  opts.on("-r", "--report STATUS", "List the status of launched tasks.") do |opt|
+    options[:report] = opt
+  end
+
+  options[:workflow_status] = FALSE
+  opts.on("-w", "--workflow_finished", "When set, logger assumes that the workflow has ended") do |opt|
+    options[:workflow_status] = TRUE
+  end
+  
+  options[:no_size] = FALSE
+  opts.on("-n", "--no_size", "When set, logger don't compute the workflow folder sizes") do |opt|
+    options[:no_size] = TRUE
+  end
+
+	options[:batch] = FALSE
+	opts.on( '-b', '--batch', 'Workflow execution using batch' ) do |opt|
+		options[:batch] = TRUE
+	end
+
+  options[:launch_failed_jobs] = FALSE
+  opts.on("-l", "--launch_failed_jobs", "Launch jobs tagges as ABORT and NOT. This option only works when the -w flag is enabled") do |opt|
+    options[:launch_failed_jobs] = TRUE
+  end 
+
+  options[:html] = FALSE
+  opts.on("-H", "--html", "Make a workflow execution full report in html format") do |opt|
+    options[:html] = TRUE
+  end  
+end.parse!
+
+#################################################################################################
+### MAIN
+#################################################################################################
+
+if !options[:start].nil?
+	add_timestamp(options[:workflow_execution],'start', options[:start])
+elsif !options[:finish].nil?
+	add_timestamp(options[:workflow_execution],'end', options[:finish])
+else
+	log_folder = File.join(options[:workflow_execution], '.wf_log')
+	job_attribs_file = File.join(options[:workflow_execution], 'wf.json')
+
+	if !Dir.exists?(log_folder) 
+		puts "Log folder not exists"
+		Process.exit
+	end
+	if !File.exists?(job_attribs_file)
+		puts "wf.json file not exists"
+		Process.exit
+	end
+
+	attribs = JSON.parse(File.open(job_attribs_file).read)
+	log = parse_log(log_folder)
+	if !options[:report].nil?
+		report_log(log, attribs, options[:report], options[:workflow_status], options[:no_size])
+	elsif options[:html]
+		report_html(log, attribs)
+	elsif options[:workflow_status] && options[:launch_failed_jobs]
+		launch_failed_jobs(log, attribs, options[:workflow_execution], options[:batch])
+	end
+end

data/lib/autoflow/logging.rb

@@ -0,0 +1,58 @@
+def parse_log(log_path)
+	log = {}
+	if Dir.exists?(log_path)
+		Dir.entries(log_path).each do |entry|
+			next if entry == '.' || entry == '..'
+			File.open(File.join(log_path, entry)).each do |line|
+				line.chomp!
+				name, status, time_int = line.split("\t")
+				time = time_int.to_i
+				query = log[name]
+				if query.nil?
+					log[name] = {status => [time]}
+				else
+					query_status = query[status]
+					if query_status.nil?
+						query[status] = [time]
+					else
+						query[status] << time
+					end
+				end
+			end
+		end
+	end
+	log.each do |task, attribs|
+		#puts "#{attribs.inspect}"
+		set_length = attribs['set'].length
+		fill_attrib(attribs, 'start', set_length)
+		fill_attrib(attribs, 'end', set_length)
+	end
+	return log
+end
+
+def fill_attrib(attribs, mode, set_length)
+	query = attribs[mode]
+	if query.nil?
+		attribs[mode] = Array.new(set_length, 0)
+	elsif query.length < set_length
+		(set_length - query.length).times do
+			query << 0
+		end
+	end
+end
+
+def write_log(log, log_path, job_relations_with_folders)
+	Dir.mkdir(log_path) if !Dir.exists?(log_path)
+	job_relations_with_folders.each do |name, folder_deps|
+		if !log[name].nil? #Control check when the wk_log folder has been deleted
+			folder, deps = folder_deps
+			f = File.open([log_path, File.basename(folder)].join('/'), 'w') 
+			log[name].each do |mode, times|
+				times.each do |time|
+					f.puts "#{name}\t#{mode}\t#{time}"
+				end
+			end
+			f.close
+		end
+	end
+end

data/lib/autoflow/queue_manager.rb

@@ -1,3 +1,5 @@
+require 'logging'
+require 'json'
 class QueueManager
 
 	def initialize(exec_folder, options, commands, persist_variables)
@@ -9,7 +11,9 @@ class QueueManager
 		@files = {}
 		@remote = options[:remote]
 		@ssh = options[:ssh]
+		@write_sh = options[:write_sh]
 		@external_dependencies = options[:external_dependencies]
+		@active_jobs = []
 	end
 
 	########################################################################################
@@ -20,7 +24,7 @@ class QueueManager
 		ObjectSpace.each_object(Class).select { |klass| klass < self }
 	end
 
-	def self.select_queue_manager(stack, options)
+	def self.select_queue_manager(exec_folder, options, jobs, persist_variables)
 		path_managers = File.join(File.dirname(__FILE__),'queue_managers')
 		Dir.glob(path_managers+'/*').each do |manager|
 			require manager
@@ -30,7 +34,7 @@ class QueueManager
 		else
 			queue_manager = select_manager(options)
 		end
-		return queue_manager.new(stack.exec_folder, options, stack.jobs, stack.persist_variables)
+		return queue_manager.new(exec_folder, options, jobs, persist_variables)
 	end
 
 	def self.select_manager(options)
@@ -60,9 +64,42 @@ class QueueManager
 		close_file('index_execution')
 	end
 
+	def init_log #TODO adapt to remote execution
+		log_path = [@exec_folder, '.wf_log'].join('/') #Join must assume linux systems so File.join canot be used for windows hosts
+		log = parse_log(log_path) #TODO modify to folder
+		job_relations_with_folders = get_relations_and_folders
+		if @write_sh
+			create_file('wf.json', @exec_folder)
+			write_file('wf.json', job_relations_with_folders.to_json)
+			close_file('wf.json')
+		end
+  		@active_jobs.each do |task|
+  			query = log[task]
+  			if query.nil?
+				log[task] = {'set' => [Time.now.to_i]}
+  			else
+				log[task]['set'] << Time.now.to_i
+  			end
+		end
+		write_log(log, log_path, job_relations_with_folders)
+	end
+
+	def get_relations_and_folders
+		relations = {}
+		@commands.each do |name, job|
+			relations[name] = [job.attrib[:exec_folder], job.dependencies]
+		end
+		return relations
+	end
+
 	def launch_all_jobs
 		buffered_jobs = []
-		sort_jobs_by_dependencies.each do |name, job|
+		sorted_jobs = sort_jobs_by_dependencies
+		sorted_jobs.each do |name, job|
+			@active_jobs << job.name if !job.attrib[:done]
+		end
+		init_log
+		sorted_jobs.each do |name, job|
 			write_file('index_execution', "#{name}\t#{job.attrib[:exec_folder]}") 
 			if job.attrib[:done]
 				next
@@ -70,7 +107,7 @@ class QueueManager
 				rm_done_dependencies(job)
 			end	
 			buffered_jobs = launch_job_in_folder(job, name, buffered_jobs)
-		end 
+		end
 	end
 
 	def sort_jobs_by_dependencies # We need job ids from queue system so we ask for each job and we give the previous queue system ids as dependencies if necessary
@@ -113,35 +150,41 @@ class QueueManager
 
 
 	def launch2queue_system(job, id, buffered_jobs)
-		# Write sh file
-		#--------------------------------
-		log_folder = File.join(@exec_folder, 'log')
 		sh_name = job.name+'.sh'
-		create_file(sh_name, job.attrib[:exec_folder])
-		write_file(sh_name, '#!/usr/bin/env bash')
-		write_file(sh_name, '##JOB_GROUP_ID='+@job_identifier)
-		write_header(id, job, sh_name)
+		if @write_sh
+			# Write sh file
+			#--------------------------------
+			create_file(sh_name, job.attrib[:exec_folder])
+			write_file(sh_name, '#!/usr/bin/env bash')
+			write_file(sh_name, '##JOB_GROUP_ID='+@job_identifier)
+			write_header(id, job, sh_name)
+		end
 
 		#Get dependencies
 		#------------------------------------
 		ar_dependencies = get_dependencies(job, id)
 		buffered_jobs.each do |id_buff_job, buff_job|
-			write_job(buff_job, sh_name)
 			ar_dependencies += get_dependencies(buff_job, id_buff_job)
-			buff_job.attrib[:exec_folder] = job.attrib[:exec_folder]
+			if @write_sh
+				write_job(buff_job, sh_name)
+				buff_job.attrib[:exec_folder] = job.attrib[:exec_folder]
+			end
 		end
 		ar_dependencies.uniq!
 
-		#Write sh body
-		#--------------------------------
-		write_file(sh_name, 'hostname')
-		write_file(sh_name, "echo -e \"STARTED   #{id}  #{job.parameters.split.first}:\\t`date`\" >> #{log_folder}")
-		write_file(sh_name, "source #{File.join(@exec_folder, 'env_file')}") if !@persist_variables.empty?
-		write_job(job, sh_name)
-		write_file(sh_name, "echo -e \"FINISHED  #{id}  #{job.parameters.split.first}:\\t`date`\" >> #{log_folder}")
-		write_file(sh_name, "echo 'General time'")
-		write_file(sh_name, "times")
-		close_file(sh_name, 0755)
+		if @write_sh
+			#Write sh body
+			#--------------------------------
+			write_file(sh_name, 'hostname')
+			log_file_path = [@exec_folder, '.wf_log', File.basename(job.attrib[:exec_folder])].join('/')
+			write_file(sh_name, "flow_logger -e #{log_file_path} -s #{job.name}")
+			write_file(sh_name, "source #{File.join(@exec_folder, 'env_file')}") if !@persist_variables.empty?
+			write_job(job, sh_name)
+			write_file(sh_name, "flow_logger -e #{log_file_path} -f #{job.name}")
+			write_file(sh_name, "echo 'General time'")
+			write_file(sh_name, "times")
+			close_file(sh_name, 0755)
+		end
 
 		#Submit node
 		#-----------------------------------
@@ -190,6 +233,17 @@ class QueueManager
 		end
 	end
 
+	def read_file(file_path)
+		content = nil
+		if @remote
+			res = @ssh.exec!("[ ! -f #{file_path} ] && echo 'Autoflow:File Not Found' || cat #{file_path}")
+			content = res if !content.include?('Autoflow:File Not Found')
+		else
+			content = File.open(file_path).read if File.exists?(file_path)
+		end
+		return content
+	end
+
 	def system_call(cmd, path = nil)
 		cmd = "cd #{path}; " + cmd if !path.nil?
 		if @remote

data/lib/autoflow/queue_managers/bash_manager.rb

@@ -4,7 +4,8 @@ class BashManager < QueueManager
 	def initialize(exec_folder, options, commands, persist_variables)
 		super
 		@queued = []
-		@last_deps = []
+		@count = 0
+		@pids = {}
 		@path2execution_script = File.join(@exec_folder, 'execution.sh') 
 		create_file('execution.sh', @exec_folder)
 		write_file('execution.sh', '#! /usr/bin/env bash')
@@ -17,22 +18,22 @@ class BashManager < QueueManager
 	end
 
 	def write_header(id, node, sh)
-		@queued << id # For dependencies purposes
+		#@queued << id # For dependencies purposes
 	end
 
 	def submit_job(job, ar_dependencies)
 		write_file('execution.sh','')
 		if !ar_dependencies.empty? 
-			deps = ar_dependencies - @last_deps
-			if !deps.empty?
-				write_file('execution.sh', 'wait') 
-				@last_deps.concat(@queued)
+			ar_dependencies.each do |dep|
+				cmd = "wait \"$pid#{@pids[dep]}\"\nif [ $? -ne 0 ]\nthen \n\techo \"#{job.name} failed\"\n\texit\nfi"
+				write_file('execution.sh', cmd) 
 			end
 		end
-		@last_deps.concat(ar_dependencies)
-		@last_deps.uniq!
 		write_file('execution.sh', "cd #{job.attrib[:exec_folder]}")
-		write_file('execution.sh', "./#{job.name}.sh &")
+		write_file('execution.sh', "./#{job.name}.sh &> task_log & pid#{@count}=$!")
+		@pids[job.name] = @count
+		@count += 1
+		@queued << job.name # For dependencies purposes
 		return nil
 	end
 

data/lib/autoflow/stack.rb

@@ -27,16 +27,17 @@ class Stack
 		@exec_folder = exec_folder #TODO move this to queue_manager
 		@do_retry = options[:retry]
 		@options = options
-		parse(options[:workflow], options[:Variables])
-		@jobs = get_jobs_relations
+		@workflow = options[:workflow]
+		@external_variables= options[:Variables] 
+		@jobs = {}
 	end
 	
-	def parse(workflow, external_variables)
+	def parse!
 		#Clean template
-		workflow.gsub!(/\#.+$/,'')	#Delete comments
-		workflow.gsub!("\t",'')		#Drop tabs
-		workflow.gsub!(/\n+/,"\n")	#Drop empty lines
-		workflow.gsub!(/^\s*/,'')
+		@workflow.gsub!(/\#.+$/,'')	#Delete comments
+		@workflow.gsub!("\t",'')		#Drop tabs
+		@workflow.gsub!(/\n+/,"\n")	#Drop empty lines
+		@workflow.gsub!(/^\s*/,'')
 
 		#Parse template
 		variables_lines = []
@@ -44,7 +45,7 @@ class Stack
 		node_lines = []
 
 		node_beg = FALSE		
-		workflow.each_line do |line|
+		@workflow.each_line do |line|
 			node_beg = TRUE if line.include?('{') 	# This check the context of a variable
 			if line.include?('}')					# if a variable is within a node,
 				if node_beg							# we consider tha is a bash variable not a static autoflow variable
@@ -62,9 +63,10 @@ class Stack
 			end
 		end
 		load_variables(variables_lines, @variables)
-		load_variables(external_variables, @variables)
+		load_variables(@external_variables, @variables)
 		load_variables(persist_variables_lines, @persist_variables)
 		parse_nodes(node_lines)
+		@jobs = get_jobs_relations
 	end
 
 	def load_variables(variables_lines, variable_type)

data/lib/autoflow/version.rb

@@ -1,3 +1,3 @@
 module Autoflow
-  VERSION = "0.6.4"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: autoflow
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.8.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-10 00:00:00.000000000 Z
+date: 2017-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.8.0
+- !ruby/object:Gem::Dependency
+  name: git
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -43,6 +59,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.7.3
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -82,6 +114,7 @@ email:
 executables:
 - AutoFlow
 - env_manager
+- flow_logger
 - flow_time
 extensions: []
 extra_rdoc_files: []
@@ -94,9 +127,11 @@ files:
 - autoflow.gemspec
 - bin/AutoFlow
 - bin/env_manager
+- bin/flow_logger
 - bin/flow_time
 - lib/autoflow.rb
 - lib/autoflow/batch.rb
+- lib/autoflow/logging.rb
 - lib/autoflow/program.rb
 - lib/autoflow/queue_manager.rb
 - lib/autoflow/queue_managers/bash_manager.rb