full_lengther_next 0.6.2 → 0.9.8

data/lib/full_lengther_next/{classes/my_worker_manager_fln.rb → my_worker_manager_fln.rb}

@@ -8,7 +8,78 @@ require 'reptrans'
 
 include FlnStats
 
-
+module ScbiMapreduce
+	class WorkManager
+		def send_next_work
+			  sleep = false
+		      # if we need to exit, send quit to workers
+		      if @@want_to_exit
+		        send_object(:quit)
+		        
+		      elsif !send_stuck_work
+		        
+		      #send stuck work
+		        objs=[]
+		        
+		        t=Time.now_us
+		        
+		        begin
+		        # prepare new data
+		        @@chunk_size.times do
+		          obj=next_work
+		          if obj.nil?
+		            break
+		          elsif obj == :sleep
+		          	#send_object(:sleep)
+		          	sleep = true
+		          	break  	
+		          else
+		            # add to obj array
+		            objs << obj
+		          end
+		        end
+		        rescue Exception => e
+		          $SERVER_LOG.error("Exception creating next_work. Worker, quit!")
+		          $SERVER_LOG.error("#{e.message}\n#{e.backtrace.join("\n")}")
+		          send_object(:sleep)
+		          self.class.global_error_received(e)
+
+		          #raise e
+		        end
+		        
+		        @@total_read_time+=(Time.now_us - t)
+		        
+		        # if new was data collected, send it
+		        if objs.count>0
+		          @@count += objs.count
+		          @@chunk_count += 1
+
+		          work_data=WorkManagerData.new(objs)
+		          send_object(work_data)
+		          @@sent_chunks+=1
+
+		          # to keep order or retry failed job, we need job status
+		          if @@keep_order || @@retry_stuck_jobs
+		            # do not remove data to be able to sent it again
+		            # work_data.data=nil
+		            @@running_jobs.push work_data
+		            # print_running_jobs
+		          end
+		        else
+		          # otherwise, 
+		          if @@running_jobs.count >0 || sleep
+		            $SERVER_LOG.info("Worker, go to sleep")
+		            send_object(:sleep)
+		            
+		          else
+		            # send a quit value indicating no more data available
+		            send_object(:quit)
+		          end
+		        end
+		      end
+		end
+	end
+end
 
 
 class MyWorkerManagerFln < ScbiMapreduce::WorkManager
@@ -20,10 +91,17 @@ class MyWorkerManagerFln < ScbiMapreduce::WorkManager
 	def self.init_work_manager(options)
 		@@stats_hash = initialize_stats_hash
 		@@stats_taxonomy = {}
+		@@stats_functional_annotation_by_seqs = {}
 		@@stats_different_prot_id = []
 		@@stats_different_prot_id_complete_seqs = []
+		@@pre_fln_seq_lengths = []
+		@@seq_lengths = []
+
+		@@map_object = {}
+
 
 		@@options = options
+
 		$verbose = options[:verbose]
 
 		input_file = options[:fasta]
@@ -45,8 +123,8 @@ class MyWorkerManagerFln < ScbiMapreduce::WorkManager
 		}
 
 		@@functional_annotations = {}
-		@@functional_annotations.merge!(load_functional_annotations(File.join(ENV['BLASTDB'], 'sp_'+options[:tax_group],'sp_'+options[:tax_group]+'.index')))
-		#@@functional_annotations.merge!(load_functional_annotations(File.join(ENV['BLASTDB'], 'tr_'+options[:tax_group],'tr_'+options[:tax_group]+'.index'))) if options[:acess_db].include?('t')
+		@@functional_annotations.merge!(load_functional_annotations(File.join(ENV['BLASTDB'], 'sp_'+options[:tax_group],'sp_'+options[:tax_group]+'.index'))) if options[:acess_db].include?('s')
+		@@functional_annotations.merge!(load_functional_annotations(File.join(ENV['BLASTDB'], 'tr_'+options[:tax_group],'tr_'+options[:tax_group]+'.index'))) if options[:acess_db].include?('t')
 
 		@@fasta_file = FastaQualFile.new(input_file,'')
 		file_head = "Query_id\tfasta_length\tSubject_id\tdb_name\tStatus\te_value\tp_ident\ts_length\tprotein_length\tWarning_msgs\tframe\tORF_start\tORF_end\ts_start\ts_end\tDescription\tgo_id\tgo_description\tkegg_id\tinterpro_id\tinterpro_description\tec_id\tpfam_id\tpfam_description\tunipathway_id"
@@ -71,13 +149,24 @@ class MyWorkerManagerFln < ScbiMapreduce::WorkManager
 		@@output_files['prot']			= File.open('fln_results/proteins.fasta', 'w') # FASTA
 		@@output_files['nts']			= File.open("fln_results/nt_seq.txt", 'w')
 		@@output_files['seqs']			= File.open('fln_results/unigenes.fasta', 'w') # FASTA
-		@@output_files['stats_html']	= File.open('fln_results/summary_stats.html', 'w')
+		@@output_files['stats_html']	= 'fln_results/summary_stats.html'
 		@@output_files['stats_txt']		= File.open('fln_results/summary_stats.txt', 'w')
 
 		@@output_files[CODING].puts file_head
 		@@output_files['db'].puts file_head
 		@@output_files[NCRNA].puts file_head
-	
+
+		if !options[:files2map].empty?
+			@@output_files['fpkm']			= File.open('fln_results/fpkm_per_transcript.txt', 'w')
+			@@output_files['coverage']		= File.open('fln_results/coverage_per_transcript.txt', 'w')
+			@@output_files['fpkm'].puts %w[Transcript_id fpkm Read_counts].join("\t")
+			@@output_files['coverage'].puts %w[seq_name mean_normalized_differences mean_max mean_coverage proportion_sequence_mapped].join("\t")	
+			if options[:remove_unmapped]
+				@@output_files[UNMAPPED]	= File.open('fln_results/unmapped.txt', 'w')
+				@@output_files[UNMAPPED].puts "Query_id\tfasta_length"
+			end
+		end
+
 		#RepTrans module
 		@@seqs_annotation_prot	= []
 		@@seqs_some_coding 		= []
@@ -87,6 +176,10 @@ class MyWorkerManagerFln < ScbiMapreduce::WorkManager
 		@@complete_sure = []
 		@@seqs_to_analyze = []
 
+		#Mapping_info
+		@@fpkm = {}
+		@@coverage_analysis = {}
+
 	end
 
 	#############################################################################################
@@ -96,9 +189,10 @@ class MyWorkerManagerFln < ScbiMapreduce::WorkManager
 	# close files
 	def self.end_work_manager
 		orf_prediction_with_transdecoder if @@options[:acess_db].include?('p') && !@@complete_sure.empty? && !@@seqs_to_analyze.empty?
-		write_summary_stats(@@stats_hash, @@stats_taxonomy, @@stats_different_prot_id, @@stats_different_prot_id_complete_seqs, @@output_files['stats_txt'], @@output_files['stats_html'])
+		write_summary_stats(@@stats_hash, @@stats_taxonomy, @@stats_functional_annotation_by_seqs, @@stats_different_prot_id, @@stats_different_prot_id_complete_seqs, @@pre_fln_seq_lengths, @@seq_lengths, @@output_files['stats_txt'], @@output_files['stats_html'])
+		write_mapping_report(@@fpkm, @@coverage_analysis, @@stats_functional_annotation_by_seqs)
 		@@output_files.each do |key, handler|
-			handler.close
+			handler.close if handler.class != String
 		end
 	end
 	
@@ -109,17 +203,66 @@ class MyWorkerManagerFln < ScbiMapreduce::WorkManager
 	# this method is called every time a worker needs new data to work. This method is executed many times like the chunk size says.
 	# Return the work data or nil if no more data is available
 	def next_work #Manage INput's worker
+		obj = nil
+		#send = 'None'
+		if !@@options[:files2map].empty? && !@@options[:ref_files].empty? #Mapping task
+		#	send = 'Map'
+			obj = send_mapping_data 
+		elsif !@@options[:files2map].empty? && @@options[:ref_files].empty? && @@map_object.length < @@options[:n_refs]
+		#	send = 'Sleep'
+			obj = :sleep
+		else
+		#	send = 'Fln'
+			obj = send_fln_data
+		end
+		#$LOG.info "Next_work: Sent => #{send}. ref_files.length = #{@@options[:ref_files].length}; n_refs = #{@@options[:n_refs]}; map_object.length = #{@@map_object.length}"
+		return obj
+	end
+
+	def send_mapping_data
+		return {mapping: @@options[:ref_files].shift}
+	end
+
+	def send_fln_data
+		obj = nil
 		n,f,q = @@fasta_file.next_seq
 		if !n.nil?
-			@@stats_hash['input_seqs'] += 1
-			return Sequence.new(n,f,q) 
-		else
-			return nil
+			seq = Sequence.new(n,f,q)
+			begin
+				
+				if !@@map_object.empty?
+					cov_analysis = @@map_object[n]
+					if cov_analysis.nil?
+						seq.fpkm = []
+						seq.coverage_analysis = []
+					else
+						seq.fpkm = cov_analysis.pop(2)
+						seq.coverage_analysis = cov_analysis
+					end
+				end
+			rescue Exception => e
+				puts e.message, e.backtrace.join("\n")
+			end
+
+			@@pre_fln_seq_lengths << f.length
+			sequence_stats(seq, @@stats_hash)
+			obj = {fln: seq} 
 		end
+		return obj
 	end
   	
 	# this method is ejecuted each time an obj is finished
 	def work_received(objs) #Manage OUTput's worker
+		task = objs.keys.first
+		data = objs.values.first
+		if task == :fln
+			receive_fln_data(data)
+		elsif task == :mapping
+			receive_mapping_data(data)
+		end
+	end 
+	
+	def receive_fln_data(objs)
 		objs.each do |seq|
 			transdecoder_keep_seq(seq)
 			repTrans_keep_seq(seq)
@@ -127,13 +270,27 @@ class MyWorkerManagerFln < ScbiMapreduce::WorkManager
 				get_taxonomy(seq.hit.definition, @@stats_taxonomy) 
 				get_functional_annotations(seq)
 			end
+			@@fpkm[seq.seq_name] = seq.fpkm if !seq.fpkm.empty?
+			@@coverage_analysis[seq.seq_name] = seq.coverage_analysis if !seq.coverage_analysis.empty?
 			write_seq(seq) if @@options[:acess_db].include?('c') || !@@options[:acess_db].include?('p') || ( seq.type != UNKNOWN && seq.type != CODING ) #Don't write Unknown or coding sequences when use transdecoder
 		end
-		@@stats_hash, @@stats_different_prot_id, @@stats_different_prot_id_complete_seqs = summary_stats(objs, @@stats_hash, @@stats_different_prot_id, @@stats_different_prot_id_complete_seqs)
-	end 
-	
+		@@stats_hash, @@stats_different_prot_id, @@stats_different_prot_id_complete_seqs, @@seq_lengths = summary_stats(objs, @@stats_hash, @@stats_different_prot_id, @@stats_different_prot_id_complete_seqs, @@seq_lengths)
+	end
+
+	def receive_mapping_data(objs)
+		data = objs
+		@@map_object.merge!(data)
+	end
+
 	def error_received(worker_error, obj)
-		puts "WARNING!!!!!. CHUNK FAILED:Error while processing object #{obj.first.seq_name}\n" + worker_error.original_exception.message + ":\n" +worker_error.original_exception.backtrace.join("\n")
+		data = nil
+		sample = obj.first 
+		if sample.class == Sequence
+			data = obj.seq_name
+		else
+			data = obj.inspect
+		end
+		puts "WARNING!!!!!. CHUNK FAILED:Error while processing object #{data}\n" + worker_error.original_exception.message + ":\n" +worker_error.original_exception.backtrace.join("\n")
 	end
 
 	def too_many_errors_received
@@ -167,6 +324,14 @@ class MyWorkerManagerFln < ScbiMapreduce::WorkManager
 			@@func_annot_type.each do |type, position|
 				annotations[type] = all_info[position]
 			end
+			annotations[:go_description].split(";").each do |annot|
+				query = @@stats_functional_annotation_by_seqs[annot]
+				if query.nil?
+					@@stats_functional_annotation_by_seqs[annot] = [seq.seq_name]
+				else
+					query << seq.seq_name 
+				end
+			end
 			seq.functional_annotations = annotations
 		end
 	end
@@ -226,7 +391,7 @@ class MyWorkerManagerFln < ScbiMapreduce::WorkManager
 		clusters_seqs_annot_prot = clustering_by_id(@@complete_sure)
 		final_seqs = select_representative(clusters_seqs_annot_prot)
 		coding_info = nil
-		Dir.chdir('temp') do
+		Dir.chdir(@@options[:temp]) do
 			orfs = get_seqs(final_seqs)
 			File.open('training_set.fasta', 'w') {|f| f.write(orfs)}
 			orfs = get_seqs(@@seqs_to_analyze)

data/lib/full_lengther_next/{classes/reptrans.rb → reptrans.rb}

@@ -186,13 +186,17 @@ def clustering_by_annot(seqs_with_hit, annotation_type)
 end
 
 def select_representative(clusters_seqs_annot_prot)
-	seqs=[]	
+	seqs = []	
 	clusters_seqs_annot_prot.each do |cluster|
-		seq=cluster.select{|s| s.type == COMPLETE}.sort{|fl1, fl2| fl2.seq_fasta <=> fl1.seq_fasta}.first # Take longest full-length, s -> sequence, fl -> full-lentgh
+		if !cluster.first.coverage_analysis.empty? # filtering by mapping coverage
+			max_transcript_mean_coverage = cluster.map{|seq| seq.coverage_analysis[3] }.max - 0.05 # Relaxed limit of 5%
+			cluster.select!{|seq| seq.coverage_analysis[3] >= max_transcript_mean_coverage}
+		end
+		seq = cluster.select{|s| s.type == COMPLETE}.sort{|fl1, fl2| fl2.seq_fasta <=> fl1.seq_fasta}.first # Take longest full-length, s -> sequence, fl -> full-lentgh
 		if seq.nil?
 			cluster.sort!{|cl1, cl2| cl2.get_pident <=> cl1.get_pident}
-			best_pident=cluster.first.get_pident
-			seq=cluster.select{|s| s.get_pident == best_pident}.sort{|s1, s2| s2.seq_fasta <=> s1.seq_fasta}.first
+			best_pident = cluster.first.get_pident
+			seq = cluster.select{|s| s.get_pident == best_pident}.sort{|s1, s2| s2.seq_fasta <=> s1.seq_fasta}.first
 		end
 		seqs << seq
 	end
@@ -207,4 +211,4 @@ def count_cpu(options)
 		cpu = options[:workers]
 	end
 	return cpu
-end
+end

data/lib/full_lengther_next/{classes/sequence.rb → sequence.rb}

@@ -5,7 +5,7 @@ require 'common_functions'
 
 class Sequence
 	
-	attr_accessor :seq_name, :seq_fasta, :fasta_length, :db_name, :seq_nt, :seq_aa, :db, :type, :status, :id, :orfs, :area_without_annotation, :save_fasta, :ignore, :hit, :t_code, :functional_annotations
+	attr_accessor :seq_name, :seq_fasta, :fasta_length, :db_name, :seq_nt, :seq_aa, :db, :type, :status, :id, :orfs, :area_without_annotation, :save_fasta, :ignore, :hit, :t_code, :functional_annotations, :fpkm, :coverage_analysis
 
 	def initialize(seq_name, seq_fasta, seq_qual='')
 		@seq_name = seq_name
@@ -22,6 +22,10 @@ class Sequence
 		@annotations=[]
 		@functional_annotations = {}
 		@orfs=[]
+
+		#Mapping_info
+		@fpkm = []
+		@coverage_analysis = []
 		
 		@area_without_annotation=FALSE
 		@save_fasta=TRUE
@@ -159,11 +163,21 @@ class Sequence
 		if @save_fasta
 			output_files['seqs'].puts get_fasta(@seq_fasta)	
 		end
+		if !@fpkm.empty?
+			output_files['fpkm'].puts "#{@seq_name}\t#{@fpkm.join("\t")}"
+		end
+		if !@coverage_analysis.empty?
+			all_coverages = @coverage_analysis.last
+			output_files['coverage'].puts "#{@seq_name}\t#{@coverage_analysis[0..@coverage_analysis.length-1].join("\t")}"
+		end
+
 		case @type
 		when OTHER
 			write_other(output_files[@type])
 		when CHIMERA
 			write_chimera(output_files[@type])
+		when UNMAPPED
+			write_unmapped(output_files[@type])
 		when MISASSEMBLED
 			write_misassembled(output_files[@type])
 		when UNKNOWN
@@ -193,6 +207,10 @@ class Sequence
 		file.puts "#{@seq_name}\t#{@fasta_length}\t#{@hit.acc}\t#{@db_name}\t#{all_warns}"
 	end
 
+	def write_unmapped(file)
+		file.puts "#{@seq_name}\t#{@fasta_length}"
+	end
+
 	def write_chimera(file) #TODO : write 'SOLVED' tag
 		@hit.each do |h|
 			file.puts "#{@seq_name}\t#{@fasta_length}\t#{h.acc}\t#{@db_name}\t#{h.q_frame}\t#{h.e_val}\t#{h.ident}\t#{h.q_beg + 1}\t#{h.q_end + 1}\t#{h.s_beg + 1}\t#{h.s_end + 1}\t#{h.definition}"
@@ -268,6 +286,8 @@ class Sequence
 				type_description = 'Failed'
 			when OTHER
 				type_description = 'Other'
+			when UNMAPPED
+				type_description = 'Unmapped'
 			when CHIMERA
 				type_description = 'Chimera'
 			when MISASSEMBLED
@@ -469,4 +489,9 @@ class Sequence
 		array_annotations.map{|annotation| @annotations << annotation.dup}
 	end
 
+	def unmapped?
+		res = FALSE
+		res = TRUE if !@coverage_analysis.empty? && @coverage_analysis[3] == 0 #3 => percentage of sequence covered by reads 
+		return res
+	end
 end

data/lib/full_lengther_next/{classes/test_code.rb → test_code.rb}

@@ -1,6 +1,6 @@
 
 require 'common_functions'
-require 'scbi_plot'
+#require 'scbi_plot'
 
 include CommonFunctions
 

data/lib/full_lengther_next/{classes/types.rb → types.rb}

@@ -1,5 +1,6 @@
-FAILED			= -4
-OTHER			= -3
+FAILED			= -5
+OTHER			= -4
+UNMAPPED		= -3
 CHIMERA			= -2
 MISASSEMBLED	= -1
 UNKNOWN			= 0

data/lib/full_lengther_next/version.rb

@@ -0,0 +1,3 @@
+module FullLengtherNext
+  VERSION = '0.9.8'
+end

data/report_templates/general_summary.erb

@@ -0,0 +1,140 @@
+<% base_table_config = {
+	header: true,
+	border: 2,		
+	attrib: {
+		'cellspacing' => 0,
+		'cellpadding' => 2
+		}
+	} 
+base_pie_config = {
+	header: true, 
+	width: '650px',
+	height: '650px',
+	responsive: false,				
+	title: nil,
+	config: {
+		'fontName' => 'Arial',
+		'showPieValues' => false,
+		'pieSegmentPrecision' => 3,
+		'maxPieSectors' => 30,
+		'legendBox' => false,
+		'legendScaleFontFactor' => 0.9
+	}
+}
+%>
+<center> 
+	<div style="float: center; font-size: 30; margin:10px">
+		<b>Full-LengtherNEXT Summary</b>
+	</div>
+	<div style="overflow: hidden; width: 1024px; height: 650px; padding: 10">
+		<div style="margin: 0; float: left; width: 400px; height: 600px;">
+			<%= table_title('General info') %>
+			<%= table(base_table_config.merge({
+					id: :general_report, 
+					cell_align: ['left', 'right', 'right']
+					}
+				))
+			%>
+		</div>
+		<div  style="margin: 0; float:rigth">
+			<%= table_title('Report guiding assembly quality') %>
+			<%= table(base_table_config.merge({
+					id: :assembly_report,
+					cell_align: ['left', 'right', 'right']
+					}
+				)) 
+			%>
+			<sup>1</sup> Percents for subclassifications of this category <br> were calculated using this line as 100% reference.
+		</div>
+	</div>
+	<div style="overflow: hidden; width: 1024px; height: 660px; padding: 10">
+		<div  style="margin: 0; float:left">
+			<%= table_title('Structural profile') %>
+			<%= stacked(
+				id: :structural_data, 
+				header: true,
+				title: nil,
+				x_label: '% sequences',
+				width: '540px',
+				config: {
+					'graphOrientation' => 'vertical',
+					'fontName' => 'Arial',
+					'setMaxX' => 80,
+					'setMinX' => 0,
+					'smpLabelScaleFontFactor' => 1.2,
+					'axisTitleScaleFontFactor' => 2,
+					'xAxis2Show' => false,
+					'xAxisExact' => true
+				})%>
+		</div>
+		<div  style="margin: 0; float: rigth">
+			<%= table_title('Status report') %>
+			<%= table(base_table_config.merge({
+					id: :status_report,
+					cell_align: ['left', 'left', 'right', 'right']
+					}
+				))
+			%>
+		</div>
+	</div>
+	<div style="overflow: hidden; width: 1024px; height: 800px; padding: 10">
+		<div  style="margin: 0; float: left">
+			<%= table_title('Taxonomy distribution on annotations') %>
+			<%= barplot(
+				id: :taxonomy, 
+				header: true,
+				width: '600px',
+				title: nil,
+				x_label: 'Number of sequences',
+				config: {
+					'fontName' => 'Arial',
+					'axisTitleScaleFontFactor' => 1.5,
+					'showLegend' => false,
+					'smpLabelScaleFontFactor' => 1
+				})%>
+		</div>
+		<div  style="margin: 0; float: rigth">
+			<%= table_title('Database usage') %>
+			<%= table(base_table_config.merge({
+					id: :database_report,
+					cell_align: ['left', 'right', 'right']
+					}
+				))
+			%>
+		</div>
+	</div>
+	<div style="overflow: hidden; width: 1320px; padding: 10">
+		<div  style="margin: 0; float: left">
+			<%= table_title('GO: Molecular function') %>
+			<%= pie(base_pie_config.merge({
+					id: :function_go,
+					row_names: true,
+					header: true
+					}
+				))
+			%>
+		</div>
+		<div  style="margin: 0; float: rigth">
+			<%= table_title('GO: Biological process') %>
+			<%= pie(base_pie_config.merge({
+					id: :process_go, 
+					row_names: true,
+					header: true
+					}
+				))
+			%>
+		</div>
+	</div>
+	<div style="overflow: hidden; width: 1320px; padding: 10">
+		<div  style="margin: 0; float: left">
+			<%= table_title('GO: Cellular component') %>
+			<%= pie(base_pie_config.merge({
+					id: :component_go, 
+					row_names: true,
+					header: true
+					}
+				))
+			%>
+		</div>
+	</div>		
+</center>