full_lengther_next 0.6.2 → 0.9.8

data/lib/full_lengther_next/go_methods.rb

@@ -0,0 +1,42 @@
+def go_for_graph(sequences_by_ontologies, fpkm = {})
+	container = {}
+	go_data = [
+		[:function_go, 'F:'],
+		[:component_go, 'C:'],
+		[:process_go, 'P:']
+	]
+
+	go_data.each do |key, prefix|		
+		go_ontology = sequences_by_ontologies.select{|go, seq_ids| go =~ /^#{prefix}/}
+		go_names = [] 
+		go_vals = []
+		go_ontology.each do |go_name, seq_names|
+			go_label = go_name.gsub(prefix, '')
+			if fpkm.empty?
+				go_vals << seq_names.length
+				go_names << go_label
+			else
+				sum = seq_names.map{|seq_name| fpkm[seq_name].first }.inject { |sum, n| sum + n }
+				if sum > 0
+					go_vals << sum 
+					go_names << go_label
+				end
+			end
+		end
+		go_table = []
+		go_names.each_with_index do |name, index|
+			go_table << [name, go_vals[index]]
+		end
+		go_table.sort!{|v1, v2| v2[1] <=> v1[1]}
+		go_table.unshift([key.to_s, 'GO'])
+		if !go_names.empty?
+			container[key] = go_table 
+		else
+			container[key] = [
+				[key.to_s, 'GO'],
+				['No_data', 1]
+			] 
+		end
+	end
+	return container
+end

data/lib/full_lengther_next/mapping.rb

@@ -0,0 +1,296 @@
+require 'scbi_zcat'
+
+class Mpileup
+	def initialize(file_path)
+		@mpileup_file = ScbiZcatFile.new(file_path)
+		@last_line = nil
+	end
+
+	def read_contig(contig_name, contig_length)
+		coverages = []
+		if !@last_line.nil?
+			if @last_line[0] != contig_name
+				return nil
+			else
+				coverages = initialize_contig(contig_length)
+			end
+		else
+			line = @mpileup_file.readline
+			if line.nil?
+				@last_line = nil
+				return nil
+			else
+				@last_line = line.chomp.split("\t")
+				if @last_line[0] != contig_name
+					return nil
+				else
+					coverages = initialize_contig(contig_length)
+				end
+			end
+		end
+		
+		while !@mpileup_file.eof
+			fields = @mpileup_file.readline.chomp.split("\t")
+			contig = fields[0]
+			if contig == contig_name
+				coverages[fields[1].to_i-1] = fields[2].to_i
+			else
+				@last_line = fields
+				break
+			end
+		end
+		return coverages
+	end
+
+	def initialize_contig(contig_length)
+		coverages = Array.new(contig_length, 0) 
+		coverages[@last_line[1].to_i-1] = @last_line[2].to_i
+		return coverages
+	end
+
+	def close
+		@mpileup_file.close
+	end
+end
+
+class Mapping
+	def initialize(user_options = {})
+
+		options = {
+			threads: 1,
+			total_reads: [],
+		}.merge!(user_options)
+		@ref_fasta_path = options[:ref_fasta_path]
+		@temp_folder = options[:temp_folder]
+		@threads = options[:threads]
+
+		@map_files = []
+		@paired = []
+		@idxstats = []
+		@mpileups = []
+		@coverage_results = {}
+		@total_reads = options[:total_reads]
+	end
+
+	def do_ref(user_options = {})
+		options = {
+			name: 'ref',
+			command: 'bowtie2-build -f --threads /THREADS/ /REF_FASTA/ /OUTPUT/',
+			log: 'reference_log',
+			force: false
+		}
+		options.merge!(user_options)
+		@ref = File.join(@temp_folder, options[:name])
+		cmd = options[:command].gsub('/THREADS/', @threads.to_s)
+		cmd.gsub!('/REF_FASTA/', @ref_fasta_path)
+		cmd.gsub!('/OUTPUT/', @ref)
+		cmd = cmd + " &> #{File.join(@temp_folder, options[:log])}"
+		system(cmd) if options[:force] || Dir.glob(@ref+'*.bt2').length == 0 
+	end
+
+	def do_samtools_ref
+		cmd = "samtools faidx #{@ref_fasta_path}"
+		system(cmd) if !File.exists?(@ref_fasta_path + '.fai')
+	end
+
+	def do_map(user_options = {})
+		options = {
+			files: [],
+			command: 'bowtie2 -p /THREADS/ -x /REFERENCE/',
+			paired_pipe: '| samtools view -bS -F 4 | samtools sort -o /OUTPUT/',
+			single_pipe: '| samtools view -bS -F 4 | samtools sort -o /OUTPUT/',
+			flag_single: '-U',
+			flags_paired: ['-1', '-2'],
+			additional_paired_flags: '',
+			flag_output: '-S',
+			output: File.join(@temp_folder, 'map_data'),
+			log: File.join(@temp_folder, 'mapping_log'),
+			force: false
+		}
+		options.merge!(user_options)
+		options[:files].each_with_index do |read_files, map_process_id|
+			cmd = options[:command].gsub('/THREADS/', @threads.to_s)
+			cmd.gsub!('/REFERENCE/', @ref)
+			if read_files.length == 1
+				cmd = cmd + " #{options[:flag_single]} #{read_files.first}"
+				@paired << false
+			elsif read_files.length == 2
+				@paired << true
+				cmd = cmd + " #{options[:additional_paired_flags]} #{options[:flags_paired].first} #{read_files.first} #{options[:flags_paired].last} #{read_files.last}" 
+			else
+				raise('Incorrect number of read files. Must be 1 (single) or 2 (paired).')
+			end
+			map_file = nil
+			if options[:paired_pipe].nil? || options[:single_pipe].nil?
+				map_file = options[:output] + "_#{map_process_id}" + '.sam'
+				cmd = cmd + " #{options[:flag_output]} #{map_file} &> #{options[:log]}_#{map_process_id}"
+			else
+				if @paired[map_process_id]
+					pipe = options[:paired_pipe]
+				else
+					pipe = options[:single_pipe]
+				end
+				map_file = options[:output] + "_#{map_process_id}" + '.bam'
+				cmd = cmd + " 2> #{options[:log]}_#{map_process_id} " + pipe.gsub('/OUTPUT/', map_file)
+			end
+			@map_files << map_file
+			system(cmd) if options[:force] || !File.exists?(map_file)
+			@total_reads << File.open("#{options[:log]}_#{map_process_id}").readlines.select{|line| /\d+ reads; of these:/ =~ line}.first.split(' ').first.to_i if File.exists?("#{options[:log]}_#{map_process_id}") && @total_reads[map_process_id].nil?
+			raise('ERROR: The mapping process has failed, please check the map folder into the temp folder') if @total_reads[map_process_id].nil? || @total_reads[map_process_id] == 0
+		end
+	end
+
+	def index(user_options = {})
+		@map_files.each do |map_file|
+			system("samtools index #{map_file}") if (map_file.include?('.bam') && !File.exists?(map_file+'.bai')) || user_options[:force]
+		end
+	end
+
+	def report
+		reports = []
+		@map_files.each do |map_file|
+			cmd = "samtools flagstat #{map_file}"
+			report = %x[#{cmd}].split("\n")
+			reports << report
+		end
+		return reports
+	end
+
+	def idxstats
+		@map_files.each_with_index do |map_file, map_process_id|
+			prefix = File.basename(map_file).gsub(/\.bam|\.sam|\.cram/, '')
+			file_path = File.join(@temp_folder, "#{prefix}_idxstats_#{map_process_id}.gz")
+			cmd = "samtools idxstats #{map_file} | gzip - -f > #{file_path}"
+			system(cmd) if !File.exists?(file_path)
+			parse_idxstats(file_path)
+		end
+	end
+
+	def mpileup(user_options = {})
+		parse_options = {
+			add_coverages: false, 
+			normalize_coverages: false,
+			cols: [1,2,4] # 1 based for cut
+		}
+		parse_options.merge!(user_options.delete(:parse_options)) if !user_options[:parse_options].nil?
+		opts = []
+		do_samtools_ref
+		user_options.each do |flag, value|
+			opts << [flag, value.to_s]
+		end
+
+		contig_list_file = File.join(@temp_folder, File.basename(@ref_fasta_path)+'.lst')
+		system("grep '>' #{@ref_fasta_path} | sed 's/>//g' > #{contig_list_file}") if !File.exists?(contig_list_file)
+		idxstats if @idxstats.empty?
+		cut = nil
+		cut = " |cut -f #{parse_options[:cols].join(',')}" if !parse_options[:cols].nil? && !parse_options[:cols].empty?
+		mpileup_files = []
+		@map_files.each_with_index do |map_file, map_process_id|
+			prefix = File.basename(map_file).gsub(/\.bam|\.sam|\.cram/, '')
+			file_path = File.join(@temp_folder, "#{prefix}_mpileup_#{map_process_id}.gz")
+			mpileup_files << file_path
+			cmd = "samtools mpileup -f #{@ref_fasta_path} #{opts.join(' ')} #{map_file}#{cut} | gzip - -f > #{file_path}" 
+			system(cmd) if !File.exists?(file_path)
+		end
+		coverage_results = {}
+
+		parse_mpileup(mpileup_files, contig_list_file) do |contig_name, contig_length, coverages|
+			mapped_reads = @idxstats.map{|info| info[contig_name][:mapped]}.inject { |sum, n| sum + n }
+			get_coverage_parameters(contig_name, contig_length, mapped_reads, coverages, parse_options, coverage_results)
+		end
+		return coverage_results
+	end
+
+	def parse_mpileup(file_paths, contig_list_file)
+		last_contig = nil
+		mpileup_files = file_paths.map{|file_path| Mpileup.new(file_path)}
+		File.open(contig_list_file).each do |contig_name|
+			contig_name.chomp!
+			contig_length = @idxstats.first[contig_name][:length]
+			all_coverages = []
+			mpileup_files.each do |mpileup_file|
+				coverages = mpileup_file.read_contig(contig_name, contig_length)
+				all_coverages << coverages if !coverages.nil? && !coverages.empty?  
+			end
+			yield(contig_name, contig_length, all_coverages)
+		end
+		mpileup_files.map{|mf| mf.close}
+	end
+
+
+	def parse_idxstats(file_path)
+		stats = {}
+		stats_file = ScbiZcatFile.new(file_path)
+		while !stats_file.eof
+	        fields = stats_file.readline.chomp.split("\t")
+	        stats[fields[0]] = {length: fields[1].to_i, mapped: fields[2].to_i, unmmapped: fields[3].to_i}
+	    end
+	    stats_file.close
+	    stats.delete('*')
+	    @idxstats << stats
+	end
+
+
+	private
+
+	def get_coverage_parameters(seq_name, contig_length, mapped_reads, mpileup_info, options, coverage_results)
+		# begin
+			mean_normalized_differences = 0
+			mean_max = 0
+			mean_coverage = 0
+			proportion_sequence_mapped = 0
+			fpkm = 0
+			if mapped_reads > 0
+				if !mpileup_info.empty?
+					if mpileup_info.length == 1
+						coverages = mpileup_info.first
+					else
+						coverages = mpileup_info.transpose.map {|x| x.reduce(:+)}
+					end
+					mean_normalized_differences, mean_max, mean_coverage, proportion_sequence_mapped, fpkm = calculate_coverage_parameters(coverages, contig_length, mapped_reads, options)
+				end
+			end
+			record = [mean_normalized_differences, mean_max, mean_coverage, proportion_sequence_mapped, fpkm, mapped_reads]
+
+			record << coverages if options[:add_coverages]
+			coverage_results[seq_name] = record
+		# rescue Exception => e
+		# 	puts "ERROR: The reference sequence: #{seq_name} has failed", 
+		# 	e.message,
+		# 	e.backtrace.join("\n")
+		# end 
+	end
+	
+	def calculate_coverage_parameters(coverages, ref_length, mapped_reads, options)
+		n_mates = 1.0
+		n_mates = 2.0 if @paired
+		millions = @total_reads.inject { |sum, n| sum + n }/1.0e6
+		mean_normalized_differences = 0
+		mean_max = 0
+		mean_coverage = 0
+		proportion_sequence_mapped = 0
+		fpkm = 0
+		
+		greater0 = coverages.select{|c| c > 0}
+		coverages_greater0 = greater0.length
+		if coverages_greater0 > 0
+			fpkm = mapped_reads/n_mates/(ref_length/1000.0)/millions
+			mean_coverage = coverages.inject { |sum, n| sum + n }.fdiv(ref_length)
+			n_max = (coverages.length/10.0).ceil
+			maximums = coverages.sort{|c1, c2| c2 <=> c1}[0..n_max-1]
+			mean_max = maximums.inject { |sum, n| sum + n }.fdiv(n_max)
+
+			mean_coverage_filtered =  greater0.inject { |sum, n| sum + n }.fdiv(coverages_greater0)
+			normalized_differences = greater0.map{|c| (c - mean_coverage_filtered).abs/mean_coverage_filtered}
+			mean_normalized_differences = normalized_differences.inject { |sum, n| sum + n } / normalized_differences.length
+			proportion_sequence_mapped =  greater0.length.fdiv(ref_length)
+		
+			if options[:normalize_coverages]
+				max = coverages.max
+				coverages.map!{|cov| cov.fdiv(max) } 
+			end
+		end	
+		return mean_normalized_differences, mean_max, mean_coverage, proportion_sequence_mapped, fpkm
+	end
+
+end

data/lib/full_lengther_next/{classes/my_worker.rb → my_worker.rb}

@@ -10,6 +10,7 @@ require 'artifacts'
 require 'blast_functions'
 require 'exonerate_result'
 require 'scbi_fasta'
+require 'mapping'
 
 require 'fl_analysis'
 include FlAnalysis
@@ -28,17 +29,67 @@ class MyWorker < ScbiMapreduce::Worker
 		$verbose = manager_options[:verbose]
 	end
 		
-	def process_object(obj_sequence)
-		# Punto de arranque de FLN
-		$WORKER_LOG.info "Processing chunk: #{obj_sequence.first.seq_name}"
-		full_lenghter2(obj_sequence)
-		return obj_sequence
+	def process_object(initial_obj)
+		task = initial_obj.first.keys.first
+		obj = nil
+		if task == :fln
+			obj = initial_obj.map{|hash| hash[:fln]}
+			# Punto de arranque de FLN
+			$WORKER_LOG.info "Task: #{task}. Processing chunk: #{obj.first.seq_name}"
+			full_lenghter2(obj)
+		elsif task == :mapping
+			obj = initial_obj.first.values.first
+			$WORKER_LOG.info "Task: #{task}. Processing chunk: #{obj}"
+			obj = map_transcriptome(obj)
+		end	
+		return {task => obj}
 	end
 		
 	def closing_worker
 		 
 	end
 
+	#####################################################################################
+	# MAPPING METHODS
+	#####################################################################################
+	def map_transcriptome(initial_obj)
+		ref_file = initial_obj
+		prefix = File.basename(ref_file, '.fasta')
+
+
+		mapping2 = Mapping.new(
+			ref_fasta_path: File.join(@options[:temp_map_folder], ref_file),
+			threads: 1,
+			temp_folder: @options[:temp_map_folder] 
+		) 
+		$WORKER_LOG.info "Do bowtie ref"
+
+		mapping2.do_ref(
+			name: "#{prefix}_ref",
+			log: "#{prefix}_reference_log"
+		)
+
+		$WORKER_LOG.info "Do bowtie mapping"
+		mapping2.do_map(
+			files: @options[:files2map], 
+			command: 'bowtie2 -p /THREADS/ -x /REFERENCE/ -a',
+			paired_pipe: '| samtools view -bS -f 2 | samtools sort -o /OUTPUT/',
+			single_pipe: '| samtools view -bS -F 4 | samtools sort -o /OUTPUT/',
+			additional_paired_flags: '--no-mixed',
+			output: File.join(@options[:temp_map_folder], "#{prefix}_map_data"),
+			log: File.join(@options[:temp_map_folder], "#{prefix}_mapping_log"),
+		)
+
+		$WORKER_LOG.info "Do samtools ref"
+		mapping2.index
+		$WORKER_LOG.info "Do idxstats"
+		mapping2.idxstats
+		$WORKER_LOG.info "Do mpileup and coverage analysis"
+		map_object2 = mapping2.mpileup('--ff' => 'UNMAP,QCFAIL')
+		$WORKER_LOG.info "Finished coverage analysis"
+		return map_object2
+	end
+
 	#####################################################################################
 	# FLN FUNCTIONS
 	#####################################################################################
@@ -49,15 +100,18 @@ class MyWorker < ScbiMapreduce::Worker
 
 	def full_lenghter2(seqs)
 		#seqs.map{|seq| seq.change_degenerated_nt!} # Clean degenerated nt
-	
+		check_seqs = seqs		
+		if !@options[:files2map].empty? && @options[:remove_unmapped]
+			check_seqs = check_mapping(seqs)
+		end 
+
 		# User database
 		#-------------------------------------------- 
 		# if the user has included his own database in the parameters entry,
 		# the location of the database is tested, and blast and the results analysis is done
-		check_seqs = seqs		
 		if @options[:user_db]
 			user_db = File.basename(@options[:user_db])
-			check_seqs = check_prot_db(seqs, @options[:user_db], 'blastx', 1, user_db, @options[:blast])
+			check_seqs = check_prot_db(check_seqs, @options[:user_db], 'blastx', 1, user_db, @options[:blast])
 		end 
  
 		# UniProt (sp)
@@ -95,6 +149,14 @@ class MyWorker < ScbiMapreduce::Worker
 	#----------------------------------------------------------------------------------
 	# END MAIN
 	#----------------------------------------------------------------------------------	
+	def check_mapping(seqs)
+		new_seqs = []
+		seqs.each do |s|
+			artifact?(s, nil, 'mapping', '', @options, new_seqs)
+		end
+		seqs.concat(new_seqs)
+		return seqs.select{|s| !s.ignore } 
+	end
 
 	def check_prot_db(seqs, db_path, blast_type, evalue, db_name, additional_blast_options)
 
@@ -136,7 +198,7 @@ class MyWorker < ScbiMapreduce::Worker
 			$WORKER_LOG.info "DB: #{File.basename(database)} #{input.length}"
 			blast = BatchBlast.new("-db #{database}", blast_type, "-evalue #{evalue} #{additional_blast_options}")
 			chunk_name = input.first.seq_name.gsub(/\W+/,'_')
-			file_path = File.join('temp', File.basename(database)+'_'+chunk_name)
+			file_path = File.join(@options[:temp], File.basename(database)+'_'+chunk_name)
 			if @options[:hdd] #Write/parse blast on Disk
 				file_name = file_path+'.blast' #Each blast is identified with database_name and first sequence's name on chunk
 				if !File.exists?(file_name)