bio-polymarker 1.3.2

data/bin/blast_triads_promoters.rb

@@ -0,0 +1,192 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'bio'
+require 'csv'
+require 'bio-blastxmlparser'
+require 'fileutils'
+require 'tmpdir'
+
+
+options = {}
+options[:identity] = 50
+options[:min_bases] = 200
+options[:split_token] = "-"
+options[:tmp_folder]  = Dir.mktmpdir
+options[:program]  = "blastn"
+options[:random_sample] = 0
+options[:cut_promoter_length] = 0
+options[:reverse] = true
+
+OptionParser.new do |opts|
+  
+  opts.banner = "Usage: filter_blat.rb [options]"
+
+  opts.on("-i", "--identity FLOAT", "Minimum percentage identity") do |o|
+    options[:identity] = o.to_f
+  end
+  opts.on("-c", "--min_bases int", "Minimum alignment length (default 200)") do |o|
+    options[:min_bases] = o.to_i
+  end
+
+  opts.on("-t", "--triads FILE", "CSV file with the gene triad names in the named columns 'A','B' and 'D' ") do |o|
+    options[:triads] = o
+  end
+
+  opts.on("-f", "--sequences FILE" , "FASTA file containing all the possible sequences. ") do |o|
+    options[:fasta] = o
+  end
+
+  opts.on("-s", "--split_token CHAR", "Character used to split the sequence name. The name will be evarything before this token on the name of the sequences") do |o|
+    options[:split_token] = o
+  end
+
+  opts.on("-p", "--program blastn|blastp", "The program to use in the alignments. Currntly only supported blastn and blastp") do |o|
+    options[:program] = o
+  end
+
+  opts.on("-r", "--random_sample INT", "Number of blast to run and keep. If set, only the number of subsets will be run") do |o|
+    options[:random_sample] = o.to_i
+  end
+
+  opts.on("-l", "--cut_promoter_length INT", "Bases to consider") do |o|
+    options[:cut_promoter_length] = o.to_i
+  end
+
+  opts.on("-v", "--reverse T|F", "Reverse the input bases") do |o|
+    if o == 'T'
+      options[:reverse] = true
+    elsif o == 'F'
+      options[:reverse] = false
+    else
+      $stderr.puts "Invalid option for reverse (should be T or F)"
+      exit -1
+    end
+  end
+end.parse!
+
+
+def blast_pair_fast(path_a, path_b, out_path, program: "blastn")
+  cmd = "#{program} -query #{path_a} -subject #{path_b} -task #{program} -out #{out_path} -outfmt '5' "
+  #puts cmd
+  executed = system cmd
+  result = []
+  blast_version = nil
+  n = Bio::BlastXMLParser::XmlIterator.new(out_path).to_enum
+  longest = nil
+  max_length = 0
+  max_pident = 0.0
+  n.each do | iter |
+    iter.each do | hit |
+      hit.each do | hsp |
+        if hsp.align_len > max_length
+          max_length = hsp.align_len
+          max_pident = 100 * hsp.identity.to_f / hsp.align_len.to_f
+        end
+      end
+    end
+  end
+  [max_length, max_pident]
+end
+
+valid_pairs_A_B = Hash.new
+valid_pairs_A_D = Hash.new
+valid_pairs_B_D = Hash.new
+
+split_token = options[:split_token]
+
+sequences = Hash.new
+sequence_count=0
+Bio::FlatFile.open(Bio::FastaFormat, options[:fasta]) do |fasta_file|
+  fasta_file.each do |entry|
+    gene_name = entry.entry_id.split(split_token)[0]  
+    seq = entry.naseq
+    seq.reverse_complement! if options[:reverse] 
+    seq = seq[0,options[:cut_promoter_length]] if options[:cut_promoter_length] > 0
+    entry.data = seq
+    sequences[gene_name] = entry unless sequences[gene_name]
+    sequences[gene_name] = entry if entry.length > sequences[gene_name].length
+    sequence_count += 1
+  end
+end
+
+$stderr.puts "#Loaded #{sequences.length} genes from #{sequence_count} sequences"
+#FileUtils.mkdir_p(options[:tmp_folder])
+$stderr.puts "TMP dir: #{options[:tmp_folder]}"
+
+a_tmp   = options[:tmp_folder] + "/A.fa"
+b_tmp   = options[:tmp_folder] + "/B.fa"
+d_tmp   = options[:tmp_folder] + "/D.fa"
+out_tmp = options[:tmp_folder] + "/out.blast"
+
+
+puts [
+  "group_id" , "query"      , "subject" , 
+  "chr_query", "chr_subject", "aln_type",
+  "length"   , "pident" , "Ns_query", "Ns_subject", "Ns_total"   ].join("\t")
+
+count_lines = File.foreach(options[:triads]).inject(0) {|c, line| c+1}
+
+probability =  options[:random_sample] / count_lines.to_f
+probability = 1 if options[:random_sample] == 0
+prng = Random.new
+#puts probability
+prom_len = options[:cut_promoter_length]
+CSV.foreach(options[:triads], headers:true ) do |row|
+   a = row['A']
+   b = row['B']
+   d = row['D']
+   triad = row['group_id'].to_i
+   triad_folder = triad/100
+
+   save = probability > prng.rand && probability < 1
+   run  = probability == 1 || save 
+   next unless run
+
+   seq_a = sequences[a]
+   seq_b = sequences[b]
+   seq_d = sequences[d]
+   File.open(a_tmp, 'w') {|f| f.write(seq_a) } if seq_a
+   File.open(b_tmp, 'w') {|f| f.write(seq_b) } if seq_b
+   File.open(d_tmp, 'w') {|f| f.write(seq_d) } if seq_d
+
+   ns_a = seq_a.seq.count('Nn') if seq_a
+   ns_b = seq_b.seq.count('Nn') if seq_b
+   ns_d = seq_d.seq.count('Nn') if seq_d
+
+   save_folder = "blast_alignments_#{prom_len}/#{triad_folder}/#{triad}"
+   
+   #if save
+    FileUtils.mkdir_p save_folder
+    FileUtils.cp(a_tmp, save_folder) if seq_a
+    FileUtils.cp(b_tmp, save_folder) if seq_b
+    FileUtils.cp(d_tmp, save_folder) if seq_d
+   #end
+
+   if seq_a and seq_b
+      to_print = [triad, a, b , "A","B","A->B"]
+      to_print << blast_pair_fast(a_tmp, b_tmp, out_tmp, program:options[:program])
+      to_print << ns_a 
+      to_print << ns_b
+      to_print << ns_a + ns_b
+      FileUtils.cp(out_tmp, "#{save_folder}/A_B.xml") #if save
+      puts to_print.join("\t")
+   end
+  if seq_a and seq_d
+      to_print = [triad, a, b , "A","D","A->D"]
+      to_print << blast_pair_fast(a_tmp, d_tmp, out_tmp, program:options[:program]) 
+      to_print << ns_a 
+      to_print << ns_d
+      to_print << ns_a + ns_d
+      FileUtils.cp(out_tmp, "#{save_folder}/A_D.xml") #if save
+      puts to_print.join("\t")
+  end
+  if seq_b and seq_d
+      to_print = [triad, a, b , "B","D","B->D"]
+      to_print << blast_pair_fast(b_tmp, d_tmp, out_tmp, program:options[:program])
+      to_print << ns_b
+      to_print << ns_d
+      to_print << ns_b + ns_d
+      FileUtils.cp(out_tmp, "#{save_folder}/B_D.xml") #if save
+      puts to_print.join("\t")
+  end
+end

data/bin/count_variations.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'bio'
+require 'rubygems'
+require 'pathname'
+require 'bio-samtools-wrapper'
+
+require 'set'
+
+$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+$: << File.expand_path('.')
+path= File.expand_path(File.dirname(__FILE__) + '/../lib/bio-polymarker.rb')
+require path
+
+puts  ARGV[0]
+
+fasta_db = Bio::DB::Fasta::FastaFile.new( {:fasta=>ARGV[0]})
+fasta_db.load_fai_entries
+bam1 =  Bio::DB::Sam.new({:fasta=>ARGV[0], :bam=>ARGV[1]})
+
+fasta_db.index.entries.each do | r |
+  #Np r.get_full_region
+  #container.process_region( { :region => r.get_full_region.to_s, :output_file => output_file } )
+  region=r.get_full_region
+  
+  
+
+  cons_1 = bam1.consensus_with_ambiguities({:region=>region, :case=>true})
+
+  snps = cons_1.count_ambiguities
+ 
+  snps_per_1k = (1000 * snps.to_f ) / region.size
+  
+  puts "#{r.id}\t#{region.size}\t#{snps}\t#{snps_per_1k}\n#{cons_1}"
+  
+end

data/bin/filter_blat_by_target_coverage.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+require 'bio'
+require 'optparse'
+$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+$: << File.expand_path('.')
+path= File.expand_path(File.dirname(__FILE__) + '/../lib/bio-polymarker.rb')
+require path
+module Bio
+	class Blat
+  		class StreamedReport < Report 
+
+  			def self.each_hit(text = '')
+  				flag = false
+  				head = []
+
+  				text.each_line do |line|
+  					if flag then
+  						yield Hit.new(line)
+  					else
+            		# for headerless data
+            			if /^\d/ =~ line then
+            				flag = true
+            				redo
+            			end
+            			line = line.chomp
+            			if /\A\-+\s*\z/ =~ line
+            				flag = true
+            			else
+            				head << line
+            			end
+            		end
+            	end
+            end
+        end
+    end
+end
+
+
+#blat_file=ARGV[0]
+
+options = {}
+options[:identity] = 95
+options[:covered] = 60
+OptionParser.new do |opts|
+  
+  opts.banner = "Usage: filter_blat_by_target_coverage.rb [options]"
+
+  opts.on("-p", "--psl FILE", "PSL file") do |o|
+    options[:blat_file] = o.upcase
+  end
+  opts.on("-i", "--identity FLOAT", "Minimum percentage identity") do |o|
+    options[:identity] = o.to_f
+  end
+  opts.on("-c", "--covered FLOAT", "Minimum percentage coverage") do |o|
+    options[:covered] = o.to_f
+  end
+  
+end.parse!
+
+
+blat_file = options[:blat_file]
+
+Bio::Blat::StreamedReport.each_hit(Bio::FlatFile.open(blat_file).to_io) do |hit|
+  if hit.percentage_covered >= options[:covered] and hit.percent_identity >= options[:identity]
+    puts hit.data.join("\t")
+  end
+end
+
+

data/bin/filter_exonerate_by_identity.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+require 'bio'
+require 'optparse'
+$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+$: << File.expand_path('.')
+path= File.expand_path(File.dirname(__FILE__) + '/../lib/bio-polymarker.rb')
+require path
+
+options = {}
+options[:identity] = 95
+options[:covered] = 90
+OptionParser.new do |opts|
+  
+  opts.banner = "Usage: filter_exonerate_by_identity.rb [options]"
+
+  opts.on("-e", "--exo FILE", "Exonerate alignment produced by polymarker or with the following ryo: 'RESULT:\\t%S\\t%pi\\t%ql\\t%tl\\t%g\\t%V\\n'") do |o|
+    options[:exo_file] = o.upcase
+  end
+  opts.on("-i", "--identity FLOAT", "Minimum percentage identity") do |o|
+    options[:identity] = o.to_f
+  end
+  opts.on("-c", "--covered FLOAT", "Minimum percentage coverage") do |o|
+    options[:covered] = o.to_f
+  end
+  
+end.parse!
+
+
+exo_file = options[:exo_file]
+min_identity = options[:identity];
+min_coverage = options[:covered]
+File.foreach(exo_file) do |line|  
+  aln = Bio::DB::Exonerate::Alignment.parse_custom(line)
+  if aln.identity > min_identity and aln.query_coverage > min_coverage
+	puts aln.line
+  end
+end
+

data/bin/find_best_blat_hit.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+require 'bio'
+
+def load_blat_alignments (blat_filename, best_aln)
+  blat_aln = Bio::Blat::Report.new(Bio::FlatFile.open(blat_filename).to_io)
+  blat_aln.each_hit() do |hit|
+    current_matches = hit.match 
+    current_name = hit.query_id
+    current_identity = hit.percent_identity
+    current_score = hit.score
+    #p current_name
+
+    best = best_aln[current_name]
+
+    if best == nil 
+      best_aln[current_name] = hit
+    else
+      if current_score > best.score
+        best_aln[current_name] = hit
+      end
+    end
+  end
+end
+
+blat_file=ARGV[0]
+best_aln = Hash.new
+
+load_blat_alignments( blat_file,best_aln)
+puts "QUERY\tTARGET"
+best_aln.each do |k, hit|
+  #puts "#{k}\t#{hit.target_id}"
+   puts hit.data.join("\t")
+end

data/bin/find_best_exonerate.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+
+found_cointigs = Set.new
+Bio::DB::Exonerate.align({:query=>temp_fasta_query, :target=>target, :model=>model, :chunk=>chunk, :total_chunks=>}) do |aln|
+  if aln.identity > min_identity
+    exo_f.puts aln.line
+    unless found_cointigs.include?(aln.target_id) #We only add once each contig. Should reduce the size of the output file. 
+      found_cointigs.add(aln.target_id)
+      entry = fasta_file.index.region_for_entry(aln.target_id)
+      raise ExonerateException.new,  "Entry not found! #{aln.target_id}. Make sure that the #{target_id}.fai was generated properly." if entry == nil
+      region = entry.get_full_region
+      seq = fasta_file.fetch_sequence(region)
+      contigs_f.puts(">#{aln.target_id}\n#{seq}")
+    end
+  end  
+end

data/bin/get_longest_hsp_blastx_triads.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'bio'
+require 'csv'
+#$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+#$: << File.expand_path('.')
+#path= File.expand_path(File.dirname(__FILE__) + '/../lib/bio-polymarker.rb')
+#require path
+
+options = {}
+options[:identity] = 50
+options[:min_bases] = 200
+options[:blastx] = "-"
+
+OptionParser.new do |opts|
+  
+  opts.banner = "Usage: filter_blat.rb [options]"
+
+  opts.on("-p", "--blastx FILE", "BLAST XML  file") do |o|
+    options[:blastx] = o
+  end
+  opts.on("-i", "--identity FLOAT", "Minimum percentage identity") do |o|
+    options[:identity] = o.to_f
+  end
+  opts.on("-c", "--min_bases int", "Minimum alignment length (default 200)") do |o|
+    options[:min_bases] = o.to_i
+  end
+
+  opts.on("-t", "--triads FILE", "CSV file with the gene triad names in the named columns 'A','B' and 'D' ") do |o|
+    options[:triads] = o
+  end
+  
+end.parse!
+
+valid_pairs_A_B = Hash.new
+valid_pairs_A_D = Hash.new
+valid_pairs_B_D = Hash.new
+
+CSV.foreach(options[:triads], headers:true ) do |row|
+  valid_pairs_A_B[row['A']] = row['B']
+  valid_pairs_A_D[row['A']] = row['D']
+  valid_pairs_B_D[row['B']] = row['D']
+end
+
+stream = ARGF
+stream = IO.open(options[:blastx]) unless  options[:blastx] == "-"
+puts "Loaded #{valid_pairs_B_D.length} triads"
+$stdout.flush
+
+blast_report = Bio::FlatFile.new(Bio::Blast::Report, stream)
+
+blast_report.each_entry do |report| 
+  puts "Hits for " + report.query_def + " against " + report.db
+  $stdout.flush
+  report.each do |hit|
+    query  = hit.query_id.split("-")[0]
+    target = hit.target_id.split("-")[0]
+    if valid_pairs_A_B[query] == target or valid_pairs_A_D[query] == target or valid_pairs_B_D[query] == target  
+      puts hit.target_id, "\t", hit.evalue, "\n" if hit.evalue < 0.001
+      puts hit.inspect
+    end
+    
+  end
+end
+
+stream.close unless  options[:blat_file] == "-"

data/bin/hexaploid_primers.rb

@@ -0,0 +1,168 @@
+#!
+require 'bio'
+require 'rubygems'
+require 'pathname'
+require 'bio-samtools-wrapper'
+
+require 'set'
+
+$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+$: << File.expand_path('.')
+path= File.expand_path(File.dirname(__FILE__) + '/../lib/bio-polymarker.rb')
+require path
+
+
+#TODO: Use temporary files somewhere in the file system and add traps to delete them/forward them as a result. 
+#TODO: Make all this parameters
+path_to_contigs="/Users/ramirezr/Documents/PHD/201305_Databases/iwgcs"
+#path_to_contigs=path_to_chromosomes
+snp_in="A"
+original_name="B"
+fasta_reference = nil
+#test_file="/Users/ramirezr/Dropbox/JIC/PrimersToTest/test_primers_nick_and_james_1.csv"
+test_file=ARGV[0]
+fasta_reference = ARGV[1] if ARGV[1]
+output_folder="#{test_file}_primer_design_#{Time.now.strftime('%Y%m%d-%H%M%S')}/"
+Dir.mkdir(output_folder)
+#TODO Make this tmp files
+temp_fasta_query="#{output_folder}to_align.fa"
+temp_contigs="#{output_folder}contigs_tmp.fa"
+exonerate_file="#{output_folder}exonerate_tmp.tab"
+primer_3_input="#{output_folder}primer_3_input_temp"
+primer_3_output="#{output_folder}primer_3_output_temp"
+exons_filename="#{output_folder}exons_genes_and_contigs.fa"
+output_primers="#{output_folder}primers.csv"
+
+primer_3_config=File.expand_path(File.dirname(__FILE__) + '/../conf/primer3_config')
+model="est2genome"
+
+
+min_identity= 92
+snps = Array.new
+
+#0. Load the fasta index 
+fasta_reference_db = nil
+if fasta_reference
+  fasta_reference_db = Bio::DB::Fasta::FastaFile.new({:fasta=>fasta_reference})
+  fasta_reference_db.load_fai_entries
+  p "Fasta reference: #{fasta_reference}"
+end
+
+
+#1. Read all the SNP files 
+#All the SNPs should be on the same chromosome as the first SNP. 
+chromosome = nil
+File.open(test_file) do | f |
+  f.each_line do | line |
+    # p line.chomp!
+    snp = nil
+    if ARGV.size == 1 #List with Sequence
+      snp = Bio::PolyploidTools::SNPSequence.parse(line)  
+    elsif ARGV.size == 2 #List and fasta file
+      snp = Bio::PolyploidTools::SNP.parse(line)
+      region = fasta_reference_db.index.region_for_entry(snp.gene).get_full_region
+      snp.template_sequence = fasta_reference_db.fetch_sequence(region)
+    else
+      raise Bio::DB::Exonerate::ExonerateException.new "Wrong number of arguments. " 
+    end
+    raise Bio::DB::Exonerate::ExonerateException.new "No SNP for line '#{line}'" if snp == nil
+    snp.snp_in = snp_in
+    snp.original_name = original_name
+    snps << snp
+    chromosome = snp.chromosome unless chromosome
+    raise Bio::DB::Exonerate::ExonerateException.new "All the snps should come from the same chromosome" if chromosome != snp.chromosome
+  end
+end
+
+#1.1 Close fasta file
+#fasta_reference_db.close() if fasta_reference_db
+#2. Generate all the fasta files
+
+written_seqs = Set.new
+file = File.open(temp_fasta_query, "w")
+snps.each do |snp|
+  unless written_seqs.include?(snp.gene)
+    written_seqs << snp.gene 
+    file.puts snp.to_fasta
+  end
+end
+file.close
+
+#3. Run exonerate on each of the possible chromosomes for the SNP
+puts chromosome
+chr_group = chromosome[0]
+exo_f = File.open(exonerate_file, "w")
+contigs_f = File.open(temp_contigs, "w")
+Dir.foreach(path_to_contigs) do |filename |
+  #puts filename
+  if  File.fnmatch("#{chr_group}*.fa", filename)
+    puts filename
+    target="#{path_to_contigs}/#{filename}"
+    
+    fasta_file = Bio::DB::Fasta::FastaFile.new({:fasta=>target})
+    fasta_file.load_fai_entries
+    Bio::DB::Exonerate.align({:query=>temp_fasta_query, :target=>target, :model=>model}) do |aln|
+      if aln.identity > min_identity
+        exo_f.puts aln.line
+        region = fasta_file.index.region_for_entry(aln.target_id).get_full_region
+        seq = fasta_file.fetch_sequence(region)
+        contigs_f.puts(">#{aln.target_id}\n#{seq}")
+      end
+      
+    end
+  end
+end
+
+exo_f.close()
+contigs_f.close()
+
+#4. Load all the results from exonerate and get the input filename for primer3
+#Custom arm selection function that only uses the first two characters. Maybe
+#we want to make it a bit more cleaver
+arm_selection = lambda do | contig_name |
+  ret = contig_name[0,2]       
+  return ret
+end
+
+container= Bio::PolyploidTools::ExonContainer.new
+container.flanking_size=100
+container.gene_models(temp_fasta_query)
+container.chromosomes(temp_contigs)
+container.add_parental({:name=>snp_in})
+container.add_parental({:name=>original_name})
+snps.each do |snp|
+  snp.container = container
+  snp.flanking_size = container.flanking_size
+  container.add_snp(snp)
+end
+container.add_alignments({:exonerate_file=>exonerate_file, :arm_selection=>arm_selection})
+
+file = File.open(exons_filename, "w")
+container.print_fasta_snp_exones(file)
+file.close
+
+file = File.open(primer_3_input, "w")
+file.puts("PRIMER_PRODUCT_SIZE_RANGE=50-150")
+file.puts("PRIMER_MAX_SIZE=25")
+file.puts("PRIMER_LIB_AMBIGUITY_CODES_CONSENSUS=1")
+file.puts("PRIMER_LIBERAL_BASE=1")
+file.puts("PRIMER_NUM_RETURN=5")
+file.puts("PRIMER_THERMODYNAMIC_PARAMETERS_PATH=#{primer_3_config}/")
+container.print_primer_3_exons(file, chromosome,snp_in)
+file.close
+
+Bio::DB::Primer3.run({:in=>primer_3_input, :out=>primer_3_output})
+
+#5. Pick the best primer and make the primer3 output
+kasp_container=Bio::DB::Primer3::KASPContainer.new
+kasp_container.line_1=snp_in
+kasp_container.line_2=original_name
+
+snps.each do |snp|
+  kasp_container.add_snp(snp) 
+end
+
+kasp_container.add_primers_file(primer_3_output)
+header = "Marker,SNP,RegionSize,SNP_type,#{snp_in},#{original_name},common,primer_type,orientation,#{snp_in}_TM,#{original_name}_TM,common_TM,selected_from,product_size"
+File.open(output_primers, 'w') { |f| f.write("#{header}\n#{kasp_container.print_primers}") }
+