bio-polymarker 1.3.2

data/bin/map_markers_to_contigs.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+require 'bio'
+require 'optparse'
+
+$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+$: << File.expand_path('.')
+path= File.expand_path(File.dirname(__FILE__) + '/../lib/bio-polymarker.rb')
+require path
+
+
+def log(msg)
+  time=Time.now.strftime("%Y-%m-%d %H:%M:%S.%L")
+  puts "#{time}: #{msg}"
+end
+
+markers = nil
+
+options = {}
+OptionParser.new do |opts|
+  
+  opts.banner = "Usage: polymarker.rb [options]"
+
+  opts.on("-c", "--chromosome CHR", "chromosome (1A, 3B, etc)") do |o|
+    options[:chromosome] = o.upcase
+  end
+  opts.on("-r", "--reference FASTA", "reference with the contigs") do |o|
+    options[:reference] = o
+  end
+  opts.on("-m", "--map CSV", "File with the map and sequence \n Header: INDEX_90K,SNP_ID,SNP_NAME,CHR,COORDINATES_CHR,MAP_ORDER,CHR_ARM,DISTANCE_CM,SEQUENCE") do |o|
+    options[:map] = o
+  end
+  
+end.parse!
+#reference="/Users/ramirezr/Documents/TGAC/references/Triticum_aestivum.IWGSP1.21.dna_rm.genome.fa"
+reference = options[:reference] if options[:reference]
+throw raise Exception.new(), "Reference has to be provided" unless reference
+
+map = Bio::PolyploidTools::ArmMap.new
+map.chromosome = options[:chromosome]
+map.global_reference(reference)
+log "Reading markers file"
+Bio::PolyploidTools::Marker.parse(options[:map]) do |marker|
+ if options[:chromosome] == marker.chr
+    map.markers[marker.snp_name] = marker 
+  end
+end
+
+
+
+fasta_tmp="markers_#{options[:chromosome]}.fa"
+contigs_tmp="contigs_#{options[:chromosome]}.fa"
+aln_tmp="align_#{options[:chromosome]}.psl"
+contigs_map="contigs_map_#{options[:chromosome]}.fa"
+map_with_contigs="contigs_map_#{options[:chromosome]}.csv"
+
+#1. Prints the sequences to print according to the chromosome to search
+log "Writing markers: #{fasta_tmp}"
+map.print_fasta_markers(fasta_tmp)
+log "Writing contigs: #{contigs_tmp}"
+map.print_fasta_contigs_from_reference(contigs_tmp)
+log "Aligning markers #{aln_tmp}" 
+map.align_markers(aln_tmp)
+log "printing contigs with markers #{contigs_map}"
+map.print_fasta_contigs_for_markers(contigs_map)
+log "printing map with contigs #{map_with_contigs}"
+map.print_map_with_contigs(map_with_contigs)

data/bin/marker_to_vcf.rb

@@ -0,0 +1,241 @@
+#!/usr/bin/env ruby
+require 'bio'
+require 'rubygems'
+require 'pathname'
+require 'bio-samtools-wrapper'
+require 'optparse'
+require 'set'
+$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+$: << File.expand_path('.')
+path= File.expand_path(File.dirname(__FILE__) + '/../lib/bio-polymarker.rb')
+require path
+
+options = {}
+options[:min_identity] = 90
+options[:filter_best]  = false
+options[:debug]  = false
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: marler_to_vcf.rb [options]"
+
+  opts.on("-c", "--contigs FILE", "File with contigs to use as database") do |o|
+    options[:path_to_contigs] = o
+  end
+  
+  opts.on("-m", "--marker_list FILE", "File with the list of markers to search from") do |o|
+    options[:marker_list] = o
+  end
+  
+  opts.on("-b", "--filter_best", "If set, only keep the best alignment for each chromosome") do 
+    options[:filter_best]  = false
+  end
+
+    opts.on("-D", "--debug", "Validate that the flanking sequences are correct") do 
+    options[:debug]  = true
+  end
+
+  opts.on("-i", "--min_identity INT", "Minimum identity to consider a hit (default 90)") do |o|
+    options[:min_identity] = o.to_i
+  end
+  
+  opts.on("-o", "--output FOLDER", "Output folder") do |o|
+    options[:output_folder] = o
+  end
+
+  opts.on("-a", "--arm_selection #{Bio::PolyploidTools::ChromosomeArm.getValidFunctions.join('|')}", "Function to decide the chromome arm") do |o|
+    options[:arm_selection] = Bio::PolyploidTools::ChromosomeArm.getArmSelection(o)
+   end
+  
+  opts.on("-A", "--aligner exonerate|blast", "Select the aligner to use. Default: blast") do |o|
+    raise "Invalid aligner" unless o == "exonerate" or o == "blast" 
+    options[:aligner] = o.to_sym
+  end
+
+  opts.on("-d", "--database PREFIX", "Path to the blast database. Only used if the aligner is blast. The default is the name of the contigs file without extension.") do |o|
+    options[:database] = o
+  end
+
+end.parse!
+options[:database] = options[:path_to_contigs] 
+p options
+p ARGV
+
+
+path_to_contigs=options[:path_to_contigs]
+
+original_name="A"
+snp_in="B"
+
+fasta_reference = nil
+test_file=options[:marker_list]
+
+output_folder="#{test_file}_primer_design_#{Time.now.strftime('%Y%m%d-%H%M%S')}" 
+output_folder= options[:output_folder] if  options[:output_folder]
+Dir.mkdir(output_folder)
+#T
+temp_fasta_query="#{output_folder}/to_align.fa"
+temp_contigs="#{output_folder}/contigs_tmp.fa"
+exonerate_file="#{output_folder}/exonerate_tmp.tab"
+vcf_file="#{output_folder}/snp_positions.vcf"
+
+min_identity= options[:min_identity]
+
+@status_file="#{output_folder}/status.txt"
+
+
+def write_status(status)
+  f=File.open(@status_file, "a")
+  f.puts "#{Time.now.to_s},#{status}"
+  f.close
+end
+
+
+snps = Hash.new
+
+fasta_reference_db=nil
+
+#if options[:debug]
+write_status "Loading Reference"
+fasta_reference_db = Bio::DB::Fasta::FastaFile.new({:fasta=>path_to_contigs})
+fasta_reference_db.load_fai_entries
+write_status "Fasta reference: #{fasta_reference}"
+#end
+
+#1. Read all the SNP files 
+#chromosome = nil
+write_status "Reading SNPs"
+
+File.open(test_file) do | f |
+  f.each_line do | line |
+    snp = Bio::PolyploidTools::SNPSequence.parse(line)  
+    snp.genomes_count = options[:genomes_count]
+    snp.snp_in = snp_in
+    snp.original_name = original_name
+    if snp.position 
+      snps[snp.gene] = snp
+    else
+      $stderr.puts "ERROR: #{snp.gene} doesn't contain a SNP"
+    end
+  end
+end
+
+#2. Generate all the fasta files
+write_status "Writing sequences to align"
+written_seqs = Set.new
+file = File.open(temp_fasta_query, "w")
+snps.each_pair do |k,snp|
+  unless written_seqs.include?(snp.gene)
+    written_seqs << snp.gene 
+    file.puts snp.to_fasta
+  end
+end
+file.close
+
+
+#3. Run exonerate on each of the possible chromosomes for the SNP
+#puts chromosome
+#chr_group = chromosome[0]
+write_status "Searching markers in genome"
+exo_f = File.open(exonerate_file, "w")
+contigs_f = File.open(temp_contigs, "w") if options[:extract_found_contigs]
+filename=path_to_contigs 
+#puts filename
+target=filename
+
+fasta_file = Bio::DB::Fasta::FastaFile.new({:fasta=>target})
+fasta_file.load_fai_entries
+found_contigs = Set.new
+
+def do_align(aln, exo_f, found_contigs, min_identity,fasta_file,options)
+  if aln.identity > min_identity
+    exo_f.puts aln.line
+  end  
+end
+
+Bio::DB::Blast.align({:query=>temp_fasta_query, :target=>options[:database]}) do |aln|
+  do_align(aln, exo_f, found_contigs,min_identity, fasta_file,options)
+end
+
+exo_f.close() 
+
+def print_positions(min_identity:90, filter_best:false, exonerate_filename:"test.exo", snps:{}, reference:nil, out:$stdout) 
+  marker_count=Hash.new { |h, k| h[k] = 1 }
+  File.open(exonerate_filename) do |f|
+    f.each_line do | line |
+      record = Bio::DB::Exonerate::Alignment.parse_custom(line)
+      next unless  record and record.identity >= min_identity
+      snp = snps[record.query_id]                           
+      next unless snp != nil and snp.position.between?( (record.query_start + 1) , record.query_end)
+      begin
+
+        position = record.query_position_on_target(snp.position)
+        q_strand = record.query_strand
+        t_strand = record.target_strand 
+        template = snp.template_sequence
+
+        vulgar = record.exon_on_gene_position(snp.position)
+        tr = vulgar.target_region
+        qr = vulgar.query_region
+        template_pre = template[qr.start - 1 .. snp.position - 1 ]
+        tr.orientation == :forward ? tr.end = position : tr.start = position
+        region = tr
+        target_seq = reference.fetch_sequence(region)
+        target_seq[-1] = target_seq[-1].upcase
+        ref_base = target_seq[-1]
+        ma = ref_base
+        alt_base = [snp.snp, snp.original].join(",")
+
+        if snp.original == ref_base
+          alt_base = snp.snp
+        elsif snp.snp == ref_base
+          alt_base = snp.original
+        end
+
+        if record.target_strand == :reverse
+          alt_base = Bio::Sequence::NA.new(alt_base)
+          ref_base = Bio::Sequence::NA.new(ref_base)
+          alt_base.complement!.upcase!
+          ref_base.complement!.upcase!
+        end
+        
+        info =  ["OR=#{record.target_strand}"]
+        info <<  "SC=#{record.score}"
+        info <<  "PI=#{record.pi}"
+        info <<  "MA=#{ma}"
+        info <<  "TS=#{target_seq}"
+        vcf_line="#{record.target_id}\t#{position}\t#{record.query_id}.path#{marker_count[record.query_id]}\t#{ref_base}\t#{alt_base}\t#{record.pi}\t.\t#{info.join(";")}"
+        #snp2 = Bio::PolyploidTools::SNP.parseVCF( vcf_line )
+        #snp2.setTemplateFromFastaFile(reference)
+        #seq2=snp2.to_polymarker_sequence(50)
+        #info << "PS=#{seq2}"
+        vcf_line="#{record.target_id}\t#{position}\t#{record.query_id}.path#{marker_count[record.query_id]}\t#{ref_base}\t#{alt_base}\t#{record.pi}\t.\t#{info.join(";")}"
+        out.puts(vcf_line)
+        
+        marker_count[record.query_id] += 1
+      rescue Bio::DB::Exonerate::ExonerateException
+        $stderr.puts "Failed for the range #{record.query_start}-#{record.query_end} for position #{snp.position}"     
+      end
+    end
+  end
+end
+
+
+write_status "Printing VCF file"
+#puts snps.inspect
+out = File.open(vcf_file, "w")
+out.puts "##fileformat=VCFv4.2"
+out.puts "##fileDate=#{Time.now.strftime("%Y%m%d")}"
+out.puts "##source=#{$0}"
+out.puts "##reference=file://#{options[:path_to_contigs]}"
+out.puts "##INFO=<ID=OR,Number=1,Type=String,Description=\"Orientation of the alignment of the marker\">"
+out.puts "##INFO=<ID=SC,Number=1,Type=Float,Description=\"Alignment score of the marker\">"
+out.puts "##INFO=<ID=PI,Number=1,Type=Float,Description=\"Percentage of identity of the alignment to the marker\">"
+out.puts "##INFO=<ID=PS,Number=1,Type=String,Description=\"SNP sequence for PolyMarker\">"
+out.puts "##INFO=<ID=MA,Number=1,Type=String,Description=\"Allele based on the original marker sequence\">"
+out.puts "##INFO=<ID=TS,Number=1,Type=String,Description=\"Target sequence before the SNP from the reference\">"
+out.puts "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO"
+print_positions(exonerate_filename:exonerate_file, min_identity:95, snps:snps, reference: fasta_reference_db, out:out)
+out.close
+write_status "DONE"
+
+

data/bin/markers_in_region.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+#This uses the map output from map_markers_to_contigs.rb 
+#You need a reference with the name of the contigs, containing the chromosome 
+#arm and a list of sequences to map. The algorithm creates a smaller reference 
+#file, so the search only spans across the contigs in the region. This should
+#allow to use a refined mapping algorithm. 
+require 'bio'
+require 'optparse'
+
+$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+$: << File.expand_path('.')
+path= File.expand_path(File.dirname(__FILE__) + '/../lib/bio-polymarker.rb')
+require path
+
+
+def log(msg)
+  time=Time.now.strftime("%Y-%m-%d %H:%M:%S.%L")
+  puts "#{time}: #{msg}"
+end
+
+markers = nil
+
+options = {}
+OptionParser.new do |opts|
+  
+  opts.banner = "Usage: markers_in_region.rb [options]"
+
+  opts.on("-c", "--chromosome CHR", "chromosome (1A, 3B, etc)") do |o|
+    options[:chromosome] = o.upcase
+  end
+  opts.on("-r", "--reference FASTA", "reference with the contigs") do |o|
+    options[:reference] = o
+  end
+  opts.on("-m", "--map CSV", "File with the map and sequence \n Header: INDEX_90K,SNP_ID,SNP_NAME,CHR,COORDINATES_CHR,MAP_ORDER,CHR_ARM,DISTANCE_CM,SEQUENCE") do |o|
+    options[:map] = o
+  end
+  
+end.parse!
+#reference="/Users/ramirezr/Documents/TGAC/references/Triticum_aestivum.IWGSP1.21.dna_rm.genome.fa"
+reference = options[:reference] if options[:reference]
+throw raise Exception.new(), "Reference has to be provided" unless reference

data/bin/mask_triads.rb

@@ -0,0 +1,169 @@
+#!/usr/bin/env ruby
+require 'optparse'
+
+require 'csv'
+require 'fileutils'
+require 'tmpdir'
+require 'bio-samtools-wrapper'
+require 'bio'
+
+$: << File.expand_path(File.dirname(__FILE__) + '/../lib')
+$: << File.expand_path('.')
+path= File.expand_path(File.dirname(__FILE__) + '/../lib/bio-polymarker.rb')
+require path
+opts = {}
+opts[:identity] = 50
+opts[:min_bases] = 200
+opts[:split_token] = "."
+opts[:tmp_folder]  = Dir.mktmpdir
+opts[:random_sample] = 0
+opts[:output_folder] = "."
+
+OptionParser.new do |o|
+
+  o.banner = "Usage: mask_triads.rb [options]"
+
+  o.on("-t", "--triads FILE", "CSV file with the gene triad names in the named columns 'A','B' and 'D' ") do |o|
+    opts[:triads] = o
+  end
+
+  o.on("-f", "--fasta FILE" , "FASTA file containing all the possible peptide sequences. ") do |o|
+    opts[:fasta] = o
+  end
+
+  o.on("-s", "--split_token CHAR", "Character used to split the sequence name. The name will be evarything before this token on the name of the sequences") do |o|
+    opts[:split_token] = o
+  end
+
+  o.on("-o", "--output_folder DIR", "Location to save the alignment masks. If the alignment exists, it is recycled to avoid calling MAFFT again") do |o|
+    opts[:output_folder] = o
+  end
+end.parse!
+
+
+split_token = opts[:split_token]
+reference_name = File.basename opts[:fasta]
+output_folder = opts[:output_folder]
+@fasta_reference_db = Bio::DB::Fasta::FastaFile.new(fasta: opts[:fasta])
+@fasta_reference_db.load_fai_entries
+#puts @fasta_reference_db.index.entries
+@cannonical = Hash.new
+@fasta_reference_db.index.entries.each do |e|
+  gene = e.id.split(split_token)[0]
+  @cannonical[gene] = e unless @cannonical[gene]
+  @cannonical[gene]  = e if   e.length > @cannonical[gene].length
+end
+
+$stderr.puts "#Loaded #{@cannonical.length} canonical sequences from #{@fasta_reference_db.index.size} in reference"
+
+$stderr.puts "TMP dir: #{opts[:tmp_folder]}"
+
+def write_fasta_from_hash(sequences, filename)
+  out = File.new(filename, "w")
+  sequences.each_pair do | chromosome, exon_seq |
+    out.puts ">#{chromosome}\n#{exon_seq}\n"
+  end
+  out.close
+end
+
+def mafft_align(a, b, d)
+  to_align = Bio::Alignment::SequenceHash.new
+  seq_a = @fasta_reference_db.fetch_sequence(@cannonical[a].get_full_region)
+  seq_b = @fasta_reference_db.fetch_sequence(@cannonical[b].get_full_region)
+  seq_d = @fasta_reference_db.fetch_sequence(@cannonical[d].get_full_region)
+  to_align[a] = seq_a
+  to_align[b] = seq_b
+  to_align[d] = seq_d
+  report = mafft.query_alignment(to_align)
+  aln = report.alignment
+  aln
+end
+
+def read_alignment(path)
+  aln = Bio::Alignment::SequenceHash.new
+  i = 0
+  Bio::FlatFile.open(Bio::FastaFormat, path) do |fasta_file|
+    fasta_file.each do |entry|
+      aln[entry.entry_id] = entry.seq if i < 3
+      i += 1
+    end
+  end
+  aln
+end
+
+
+mafft_opts = ['--maxiterate', '1000', '--localpair', '--quiet']
+mafft = Bio::MAFFT.new( "mafft" , mafft_opts)
+header_printed = false
+stats     = File.open("#{output_folder}/#{reference_name}.identity_stats.csv", "w")
+distances = File.open("#{output_folder}/#{reference_name}.distance_between_snps.csv.gz", "w")
+gz = Zlib::GzipWriter.new(distances)
+gz.write "triad,gene,genome,reference,type,distance\n"
+#gz.close
+
+def write_distances(distances, triad, gene, genome, reference, type, out)
+  distances.each { |e| out.write "#{triad},#{gene},#{genome},#{reference},#{type},#{e}\n" }
+end
+
+i = 0
+CSV.foreach(opts[:triads], headers:true ) do |row|
+  next unless row["cardinality_abs"] == "1:1:1" and row["HC.LC"] == "HC-only"
+   a = row['A']
+   b = row['B']
+   d = row['D']
+   triad = row['group_id']
+   cent_triad = triad.to_i / 100
+   folder = "#{output_folder}/alignments/#{reference_name}/#{cent_triad}/"
+   save_cds = "#{folder}/#{triad}.fa"
+   aligned = File.file?(save_cds)
+   aln = aligned ? read_alignment(save_cds)  : mafft_align(a,b,d)
+   folder = "#{output_folder}/alignments_new/#{reference_name}/#{cent_triad}/" if aligned
+   FileUtils.mkdir_p folder
+   save_cds = "#{folder}/#{triad}.fa"
+
+   aln2 = Bio::Alignment.new aln
+   seq_start = Bio::PolyploidTools::Mask.find_start(aln)
+   seq_end   = Bio::PolyploidTools::Mask.find_end(aln)
+   #puts "#{triad}: #{seq_start}-#{seq_end}"
+
+
+   aln2.add_seq(Bio::PolyploidTools::Mask.get(aln,seq_start: seq_start, seq_end: seq_end, target: a), "A")
+   aln2.add_seq(Bio::PolyploidTools::Mask.get(aln,seq_start: seq_start, seq_end: seq_end, target: b), "B")
+   aln2.add_seq(Bio::PolyploidTools::Mask.get(aln,seq_start: seq_start, seq_end: seq_end, target: d), "D")
+
+   a_stats =  Bio::PolyploidTools::Mask.stats(aln2["A"], triad, a, "A", reference_name)
+   b_stats =  Bio::PolyploidTools::Mask.stats(aln2["B"], triad, b, "B", reference_name)
+   d_stats =  Bio::PolyploidTools::Mask.stats(aln2["D"], triad, d, "D", reference_name)
+  
+   write_distances(a_stats[:specific], triad, a, "A", reference_name, "specific", gz)
+   write_distances(b_stats[:specific], triad, b, "B", reference_name, "specific", gz)
+   write_distances(d_stats[:specific], triad, d, "D", reference_name, "specific", gz)
+
+   write_distances(a_stats[:semispecific], triad, a, "A", reference_name, "semispecific", gz)
+   write_distances(b_stats[:semispecific], triad, b, "B", reference_name, "semispecific", gz)
+   write_distances(d_stats[:semispecific], triad, d, "D", reference_name, "semispecific", gz)
+   
+   a_stats.delete(:semispecific)
+   b_stats.delete(:semispecific)
+   d_stats.delete(:semispecific)
+   
+   a_stats.delete(:specific)
+   b_stats.delete(:specific)
+   d_stats.delete(:specific)
+
+   a_stats[:length] = @cannonical[a].length
+   b_stats[:length] = @cannonical[b].length
+   d_stats[:length] = @cannonical[d].length
+
+   stats.puts a_stats.keys.join(",") unless header_printed
+   stats.puts a_stats.values.join(",")
+   stats.puts b_stats.values.join(",")
+   stats.puts d_stats.values.join(",")
+   header_printed = true
+
+   write_fasta_from_hash(aln2, save_cds)
+   i += 1
+end
+gz.close
+distances.close
+stats.close