miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Scripts/Taxonomy.silva2ncbi.rb

@@ -0,0 +1,227 @@
+#!/usr/bin/env ruby
+
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Feb-06-2015
+# @license artistic license 2.0
+#
+
+require 'optparse'
+
+$opts = {:warns=>false}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opt|
+   opt.separator "Re-formats Silva taxonomy into NCBI-like taxonomy dump files."
+   opt.separator ""
+   opt.separator "Mandatory arguments"
+   opt.on("-k", "--silvaranks FILE", "Input Silva ranks file (e.g., tax_ranks_ssu_115.txt)."){ |v| $opts[:silvaranks]=v }
+   opt.on("-f", "--silvaref FILE", "Input Silva ref alignment file (e.g., SSURef_NR99_115_tax_silva_full_align_trunc.fasta)."){ |v| $opts[:silvaref]=v }
+   opt.separator ""
+   opt.separator "Additional options"
+   opt.on("-p", "--patch FILE", "If passed, it replaces the paths specified in the patch."){ |v| $opts[:patch]=v }
+   opt.on("-s", "--seqinfo FILE", "If passed, it creates a CSV seq-info file compatible with taxtastic."){ |v| $opts[:seqinfo]=v }
+   opt.on("-t", "--taxfile FILE", "If passed, it creates a simple TSV taxonomy file."){ |v| $opts[:taxfile]=v }
+   opt.on("-n", "--ncbi FILE", "If passed, output folder for the NCBI dump files (e.g., taxdmp)."){ |v| $opts[:ncbi]=v }
+   opt.on("-w", "--warns", "Verbously display warnings."){ $opts[:warns]=true }
+   opt.on("-h", "--help","Display this screen") do
+      puts opt
+      exit
+   end
+   opt.separator ""
+end.parse!
+abort "-k/--silvaranks is mandatory." if $opts[:silvaranks].nil?
+abort "-k/--silvaranks must exist." unless File.exists? $opts[:silvaranks]
+abort "-f/--silvaref is mandatory." if $opts[:silvaref].nil?
+abort "-f/--silvaref must exist." unless File.exists? $opts[:silvaref]
+
+class Node
+   attr_accessor :id, :tax, :leaf, :name_type
+   attr_reader :name, :rank, :parent, :children
+   def initialize(name, rank=nil)
+      @name = name
+      @rank = rank.nil? ? "no rank" : rank
+      @children = []
+      @leaf = false
+      @name_type = "scientific name";
+   end
+   def parent=(node)
+      @parent=node
+      node.add_child(self)
+   end
+   def add_child(node)
+      @children << node
+   end
+   def ncbirank
+      ncbirank =
+	 self.rank == "superkingdom" ? "no rank" :
+	 self.rank == "domain" ? "superkingdom" :
+	 self.rank == "major_clade" ? "no rank" : self.rank
+      return ncbirank
+   end
+   def path
+      if self.parent.nil?
+         self.name
+      else
+         "#{self.parent.path};#{self.name}"
+      end
+   end
+   def each_desc internals, leaves, &blk
+      blk[self] if (leaves and self.leaf) or (internals and not self.leaf)
+      self.children.each {|child| child.each_desc internals, leaves, &blk}
+   end
+   def to_s
+      "#{self.name} (#{self.rank})"
+   end
+end
+
+class Taxonomy
+   attr_reader :root, :next_id
+   def initialize
+      @root = Node.new('root')
+      @root.id = 1
+      @next_id = 2
+   end
+   def register(node)
+      node.id = self.next_id
+      node.parent = self.root if node.parent.nil?
+      @next_id += 1
+   end
+   def node(path)
+      node = self.root
+      path.each do |level|
+         node.children.each do |child|
+	    if child.name == level
+	       node = child
+	       break
+	    end
+	 end
+	 unless node.name == level
+	    $stderr.puts "Warning: Impossible to find #{level} at #{node.to_s}, making it up." if $opts[:warns]
+	    child = Node.new(level)
+	    child.parent = node
+	    self.register(child)
+	    node = child
+	 end
+      end
+      node
+   end
+   def each_node &blk
+      self.root.each_desc true, true, &blk
+   end
+   def each_leaf &blk
+      self.root.each_desc false, true, &blk
+   end
+   def each_internal &blk
+      self.root.each_desc true, false, &blk
+   end
+end
+
+begin
+   taxo = Taxonomy.new()
+   
+   ## Read patch
+   patch = {}
+   unless $opts[:patch].nil?
+      $stderr.puts "Reading patch: #{$opts[:patch]}"
+      f = File.open($opts[:patch], "r")
+      while(ln = f.gets)
+	 m = ln.chomp.split(/\t/)
+	 patch[ m[0] ] = m[1]
+      end
+   end
+   
+   ## Read the Silva ranks
+   $stderr.puts "Reading Silva ranks: #{$opts[:silvaranks]}"
+   f = File.open($opts[:silvaranks], "r")
+   f.gets # header
+   while(ln = f.gets)
+      m = ln.chomp.split(/\t/)
+      m[0] = patch[ m[0] ] unless patch[ m[0] ].nil?
+      p = m[0].split(/;/)
+      raise "Inconsistent path and node name at line #{$.}: #{ln}." unless m[1] == p.pop
+      if m[3] != "w"
+	 node = Node.new(m[1], m[2])
+	 node.name_type = "common name" if m[3] == "a"
+	 node.parent = taxo.node(p)
+	 taxo.register(node)
+      end
+   end
+   f.close
+
+   $stderr.puts "  Top taxa:"
+   taxo.root.children.each do |top|
+      $stderr.puts "    o #{top.to_s} has #{top.children.length} children."
+   end
+
+   ## Read the Silva ref alignment
+   $stderr.puts "Reading Silva ref alignment: #{$opts[:silvaref]}"
+   i = 0
+   f = File.open($opts[:silvaref], "r")
+   while(ln = f.gets)
+      m = />([^\s]+)\s(.*)/.match(ln)
+      next unless m
+      # Patch
+      pm = /(.+);([^;]+)/.match(m[2])
+      path = "#{patch[ pm[1] ].nil? ? pm[1] : patch[ pm[1] ]};#{pm[2]}".split(/;/)
+      # Register
+      node = taxo.node(path)
+      taxo.register(node)
+      refseq = Node.new(m[1], 'refseq')
+      refseq.parent = node
+      refseq.leaf = true
+      taxo.register(refseq)
+      i += 1
+   end
+   f.close
+   $stderr.puts "  Saved #{i} leaves."
+
+   ### NCBI
+   unless $opts[:ncbi].nil?
+      ## Create taxonomy .dmp files
+      $stderr.puts "Creating NCBI-like files: #{$opts[:ncbi]}"
+      Dir.mkdir($opts[:ncbi]) unless Dir.exists?($opts[:ncbi]);
+      # merged.dmp
+      $stderr.puts "  o Creating merged.dmp"
+      File.open(File.join($opts[:ncbi], 'merged.dmp'), 'w'){}
+      # names.dmp
+      $stderr.puts "  o Creating names.dmp"
+      f = File.open(File.join($opts[:ncbi], 'names.dmp'), 'w')
+      taxo.each_internal do |n|
+	 f.puts [n.id, n.name, "", n.name_type].join("\t|\t")+"\t|"
+      end
+      f.close
+      # nodes.dmp
+      $stderr.puts "  o Creating nodes.dmp"
+      f = File.open(File.join($opts[:ncbi], 'nodes.dmp'), 'w')
+      taxo.each_internal do |n|
+	 f.puts ([n.id, n.parent.nil? ? n.id : n.parent.id, n.ncbirank, ""] << Array.new(8,0) << "").join("\t|\t")+"\t|"
+      end
+      f.close
+   end
+
+   ## Taxtastic
+   unless $opts[:seqinfo].nil?
+      $stderr.puts "Creating seq-info file: #{$opts[:seqinfo]}"
+      f = File.open($opts[:seqinfo], 'w')
+      f.puts "\"seqname\",\"tax_id\",\"group_name\""
+      taxo.each_leaf { |n| f.puts "\"#{n.name}\",\"#{n.parent.id}\",\"#{n.parent.name}\"" }
+      f.close
+   end
+   
+   ## Misc
+   unless $opts[:taxfile].nil?
+      $stderr.puts "Creating taxonomy file: #{$opts[:taxfile]}"
+      f = File.open($opts[:taxfile], 'w')
+      f.puts "tax_id\tparent_id\trank\ttax_name"
+      taxo.each_internal do |n|
+	 f.puts [n.id, n.parent.nil? ? n.id : n.parent.id, n.rank, n.name].join("\t")
+      end
+      f.close
+   end
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/VCF.KaKs.rb

@@ -0,0 +1,147 @@
+#!/usr/bin/env ruby
+
+# @author  Luis M. Rodriguez-R
+# @license Artistic-2.0
+
+$:.push File.expand_path("../lib", __FILE__)
+require "enveomics_rb/enveomics"
+require "enveomics_rb/vcf"
+
+o = {}
+OptionParser.new do |opt|
+  opt.banner = "
+  Estimates the Ka/Ks ratio from the SNPs in a VCF file. Ka and Ks are corrected
+  using pseudo-counts, but no corrections for multiple substitutions are
+  applied.
+  
+  Usage: #{$0} [options]".gsub(/^ +/,"")
+  opt.separator ""
+  opt.separator "Mandatory"
+  opt.on("-i", "--input FILE",
+    "Input file in Variant Call Format (VCF)."){ |v| o[:file] = v}
+  opt.on("-s", "--seqs FILE",
+    "Input gene sequences (nucleotides) in FastA format."){ |v| o[:seqs] = v}
+  opt.separator ""
+  opt.separator "Parameters"
+  opt.on("-f", "--syn-frx FLOAT",
+    "Fraction of synonymous substitutions. If passed, the number of sites are",
+    "estimated (not counted per gene), speeding up the computation ~10X."
+    ){ |v| o[:syn_frx] = v.to_f }
+  opt.on("-b", "--syn-bacterial-code",
+    "Sets --syn-frx to 0.760417, approximately the proportion of synonymous",
+    "substitutions in the bacterial code."){ o[:syn_frx] = 0.760417 }
+  opt.separator ""
+  opt.separator "Miscellaneous"
+  opt.on("-c", "--codon-file FILE",
+    "Output file including the codons of substitution variants."
+    ){ |v| o[:codon_file] = v }
+  opt.on("-h", "--help", "Display this screen.") do
+    puts opt
+    exit
+  end
+  opt.separator ""
+end.parse!
+
+abort "--input is mandatory" if o[:file].nil?
+abort "--seqs is mandatory"  if o[:seqs].nil?
+
+# Codon table (11. The Bacterial, Archaeal and Plant Plastid Code)
+# https://www.ncbi.nlm.nih.gov/Taxonomy/Utils/wprintgc.cgi#SG11
+t = {
+    AAs: "FFLLSSSSYY**CC*WLLLLPPPPHHQQRRRRIIIMTTTTNNKKSSRRVVVVAAAADDEEGGGG",
+ Starts: "---M------**--*----M------------MMMM---------------M------------",
+  Base1: "TTTTTTTTTTTTTTTTCCCCCCCCCCCCCCCCAAAAAAAAAAAAAAAAGGGGGGGGGGGGGGGG",
+  Base2: "TTTTCCCCAAAAGGGGTTTTCCCCAAAAGGGGTTTTCCCCAAAAGGGGTTTTCCCCAAAAGGGG",
+  Base3: "TCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAG"
+}
+$codon_aa = {}
+$codon_st = {}
+(0 .. (t[:Base1].size-1)).each do |i|
+  cod = [:Base1, :Base2, :Base3].map{ |k| t[k][i] }.join
+  $codon_aa[cod] = t[:AAs][i]
+  $codon_st[cod] = t[:Starts][i]
+end
+
+##
+# Is the change +cod+ to +cod_alt+ synonymous? +start_codon+ indicates if the
+# codon the first in the gene.
+def syn?(cod, cod_alt, start_codon=false)
+  start_codon ?
+    ( $codon_st[cod] == $codon_st[cod_alt] ) :
+    ( $codon_aa[cod] == $codon_aa[cod_alt] )
+end
+
+##
+# Estimates the fraction of times that the substitutions in the sequence +seq+
+# result in synonymous mutations from those in position +pos+ by any of the
+# nucleotides in +alts+.
+def syn_fraction(seq, pos, alts)
+  cod_let = (pos-1)%3
+  cod_pos = (pos-1) - cod_let
+  cod = seq[cod_pos .. (cod_pos+2)]
+  syn = 0
+  cod_alts = alts.map do |alt|
+    cod_alt = "#{cod}"
+    cod_alt[cod_let] = alt
+    cod_alt
+  end
+  syn = cod_alts.map{ |i| syn?(cod, i, pos<=3) ? 1 : 0 }.inject(0,:+)
+  $codon_fh.puts [syn, cod, cod_alts.join(",")].join("\t") unless $codon_fh.nil?
+  syn.to_f/alts.size
+end
+
+# Read sequences
+seqs = {}
+File.open(o[:seqs], "r") do |fh|
+  id = ""
+  fh.each_line do |ln|
+    if ln =~ /^>(\S+)/
+      id = $1
+      seqs[id] = ""
+    else
+      seqs[id] += ln.chomp.gsub(/[^A-Za-z]/, "")
+    end
+  end
+end
+
+# Process variants
+$codon_fh = nil
+unless o[:codon_file].nil?
+  $codon_fh = File.open(o[:codon_file], "w")
+  $codon_fh.puts "#" + %w[Syn Ref Alt].join("\t")
+end
+vcf = VCF.new(o[:file])
+gen = {}
+vcf.each_variant do |v|
+  next if v.indel?
+  raise "REF doesn't match VCF:\n#{v}" unless seqs[v.chrom][v.pos-1] == v.ref
+  gen[v.chrom] ||= [0.0, 0.0]
+  alts = v.alt.split(",")
+  syn = syn_fraction(seqs[v.chrom], v.pos, alts)
+  gen[v.chrom][0] += 1.0-syn
+  gen[v.chrom][1] += syn
+end
+$codon_fh.close unless $codon_fh.nil?
+$codon_fh = nil
+
+# Ka/Ks
+puts "#" +
+  "SeqID KaKs Ka Ks NonSynSubs SynSubs NonSynSites SynSites".tr(" ","\t")
+gen.each do |k,v|
+  if o[:syn_frx].nil?
+    v[2,3] = [0.0,0.0]
+    (1 .. seqs[k].size).each do |pos|
+      alts = %w(A C T G) - [seqs[k][pos-1]]
+      syn = syn_fraction(seqs[k], pos, alts)
+      v[2] += 1.0-syn
+      v[3] += syn
+    end
+  else
+    v[2] = seqs[k].size.to_f*o[:syn_frx]
+    v[3] = seqs[k].size.to_f*(1.0-o[:syn_frx])
+  end
+  ka = (v[0] + 1) / (v[2] + 2)
+  ks = (v[1] + 1) / (v[3] + 2)
+  puts ([k, ka/ks, ka, ks] + v).join("\t")
+end
+

data/utils/enveomics/Scripts/VCF.SNPs.rb

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+
+# @author  Luis M. Rodriguez-R
+# @license Artistic-2.0
+
+$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
+require "enveomics_rb/enveomics"
+require "enveomics_rb/vcf"
+
+o = {min_dp:4, max_dp:Float::INFINITY, min_ref_dp:2, min_alt_dp:2, min_qual:0.0,
+  indels:false, min_ic:0.0}
+OptionParser.new do |opt|
+  opt.banner = "
+  Counts the number of Single-Nucleotide Polymorphisms (SNPs) in a VCF file.
+  
+  Usage: #{$0} [options]".gsub(/^ +/,"")
+  opt.separator ""
+  opt.separator "Mandatory"
+  opt.on("-i", "--input FILE",
+    "Input file in Variant Call Format (VCF)."){ |v| o[:file] = v}
+  opt.separator ""
+  opt.separator "Parameters"
+  opt.on("-o", "--out FILE",
+    "Output (filtered) file in Variant Call Format (VCF)."){ |v| o[:out] = v}
+  opt.on("-m", "--min-dp INT",
+    "Minimum number of reads covering the position. By default: #{o[:min_dp]}."
+    ){ |v| o[:min_dp] = v.to_i }
+  opt.on("-M", "--max-dp INT",
+    "Maximum number of reads covering the position. By default: #{o[:max_dp]}."
+    ){ |v| o[:max_dp] = (v=="Infinity" ? Float::INFINITY : v.to_i) }
+  opt.on("-r", "--min-ref-dp INT",
+    "Minimum number of reads supporting allele REF. " +
+    "By default: #{o[:min_ref_dp]}."
+    ){ |v| o[:min_ref_dp] = v.to_i }
+  opt.on("-a", "--min-alt-dp INT",
+    "Minimum number of reads supporting allele ALT. " +
+    "By default: #{o[:min_alt_dp]}."
+    ){ |v| o[:min_alt_dp] = v.to_i }
+  opt.on("-q", "--min-quality FLOAT",
+    "Minimum quality of the position mapping. By default: #{o[:min_qual]}."
+    ){ |v| o[:max_dp] = v.to_f }
+  opt.on("-s", "--min-shannon FLOAT",
+    "Minimum information content (in bits, from 0 to 1). " +
+    "By default: #{o[:min_ic]}"){ |v| o[:min_ic] = v.to_f }
+  opt.on("--[no-]indels",
+    "Process (or ignore) indels. By default: ignore."
+    ){ |v| o[:indels] = v }
+  opt.on("-h", "--help", "Display this screen.") do
+    puts opt
+    exit
+  end
+  opt.separator ""
+end.parse!
+
+abort "--input is mandatory" if o[:file].nil?
+
+vcf = VCF.new(o[:file])
+c = 0
+dp = 0
+ref_dp = 0
+alt_dp = 0
+h = 0
+unless o[:out].nil?
+  ofh = File.open(o[:out], "w")
+  vcf.each_header{ |h| ofh.print h }
+end
+vcf.each_variant do |v|
+  next if v.indel? and not o[:indels]
+  next if v.dp < o[:min_dp]
+  next if v.dp > o[:max_dp]
+  next if v.ref_dp < o[:min_ref_dp]
+  next if v.alt_dp < o[:min_alt_dp]
+  next if v.qual < o[:min_qual]
+  next if v.shannon < o[:min_ic]
+  c += 1
+  dp += v.dp
+  ref_dp += v.ref_dp
+  alt_dp += v.alt_dp
+  h += v.shannon
+  ofh.print v.to_s unless o[:out].nil?
+end
+ofh.close unless o[:out].nil?
+
+puts "SNPs: #{c}", "Information content: #{h}",
+  "Average SNP depth: #{dp.to_f/c}",
+  "Average REF allele depth: #{ref_dp.to_f/c}",
+  "Average ALT allele depth: #{alt_dp.to_f/c}"
+