miga-base 1.2.17.0 → 1.2.17.2

data/utils/enveomics/Scripts/Table.prefScore.R

@@ -0,0 +1,60 @@
+#!/usr/bin/env Rscript
+
+#= Load stuff
+args <- commandArgs(trailingOnly = FALSE)  
+enveomics_R <- file.path(
+  dirname(sub('^--file=', '', args[grep('^--file=', args)])),
+  'lib',
+  'enveomics.R'
+)
+for(file in c('cliopts.R','utils.R','prefscore.R'))
+  source(file.path(enveomics_R, 'R', file))
+
+#= Generate interface
+opt <- enve.cliopts(
+  enve.prefscore,
+  file.path(enveomics_R, 'man', 'enve.prefscore.Rd'),
+  positional_arguments = c(1, 4),
+  usage = 'usage: %prog [options] output.tsv [output.pdf [width height]]',
+  mandatory = c('x', 'set'),
+  number = c('signif.thr'),
+  ignore = c('plot'),
+  o_desc = list(
+    x = 'A tab-delimited table of presence/absence (1/0) with species as rows and samples as columns.',
+    set = 'A list of sample names that constitute the test set, one per line',
+    ignore = 'A list of species to exclude from the analysis, one per line'
+  )
+)
+
+#= Set output files
+opt$options[['x']] <- read.table(
+  opt$options[['x']],
+  header = TRUE,
+  row.names = 1,
+  sep = '\t'
+)
+opt$options[['set']] <- read.table(
+  opt$options[['set']],
+  header = FALSE,
+  sep = '\t',
+  as.is = TRUE
+)[,1]
+if(!is.null(opt$options[['ignore']]))
+  opt$options[['ignore']] <- read.table(
+    opt$options[['ignore']],
+    header = FALSE,
+    sep = '\t',
+    as.is = TRUE
+  )[,1]
+if(length(opt$args) > 1) {
+  args <- as.list(opt$args[-1])
+  for(i in 2:3) if(length(args) >= i) args[[i]] <- as.numeric(args[[i]])
+  do.call('pdf', args)
+} else {
+  opt$options[['plot']] <- FALSE
+}
+
+#= Run it!
+y <- do.call('enve.prefscore', opt$options)
+write.table(y, opt$args[1], quote = FALSE, sep = '\t', col.names = FALSE)
+if(length(opt$args)>1) ttt <- dev.off()

data/utils/enveomics/Scripts/Table.replace.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update  Feb 01 2016
+# @license artistic license 2.0
+#
+
+require "optparse"
+
+o = {delimiter: "\t", key: 1, default: ""}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "\nReplaces a field in a table using a mapping file."
+   opts.on("-m", "--map FILE",
+      "Mapping file with two columns (key and replacement)."){ |v| o[:map] = v }
+   opts.on("-i", "--in FILE", "Input table."){ |v| o[:in] = v }
+   opts.on("-o", "--out FILE", "Output table."){ |v| o[:out] = v }
+   opts.on("-k", "--key INT",
+      "Column to replace in --in. By deafult: 1."){ |v| o[:key] = v.to_i }
+   opts.on("-u", "--unknown STR",
+      "String to use whenever the key is not found in --map."
+      ){ |v| o[:default] = v }
+   opts.on("-d", "--delimiter STR",
+      "String delimiting columns. By default, tabulation."
+      ){ |v| o[:delimiter] = v }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-m is mandatory" if o[:map].nil?
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+
+class String
+   def is_number?
+      true if Float(self) rescue false
+   end
+end
+
+begin
+   # Read mapping file
+   ifh = File.open(o[:map], "r")
+   map = {}
+   while(ln = ifh.gets)
+      row = ln.chomp.split(o[:delimiter])
+      map[ row[0] ] = row[1]
+   end
+   ifh.close
+   # Process table
+   ifh = File.open(o[:in], "r")
+   ofh = File.open(o[:out], "w")
+   while(ln = ifh.gets)
+      row = ln.chomp.split(o[:delimiter])
+      k = row[ o[:key]-1 ]
+      v = map[ k ]
+      v = o[:default] if v.nil?
+      row[ o[:key]-1 ] = v
+      ofh.puts(row.join(o[:delimiter]))
+   end
+   ifh.close
+   ofh.close
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts " - " + l + "\n" }
+   err
+end

data/utils/enveomics/Scripts/Table.round.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Feb 04 2015
+# @license: artistic license 2.0
+#
+
+require 'optparse'
+
+o = {:ndigits=>0, :action=>:round, :delimiter=>"\t"}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "\nRounds numbers in a table."
+   opts.on("-i", "--in FILE", "Input table."){ |v| o[:in] = v}
+   opts.on("-o", "--out FILE", "Output table."){ |v| o[:out] = v }
+   opts.on("-n", "--ndigits INT", "Number of decimal digits. By default: #{o[:ndigits]}"){ |v| o[:ndigits] = v.to_i }
+   opts.on("-f", "--floor", "Floors the values instead of rounding them. Ignores -n."){ o[:action] = :floor }
+   opts.on("-c", "--ceil", "Ceils the values instead of rounding them. Ignores -n."){ o[:action] = :ceil }
+   opts.on("-d", "--delimiter STR", "String delimiting columns. By default, tabulation."){ |v| o[:delimiter] = v }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+
+class String
+   def is_number?
+      true if Float(self) rescue false
+   end
+end
+
+begin
+   ifh = File.open(o[:in], "r")
+   ofh = File.open(o[:out], "w")
+   while(ln = ifh.gets)
+      ln.chomp!
+      row = []
+      ln.split(o[:delimiter]).each do |value|
+	 if value.is_number?
+	    case o[:action]
+	    when :round
+	       value = value.to_f.round(o[:ndigits])
+	    when :floor
+	       value = value.to_f.floor
+	    when :ceil
+	       value = value.to_f.ceil
+	    end
+	 end
+	 row.push value.to_s
+      end
+      ofh.puts(row.join(o[:delimiter]))
+   end
+   ifh.close
+   ofh.close
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts " - " + l + "\n" }
+   err
+end

data/utils/enveomics/Scripts/Table.split.pl

@@ -0,0 +1,57 @@
+#!/usr/bin/env perl
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update Feb-01-2016
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+use Symbol;
+
+my %o;
+getopts('i:o:d:e:h', \%o);
+my $file = shift @ARGV;
+
+($file and not $o{h}) or die "
+.Description:
+   Split a file with multiple columns into multiple two-columns lists.
+
+.Usage:
+   $0 [options] file
+   
+   Options:
+      -i <str>	Input field-delimiter.  By default: tabulation (\"\\t\").
+      -o <str>	Prefix of the output files.  By default: no prefix (\"\").
+      -d <str>	Output directory. By default: current directory (\"\").
+
+";
+$o{i} ||= "\t";
+$o{o} ||= "";
+$o{o} = $o{d}."/".$o{o} if $o{d};
+
+my $open=0;
+my @fhs=();
+open IN, "<", $file or die "Cannot read file: $file: $!\n";
+while(<IN>){
+   chomp;
+   my @row = split $o{i};
+   my $h = shift @row;
+   if($open){
+      for my $i (0 .. $#row){
+         print { qualify_to_ref $fhs[$i] } $h.$o{i}.$row[$i]."\n" if $row[$i];
+      }
+   }else{
+      $open++;
+      for my $l (@row){
+         $l =~ s/[\.\/:]/_/g;
+	 my $gs = gensym;
+	 open($gs, '>', $o{o}.$l.".txt") or die "Cannot create file: $o{o}$l.txt: $!\n";
+	 push @fhs, $gs;
+      }
+   }
+}
+close IN;
+close $_ for @fhs;
+

data/utils/enveomics/Scripts/Taxonomy.silva2ncbi.rb

@@ -0,0 +1,227 @@
+#!/usr/bin/env ruby
+
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Feb-06-2015
+# @license artistic license 2.0
+#
+
+require 'optparse'
+
+$opts = {:warns=>false}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opt|
+   opt.separator "Re-formats Silva taxonomy into NCBI-like taxonomy dump files."
+   opt.separator ""
+   opt.separator "Mandatory arguments"
+   opt.on("-k", "--silvaranks FILE", "Input Silva ranks file (e.g., tax_ranks_ssu_115.txt)."){ |v| $opts[:silvaranks]=v }
+   opt.on("-f", "--silvaref FILE", "Input Silva ref alignment file (e.g., SSURef_NR99_115_tax_silva_full_align_trunc.fasta)."){ |v| $opts[:silvaref]=v }
+   opt.separator ""
+   opt.separator "Additional options"
+   opt.on("-p", "--patch FILE", "If passed, it replaces the paths specified in the patch."){ |v| $opts[:patch]=v }
+   opt.on("-s", "--seqinfo FILE", "If passed, it creates a CSV seq-info file compatible with taxtastic."){ |v| $opts[:seqinfo]=v }
+   opt.on("-t", "--taxfile FILE", "If passed, it creates a simple TSV taxonomy file."){ |v| $opts[:taxfile]=v }
+   opt.on("-n", "--ncbi FILE", "If passed, output folder for the NCBI dump files (e.g., taxdmp)."){ |v| $opts[:ncbi]=v }
+   opt.on("-w", "--warns", "Verbously display warnings."){ $opts[:warns]=true }
+   opt.on("-h", "--help","Display this screen") do
+      puts opt
+      exit
+   end
+   opt.separator ""
+end.parse!
+abort "-k/--silvaranks is mandatory." if $opts[:silvaranks].nil?
+abort "-k/--silvaranks must exist." unless File.exists? $opts[:silvaranks]
+abort "-f/--silvaref is mandatory." if $opts[:silvaref].nil?
+abort "-f/--silvaref must exist." unless File.exists? $opts[:silvaref]
+
+class Node
+   attr_accessor :id, :tax, :leaf, :name_type
+   attr_reader :name, :rank, :parent, :children
+   def initialize(name, rank=nil)
+      @name = name
+      @rank = rank.nil? ? "no rank" : rank
+      @children = []
+      @leaf = false
+      @name_type = "scientific name";
+   end
+   def parent=(node)
+      @parent=node
+      node.add_child(self)
+   end
+   def add_child(node)
+      @children << node
+   end
+   def ncbirank
+      ncbirank =
+	 self.rank == "superkingdom" ? "no rank" :
+	 self.rank == "domain" ? "superkingdom" :
+	 self.rank == "major_clade" ? "no rank" : self.rank
+      return ncbirank
+   end
+   def path
+      if self.parent.nil?
+         self.name
+      else
+         "#{self.parent.path};#{self.name}"
+      end
+   end
+   def each_desc internals, leaves, &blk
+      blk[self] if (leaves and self.leaf) or (internals and not self.leaf)
+      self.children.each {|child| child.each_desc internals, leaves, &blk}
+   end
+   def to_s
+      "#{self.name} (#{self.rank})"
+   end
+end
+
+class Taxonomy
+   attr_reader :root, :next_id
+   def initialize
+      @root = Node.new('root')
+      @root.id = 1
+      @next_id = 2
+   end
+   def register(node)
+      node.id = self.next_id
+      node.parent = self.root if node.parent.nil?
+      @next_id += 1
+   end
+   def node(path)
+      node = self.root
+      path.each do |level|
+         node.children.each do |child|
+	    if child.name == level
+	       node = child
+	       break
+	    end
+	 end
+	 unless node.name == level
+	    $stderr.puts "Warning: Impossible to find #{level} at #{node.to_s}, making it up." if $opts[:warns]
+	    child = Node.new(level)
+	    child.parent = node
+	    self.register(child)
+	    node = child
+	 end
+      end
+      node
+   end
+   def each_node &blk
+      self.root.each_desc true, true, &blk
+   end
+   def each_leaf &blk
+      self.root.each_desc false, true, &blk
+   end
+   def each_internal &blk
+      self.root.each_desc true, false, &blk
+   end
+end
+
+begin
+   taxo = Taxonomy.new()
+   
+   ## Read patch
+   patch = {}
+   unless $opts[:patch].nil?
+      $stderr.puts "Reading patch: #{$opts[:patch]}"
+      f = File.open($opts[:patch], "r")
+      while(ln = f.gets)
+	 m = ln.chomp.split(/\t/)
+	 patch[ m[0] ] = m[1]
+      end
+   end
+   
+   ## Read the Silva ranks
+   $stderr.puts "Reading Silva ranks: #{$opts[:silvaranks]}"
+   f = File.open($opts[:silvaranks], "r")
+   f.gets # header
+   while(ln = f.gets)
+      m = ln.chomp.split(/\t/)
+      m[0] = patch[ m[0] ] unless patch[ m[0] ].nil?
+      p = m[0].split(/;/)
+      raise "Inconsistent path and node name at line #{$.}: #{ln}." unless m[1] == p.pop
+      if m[3] != "w"
+	 node = Node.new(m[1], m[2])
+	 node.name_type = "common name" if m[3] == "a"
+	 node.parent = taxo.node(p)
+	 taxo.register(node)
+      end
+   end
+   f.close
+
+   $stderr.puts "  Top taxa:"
+   taxo.root.children.each do |top|
+      $stderr.puts "    o #{top.to_s} has #{top.children.length} children."
+   end
+
+   ## Read the Silva ref alignment
+   $stderr.puts "Reading Silva ref alignment: #{$opts[:silvaref]}"
+   i = 0
+   f = File.open($opts[:silvaref], "r")
+   while(ln = f.gets)
+      m = />([^\s]+)\s(.*)/.match(ln)
+      next unless m
+      # Patch
+      pm = /(.+);([^;]+)/.match(m[2])
+      path = "#{patch[ pm[1] ].nil? ? pm[1] : patch[ pm[1] ]};#{pm[2]}".split(/;/)
+      # Register
+      node = taxo.node(path)
+      taxo.register(node)
+      refseq = Node.new(m[1], 'refseq')
+      refseq.parent = node
+      refseq.leaf = true
+      taxo.register(refseq)
+      i += 1
+   end
+   f.close
+   $stderr.puts "  Saved #{i} leaves."
+
+   ### NCBI
+   unless $opts[:ncbi].nil?
+      ## Create taxonomy .dmp files
+      $stderr.puts "Creating NCBI-like files: #{$opts[:ncbi]}"
+      Dir.mkdir($opts[:ncbi]) unless Dir.exists?($opts[:ncbi]);
+      # merged.dmp
+      $stderr.puts "  o Creating merged.dmp"
+      File.open(File.join($opts[:ncbi], 'merged.dmp'), 'w'){}
+      # names.dmp
+      $stderr.puts "  o Creating names.dmp"
+      f = File.open(File.join($opts[:ncbi], 'names.dmp'), 'w')
+      taxo.each_internal do |n|
+	 f.puts [n.id, n.name, "", n.name_type].join("\t|\t")+"\t|"
+      end
+      f.close
+      # nodes.dmp
+      $stderr.puts "  o Creating nodes.dmp"
+      f = File.open(File.join($opts[:ncbi], 'nodes.dmp'), 'w')
+      taxo.each_internal do |n|
+	 f.puts ([n.id, n.parent.nil? ? n.id : n.parent.id, n.ncbirank, ""] << Array.new(8,0) << "").join("\t|\t")+"\t|"
+      end
+      f.close
+   end
+
+   ## Taxtastic
+   unless $opts[:seqinfo].nil?
+      $stderr.puts "Creating seq-info file: #{$opts[:seqinfo]}"
+      f = File.open($opts[:seqinfo], 'w')
+      f.puts "\"seqname\",\"tax_id\",\"group_name\""
+      taxo.each_leaf { |n| f.puts "\"#{n.name}\",\"#{n.parent.id}\",\"#{n.parent.name}\"" }
+      f.close
+   end
+   
+   ## Misc
+   unless $opts[:taxfile].nil?
+      $stderr.puts "Creating taxonomy file: #{$opts[:taxfile]}"
+      f = File.open($opts[:taxfile], 'w')
+      f.puts "tax_id\tparent_id\trank\ttax_name"
+      taxo.each_internal do |n|
+	 f.puts [n.id, n.parent.nil? ? n.id : n.parent.id, n.rank, n.name].join("\t")
+      end
+      f.close
+   end
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/VCF.KaKs.rb

@@ -0,0 +1,147 @@
+#!/usr/bin/env ruby
+
+# @author  Luis M. Rodriguez-R
+# @license Artistic-2.0
+
+$:.push File.expand_path("../lib", __FILE__)
+require "enveomics_rb/enveomics"
+require "enveomics_rb/vcf"
+
+o = {}
+OptionParser.new do |opt|
+  opt.banner = "
+  Estimates the Ka/Ks ratio from the SNPs in a VCF file. Ka and Ks are corrected
+  using pseudo-counts, but no corrections for multiple substitutions are
+  applied.
+  
+  Usage: #{$0} [options]".gsub(/^ +/,"")
+  opt.separator ""
+  opt.separator "Mandatory"
+  opt.on("-i", "--input FILE",
+    "Input file in Variant Call Format (VCF)."){ |v| o[:file] = v}
+  opt.on("-s", "--seqs FILE",
+    "Input gene sequences (nucleotides) in FastA format."){ |v| o[:seqs] = v}
+  opt.separator ""
+  opt.separator "Parameters"
+  opt.on("-f", "--syn-frx FLOAT",
+    "Fraction of synonymous substitutions. If passed, the number of sites are",
+    "estimated (not counted per gene), speeding up the computation ~10X."
+    ){ |v| o[:syn_frx] = v.to_f }
+  opt.on("-b", "--syn-bacterial-code",
+    "Sets --syn-frx to 0.760417, approximately the proportion of synonymous",
+    "substitutions in the bacterial code."){ o[:syn_frx] = 0.760417 }
+  opt.separator ""
+  opt.separator "Miscellaneous"
+  opt.on("-c", "--codon-file FILE",
+    "Output file including the codons of substitution variants."
+    ){ |v| o[:codon_file] = v }
+  opt.on("-h", "--help", "Display this screen.") do
+    puts opt
+    exit
+  end
+  opt.separator ""
+end.parse!
+
+abort "--input is mandatory" if o[:file].nil?
+abort "--seqs is mandatory"  if o[:seqs].nil?
+
+# Codon table (11. The Bacterial, Archaeal and Plant Plastid Code)
+# https://www.ncbi.nlm.nih.gov/Taxonomy/Utils/wprintgc.cgi#SG11
+t = {
+    AAs: "FFLLSSSSYY**CC*WLLLLPPPPHHQQRRRRIIIMTTTTNNKKSSRRVVVVAAAADDEEGGGG",
+ Starts: "---M------**--*----M------------MMMM---------------M------------",
+  Base1: "TTTTTTTTTTTTTTTTCCCCCCCCCCCCCCCCAAAAAAAAAAAAAAAAGGGGGGGGGGGGGGGG",
+  Base2: "TTTTCCCCAAAAGGGGTTTTCCCCAAAAGGGGTTTTCCCCAAAAGGGGTTTTCCCCAAAAGGGG",
+  Base3: "TCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAGTCAG"
+}
+$codon_aa = {}
+$codon_st = {}
+(0 .. (t[:Base1].size-1)).each do |i|
+  cod = [:Base1, :Base2, :Base3].map{ |k| t[k][i] }.join
+  $codon_aa[cod] = t[:AAs][i]
+  $codon_st[cod] = t[:Starts][i]
+end
+
+##
+# Is the change +cod+ to +cod_alt+ synonymous? +start_codon+ indicates if the
+# codon the first in the gene.
+def syn?(cod, cod_alt, start_codon=false)
+  start_codon ?
+    ( $codon_st[cod] == $codon_st[cod_alt] ) :
+    ( $codon_aa[cod] == $codon_aa[cod_alt] )
+end
+
+##
+# Estimates the fraction of times that the substitutions in the sequence +seq+
+# result in synonymous mutations from those in position +pos+ by any of the
+# nucleotides in +alts+.
+def syn_fraction(seq, pos, alts)
+  cod_let = (pos-1)%3
+  cod_pos = (pos-1) - cod_let
+  cod = seq[cod_pos .. (cod_pos+2)]
+  syn = 0
+  cod_alts = alts.map do |alt|
+    cod_alt = "#{cod}"
+    cod_alt[cod_let] = alt
+    cod_alt
+  end
+  syn = cod_alts.map{ |i| syn?(cod, i, pos<=3) ? 1 : 0 }.inject(0,:+)
+  $codon_fh.puts [syn, cod, cod_alts.join(",")].join("\t") unless $codon_fh.nil?
+  syn.to_f/alts.size
+end
+
+# Read sequences
+seqs = {}
+File.open(o[:seqs], "r") do |fh|
+  id = ""
+  fh.each_line do |ln|
+    if ln =~ /^>(\S+)/
+      id = $1
+      seqs[id] = ""
+    else
+      seqs[id] += ln.chomp.gsub(/[^A-Za-z]/, "")
+    end
+  end
+end
+
+# Process variants
+$codon_fh = nil
+unless o[:codon_file].nil?
+  $codon_fh = File.open(o[:codon_file], "w")
+  $codon_fh.puts "#" + %w[Syn Ref Alt].join("\t")
+end
+vcf = VCF.new(o[:file])
+gen = {}
+vcf.each_variant do |v|
+  next if v.indel?
+  raise "REF doesn't match VCF:\n#{v}" unless seqs[v.chrom][v.pos-1] == v.ref
+  gen[v.chrom] ||= [0.0, 0.0]
+  alts = v.alt.split(",")
+  syn = syn_fraction(seqs[v.chrom], v.pos, alts)
+  gen[v.chrom][0] += 1.0-syn
+  gen[v.chrom][1] += syn
+end
+$codon_fh.close unless $codon_fh.nil?
+$codon_fh = nil
+
+# Ka/Ks
+puts "#" +
+  "SeqID KaKs Ka Ks NonSynSubs SynSubs NonSynSites SynSites".tr(" ","\t")
+gen.each do |k,v|
+  if o[:syn_frx].nil?
+    v[2,3] = [0.0,0.0]
+    (1 .. seqs[k].size).each do |pos|
+      alts = %w(A C T G) - [seqs[k][pos-1]]
+      syn = syn_fraction(seqs[k], pos, alts)
+      v[2] += 1.0-syn
+      v[3] += syn
+    end
+  else
+    v[2] = seqs[k].size.to_f*o[:syn_frx]
+    v[3] = seqs[k].size.to_f*(1.0-o[:syn_frx])
+  end
+  ka = (v[0] + 1) / (v[2] + 2)
+  ks = (v[1] + 1) / (v[3] + 2)
+  puts ([k, ka/ks, ka, ks] + v).join("\t")
+end
+