miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Scripts/AAsubs.log2ratio.rb

@@ -0,0 +1,171 @@
+#!/usr/bin/env ruby
+#
+# @author  Luis M. Rodriguez-R
+# @update  Dec-21-2015
+# @license artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
+require "enveomics_rb/enveomics"
+
+o = {permutations: 1000, bootstraps: 1000, overwrite: false}
+OptionParser.new do |opt|
+   opt.banner = "
+   Estimates the log2-ratio of different amino acids in homologous sites using
+   an AAsubs file (see BlastPairwise.AAsubs.pl). It provides the point
+   estimation (.obs file), the bootstrap of the estimation (.boot file) and the
+   null model based on label-permutation (.null file).
+
+   Usage: #{$0} [options]".gsub(/^ +/,"")
+   opt.separator ""
+   opt.separator "Mandatory"
+   opt.on("-i", "--input FILE",
+      "Input file in AAsubs format (see BlastPairwise.AAsubs.pl)."
+      ){ |v| o[:file] = v}
+   opt.separator ""
+   opt.separator "Output files"
+   opt.on("-O", "--obs-file FILE",
+      "Output file with the log2-ratios per amino acid.",
+      "By default, '--input value'.obs."
+      ){ |v| o[:obs] = v }
+   opt.on("-B", "--bootstrap-file FILE",
+      "Output file with the bootstrap results of log2-ratios per amino acid.",
+      "By default, '--input value'.boot."
+      ){ |v| o[:boot] = v }
+   opt.on("-N", "--null-file FILE",
+      "Output file with the permutation results of log2-ratios per amino acid.",
+      "By default, '--input value'.null."
+      ){ |v| o[:null] = v }
+   opt.on("--overwrite",
+      "Overwrite existing files. By default, skip steps if the files already" +
+      " exist."){ |v| o[:overwrite] = v }
+   opt.separator ""
+   opt.separator "Parameters"
+   opt.on("-b", "--bootstraps INT",
+      "Number of bootstraps to run. By default: #{o[:bootstraps]}."
+      ){ |v| o[:bootstraps] = v.to_i }
+   opt.on("-p", "--permutations INT",
+      "Number of permutations to run. By default: #{o[:permutations]}."
+      ){ |v| o[:permutations] = v.to_i }
+   opt.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = TRUE }
+   opt.on("-h", "--help", "Display this screen.") do
+      puts opt
+      exit
+   end
+   opt.separator ""
+end.parse!
+
+# Initialize
+abort "--input is mandatory" if o[:file].nil?
+ALPHABET = %w(A C D E F G H I K L M N P Q R S T V W Y X)
+o[:obs] ||= "#{o[:file]}.obs"
+o[:boot] ||= "#{o[:file]}.boot"
+o[:null] ||= "#{o[:file]}.null"
+
+# Functions
+def dist_summary(a,b)
+   ALPHABET.map do |i|
+      Math.log(a[i].reduce(0,:+).to_f/b[i].reduce(0,:+), 10)
+   end
+end
+def empty_sample
+   Hash[ALPHABET.map{|k| [k, []]}]
+end
+
+# Initialize
+$stderr.puts "Initializing." unless o[:q]
+sample_A = empty_sample
+sample_B = empty_sample
+last_label = nil
+prot_index = -1
+
+# Read file
+$stderr.puts "Reading input file." unless o[:q]
+ifh = File.open(o[:file], "r")
+ifh.each do |l|
+   r = l.chomp.split /\t/
+   if r.first != last_label
+      prot_index +=1
+      last_label = r.first
+      ALPHABET.each do |a|
+         sample_A[a][prot_index] = 0
+         sample_B[a][prot_index] = 0
+      end
+   end
+   [1,2].each do |ds|
+      unless %w(- *).include? r[ds]
+	 abort "Unknown amino acid in line #{$.}: '#{r[ds]}'." unless
+	    ALPHABET.include? r[ds]
+	 sample_A[ r[ds] ][ prot_index ] += 1 if ds==1
+	 sample_B[ r[ds] ][ prot_index ] += 1 if ds==2
+      end
+   end
+end
+ifh.close
+$stderr.puts "  > Found #{prot_index+1} proteins." unless o[:q]
+$stderr.puts "  > Saving #{o[:obs]}" unless o[:q]
+sum = dist_summary(sample_A, sample_B)
+File.open(o[:obs], "w") do |fh|
+   fh.puts ["AA", "log10_AB"].join("\t")
+   ALPHABET.each do |i|
+      fh.puts [i, sum.shift].join("\t")
+   end
+end
+
+# Permutations
+if File.size? o[:null] and not o[:overwrite]
+   $stderr.puts "Skipping permutations." unless o[:q]
+else
+   $stderr.puts "Permutating." unless o[:q]
+   permut_sum = []
+   o[:permutations].times do |i|
+      permut_A = empty_sample
+      permut_B = empty_sample
+      (0 .. prot_index).each do |j|
+	 # Copy counts of the protein
+	 ALPHABET.each do |k|
+	    permut_A[k][j] = sample_A[k][j]
+	    permut_B[k][j] = sample_B[k][j]
+	 end
+	 # Swap labels at random
+	 permut_A,permut_B = permut_B,permut_A if rand(2)==1
+      end
+      permut_sum << dist_summary(permut_A, permut_B)
+   end
+   $stderr.puts "  > Performed #{o[:permutations]} permutations." unless o[:q]
+   $stderr.puts "  > Saving #{o[:null]}" unless o[:q]
+   File.open(o[:null], "w") do |fh|
+      fh.puts ALPHABET.join("\t")
+      permut_sum.each{ |s| fh.puts s.join("\t") }
+   end
+end
+
+# Bootstraps
+if File.size? o[:boot] and not o[:overwrite]
+   $stderr.puts "Skipping bootstraps." unless o[:q]
+else
+   $stderr.puts "Bootstrapping." unless o[:q]
+   boot_sum = []
+   o[:bootstraps].times do |i|
+      boot_A = empty_sample
+      boot_B = empty_sample
+      (0 .. prot_index).each do |j|
+	 # Sample randomly with replacement
+	 jr = rand(prot_index+1)
+	 # Copy counts of the protein
+	 ALPHABET.each do |k|
+	    boot_A[k][j] = sample_A[k][jr]
+	    boot_B[k][j] = sample_B[k][jr]
+	 end
+      end
+      boot_sum << dist_summary(boot_A, boot_B)
+   end
+   $stderr.puts "  > Performed #{o[:bootstraps]} bootstraps." unless o[:q]
+   $stderr.puts "  > Saving #{o[:boot]}" unless o[:q]
+   File.open(o[:boot], "w") do |fh|
+      fh.puts ALPHABET.join("\t")
+      boot_sum.each{ |s| fh.puts s.join("\t") }
+   end
+end
+
+$stderr.puts "Done. Yayyy!" unless o[:q]

data/utils/enveomics/Scripts/Aln.cat.rb

@@ -0,0 +1,221 @@
+#!/usr/bin/env ruby
+
+# @author  Luis M. Rodriguez-R
+# @license artistic license 2.0
+
+$VERSION = 1.0
+$:.push File.expand_path('../lib', __FILE__)
+require 'enveomics_rb/enveomics'
+
+o = {
+  q: false, missing: '-', model: 'AUTO', removeinvar: false, undefined: '-.Xx?'
+}
+
+OptionParser.new do |opt|
+  cmd = File.basename($0)
+  opt.banner = <<~BANNER
+
+    [Enveomics Collection: #{cmd} v#{$VERSION}]
+
+    Concatenates several multiple alignments in FastA format into a single
+    multiple alignment.  The IDs of the sequences (or the ID prefixes, if using
+    --ignore-after) must coincide across files.
+  
+    Usage: #{cmd} [options] aln1.fa aln2.fa ... > aln.fa
+
+  BANNER
+  opt.on(
+    '-c', '--coords FILE',
+    'Output file of coordinates in RAxML-compliant format'
+  ) { |v| o[:coords] = v }
+  opt.on(
+    '-i', '--ignore-after STRING',
+    'Remove everything in the IDs after the specified string'
+  ) { |v| o[:ignoreafter] = v }
+  opt.on(
+    '-I', '--remove-invariable', 'Remove invariable sites',
+    'Note: Invariable sites are defined as columns with only one state and',
+    'undefined characters.  Additional ambiguous characters may exist and',
+    'should be declared using --undefined'
+  ) { |v| o[:removeinvar] = v }
+  opt.on(
+    '-u', '--missing-char CHAR',
+    "Character denoting missing data. By default: '#{o[:missing]}'"
+  ) do |v|
+    if v.length != 1
+      abort "-missing-char can only be denoted by single characters: #{v}"
+    end
+    o[:missing] = v
+  end
+  opt.on(
+    '-m', '--model STRING',
+    'Name of the model to use if --coords is used. See RAxML docs;',
+    'supported values in v8+ include:',
+    '~ For DNA alignments:',
+    '  "DNA[F|X]", or "DNA[F|X]/3" (to estimate rates per codon position,',
+    '  particular notation for this script)',
+    '~ General protein alignments:',
+    '  "AUTO" (default in this script), "DAYHOFF" (1978), "DCMUT" (MBE 2005;',
+    '  22(2):193-199), "JTT" (Nat 1992;358:86-89), "VT" (JCompBiol 2000;',
+    '  7(6):761-776), "BLOSUM62" (PNAS 1992;89:10915), and "LG" (MBE 2008;',
+    '  25(7):1307-1320)',
+    '~ Specialized protein alignments:',
+    '  "MTREV" (mitochondrial, JME 1996;42(4):459-468), "WAG" (globular, MBE',
+    '  2001;18(5):691-699), "RTREV" (retrovirus, JME 2002;55(1):65-73),',
+    '  "CPREV" (chloroplast, JME 2000;50(4):348-358), and "MTMAM" (nuclear',
+    '  mammal proteins, JME 1998;46(4):409-418)'
+  ) { |v| o[:model] = v }
+  opt.on(
+    '--undefined STRING',
+    'All characters to be regarded as "undefined". It should include all',
+    'ambiguous and missing data chars.  Ignored unless --remove-invariable',
+    "By default: '#{o[:undefined]}'"
+  ) { |v| o[:undefined] = v }
+  opt.on('-q', '--quiet', 'Run quietly (no STDERR output)') { o[:q] = true }
+  opt.on('-V', '--version', 'Returns version') { puts $VERSION ; exit }
+  opt.on('-h', '--help', 'Display this screen') { puts opt ; exit }
+  opt.separator ''
+end.parse!
+files = ARGV
+abort 'Alignment files are mandatory' if files.nil? || files.empty?
+$QUIET = o[:q]
+
+# Read individual gene alignments and return them as a single hash with genome
+# IDs as keys and arrays of single-line strings as values
+#
+# IDs are trimmed after the first occurrence of +ignoreafter+, if defined
+def read_alignments(files, ignoreafter = nil)
+  aln = {}
+  files.each_with_index do |file, i|
+    key = nil
+    File.open(file, 'r').each do |ln|
+      ln.chomp!
+      if ln =~ /^>(\S+)/
+        key = $1
+        key.sub!(/#{ignoreafter}.*/, '') if ignoreafter
+        aln[key] ||= []
+        aln[key][i] = ''
+      else
+        if key.nil?
+          abort "Invalid FastA file: #{file}: Leading line not a def-line"
+        end
+        ln.gsub!(/\s/, '')
+        aln[key][i] += ln
+      end
+    end
+    abort "Empty alignment file: #{file}" if key.nil?
+  end
+  aln
+end
+
+# Remove invariable sites from the alignment hash +aln+, using +undefined+ as
+# a string including all characters representing undefined positions (e.g., X)
+#
+# Returns number of columns removed
+def remove_invariable(aln, undefined)
+  invs = 0
+  lengths = aln.values.first.map(&:length)
+  undef_chars = undefined.chars
+
+  lengths.each_with_index do |len, i|
+    (0 .. len - 1).each do |pos|
+      chr = nil
+      inv = true
+      aln.each_key do |key|
+        next if aln[key][i].nil?
+        chr = aln[key][i][pos] if chr.nil? || undefined.chars.include?(chr)
+        if chr != aln[key][i][pos] && !undef_chars.include?(aln[key][i][pos])
+          inv = false
+          break
+        end
+      end
+      if inv
+        aln.each_key { |key| aln[key][i][pos] = '!' unless aln[key][i].nil? }
+        lengths[i] -= 1
+        invs += 1
+      end
+    end
+    aln.each_key { |key| aln[key][i].gsub!('!', '') unless aln[key][i].nil? }
+  end
+  invs
+end
+
+# Concatenate the alignments hash +aln+ using the character +missing+ to
+# indicate missing alignments, and send each entry in the concatenated alignment
+# to +blk+ as two variables: key (name) and value (alignment string)
+#
+# Returns an array with the lengths of each individual alignment
+def concatenate(aln, missing, &blk)
+  say 'Concatenating'
+  lengths = aln.values.first.map(&:length)
+  aln.each_key do |key|
+    # Pad missing entries
+    lengths.each_with_index { |len, i| aln[key][i] ||= missing * len }
+
+    # Check length
+    obs_len = aln[key].map(&:length)
+    unless lengths == obs_len
+      abort "Inconsistent lengths in '#{key}'\nexp: #{lengths}\nobs: #{obs_len}"
+    end
+
+    # Pass entry to the block and remove from alignment hash
+    blk[key, aln[key].join('')]
+    aln.delete(key)
+  end
+  lengths
+end
+
+# Save the coordinates in +file+ based on +files+ paths (for the names), and
+# using +lengths+ individual alignment lengths
+#
+# The saved format is RAxML coords, including the +model+ for each alignment
+def save_coords(file, names, lengths, model)
+  File.open(file, 'w') do |fh|
+    s = 0
+    names.each_with_index do |name, i|
+      l = lengths[i]
+      next unless l > 0
+      name += "_#{i}" while names.count(name) > 1
+      if model =~ /(DNA.?)\/3/
+        fh.puts "#{$1}, #{name}codon1 = #{s + 1}-#{s + l}\\3"
+        fh.puts "#{$1}, #{name}codon2 = #{s + 2}-#{s + l}\\3"
+        fh.puts "#{$1}, #{name}codon3 = #{s + 3}-#{s + l}\\3"
+      else
+        fh.puts "#{model}, #{name} = #{s + 1}-#{s + l}"
+      end
+      s += l
+    end
+  end
+end
+
+# ------ MAIN ------
+begin
+  say 'Reading'
+  alignments = read_alignments(files, o[:ignoreafter])
+
+  if o[:removeinvar]
+    say 'Removing invariable sites'
+    inv = remove_invariable(alignments, o[:undefined])
+    say "  Removed #{inv} sites"
+  end
+
+  lengths = concatenate(alignments, o[:missing]) do |name, seq|
+    puts ">#{name}", seq.gsub(/(.{1,60})/, "\\1\n")
+  end
+  say "  #{lengths.inject(:+)} columns"
+
+  unless o[:coords].nil?
+    say 'Generating coordinates'
+    names = files.map do |i|
+      File.basename(i).gsub(/\..*/, '').gsub(/[^A-Za-z0-9_]/, '_')
+    end
+    save_coords(o[:coords], names, lengths, o[:model])
+  end
+
+  $stderr.puts 'Done' unless o[:q] 
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end
+

data/utils/enveomics/Scripts/Aln.convert.pl

@@ -0,0 +1,35 @@
+#!/usr/bin/env perl
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+use Bio::AlignIO;
+
+my($iformat,$oformat) = @ARGV;
+($iformat and $oformat) or die "
+Usage:
+   $0 in-format out-format < in_file > output_file
+
+   in-format	Input file's format.
+   out-format	Output file's format.
+   in_file	Input file.
+   out_file	Output file.
+
+Example:
+   # Re-format example.fa into Stockholm
+   $0 fasta stockholm < example.fa > example.stk
+
+Supported formats are:
+   bl2seq, clustalw, emboss, fasta, maf, mase, mega,
+   meme, metafasta, msf, nexus, pfam, phylip, po,
+   prodom, psi, selex, stockholm, XMFA, arp
+
+";
+
+$in  = Bio::AlignIO->new(-fh => \*STDIN, -format => $iformat);
+$out = Bio::AlignIO->new(-fh => \*STDOUT, -format => $oformat);
+while ( my $aln = $in->next_aln ) { $out->write_aln($aln) }
+

data/utils/enveomics/Scripts/AlphaDiversity.pl

@@ -0,0 +1,152 @@
+#!/usr/bin/env perl
+
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Nov-29-2015
+# @license: artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+use List::Util qw/min max sum/;
+
+sub VERSION_MESSAGE(){print "Alpha-diversity indices (enveomics)\n"}
+sub HELP_MESSAGE(){die "
+Description:
+   Takes a table of OTU abundance in one or more samples and calculates the Rao
+   (Q_alpha), Rao-Jost (Q_alpha_eqv), Shannon (Hprime), and inverse Simpson
+   (1_lambda) indices of alpha diversity for each sample.
+   
+   To use it with Qiime OTU Tables, run it as:
+   $0 -i OTU_Table.txt -h
+
+Usage:
+   $0 [opts]
+
+   -i <str>	* Input table (columns:samples, rows:OTUs, first column:OTU
+		names).
+   -r <int>	Number of rows to ignore.  By default: 0.
+   -c <int>	Number of columns to ignore after the first column (i.e.,
+		between the first column, containing the name of the categories,
+		and the first column of abundance values).  By default: 0.
+   -C <int>	Number of columns to ignore at the end of each line.
+		By default: 0.
+   -d <str>	Delimiter.  Supported escaped characters are: \"\\t\"
+		(tabulation), and \"\\0\" (null bit).  By default: \"\\t\".
+   -h		If set, the first row is assumed to have the names of the
+		samples.
+   -D <str>	Distances file.  A squared matrix (or bottom-left half matrix)
+		with the distances between categories (OTUs or functions).  The
+		first column must contain the names of the categories, and it
+		shouldn't have headers.  If not set, all distances are assumed
+		to be one.  Only used for Rao.
+   -R		Do not calculate Rao indices.  This significantly decreases the
+		total running time. Note that Rao indices are highly susceptible
+		to precision errors, and shouldn't be trusted for very big
+		numbers.
+   -q <int>	Estimate the qD index (true diversity order q).  By default: 0.
+   --help	This help message.
+
+   * Mandatory.
+
+"}
+
+# Input arguments
+my %o;
+getopts('i:c:C:d:r:hD:Rq:', \%o);
+
+#$o{B} and (eval("use bignum; 1") or die "Cannot use bignum.\n");
+&HELP_MESSAGE() unless $o{i};
+$o{c} ||= 0;
+$o{C} ||= 0;
+$o{r} ||= 0;
+$o{d} ||= "\\t";
+$o{q} ||= 0;
+
+$o{d}="\t" if $o{d} eq "\\t";
+$o{d}="\0" if $o{d} eq "\\0";
+
+# Distance matrix
+my $D = {};
+if($o{D} and not $o{R}){
+   my @Didx = ();
+   open DIST, "<", $o{D} or die "Cannot read file: $o{D}: $!\n";
+   while(<DIST>){
+      chomp;
+      my @d = split /\t/;
+      my $idx = shift @d;
+      push  @Didx,  $idx;
+      $D->{ $idx } ||= {};
+      $D->{ $idx }->{ $Didx[$_] } = $d[$_] for(0 .. $#d);
+   }
+   close DIST;
+   undef @Didx;
+}
+
+# Abundance matrix
+my @names  = ();
+my @cats   = ();
+my @values = ();
+open TABLE, "<", $o{i} or die "Cannot open file: ".$o{i}.": $!\n";
+<TABLE> for (1 .. $o{r});
+if($o{h}){
+   my $h = <TABLE>;
+   $h or die "Empty table!\n";
+   chomp $h;
+   @names = split $o{d}, $h;
+   shift @names for (0 .. $o{c});
+}
+
+while(<TABLE>){
+   chomp;
+   my @ln = split $o{d};
+   push @cats, shift(@ln);
+   shift @ln for (1 .. $o{c});
+   pop @ln for (1 .. $o{C});
+   push @{$values[$_] ||= []}, $ln[$_] for (0 .. $#ln);
+   push @{$values[$#ln+1]}, sum(@ln);
+}
+close TABLE;
+$names[$#values] = "gamma";
+
+if($o{R}){
+   print "".join($o{d}, qw/Sample Hprime 1_lambda qD/)."\n";
+}else{
+   print "".join($o{d}, qw/Sample Q_alpha Q_alpha_eqv Hprime 1_lambda qD/)."\n";
+}
+for my $i (0 .. $#values){
+   print "".(exists $names[$i] ? $names[$i] : $i).$o{d};
+   my $N = sum @{$values[$i]};
+   my $Q = 0;
+   my $H = 0;
+   my $l = 0;
+   my $qD = 0 unless $o{q}==1;
+   for my $ik (0 .. $#{$values[$i]}){
+      unless($o{R}){
+	 my $Qi = 0;
+	 for my $jk (0 .. $#{$values[$i]}){
+	    my $dij = (!$o{D}) ? 1 : 
+	       exists $D->{ $cats[$ik] }->{ $cats[$jk] } ?
+	       $D->{ $cats[$ik] }->{ $cats[$jk] } :
+	       exists $D->{ $cats[$jk] }->{ $cats[$ik] } ?
+	       $D->{ $cats[$jk] }->{ $cats[$ik] } :
+	       die "Cannot find distance between ".$cats[$ik].
+		  " and ".$cats[$jk].".\n";
+	    $Qi += $dij * ($values[$i]->[$ik]/$N) * ($values[$i]->[$jk]/$N);
+	 }
+	 $Q += $Qi;
+      }
+      my $pi = $N ? $values[$i]->[$ik]/$N : 0;
+      $H  -= $pi * log($pi) if $pi;
+      $l  += $pi**2;
+      $qD += $pi * ($pi**($o{q}-1)) unless $o{q}==1 or not $pi;
+   }
+   $qD = $o{q}==1 ? exp($H) : 1/($qD**(1/($o{q}-1)));
+   if($o{R}){
+      print "".join($o{d}, $H, $l ? 1/$l : "Inf", $qD)."\n";
+   }else{
+      print "".join($o{d}, $Q, ($Q==1 ? "NA" : 1/(1-$Q)), $H, 1/$l, $qD)."\n";
+   }
+}
+