miga-base 1.2.17.0 → 1.2.17.2

data/utils/enveomics/Scripts/AlphaDiversity.pl

@@ -0,0 +1,152 @@
+#!/usr/bin/env perl
+
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Nov-29-2015
+# @license: artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+use List::Util qw/min max sum/;
+
+sub VERSION_MESSAGE(){print "Alpha-diversity indices (enveomics)\n"}
+sub HELP_MESSAGE(){die "
+Description:
+   Takes a table of OTU abundance in one or more samples and calculates the Rao
+   (Q_alpha), Rao-Jost (Q_alpha_eqv), Shannon (Hprime), and inverse Simpson
+   (1_lambda) indices of alpha diversity for each sample.
+   
+   To use it with Qiime OTU Tables, run it as:
+   $0 -i OTU_Table.txt -h
+
+Usage:
+   $0 [opts]
+
+   -i <str>	* Input table (columns:samples, rows:OTUs, first column:OTU
+		names).
+   -r <int>	Number of rows to ignore.  By default: 0.
+   -c <int>	Number of columns to ignore after the first column (i.e.,
+		between the first column, containing the name of the categories,
+		and the first column of abundance values).  By default: 0.
+   -C <int>	Number of columns to ignore at the end of each line.
+		By default: 0.
+   -d <str>	Delimiter.  Supported escaped characters are: \"\\t\"
+		(tabulation), and \"\\0\" (null bit).  By default: \"\\t\".
+   -h		If set, the first row is assumed to have the names of the
+		samples.
+   -D <str>	Distances file.  A squared matrix (or bottom-left half matrix)
+		with the distances between categories (OTUs or functions).  The
+		first column must contain the names of the categories, and it
+		shouldn't have headers.  If not set, all distances are assumed
+		to be one.  Only used for Rao.
+   -R		Do not calculate Rao indices.  This significantly decreases the
+		total running time. Note that Rao indices are highly susceptible
+		to precision errors, and shouldn't be trusted for very big
+		numbers.
+   -q <int>	Estimate the qD index (true diversity order q).  By default: 0.
+   --help	This help message.
+
+   * Mandatory.
+
+"}
+
+# Input arguments
+my %o;
+getopts('i:c:C:d:r:hD:Rq:', \%o);
+
+#$o{B} and (eval("use bignum; 1") or die "Cannot use bignum.\n");
+&HELP_MESSAGE() unless $o{i};
+$o{c} ||= 0;
+$o{C} ||= 0;
+$o{r} ||= 0;
+$o{d} ||= "\\t";
+$o{q} ||= 0;
+
+$o{d}="\t" if $o{d} eq "\\t";
+$o{d}="\0" if $o{d} eq "\\0";
+
+# Distance matrix
+my $D = {};
+if($o{D} and not $o{R}){
+   my @Didx = ();
+   open DIST, "<", $o{D} or die "Cannot read file: $o{D}: $!\n";
+   while(<DIST>){
+      chomp;
+      my @d = split /\t/;
+      my $idx = shift @d;
+      push  @Didx,  $idx;
+      $D->{ $idx } ||= {};
+      $D->{ $idx }->{ $Didx[$_] } = $d[$_] for(0 .. $#d);
+   }
+   close DIST;
+   undef @Didx;
+}
+
+# Abundance matrix
+my @names  = ();
+my @cats   = ();
+my @values = ();
+open TABLE, "<", $o{i} or die "Cannot open file: ".$o{i}.": $!\n";
+<TABLE> for (1 .. $o{r});
+if($o{h}){
+   my $h = <TABLE>;
+   $h or die "Empty table!\n";
+   chomp $h;
+   @names = split $o{d}, $h;
+   shift @names for (0 .. $o{c});
+}
+
+while(<TABLE>){
+   chomp;
+   my @ln = split $o{d};
+   push @cats, shift(@ln);
+   shift @ln for (1 .. $o{c});
+   pop @ln for (1 .. $o{C});
+   push @{$values[$_] ||= []}, $ln[$_] for (0 .. $#ln);
+   push @{$values[$#ln+1]}, sum(@ln);
+}
+close TABLE;
+$names[$#values] = "gamma";
+
+if($o{R}){
+   print "".join($o{d}, qw/Sample Hprime 1_lambda qD/)."\n";
+}else{
+   print "".join($o{d}, qw/Sample Q_alpha Q_alpha_eqv Hprime 1_lambda qD/)."\n";
+}
+for my $i (0 .. $#values){
+   print "".(exists $names[$i] ? $names[$i] : $i).$o{d};
+   my $N = sum @{$values[$i]};
+   my $Q = 0;
+   my $H = 0;
+   my $l = 0;
+   my $qD = 0 unless $o{q}==1;
+   for my $ik (0 .. $#{$values[$i]}){
+      unless($o{R}){
+	 my $Qi = 0;
+	 for my $jk (0 .. $#{$values[$i]}){
+	    my $dij = (!$o{D}) ? 1 : 
+	       exists $D->{ $cats[$ik] }->{ $cats[$jk] } ?
+	       $D->{ $cats[$ik] }->{ $cats[$jk] } :
+	       exists $D->{ $cats[$jk] }->{ $cats[$ik] } ?
+	       $D->{ $cats[$jk] }->{ $cats[$ik] } :
+	       die "Cannot find distance between ".$cats[$ik].
+		  " and ".$cats[$jk].".\n";
+	    $Qi += $dij * ($values[$i]->[$ik]/$N) * ($values[$i]->[$jk]/$N);
+	 }
+	 $Q += $Qi;
+      }
+      my $pi = $N ? $values[$i]->[$ik]/$N : 0;
+      $H  -= $pi * log($pi) if $pi;
+      $l  += $pi**2;
+      $qD += $pi * ($pi**($o{q}-1)) unless $o{q}==1 or not $pi;
+   }
+   $qD = $o{q}==1 ? exp($H) : 1/($qD**(1/($o{q}-1)));
+   if($o{R}){
+      print "".join($o{d}, $H, $l ? 1/$l : "Inf", $qD)."\n";
+   }else{
+      print "".join($o{d}, $Q, ($Q==1 ? "NA" : 1/(1-$Q)), $H, 1/$l, $qD)."\n";
+   }
+}
+

data/utils/enveomics/Scripts/BedGraph.tad.rb

@@ -0,0 +1,138 @@
+#!/usr/bin/env ruby
+
+$VERSION = 1.01
+$:.push File.expand_path('../lib', __FILE__)
+require 'enveomics_rb/enveomics'
+
+o = { range: 0.5, perseq: false, length: false, o: '-' }
+OptionParser.new do |opts|
+  opts.version = $VERSION
+  banner = <<~BANNER
+    Estimates the truncated average sequencing depth (TAD) from a BedGraph file
+
+    IMPORTANT: This script doesn't consider zero-coverage positions if missing
+    from the file. If you produce your BedGraph file with bedtools genomecov and
+    want to consider zero-coverage position, be sure to use -bga (not -bg).
+  BANNER
+  Enveomics.opt_banner(opts, banner, "#{File.basename($0)} [options]")
+
+  opts.separator 'Mandatory'
+  opts.on(
+    '-i', '--input PATH',
+    'Input BedGraph file',
+    'Supports compression with .gz extension, use - for STDIN'
+  ) { |v| o[:i] = v }
+
+  opts.separator ''
+  opts.separator 'Other Options'
+  opts.on(
+    '-o', '--out PATH',
+    'Output tab-delimited values (by default, STDOUT)',
+    'Supports compression with .gz extension, use - for STDOUT'
+  ) { |v| o[:o] = v }
+  opts.on(
+    '-r', '--range FLOAT', Float,
+    'Central range to consider, between 0 and 1',
+    "By default: #{o[:range]} (inter-quartile range)"
+  ) { |v| o[:range] = v }
+  opts.on(
+    '-n', '--name STRING',
+    'Name (ID) of the sequence (added as first column)'
+  ) { |v| o[:name] = v }
+  opts.on(
+    '-s', '--per-seq',
+    'Calculate averages per reference sequence, not total',
+    'Assumes a sorted BedGraph file'
+  ) { |v| o[:perseq] = v }
+  opts.on(
+    '-l', '--length',
+    'Add sequence length to the output'
+  ) { |v| o[:length] = v }
+  opts.on(
+    '-b', '--breadth',
+    'Add sequencing breadth to the output'
+  ) { |v| o[:breadth] = v }
+  opts.on('-h', '--help', 'Display this screen') do
+    puts opts
+    exit
+  end
+  opts.separator ''
+end.parse!
+raise Enveomics::OptionError.new('-i is mandatory') if o[:i].nil?
+
+##
+# Pad an array to include all index values up to +r+ entries:
+# - d:   Array of [ depth => counts ]
+# - idx: Array of [ depth, depth, ... ]
+# - r:   Expected number of entries in the array
+def pad(d, idx, r)
+  idx.each do |i|
+    next if d[i].nil?
+
+    d[i] -= r
+    break unless d[i] < 0
+
+    r = -d[i]
+    d[i] = nil
+  end
+  d
+end
+
+##
+# Report the results for:
+# - sq: Contig ID
+# - d:  Array of [ depth => counts ]
+# - ln: Length of the sequence
+# - o:  CLI Options
+def report(sq, d, ln, o)
+  # Estimate padding ranges
+  pad = (1.0 - o[:range]) / 2.0
+  r = (pad * ln).round
+  zeroes = d[0].to_i
+
+  # Pad (truncation)
+  d = pad(d, d.each_index.to_a, r + 0)
+  d = pad(d, d.each_index.to_a.reverse, r + 0)
+
+  # Average
+  y = [0.0]
+  unless d.compact.empty?
+    s = d.each_with_index.to_a.map { |v, i| v.nil? ? 0 : i * v }.inject(0, :+)
+    y[0] = s.to_f / d.compact.inject(:+)
+  end
+
+  # Report
+  y.unshift(sq) if o[:perseq]
+  y.unshift(o[:name]) if o[:name]
+  y << ln if o[:length]
+  y << (ln - zeroes).to_f / ln if o[:breadth]
+  y.join("\t")
+end
+
+# Read BedGraph
+d  = [] # [ depth => count ]
+ln = 0
+pre_sq = nil
+ifh = reader(o[:i])
+ofh = writer(o[:o])
+ifh.each_line do |i|
+  next if i =~ /^#/
+  r  = i.chomp.split("\t")
+  sq = r.shift # Contig ID
+  if o[:perseq] && !pre_sq.nil? && pre_sq != sq
+    ofh.puts(report(pre_sq, d, ln, o))
+    d  = []
+    ln = 0
+  end
+  r.map!(&:to_i) # From, To, Depth
+  l = r[1] - r[0] # Window length: To - From
+  d[ r[2] ] ||= 0
+  d[ r[2] ]  += l # Add these "l" positions with depth "Depth"
+  ln += l
+  pre_sq = sq
+end
+ofh.puts(report(pre_sq, d, ln, o))
+
+ifh.close
+ofh.close
+

data/utils/enveomics/Scripts/BedGraph.window.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+o = {win: 1000}
+ARGV << '-h' if ARGV.empty?
+OptionParser.new do |opt|
+  opt.banner = "
+  Estimates the sequencing depth per windows from a BedGraph file.
+
+  IMPORTANT: This script doesn't consider zero-coverage positions if missing
+  from the file. If you produce your BedGraph file with bedtools genomecov and
+  want to consider zero-coverage position, be sure to use -bga (not -bg).
+
+  Usage: #{$0} [options]"
+  opt.separator ''
+  opt.on('-i', '--input PATH',
+    'Input BedGraph file (mandatory).'){ |v| o[:i]=v }
+  opt.on('-w', '--win INT',
+    'Window size, in base pairs.', "By default: #{o[:win]}."
+    ){ |v| o[:win]=v.to_i }
+  opt.on('-h', '--help', 'Display this screen.') do
+    puts opt
+    exit
+  end
+  opt.separator ''
+end.parse!
+abort '-i is mandatory.' if o[:i].nil?
+
+def report(d, a, b, seqs)
+  # Average
+  y = 0.0
+  unless d.compact.empty?
+    s = d.each_with_index.to_a.map{ |v,i| v.nil? ? 0 : i*v }.inject(0,:+)
+    y = s.to_f/d.compact.inject(:+)
+  end
+
+  # Report
+  puts [a, b, y, seqs.keys.join(",")].join("\t")
+end
+
+# Read BedGraph
+d  = []
+ln = 0
+a = 1
+seqs = {}
+b = o[:win]
+File.open(o[:i], "r") do |ifh|
+  ifh.each_line do |i|
+    next if i =~ /^#/
+    r  = i.chomp.split("\t")
+    sq = r.shift
+    seqs[sq] = 1
+    r.map!{ |j| j.to_i }
+    l = r[1]-r[0]
+    d[ r[2] ] ||= 0
+    d[ r[2] ]  += l
+    ln += l
+    while ln >= b
+      d[ r[2] ] -= (ln-b)
+      report(d, a, b, seqs)
+      seqs = {}
+      seqs[ sq ] = 1 if ln > b
+      d = []
+      d[ r[2] ] = (ln-b)
+      a = b + 1
+      b = a + o[:win] - 1
+    end
+  end
+end
+

data/utils/enveomics/Scripts/BlastPairwise.AAsubs.pl

@@ -0,0 +1,102 @@
+#!/usr/bin/env perl
+# 
+# @authors Konstantinos Konstantinidis (initial version)
+#          modified to work with the BLASTp 2.2.25+ m0 output by
+#          Despina Tsementzi & Luis M. Rodriguez-R
+# @updated Dec-21-2015
+#
+
+
+$/ = "Lambda     ";
+use strict;
+my %hash_depth;
+
+my @query;
+my @subject;
+my @similarity;
+my $length = "0";
+
+my($cigar_chr, $blast) = @ARGV;
+
+($cigar_chr and $blast) or die "
+.Description:
+   Counts the different AA substitutions in the best hit blast alignments, from
+   a BLASTP pairwise format output (-outfmt 0 in BLAST+, -m 0 in legacy BLAST).
+
+.Usage: $0 cigar_char blast.m0.txt > aa-subs.list
+
+   cigar_char    Use '+' for similar substitutions, use '_' for non similar
+                 substitutions
+   blast.m0.txt  Blast in 'text' format (-outfmt/-m 0).
+   aa-subs.list  A tab-delimited raw file with one substitution per row and
+                 columns:
+                 (1) Name-of-query_Name-of-subject
+		 (2) AA-in-subject
+		 (3) AA-in-query
+		 (4) Total-Align-Length
+
+";
+
+# For each blast result (i.e., for each query)
+open BLAST, "<", $blast or die "Cannot read file: $blast: $!\n";
+while(my $data=<BLAST>) {
+   $data =~ s/\r//g;
+   my ($data_q, @array_matches) = split(/>/,$data);
+   my ($name_query) = ($data_q =~ /Query\= (\S+?)(?:_GENE|\s)/);
+   my ($length_query) = ($data_q =~ /\(([\d,]+) letters/ );
+   ($length_query) = ($data_q =~ /Length=([\d,]+)/) unless $length_query;
+   $length_query =~ tr/,//d;
+   
+   # For each alignment (i.e., for each HSP),
+   # note the "last" at the end of the block,
+   # so only the best match is considered
+   foreach my $data_f (@array_matches) {
+      # Capture statistics
+      my ($length_match) = ($data_f =~ /Identities = \d+\/(\d+)/);
+      my ($identity_match) = ($data_f =~ /Identities = \d+\/\d+ \((\d+)%/);
+      my ($target_name) = ($data_f =~ /^\s?(\S+)/);
+
+      # If the alignment meets minimum requirements
+      if ($length_query >30 && ($length_match/$length_query > 0.7) && $identity_match > 60) {
+	 $data_f =~ tr/ /_/;
+	 my @array = split ("\n", $data_f);
+	 my $blanks = 0;
+	 my $prefix_size = 0;
+
+	 # For each line in the alignment
+	 for my $data_fff (@array) {	
+	    if ($data_fff =~ /(Query[:_]_+\d+_+)([^_]+)/){
+	       # Query lines
+	       $prefix_size = length($1);
+	       $length = length($2);
+	       @query = split (//, $2);
+	    }elsif ($data_fff =~ /^_{11}/){
+	       # Cigar lines
+	       @similarity = split(//, substr($data_fff, $prefix_size, $length));
+	    }elsif ($data_fff =~ /Sbjct[:_]_+\d+_+([^_]+)/){
+	       # Subject lines
+	       @subject = split(//, $1);
+	       # For each alignment column
+	       for(my $i=0; $i <= $length; $i++){
+		  if ($similarity[$i] eq $cigar_chr) {
+		     print "$name_query\_$target_name\t$subject[$i]\t$query[$i]\t$length_match\n";
+		  }
+	       }
+	       undef @query;
+	       undef @similarity;
+	       undef @subject; 
+	    }
+	    
+	    # Remove secondary alignments
+	    if ($data_fff =~ /^$/){
+	       $blanks++;
+	       last if $blanks >= 3;
+	    }else{
+	       $blanks=0;
+	    }
+	 } # for my $data_fff (@array)
+      } # if ($length_query >30 ...
+      last; # <---- So it takes only the best match!
+   } # foreach my $data_f (@array_matches)
+} # while(my $data=<>)
+

data/utils/enveomics/Scripts/BlastTab.addlen.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @license: artistic license 2.0
+#
+
+require 'optparse'
+
+o = { sbj: false, q: false }
+ARGV << '-h' if ARGV.size == 0
+OptionParser.new do |opts|
+  opts.banner = "
+Appends an extra column to a BLAST with the length of the query or the subject
+sequence. You can pipe two instances to add both:
+  cat input.blast | #{$0} -f queries.fa | #{$0} -f subjects.fa -s > output.blast
+
+Usage: #{$0} [options] < input.blast > output.blast"
+  opts.separator ''
+  opts.separator 'Mandatory'
+  opts.on('-f', '--fasta FILE', 'Path to the FastA file'){ |v| o[:fasta] = v }
+  opts.separator ''
+  opts.separator 'Options'
+  opts.on('-s', '--subject',
+    'Use the subject column of the BLAST, by default the query column is used'
+    ){ o[:sbj] = true }
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)'){ o[:q] = true }
+  opts.on('-h', '--help', 'Display this screen') do
+    puts opts
+    exit
+  end
+  opts.separator ''
+end.parse!
+abort '-f is mandatory' if o[:fasta].nil?
+
+len = {}
+id  = ''
+$stderr.puts "Reading FastA file: #{o[:fasta]}" unless o[:q]
+fh = File.open(o[:fasta], 'r')
+fh.each_line do |ln|
+   defline = /^>(\S+)/.match(ln)
+   if defline.nil?
+      ln.gsub! /[^A-Za-z]/, ''
+      abort 'Error: Unsupported format, expecting FastA' if len[id].nil?
+      len[id] = len[id] + ln.size
+   else
+      id = defline[1]
+      len[id] = 0
+   end
+end
+fh.close
+
+unless o[:q]
+  $stderr.puts 'Appending %s length column' % (o[:sbj] ? 'subject' : 'query')
+end
+ARGF.each_line do |ln|
+  ln.chomp!
+  row = ln.split /\t/
+  id = o[:sbj] ? row[1] : row[0]
+  abort "Impossible to find sequence of #{id}" if len[id].nil?
+  puts "#{ln}\t#{len[id]}"
+end
+

data/utils/enveomics/Scripts/BlastTab.advance.bash

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+#
+# @author  Luis M. Rodriguez-R
+# @update  Mar-23-2016
+# @license artistic license 2.0
+#
+
+if [[ ! $2 ]] ; then
+   echo "
+.DESCRIPTION
+   Calculates the percentage of a partial BLAST result. The
+   value produced slightly subestimates the actual advance,
+   due to un-flushed output and trailing queries that could
+   be processed but generate no results.
+
+.USAGE
+   $0 blast.txt qry.fasta
+
+   blast.txt	Incomplete Tabular BLAST output.
+   qry.fasta	FastA file with query sequences.
+";
+   exit 1;
+fi
+
+if [[ ! -r $1 ]]; then
+   echo "Cannot open file: $1";
+   exit 1;
+fi
+
+if [[ ! -r $2 ]]; then
+   echo "Cannot open file: $2";
+   exit 1;
+fi
+
+LAST_Q=`tail -n 2 $1 | head -n 1 | awk '{print $1}'`
+LAST_Q_NO=`grep -n "^>$LAST_Q\\( \\|$\\)" $2 | sed -e 's/:.*//'`
+if [[ ! $LAST_Q_NO ]]; then
+   echo "Cannot find sequence: $LAST_Q";
+   echo "Make sure you are providing the right query file.";
+   exit 1;
+fi
+TOTAL_Q_NO=`cat $2 | wc -l | sed -e 's/ *//'`
+let PERC=100*$LAST_Q_NO/$TOTAL_Q_NO
+
+echo "$PERC%: $LAST_Q_NO / $TOTAL_Q_NO"
+exit 0;
+

data/utils/enveomics/Scripts/BlastTab.best_hit_sorted.pl

@@ -0,0 +1,55 @@
+#!/usr/bin/env perl
+
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license: artistic license 2.0
+# @last_update: Mar-23-2015
+#
+
+use strict;
+use warnings;
+
+die "
+Usage:
+   sort blast.txt ... | $0 > blast.bh.txt
+   $0 blast_sorted.txt ... > blast.bh.txt
+   $0 -h|--help|-?
+
+   blast.txt ...	One or more files in Tabular BLAST format.
+   blast_sorted.txt ...	One or more files in Tabular BLAST format pre-sorted.
+   blast.bh.txt		Output file in BLAST format containing best-hits only.
+   -h|--help|-?		Any of these flags trigger this help message and exits.
+
+   NOTE: This script assumes that the BLAST is sorted.  Because it can read
+   from the STDIN, calling this script without arguments cause it to still until
+   killed or until an EOF (^D) is presented.
+
+" if exists $ARGV[0] and $ARGV[0] =~ /^\-?\-(h(elp)?|\?)/i;
+
+my $last_qry = '';
+my @best_res;
+
+sub best_result($$){
+   my($r1, $r2)=@_;
+   return $r1 unless $r2;
+   return $r1->[11] > $r2->[11] ? @$r1 : @$r2;
+}
+
+my $i=0;
+while(<>){
+   chomp;
+   #print STDERR " Reading entry $i...                             \r" unless $i%1000;
+   my @res = split /\t/;
+   die "\nCannot parse BLAST line $.: $_\n" unless exists $res[1];
+   if($last_qry eq $res[0]){
+      @best_res = &best_result(\@res, \@best_res);
+   }else{
+      print join("\t", @best_res), "\n" if $#best_res>0;
+      @best_res = @res;
+      $last_qry = $res[0];
+   }
+}
+print join("\t", @best_res), "\n" if @best_res;
+
+
+