miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Scripts/BlastTab.seqdepth_nomedian.pl

@@ -0,0 +1,86 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license: artistic license 2.0
+# @update: Mar-23-2015
+#
+
+use strict;
+use warnings;
+use List::Util qw/min max sum/;
+
+my $fna = shift @ARGV;
+$fna or die "
+Usage:
+   cat blast1... | $0 genes_or_ctgs.fna > genes_or_ctgs.cov
+
+   blast1...		One or more Tabular BLAST files of reads vs genes (or contigs).
+   genes_or_ctgs.fna	A FastA file containing the genes or the contigs (db).
+   genes_or_ctgs.cov	The output file.
+
+Output:
+   A tab-delimited file with the following columns:
+   1. Subject ID
+   2. Average sequencing depth
+   3. Number of mapped reads
+   4. Length of the subject sequence
+
+Note:
+   The values reported by this script may differ from those of BlastTab.seqdepth.pl,
+   because this script uses the aligned length of the read while BlastTab.seqdepth.pl
+   uses the aligned length of the subject sequence. 
+
+";
+
+my $size  = {};
+my $gene  = {};
+my $reads = {};
+
+SIZE:{
+   local $/=">";
+   print STDERR "== Reading fasta\n";
+   open FNA, "<", $fna or die "Cannot read the file: $fna: $!\n";
+   my $i=0;
+   while(<FNA>){
+      chomp;
+      my @g = split /\n/, $_, 2;
+      next unless $g[1];
+      #$g[1] =~ s/[^A-Za-z]//g;
+      #$size->{$g[0]} = length $g[1];
+      $g[0] =~ s/\s.*//;
+      $size->{$g[0]} = ( $g[1] =~ tr/[A-Za-z]// );
+      print STDERR " Measuring sequence ".($i).": $g[0]      \r" unless ++$i%500;
+   }
+   close FNA;
+   print STDERR " Found $i sequences".(" "x30)."\n";
+}
+
+MAP:{
+   print STDERR "== Reading mapping\n";
+   my $i=0;
+   while(<>){
+      my @ln = split /\t/;
+      $gene->{$ln[1]} ||= 0;
+      $gene->{$ln[1]} += abs($ln[6]-$ln[7])+1;
+      ($reads->{$ln[1]} ||= 0)++;
+      print STDERR " Saving hit ".($i).": $ln[1]      \r" unless ++$i%5000;
+   }
+   print STDERR " Found $i hits".(" "x30)."\n";
+}
+
+OUT:{
+   print STDERR "== Creating output\n";
+   my $i=0;
+   for my $g (keys %$gene){
+      die "Cannot find gene in $fna: $g.\n" unless exists $size->{$g};
+      printf "%s\t%.6f\t%d\t%d\n", $g,
+	   $gene->{$g}/$size->{$g},
+	   $reads->{$g},
+	   $size->{$g};
+      print STDERR " Saving sequence $g:".($i)."\r" unless ++$i%500;
+   }
+   print STDERR " Saved $i sequences".(" "x30)."\n";
+}
+
+print STDERR " done.\n";
+

data/utils/enveomics/Scripts/BlastTab.subsample.pl

@@ -0,0 +1,47 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update  Mar-23-2015
+#
+
+use strict;
+use warnings;
+
+my($blast, $fasta) = @ARGV;
+($blast and $fasta) or die "
+Description:
+   Filters a BLAST output including only the hits produced by
+   any of the given sequences as query.
+
+Usage:
+   $0 blast.tab sample.fa > out.tab
+
+   blast.tab	BLAST output to be filtered (tabular format).
+   sample.fa	Sequences to use as query.
+   out.tab	The filtered BLAST output (tabular format).
+
+";
+
+print STDERR "== Reading sequences\n";
+my $seq = {};
+open FASTA, "<", $fasta or die "Cannot read the file: $fasta: $!\n";
+while(<FASTA>){
+   next unless /^>(\S+)/;
+   $seq->{$1} = 1;
+}
+close FASTA;
+print STDERR " ".(scalar keys %$seq)." sequences to be used as query.\n";
+
+print STDERR "== Reading BLAST\n";
+my ($N,$n)=(0,0);
+open BLAST, "<", $blast or die "Cannot read the file: $blast: $!\n";
+while(my $ln = <BLAST>){
+   next if $ln=~/^#/;
+   $N++; my ($qry) = split /\t/, $ln;
+   next unless exists $seq->{$qry};
+   $n++; print $ln;
+}
+close BLAST;
+print STDERR " Reported $n entries out of $N.\n";
+

data/utils/enveomics/Scripts/BlastTab.sumPerHit.pl

@@ -0,0 +1,114 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update  Mar-23-2016
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+sub HELP_MESSAGE {
+die "
+.Description
+   Sums the weights of all the queries hitting each subject.  Often (but not
+   necessarily) the BLAST files contain only best matches.  The weights can be
+   any number, but a common use of this Script is to add up counts (weights are
+   integers).  For example, in a BLAST of predicted genes vs some annotation
+   source, the weights could be the number of reads recruited by each gene.
+
+.Usage:
+   $0 [options] blast... > out-file
+
+   blast... *	One or more BLAST files.
+   out-file	A two-columns tab-delimited file containing the summed weights
+   		per hit.
+   
+   -w <str>	Weights file: A two-columns tab-delimited file containing the
+   		name (column 1) and the weight (column 2) of each query.
+   -s <float>	Minimum score.  By default: 0.
+   -i <float>	Minimum identity (in percentage).  By default: 0.
+   -m <int>	Maximum number of queries.  Set to 0 for all.  By default: 0.
+   -n		Normalize weights by the number of hits per query.
+   -z		Add zero when weight is not found (by default: doesn't list
+   		them).
+   -q		Run quietly.
+   -h		Display this message and exit.
+
+   * Mandatory
+
+.Note:
+   The weights (-w parameter) are optional, but its use is encouraged.  When
+   weights are not passed, the script simply assumes all queries to be equally
+   weighted (unity), a result that can be faster to compute with, for example:
+      cat blast | cut -f 2 | sort | uniq -c | awk '{print \$2\"\\t\"\$1}' > out
+   It is equivalent to simply count the number of times that each subject
+   occurs.
+"
+}
+
+my %o = ();
+getopts('w:s:i:m:znqh', \%o);
+$o{h} and &HELP_MESSAGE;
+$o{s}||=0;
+$o{i}||=0;
+$o{m}||=0;
+
+my %count;
+if($o{w}){
+   print STDERR "Reading counts.\n" unless $o{q};
+   open COUNT, "<", $o{w} or die "Cannot open file: $o{w}: $!\n";
+   %count = map {split /\t/} <COUNT>;
+   close COUNT;
+}
+
+print STDERR "Reading BLASTs.\n" unless $o{q};
+my $qry = '';
+my $hits = 0;
+my @buf = ();
+my $qries = 0;
+my $noQry = 0;
+my $ln1   = 0;
+my %out = ();
+BFILE: for my $blast (@ARGV){
+   print STDERR " o $blast\n" unless $o{q};
+   open BLAST, "<", $blast or die "Cannot open file: $blast: $!\n";
+   BLINE: while(<BLAST>){
+      chomp;
+      my @ln = split /\t/;
+      $ln1 ||= $#ln;
+      die "Bad line $.: $_\n" unless $#ln==$ln1;
+      next if ($o{s} and $ln[11]<$o{s}) or ($o{i} and $ln[2]<$o{i});
+      unless(exists $count{$ln[0]}){
+	 $noQry++;
+	 if(not $o{w}){
+	    $count{$ln[0]}=1;
+	 }elsif($o{z}){
+	    $count{$ln[0]}=0;
+	 }else{
+	    next BLINE;
+	 }
+      }
+      
+      if($qry ne $ln[0]){
+	 $qries++;
+	 ($out{$_->[0]}||=0) += ($_->[1]/($o{n}?$hits:1)) for @buf;
+	 last BFILE if $o{m} and $qries >= $o{m};
+	 @buf = ();
+	 $qry  = $ln[0];
+	 $hits = 0;
+      }
+      
+      push @buf, [$ln[1], $count{$ln[0]}];
+      $hits++;
+   }
+   ($out{$_->[0]}||=0) += ($_->[1]/($o{n}?$hits:1)) for @buf;
+   close BLAST;
+}
+print STDERR "Warning: Couldn't find $noQry queries\n" if $noQry and $o{w};
+
+for my $h (keys %out){
+   print "$h\t".$out{$h}."\n";
+}
+

data/utils/enveomics/Scripts/BlastTab.taxid2taxrank.pl

@@ -0,0 +1,90 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic 2.0
+# @update  Oct-13-2015
+#
+
+use warnings;
+use strict;
+
+my($blast, $nodes, $names, $rank, $bh) = @ARGV;
+($blast and $nodes and $names) or die <<HELP
+
+Takes a BLAST with NCBI Taxonomy IDs as subjects and replaces them by names at a
+given taxonomic rank.
+
+Usage:
+   $0 tax_blast.txt nodes.dmp names.dmp[ rank[ best-hit]] > taxrank_blast.txt
+
+   tax_blast.txt	BLAST output, where subject IDs are NCBI Taxonomy IDs.
+   nodes.dmp		Nodes file from NCBI Taxonomy*.
+   names.dmp		Names file from NCBI Taxonomy*.
+   rank			The rank to be reported.  All the reported nodes will
+   			have the same rank.  By default, genus.  To see
+			supported values, run:
+			cut -f 5 nodes.dmp | sort -u
+   best-hit		A word (yes or no) telling the program whether or not it
+   			should take into account the best hit per query only.
+			By default: yes.
+   taxrank_list.txt	BLAST-like output, where subject IDs are Taxonomy names.
+
+   * Download from ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz
+
+HELP
+;
+$rank ||= "genus";
+$bh   ||= "yes";
+
+# %nodes structure:
+#   taxid => [parent's taxid, rank, nil, name, name type]
+
+print STDERR "Reading $nodes.\n";
+open NODES, "<", $nodes or die "Cannot read file: $nodes: $!\n";
+my %nodes = map { my @a=split /\t\|\t/; ($a[0] => [$a[1], $a[2]]) } <NODES>;
+close NODES;
+
+print STDERR "Reading $names.\n";
+open NAMES, "<", $names or die "Cannot read file: $names: $!\n";
+while(<NAMES>){
+   my @a=split /\t\|\t/;
+   next if exists $nodes{$a[0]}->[3] and
+      $nodes{$a[0]}->[4] eq "scientific name";
+   next if exists $nodes{$a[0]}->[3] and
+      $a[3] ne "scientific name";
+   $nodes{$a[0]}->[3] = $a[1];
+   $nodes{$a[0]}->[4] = $a[3];
+}
+close NAMES;
+
+my $i     = 0;
+my $nomap = 0;
+my $qry   = "";
+open BLAST, "<", $blast or die "Cannot read file: $blast: $!\n";
+HIT:while(<BLAST>){
+   if(/^#/){
+      print $_;
+      next;
+   }
+   chomp;
+   my @row = split /\t/;
+   next if $bh eq "yes" and $row[0] eq $qry;
+   $i++;
+   print STDERR " Mapping hit $i\r" unless $i%10;
+   exists $nodes{$row[1]} or die "Cannot find Taxonomy node: $row[1].\n";
+   my $n = $nodes{$row[1]};
+   while($n->[1] ne $rank){
+      if($n->[0] eq $nodes{$n->[0]}->[0]){
+	 $nomap++;
+	 next HIT;
+      }
+      $n = $nodes{$n->[0]};
+   }
+   $row[1] = $n->[3];
+   print "".join("\t", @row)."\n";
+}
+close BLAST;
+print STDERR " Mapped $i hits\n";
+print STDERR
+   " WARNING: $nomap hits above rank or in a lineage without rank.\n" if $nomap;
+

data/utils/enveomics/Scripts/BlastTab.topHits_sorted.rb

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+
+#
+# @author:  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update:  Dec-11-2015
+# @license: artistic license 2.0
+#
+
+require "optparse"
+
+$opts = {n:5, sortby:"bitscore", q:false}
+$cols = {"bitscore"=>11, "evalue"=>10, "identity"=>2, "length"=>3}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "Reports the top-N best hits of a BLAST, pre-sorted by query."
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-i", "--blast FILE",
+      "Path to the BLAST file."){ |v| $opts[:blast]=v }
+   opts.separator ""
+   opts.separator "Optional"
+   opts.on("-n", "--top INTEGER",
+      "Maximum number of hits to report for each query.",
+      "By default: #{$opts[:n]}"){ |v| $opts[:n]=v.to_i }
+   opts.on("-s", "--sort-by STRING",
+      "Parameter used to detect the 'best' hits.",
+      "Any of: bitscore (default), evalue, identity, length."
+      ){ |v| $opts[:sortby]=v }
+   opts.on("-q", "--quiet", "Run quietly."){ $opts[:q]=true }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+
+abort "-i/--blast is mandatory." if $opts[:blast].nil?
+abort "Unrecognized value for -s/--sortby: #{$opts[:sortby]}." if
+   $cols[ $opts[:sortby] ].nil?
+
+class Hit
+   attr_reader :blast_line
+   def initialize(blast_line)
+      @blast_line = blast_line.chomp.split(/\t/)
+   end
+   def col(i)
+      @blast_line[i]
+   end
+   def <=>(other)
+      ans = self.col( $cols[ $opts[:sortby] ] ).to_f <=> other.col( $cols[ $opts[:sortby] ] ).to_f
+      ans = ans * -1 unless $opts[:sortby] == "evalue"
+      return ans
+   end
+   def to_s
+      @blast_line.join("\t")
+   end
+end
+
+class HitSet
+   attr_reader :query, :hits
+   def initialize
+      @hits = []
+      @query = nil
+   end
+   def <<(hit)
+      @query = hit.col(0) if @query.nil?
+      raise "Inconsistent query, expecting #{self.query}" unless
+	 self.query == hit.col(0)
+      @hits << hit
+   end
+   def empty?
+      self.hits.length == 0
+   end
+   def filter!
+      @hits.sort!
+      @hits.slice!($opts[:n], @hits.length)
+   end
+   def to_s
+      @hits.join("\n")
+   end
+end
+
+$stderr.puts "Parsing BLAST." unless $opts[:q]
+fh = File.open $opts[:blast], "r"
+hs = HitSet.new
+while ln=fh.gets
+   hit = Hit.new( ln )
+   if hs.query != hit.col(0)
+      hs.filter!
+      puts hs unless hs.empty?
+      hs = HitSet.new
+      $stderr.print "Parsing line #{$.}... \r" unless $opts[:q]
+   end
+   hs << hit
+end
+$stderr.print "Parsed #{$.} lines.   \n" unless $opts[:q]
+fh.close
+
+hs.filter!
+puts hs unless hs.empty?
+

data/utils/enveomics/Scripts/Chao1.pl

@@ -0,0 +1,97 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update  Dec-29-2015
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+use List::Util qw/min max/;
+
+sub VERSION_MESSAGE(){print "Home-made Chao1 (enveomics)\n"}
+sub HELP_MESSAGE(){die "
+Description:
+   Takes a table of OTU abundance in one or more samples and calculates the
+   chao1 index (with 95% Confidence Interval) for each sample.  To use it with
+   Qiime OTU Tables, run it as:
+   $0 -i OTU_Table.txt -c 1 -h
+
+Usage:
+   $0 [opts]
+
+   -i <str>	* Input table (columns:samples, rows:OTUs).
+   -r <int>	Number of rows to ignore.  By default: 0.
+   -c <int>	Number of columns to ignore.  By default: 0.
+   -C <int>	Number of columns to ignore at the end. By default: 0.
+   -d <str>	Delimiter.  Supported escaped characters are: \"\\t\"
+		(tabulation), and \"\\0\" (null bit).  By default: \"\\t\".
+   -h		If set, the first row is assumed to have the names of the
+		samples.
+   --help	This help message.
+
+   * Mandatory.
+
+To improve:
+   o Account for n1==0 and n2==0 cases.  See http://www.mothur.org/wiki/Chao.
+
+"}
+
+my %o;
+getopts('i:c:C:r:d:h', \%o);
+
+&HELP_MESSAGE() unless $o{i};
+$o{c} ||= 0;
+$o{C} ||= 0;
+$o{r} ||= 0;
+$o{d} ||= "\\t";
+
+$o{d}="\t" if $o{d} eq "\\t";
+$o{d}="\0" if $o{d} eq "\\0";
+
+my @names = ();
+my @values = ();
+open TABLE, "<", $o{i} or die "Cannot open file: ".$o{i}.": $!\n";
+<TABLE> for (1 .. $o{r});
+if($o{h}){
+   my $h = <TABLE>;
+   $h or die "Empty table!\n";
+   chomp $h;
+   @names = split $o{d}, $h;
+   shift @names for (1 .. $o{c});
+   pop @names for (1 .. $o{C});
+}
+while(<TABLE>){
+   chomp;
+   my @ln = split $o{d};
+   shift @ln for (1 .. $o{c});
+   pop @ln for (1 .. $o{C});
+   push @{$values[$_] ||= []}, $ln[$_] for (0 .. $#ln);
+}
+close TABLE;
+
+print "Sample\tObs\tChao1\tChao1_LB\tChao1_UL\n";
+for my $i (0 .. $#values){
+   print "".(exists $names[$i] ? $names[$i] : $i).$o{d};
+   my $n1=0;
+   my $n2=0;
+   my $ob=0;
+   for my $v (@{$values[$i]}){
+      $n1++ if $v==1;
+      $n2++ if $v==2;
+      $ob++ if $v>=1;
+   }
+   if($ob and $n1 and $n2){
+      my $m   = $n1/$n2;
+      my $ch  = $ob + (($n1**2)/(2*$n2));
+      my $var = ($n2*((($m**4)/4) + ($m**3) + (($m**2)/2)));
+      my $c   = exp(1.96*sqrt(log(1+ $var/(($ch-$ob)**2))));
+      my $lc  = max($ob + ($ch-$ob)/$c, $ob);
+      my $uc  = $ob + $c*($ch-$ob);
+      print "".join($o{d}, $ob, $ch, $lc, $uc)."\n"
+   }else{
+      print "".join($o{d}, $ob, $ob, 0, 0)."\n"
+   }
+}
+