miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Scripts/M5nr.getSequences.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Feb-06-2015
+# @license: artistic license 2.0
+#
+
+require 'optparse'
+require 'rubygems'
+require 'restclient'
+require 'open-uri'
+require 'JSON'
+
+o = {:q=>FALSE, :url=>'http://api.metagenomics.anl.gov/m5nr', :max=>0, :recover=>FALSE}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "
+Downloads a set of sequences from M5nr with a given functional annotation.
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-f", "--function STR", "Functional annotation."){ |v| o[:function] = v }
+   opts.separator ""
+   opts.separator "Options"
+   opts.on("-m", "--max INT", "Maximum number of sequences to download.  By default: all (0)."){ |v| o[:max] = v.to_i }
+   opts.on("-r", "--recover", "If set, tries to recover a previous FastA."){ o[:recover]=TRUE }
+   opts.on("-n", "--url STR", "URL for M5nr API.  By default: #{o[:url]}."){ |v| o[:url] = v }
+   opts.on("-o", "--out FILE", "File containing the sequences.  By default: value of -f appended with .fa."){ |v| o[:out] = v }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)"){ o[:q] = TRUE }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-f is mandatory" if o[:function].nil?
+o[:out] = "#{o[:function].gsub(/ /,'_')}.fa" if o[:out].nil?
+uri_fun = URI::encode(o[:function])
+
+ignore = {}
+if o[:recover] and File.exists? o[:out]
+   ih = File.open(o[:out], "r")
+   ih.each_line do |ln|
+      id = /^>(\S+)\s/.match(ln)
+      unless id.nil?
+         ignore[id[1]] = 1
+      end
+   end
+   ih.close
+   of = File.open(o[:out], "a+")
+else
+   of = File.open(o[:out], "w")
+end
+
+next_url = "#{o[:url]}/function/#{uri_fun}"
+i = 0
+loop do
+   $stderr.print "Downloading sequence #{i+1}.   \r" unless o[:q]
+   res_fun = RestClient.get next_url
+   abort "Unable to reach MG-RAST M5nr API, error code #{res_fun.code}." unless res_fun.code == 200
+   fun = JSON.parse(res_fun.to_str)
+   fun["data"].each do |datum|
+      if ignore["#{datum["source"]}:#{datum["accession"]}"].nil?
+	 res_seq = RestClient.get "#{o[:url]}/md5/#{datum["md5"]}", {:params=>{:sequence=>1}}
+	 abort "Unable to reach MG-RAST M5nr API, error code #{res_seq.code}." unless res_seq.code == 200
+	 seq = JSON.parse(res_seq.to_str)
+	 of.puts ">#{datum["source"]}:#{datum["accession"]} #{datum["function"]} [#{datum["organism"]} taxid:#{datum["ncbi_tax_id"]}]"
+	 of.puts seq["data"]["sequence"].scan(/.{80}|.+/).map{ |x| x.strip }.join($/)
+      end
+      i += 1
+      break if o[:max]>0 and i >= o[:max]
+   end # |datum|
+   next_url = fun["next"]
+   break if next_url.nil? or (o[:max] > 0 and i >= o[:max])
+end
+of.close
+
+$stderr.puts "Downloaded #{i} sequences." unless o[:q]
+

data/utils/enveomics/Scripts/MeTaxa.distribution.pl

@@ -0,0 +1,198 @@
+#!/usr/bin/env perl
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update Mar-23-2015
+#
+use warnings;
+use strict;
+use Symbol;
+use Getopt::Std;
+use List::Util qw/max/;
+
+sub HELP_MESSAGE { die "
+Usage:
+   $0 [args]
+
+Mandatory:
+   -m <str>	MyTaxa output.
+
+Optional:
+   -g <str>	Genes predicted in the format defined by -f.  If not passed, abundance is assumed to be based
+   		on contigs.
+   -f <str>	Format of the predicted genes.  One of:
+   		o gff2: GFF v2 as produced by MetaGeneMark.hmm (default).
+		o gff3: GFF v3, including the field id in the last column (with the Gene ID).
+		o tab: A tab-delimited file with the gene ID (col #1), the length of the gene in bp (col #2),
+		  and the ID of the corresponding contig (col #3). The length of the gene (col #2) isn't used
+		  (and it can be empty),  but the column must exist (i.e., 2 tabs per line) for compatibility
+		  with BlastTab.metaxaPrep.pl
+   -c <str>	Counts file: Sequence IDs (genes if -g is provided, contigs otherwise) and reads per sequence
+   		in a tab-delimited file.  If not provided, each sequence counts as 1.
+   -O <str>	Prefix of the output files to be generated.  By default, the value of -m.
+   -I <str>	File containing the complete classification of all the contigs identified as Innominate taxa.
+		By default, this file is not created.
+   -G <str>	File containing the classification of each gene.  By default, this file is not created.  This
+   		requires -g to be set.  Note: This option requires extra RAM.
+   -K <str>	File containing a krona input file.  By default, this file is not created.
+   -k <str>	List of ranks to include in the Krona file, delimited by comma.   It MUST be decreasing rank.
+   		By default: 'superkingdom,phylum,class,family,genus,species'.  This is ignored unless -K also
+		is passed.
+   -R <str>	List of taxonomic ranks for which individual reports should be generated, delimited by comma.
+   		It MUST be decreasing rank.  By default: 'phylum,genus,species'.
+   -r		If set, reports raw counts.  Otherwise, reports permil of the rank.
+   -u		Report Unknown taxa.
+   -q		Run quietly.
+   -h		Display this help message and exits.
+
+" }
+
+my %o;
+getopts('g:f:c:m:O:I:G:K:k:R:ruqh', \%o);
+$o{h} and &HELP_MESSAGE;
+$o{m} or  &HELP_MESSAGE;
+$o{O} ||= $o{m};
+$o{f} ||= "gff2";
+$o{k} ||= "superkingdom,phylum,class,family,genus,species";
+my @K = split /,/, lc $o{k};
+$o{R} ||= "phylum,genus,species";
+my @R = split /,/, lc $o{R};
+($o{G} and not $o{g}) and die "-G requires -g to be set.\n";
+
+
+my %gene;
+my %count;
+my %ctg=();
+if($o{g}){
+   print STDERR "Reading genes collection.\n" unless $o{q};
+   open GFF, "<", $o{g} or die "Cannot read file: $o{g}: $!\n";
+   while(<GFF>){
+      next if /^#/;
+      next if /^\s*$/;
+      chomp;
+      my($id,$ctg);
+      my @ln = split /\t/;
+      if($o{f} eq 'gff2'){
+	 exists $ln[8] or die "Cannot parse line $., expecting 9 columns: $_\n";
+	 $id = $ln[8];
+	 $id =~ s/gene_id /gene_id_/;
+	 $ctg=$ln[0];
+      }elsif($o{f} eq 'gff3'){
+	 exists $ln[8] or die "Cannot parse line $., expecting 9 columns: $_\n";
+	 $ln[8] =~ /id=([^;]+)/ or die "Cannot parse line $.: $_\n";
+	 $id = $1;
+	 $ctg = $ln[0];
+      }elsif($o{f} eq 'tab'){
+         exists $ln[2] or die "Cannot parse line $., expecting 3 columns: $_\n";
+	 $id = $ln[0];
+	 $ctg = $ln[2];
+      }else{
+         die "Unsupported format: ".$o{f}.".\n";
+      }
+      $ctg =~ s/ .*//;
+      if($o{c}){
+	 $gene{$id} = $ctg;
+      }else{
+         $count{$ctg}++;
+      }
+      push( @{$ctg{$ctg}||=[]}, $id ) if $o{G};
+   }
+   close GFF;
+   print STDERR " Found ".(scalar(keys %gene))." genes.\n" unless $o{q};
+}
+
+my $Nreads = 0;
+if($o{c}){
+   print STDERR "Reading read-counts.\n" unless $o{q};
+   open COUNT, "<", $o{c} or die "Cannot read file: $o{c}: $!\n";
+   while(<COUNT>){
+      chomp;
+      my @l = split /\t/;
+      if($o{g}){
+	 exists $gene{$l[0]} or die "Cannot find gene's contig: $l[0].\n";
+	 $count{ $gene{$l[0]} } += $l[1];
+	 delete $gene{$l[0]};
+      }else{
+	 $count{ $l[0] } += $l[1];
+      }
+      $Nreads += $l[1];
+   }
+   close COUNT;
+   print STDERR " Found ".scalar(keys %gene)." genes without reads.\n" if scalar(keys %gene) and not $o{q};
+   $count{$_}+=0 for values %gene;
+   print STDERR " Found ".scalar(keys %count)." sequences and $Nreads reads.\n" unless $o{q};
+}
+
+print STDERR "Reading Metaxa results.\n";
+open METAXA, "<", $o{m} or die "Cannot read file: $o{m}: $!\n";
+my $ctg;
+my $rank;
+my @ofh = ();
+my @n   = (0,0,0);
+my @out = ({},{},{});
+my @rank_name = map { ucfirst } ('unknown', @R);
+my %rank = map { ($rank_name[$_]=>$_) } 0 .. $#rank_name;
+my @rank_tag  = ("NA", map { "<$_>" } @R);
+$o{I} and (open OUT_I, ">", $o{I} or die "Cannot create file: $o{I}: $!\n");
+$o{K} and (open OUT_K, ">", $o{K} or die "Cannot create file: $o{K}: $!\n");
+$o{G} and (open OUT_G, ">", $o{G} or die "Cannot create file: $o{G}: $!\n");
+
+my $Nreads_class = 0;
+my $Nno_read_ctg = 0;
+while(not eof(METAXA)){
+   my @h=split /\t/, <METAXA>;
+   my $t=<METAXA>; chomp $t;
+   exists $h[3] or die "Cannot parse MyTaxa file, line $.: $_\n";
+   my $count_h;
+   if($o{c} or $o{g}){
+      unless(exists $count{$h[0]}){
+         $Nno_read_ctg++;
+	 next;
+      }
+      $count_h = $count{$h[0]};
+   }else{
+      $count_h = 1;
+   }
+   if($o{G}){ print OUT_G "$_\t$t\n" for @{$ctg{$h[0]}} }
+   next unless $count_h;
+   my $last = 'organism';
+   $n[0] += $count_h;
+   for my $r (1 .. max(values %rank)){
+      if($rank{$h[1]} >= $r){
+	 if($t =~ m/$rank_tag[$r]([^;]*)/){
+	    $last = $1 if $1;
+	 }else{
+	    $last = $last=~/^Innominate / ? $last : "Innominate $last";
+	    $o{I} and print OUT_I "$h[0]\t$rank_name[$r]\t$last\t$t\n";
+	 }
+	 $out[$r]->{$last} += $count_h;
+	 $n[$r] += $count_h;
+      }else{
+         $out[$r]->{"Unknown $last"} += $count_h if $o{u};
+      }
+   }
+   if($o{K}){
+      my $ln = $count_h;
+      for my $r (@K){ $ln.= "\t".($t=~m/<$r>([^;]+)/?$1:'') }
+      print OUT_K "$ln\n";
+   }
+   $Nreads_class+= $count_h;
+}
+print OUT_K "".($Nreads-$Nreads_class)."\n" if $o{K} and $Nreads>$Nreads_class;
+close METAXA;
+$o{I} and close OUT_I;
+$o{K} and close OUT_K;
+$o{G} and close OUT_G;
+print " Found $n[0] reads.\n" unless $o{q};
+print " Couldn't find counts for $Nno_read_ctg contigs.\n" if $Nno_read_ctg;
+unless($o{q}){ print " Found $n[$_] classified reads at ".$rank_name[$_]." level.\n" for (1 .. max(values %rank)) }
+
+print STDERR "Generating output.\n" unless $o{q};
+for my $rank (1 .. max(values %rank)){
+   open OUT, ">", "$o{O}.".$rank_name[$rank].".txt" or die "Cannot create file: $o{O}.".$rank_name[$rank].".txt: $!\n";
+   for my $class (keys %{$out[$rank]}){
+      printf OUT "%s\t%.20f\n", $class, ($out[$rank]->{$class}*($o{r}?1:1000/$n[$rank]));
+   }
+   close OUT;
+}
+

data/utils/enveomics/Scripts/MyTaxa.fragsByTax.pl

@@ -0,0 +1,35 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+use warnings;
+use strict;
+
+my($file,$tax,$rank) = @ARGV;
+($file and $tax) or die "
+.Usage:
+   $0 file.txt taxon[ rank]
+
+   file.txt	MyTaxa output.
+   taxon	Taxon to look for.
+   rank		Rank of taxon (optional). By default: any rank.
+
+";
+$rank ||= ".*";
+$rank = lc $rank;
+
+open MT, "<", $file or die "Cannot read file: $file: $!\n";
+my $last = '';
+while(my $ln=<MT>){
+   chomp $ln;
+   if($ln =~ /<$rank>$tax(;|$)/){
+      $last =~ s/\t.*//;
+      print $last, "\n";
+   }
+   $last = $ln;
+}
+close MT;
+

data/utils/enveomics/Scripts/MyTaxa.seq-taxrank.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Feb-06-2015
+# @license artistic license 2.0
+#
+
+require 'optparse'
+
+opts = {:rank=>'genus', :quiet=>FALSE}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opt|
+   opt.separator "Generates a simple tabular file with the classification of each sequence at a given taxonomic rank from a MyTaxa output."
+   opt.separator ""
+   opt.on("-i", "--mytaxa FILE", "Input MyTaxa file."){ |v| opts[:mytaxa]=v }
+   opt.on("-r", "--rank STR", "Taxonomic rank.  By default: #{opts[:rank]}."){ |v| opts[:rank] = v.downcase }
+   opt.on("-q","--quiet","Run quietly.") { opts[:quiet]=TRUE }
+   opt.on("-h","--help","Display this screen.") do
+      puts opt
+      exit
+   end
+   opt.separator ""
+end.parse!
+abort "-i/--mytaxa is mandatory." if opts[:mytaxa].nil?
+abort "-i/--mytaxa must exist." unless File.exists? opts[:mytaxa]
+
+begin
+   f = File.open(opts[:mytaxa], "r")
+   ctg = nil;
+   while(ln = f.gets)
+      m = /^(.+)(\t.+){3}/.match(ln)
+      if m
+	 raise "Couldn't find classification for contig #{ctg}" unless ctg.nil?
+	 ctg = m[1]
+      else
+	 raise "Couldn't find the contig name at line #{$.}" if ctg.nil?
+	 m = /<#{opts[:rank]}>([^;]+)/.match(ln)
+	 puts "#{ctg}\t#{m ? m[1] : "Unclassified"}"
+	 ctg = nil
+      end
+   end
+   f.close
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+

data/utils/enveomics/Scripts/NCBIacc2tax.rb

@@ -0,0 +1,92 @@
+#!/usr/bin/env ruby
+
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
+require "enveomics_rb/remote_data"
+use "nokogiri"
+
+#================================[ Options parsing ]
+o = {
+  :q=>false, :accs=>[], :dbfrom=>"nuccore", :header=>true,
+  :no_nil=>false, :ret=>"ScientificName",
+  :ranks=>%w(superkingdom phylum class order family genus species)}
+OptionParser.new do |opt|
+  opt.banner = "
+  Maps a list of NCBI accessions to their corresponding taxonomy using the NCBI
+  EUtilities. Avoid using this script on millions of entries at a time, since
+  each entry elicits two requests to NCBI's servers.
+
+  Usage: #{$0} [options]".gsub(/^ +/,"")
+  opt.separator ""
+  opt.on("-a", "--acc acc1,acc2.ver,...", Array,
+    "Comma-separated list of accessions. Required unless -i is passed."
+    ){ |v| o[:accs]=v }
+  opt.on("-i", "--infile FILE",
+    "Raw text file containing the list of accessions, one per line.",
+    "Required unless -g is passed."){ |v| o[:infile]=v }
+  opt.on("-p", "--protein",
+    "Use if the accessions are proteins. Otherwise, accessions are assumed " +
+    "to be from the Nuccore Database."){ o[:dbfrom]="protein" }
+  opt.on("-r", "--ranks RANK1,RANK2,...", Array,
+    "Taxonomic ranks to report. By default: #{o[:ranks].join(",")}."
+    ){ |v| o[:ranks]=v }
+  opt.on("-n", "--noheader",
+    "Do not include a header in the output."){ o[:header]=false }
+  opt.on("-t", "--taxids",
+    "Return Taxonomy IDs instead of scientific names."){ o[:ret]="TaxId" }
+  opt.on("--ignore-missing",
+    "Does not report missing accessions in the output file.",
+    "By default, it reports accessions and empty values for all other columns."
+    ){ |v| o[:no_nil]=v }
+  opt.on("-q", "--quiet", "Run quietly."){ |v| o[:q]=true }
+  opt.on("-h", "--help","Display this screen") do
+    puts opt
+    exit
+  end
+  opt.separator ""
+end.parse!
+
+#================================[ Functions ]
+def acc2taxid(db, acc)
+  doc = Nokogiri::XML( RemoteData.elink({:dbfrom=>db,
+    :db=>"taxonomy", :id=>acc, :idtype=>"acc"}) )
+  doc.at_xpath("/eLinkResult/LinkSet/LinkSetDb/Link/Id")
+end
+#================================[ Main ]
+begin
+  o[:accs] += File.readlines(o[:infile]).map{ |l| l.chomp } unless
+    o[:infile].nil?
+  o[:ranks].map!{ |r| r.downcase }
+  puts (["Acc", "TaxId"] + o[:ranks].map{ |r| r.capitalize }).join("\t") if
+    o[:header]
+  o[:accs].each do |acc|
+    taxid = acc2taxid(o[:dbfrom], acc)
+    status = ""
+    if taxid.nil?
+      warn "Cannot find link to taxonomy: #{acc} #{status}"
+      puts ([acc, ""] + o[:ranks].map{ |i| "" }).join("\t") unless o[:no_nil]
+      next
+    end
+    taxonomy = {}
+    unless taxid.nil?
+      doc = Nokogiri::XML( RemoteData.efetch({:db=>"taxonomy",
+        :id=>taxid.content}) )
+      taxonomy[ doc.at_xpath("/TaxaSet/Taxon/Rank").content ] =
+        doc.at_xpath("/TaxaSet/Taxon/#{o[:ret]}").content
+      doc.xpath("/TaxaSet/Taxon/LineageEx/Taxon").each do |taxon|
+        taxonomy[ taxon.at_xpath("./Rank").content ] =
+          taxon.at_xpath("./#{o[:ret]}").content
+      end
+    end
+    puts ([acc, taxid.content] +
+      o[:ranks].map{ |rank| taxonomy[ rank ] ||= "" }).join("\t")
+  end
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end