miga-base 0.7.24.0 → 0.7.26.0

data/utils/enveomics/Scripts/M5nr.getSequences.rb

@@ -1,81 +0,0 @@
-#!/usr/bin/env ruby
-
-#
-# @author: Luis M. Rodriguez-R
-# @update: Feb-06-2015
-# @license: artistic license 2.0
-#
-
-require 'optparse'
-require 'rubygems'
-require 'restclient'
-require 'open-uri'
-require 'JSON'
-
-o = {:q=>FALSE, :url=>'http://api.metagenomics.anl.gov/m5nr', :max=>0, :recover=>FALSE}
-ARGV << '-h' if ARGV.size==0
-OptionParser.new do |opts|
-   opts.banner = "
-Downloads a set of sequences from M5nr with a given functional annotation.
-
-Usage: #{$0} [options]"
-   opts.separator ""
-   opts.separator "Mandatory"
-   opts.on("-f", "--function STR", "Functional annotation."){ |v| o[:function] = v }
-   opts.separator ""
-   opts.separator "Options"
-   opts.on("-m", "--max INT", "Maximum number of sequences to download.  By default: all (0)."){ |v| o[:max] = v.to_i }
-   opts.on("-r", "--recover", "If set, tries to recover a previous FastA."){ o[:recover]=TRUE }
-   opts.on("-n", "--url STR", "URL for M5nr API.  By default: #{o[:url]}."){ |v| o[:url] = v }
-   opts.on("-o", "--out FILE", "File containing the sequences.  By default: value of -f appended with .fa."){ |v| o[:out] = v }
-   opts.on("-q", "--quiet", "Run quietly (no STDERR output)"){ o[:q] = TRUE }
-   opts.on("-h", "--help", "Display this screen") do
-      puts opts
-      exit
-   end
-   opts.separator ""
-end.parse!
-abort "-f is mandatory" if o[:function].nil?
-o[:out] = "#{o[:function].gsub(/ /,'_')}.fa" if o[:out].nil?
-uri_fun = URI::encode(o[:function])
-
-ignore = {}
-if o[:recover] and File.exists? o[:out]
-   ih = File.open(o[:out], "r")
-   ih.each_line do |ln|
-      id = /^>(\S+)\s/.match(ln)
-      unless id.nil?
-         ignore[id[1]] = 1
-      end
-   end
-   ih.close
-   of = File.open(o[:out], "a+")
-else
-   of = File.open(o[:out], "w")
-end
-
-next_url = "#{o[:url]}/function/#{uri_fun}"
-i = 0
-loop do
-   $stderr.print "Downloading sequence #{i+1}.   \r" unless o[:q]
-   res_fun = RestClient.get next_url
-   abort "Unable to reach MG-RAST M5nr API, error code #{res_fun.code}." unless res_fun.code == 200
-   fun = JSON.parse(res_fun.to_str)
-   fun["data"].each do |datum|
-      if ignore["#{datum["source"]}:#{datum["accession"]}"].nil?
-	 res_seq = RestClient.get "#{o[:url]}/md5/#{datum["md5"]}", {:params=>{:sequence=>1}}
-	 abort "Unable to reach MG-RAST M5nr API, error code #{res_seq.code}." unless res_seq.code == 200
-	 seq = JSON.parse(res_seq.to_str)
-	 of.puts ">#{datum["source"]}:#{datum["accession"]} #{datum["function"]} [#{datum["organism"]} taxid:#{datum["ncbi_tax_id"]}]"
-	 of.puts seq["data"]["sequence"].scan(/.{80}|.+/).map{ |x| x.strip }.join($/)
-      end
-      i += 1
-      break if o[:max]>0 and i >= o[:max]
-   end # |datum|
-   next_url = fun["next"]
-   break if next_url.nil? or (o[:max] > 0 and i >= o[:max])
-end
-of.close
-
-$stderr.puts "Downloaded #{i} sequences." unless o[:q]
-

data/utils/enveomics/Scripts/MeTaxa.distribution.pl

@@ -1,198 +0,0 @@
-#!/usr/bin/env perl
-#
-# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @license artistic license 2.0
-# @update Mar-23-2015
-#
-use warnings;
-use strict;
-use Symbol;
-use Getopt::Std;
-use List::Util qw/max/;
-
-sub HELP_MESSAGE { die "
-Usage:
-   $0 [args]
-
-Mandatory:
-   -m <str>	MyTaxa output.
-
-Optional:
-   -g <str>	Genes predicted in the format defined by -f.  If not passed, abundance is assumed to be based
-   		on contigs.
-   -f <str>	Format of the predicted genes.  One of:
-   		o gff2: GFF v2 as produced by MetaGeneMark.hmm (default).
-		o gff3: GFF v3, including the field id in the last column (with the Gene ID).
-		o tab: A tab-delimited file with the gene ID (col #1), the length of the gene in bp (col #2),
-		  and the ID of the corresponding contig (col #3). The length of the gene (col #2) isn't used
-		  (and it can be empty),  but the column must exist (i.e., 2 tabs per line) for compatibility
-		  with BlastTab.metaxaPrep.pl
-   -c <str>	Counts file: Sequence IDs (genes if -g is provided, contigs otherwise) and reads per sequence
-   		in a tab-delimited file.  If not provided, each sequence counts as 1.
-   -O <str>	Prefix of the output files to be generated.  By default, the value of -m.
-   -I <str>	File containing the complete classification of all the contigs identified as Innominate taxa.
-		By default, this file is not created.
-   -G <str>	File containing the classification of each gene.  By default, this file is not created.  This
-   		requires -g to be set.  Note: This option requires extra RAM.
-   -K <str>	File containing a krona input file.  By default, this file is not created.
-   -k <str>	List of ranks to include in the Krona file, delimited by comma.   It MUST be decreasing rank.
-   		By default: 'superkingdom,phylum,class,family,genus,species'.  This is ignored unless -K also
-		is passed.
-   -R <str>	List of taxonomic ranks for which individual reports should be generated, delimited by comma.
-   		It MUST be decreasing rank.  By default: 'phylum,genus,species'.
-   -r		If set, reports raw counts.  Otherwise, reports permil of the rank.
-   -u		Report Unknown taxa.
-   -q		Run quietly.
-   -h		Display this help message and exits.
-
-" }
-
-my %o;
-getopts('g:f:c:m:O:I:G:K:k:R:ruqh', \%o);
-$o{h} and &HELP_MESSAGE;
-$o{m} or  &HELP_MESSAGE;
-$o{O} ||= $o{m};
-$o{f} ||= "gff2";
-$o{k} ||= "superkingdom,phylum,class,family,genus,species";
-my @K = split /,/, lc $o{k};
-$o{R} ||= "phylum,genus,species";
-my @R = split /,/, lc $o{R};
-($o{G} and not $o{g}) and die "-G requires -g to be set.\n";
-
-
-my %gene;
-my %count;
-my %ctg=();
-if($o{g}){
-   print STDERR "Reading genes collection.\n" unless $o{q};
-   open GFF, "<", $o{g} or die "Cannot read file: $o{g}: $!\n";
-   while(<GFF>){
-      next if /^#/;
-      next if /^\s*$/;
-      chomp;
-      my($id,$ctg);
-      my @ln = split /\t/;
-      if($o{f} eq 'gff2'){
-	 exists $ln[8] or die "Cannot parse line $., expecting 9 columns: $_\n";
-	 $id = $ln[8];
-	 $id =~ s/gene_id /gene_id_/;
-	 $ctg=$ln[0];
-      }elsif($o{f} eq 'gff3'){
-	 exists $ln[8] or die "Cannot parse line $., expecting 9 columns: $_\n";
-	 $ln[8] =~ /id=([^;]+)/ or die "Cannot parse line $.: $_\n";
-	 $id = $1;
-	 $ctg = $ln[0];
-      }elsif($o{f} eq 'tab'){
-         exists $ln[2] or die "Cannot parse line $., expecting 3 columns: $_\n";
-	 $id = $ln[0];
-	 $ctg = $ln[2];
-      }else{
-         die "Unsupported format: ".$o{f}.".\n";
-      }
-      $ctg =~ s/ .*//;
-      if($o{c}){
-	 $gene{$id} = $ctg;
-      }else{
-         $count{$ctg}++;
-      }
-      push( @{$ctg{$ctg}||=[]}, $id ) if $o{G};
-   }
-   close GFF;
-   print STDERR " Found ".(scalar(keys %gene))." genes.\n" unless $o{q};
-}
-
-my $Nreads = 0;
-if($o{c}){
-   print STDERR "Reading read-counts.\n" unless $o{q};
-   open COUNT, "<", $o{c} or die "Cannot read file: $o{c}: $!\n";
-   while(<COUNT>){
-      chomp;
-      my @l = split /\t/;
-      if($o{g}){
-	 exists $gene{$l[0]} or die "Cannot find gene's contig: $l[0].\n";
-	 $count{ $gene{$l[0]} } += $l[1];
-	 delete $gene{$l[0]};
-      }else{
-	 $count{ $l[0] } += $l[1];
-      }
-      $Nreads += $l[1];
-   }
-   close COUNT;
-   print STDERR " Found ".scalar(keys %gene)." genes without reads.\n" if scalar(keys %gene) and not $o{q};
-   $count{$_}+=0 for values %gene;
-   print STDERR " Found ".scalar(keys %count)." sequences and $Nreads reads.\n" unless $o{q};
-}
-
-print STDERR "Reading Metaxa results.\n";
-open METAXA, "<", $o{m} or die "Cannot read file: $o{m}: $!\n";
-my $ctg;
-my $rank;
-my @ofh = ();
-my @n   = (0,0,0);
-my @out = ({},{},{});
-my @rank_name = map { ucfirst } ('unknown', @R);
-my %rank = map { ($rank_name[$_]=>$_) } 0 .. $#rank_name;
-my @rank_tag  = ("NA", map { "<$_>" } @R);
-$o{I} and (open OUT_I, ">", $o{I} or die "Cannot create file: $o{I}: $!\n");
-$o{K} and (open OUT_K, ">", $o{K} or die "Cannot create file: $o{K}: $!\n");
-$o{G} and (open OUT_G, ">", $o{G} or die "Cannot create file: $o{G}: $!\n");
-
-my $Nreads_class = 0;
-my $Nno_read_ctg = 0;
-while(not eof(METAXA)){
-   my @h=split /\t/, <METAXA>;
-   my $t=<METAXA>; chomp $t;
-   exists $h[3] or die "Cannot parse MyTaxa file, line $.: $_\n";
-   my $count_h;
-   if($o{c} or $o{g}){
-      unless(exists $count{$h[0]}){
-         $Nno_read_ctg++;
-	 next;
-      }
-      $count_h = $count{$h[0]};
-   }else{
-      $count_h = 1;
-   }
-   if($o{G}){ print OUT_G "$_\t$t\n" for @{$ctg{$h[0]}} }
-   next unless $count_h;
-   my $last = 'organism';
-   $n[0] += $count_h;
-   for my $r (1 .. max(values %rank)){
-      if($rank{$h[1]} >= $r){
-	 if($t =~ m/$rank_tag[$r]([^;]*)/){
-	    $last = $1 if $1;
-	 }else{
-	    $last = $last=~/^Innominate / ? $last : "Innominate $last";
-	    $o{I} and print OUT_I "$h[0]\t$rank_name[$r]\t$last\t$t\n";
-	 }
-	 $out[$r]->{$last} += $count_h;
-	 $n[$r] += $count_h;
-      }else{
-         $out[$r]->{"Unknown $last"} += $count_h if $o{u};
-      }
-   }
-   if($o{K}){
-      my $ln = $count_h;
-      for my $r (@K){ $ln.= "\t".($t=~m/<$r>([^;]+)/?$1:'') }
-      print OUT_K "$ln\n";
-   }
-   $Nreads_class+= $count_h;
-}
-print OUT_K "".($Nreads-$Nreads_class)."\n" if $o{K} and $Nreads>$Nreads_class;
-close METAXA;
-$o{I} and close OUT_I;
-$o{K} and close OUT_K;
-$o{G} and close OUT_G;
-print " Found $n[0] reads.\n" unless $o{q};
-print " Couldn't find counts for $Nno_read_ctg contigs.\n" if $Nno_read_ctg;
-unless($o{q}){ print " Found $n[$_] classified reads at ".$rank_name[$_]." level.\n" for (1 .. max(values %rank)) }
-
-print STDERR "Generating output.\n" unless $o{q};
-for my $rank (1 .. max(values %rank)){
-   open OUT, ">", "$o{O}.".$rank_name[$rank].".txt" or die "Cannot create file: $o{O}.".$rank_name[$rank].".txt: $!\n";
-   for my $class (keys %{$out[$rank]}){
-      printf OUT "%s\t%.20f\n", $class, ($out[$rank]->{$class}*($o{r}?1:1000/$n[$rank]));
-   }
-   close OUT;
-}
-

data/utils/enveomics/Scripts/MyTaxa.fragsByTax.pl

@@ -1,35 +0,0 @@
-#!/usr/bin/env perl
-#
-# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @update: Mar-23-2015
-# @license: artistic license 2.0
-#
-
-use warnings;
-use strict;
-
-my($file,$tax,$rank) = @ARGV;
-($file and $tax) or die "
-.Usage:
-   $0 file.txt taxon[ rank]
-
-   file.txt	MyTaxa output.
-   taxon	Taxon to look for.
-   rank		Rank of taxon (optional). By default: any rank.
-
-";
-$rank ||= ".*";
-$rank = lc $rank;
-
-open MT, "<", $file or die "Cannot read file: $file: $!\n";
-my $last = '';
-while(my $ln=<MT>){
-   chomp $ln;
-   if($ln =~ /<$rank>$tax(;|$)/){
-      $last =~ s/\t.*//;
-      print $last, "\n";
-   }
-   $last = $ln;
-}
-close MT;
-

data/utils/enveomics/Scripts/MyTaxa.seq-taxrank.rb

@@ -1,49 +0,0 @@
-#!/usr/bin/env ruby
-
-#
-# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @update: Feb-06-2015
-# @license artistic license 2.0
-#
-
-require 'optparse'
-
-opts = {:rank=>'genus', :quiet=>FALSE}
-ARGV << '-h' if ARGV.size==0
-OptionParser.new do |opt|
-   opt.separator "Generates a simple tabular file with the classification of each sequence at a given taxonomic rank from a MyTaxa output."
-   opt.separator ""
-   opt.on("-i", "--mytaxa FILE", "Input MyTaxa file."){ |v| opts[:mytaxa]=v }
-   opt.on("-r", "--rank STR", "Taxonomic rank.  By default: #{opts[:rank]}."){ |v| opts[:rank] = v.downcase }
-   opt.on("-q","--quiet","Run quietly.") { opts[:quiet]=TRUE }
-   opt.on("-h","--help","Display this screen.") do
-      puts opt
-      exit
-   end
-   opt.separator ""
-end.parse!
-abort "-i/--mytaxa is mandatory." if opts[:mytaxa].nil?
-abort "-i/--mytaxa must exist." unless File.exists? opts[:mytaxa]
-
-begin
-   f = File.open(opts[:mytaxa], "r")
-   ctg = nil;
-   while(ln = f.gets)
-      m = /^(.+)(\t.+){3}/.match(ln)
-      if m
-	 raise "Couldn't find classification for contig #{ctg}" unless ctg.nil?
-	 ctg = m[1]
-      else
-	 raise "Couldn't find the contig name at line #{$.}" if ctg.nil?
-	 m = /<#{opts[:rank]}>([^;]+)/.match(ln)
-	 puts "#{ctg}\t#{m ? m[1] : "Unclassified"}"
-	 ctg = nil
-      end
-   end
-   f.close
-rescue => err
-   $stderr.puts "Exception: #{err}\n\n"
-   err.backtrace.each { |l| $stderr.puts l + "\n" }
-   err
-end
-

data/utils/enveomics/Scripts/NCBIacc2tax.rb

@@ -1,92 +0,0 @@
-#!/usr/bin/env ruby
-
-#
-# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @license artistic license 2.0
-#
-
-$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
-require "enveomics_rb/remote_data"
-use "nokogiri"
-
-#================================[ Options parsing ]
-o = {
-  :q=>false, :accs=>[], :dbfrom=>"nuccore", :header=>true,
-  :no_nil=>false, :ret=>"ScientificName",
-  :ranks=>%w(superkingdom phylum class order family genus species)}
-OptionParser.new do |opt|
-  opt.banner = "
-  Maps a list of NCBI accessions to their corresponding taxonomy using the NCBI
-  EUtilities. Avoid using this script on millions of entries at a time, since
-  each entry elicits two requests to NCBI's servers.
-
-  Usage: #{$0} [options]".gsub(/^ +/,"")
-  opt.separator ""
-  opt.on("-a", "--acc acc1,acc2.ver,...", Array,
-    "Comma-separated list of accessions. Required unless -i is passed."
-    ){ |v| o[:accs]=v }
-  opt.on("-i", "--infile FILE",
-    "Raw text file containing the list of accessions, one per line.",
-    "Required unless -g is passed."){ |v| o[:infile]=v }
-  opt.on("-p", "--protein",
-    "Use if the accessions are proteins. Otherwise, accessions are assumed " +
-    "to be from the Nuccore Database."){ o[:dbfrom]="protein" }
-  opt.on("-r", "--ranks RANK1,RANK2,...", Array,
-    "Taxonomic ranks to report. By default: #{o[:ranks].join(",")}."
-    ){ |v| o[:ranks]=v }
-  opt.on("-n", "--noheader",
-    "Do not include a header in the output."){ o[:header]=false }
-  opt.on("-t", "--taxids",
-    "Return Taxonomy IDs instead of scientific names."){ o[:ret]="TaxId" }
-  opt.on("--ignore-missing",
-    "Does not report missing accessions in the output file.",
-    "By default, it reports accessions and empty values for all other columns."
-    ){ |v| o[:no_nil]=v }
-  opt.on("-q", "--quiet", "Run quietly."){ |v| o[:q]=true }
-  opt.on("-h", "--help","Display this screen") do
-    puts opt
-    exit
-  end
-  opt.separator ""
-end.parse!
-
-#================================[ Functions ]
-def acc2taxid(db, acc)
-  doc = Nokogiri::XML( RemoteData.elink({:dbfrom=>db,
-    :db=>"taxonomy", :id=>acc, :idtype=>"acc"}) )
-  doc.at_xpath("/eLinkResult/LinkSet/LinkSetDb/Link/Id")
-end
-#================================[ Main ]
-begin
-  o[:accs] += File.readlines(o[:infile]).map{ |l| l.chomp } unless
-    o[:infile].nil?
-  o[:ranks].map!{ |r| r.downcase }
-  puts (["Acc", "TaxId"] + o[:ranks].map{ |r| r.capitalize }).join("\t") if
-    o[:header]
-  o[:accs].each do |acc|
-    taxid = acc2taxid(o[:dbfrom], acc)
-    status = ""
-    if taxid.nil?
-      warn "Cannot find link to taxonomy: #{acc} #{status}"
-      puts ([acc, ""] + o[:ranks].map{ |i| "" }).join("\t") unless o[:no_nil]
-      next
-    end
-    taxonomy = {}
-    unless taxid.nil?
-      doc = Nokogiri::XML( RemoteData.efetch({:db=>"taxonomy",
-        :id=>taxid.content}) )
-      taxonomy[ doc.at_xpath("/TaxaSet/Taxon/Rank").content ] =
-        doc.at_xpath("/TaxaSet/Taxon/#{o[:ret]}").content
-      doc.xpath("/TaxaSet/Taxon/LineageEx/Taxon").each do |taxon|
-        taxonomy[ taxon.at_xpath("./Rank").content ] =
-          taxon.at_xpath("./#{o[:ret]}").content
-      end
-    end
-    puts ([acc, taxid.content] +
-      o[:ranks].map{ |rank| taxonomy[ rank ] ||= "" }).join("\t")
-  end
-rescue => err
-  $stderr.puts "Exception: #{err}\n\n"
-  err.backtrace.each { |l| $stderr.puts l + "\n" }
-  err
-end