miga-base 0.7.26.0 → 0.7.26.1

data/utils/enveomics/Scripts/BlastTab.cogCat.rb

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update  Mar-23-2016
+# @license artistic license 2.0
+#
+
+require "optparse"
+
+o = {:cog=>false, :desc=>false, :q=>false, :w=>true}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "Replaces the COG gene IDs in a BLAST for the COG category."
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-w", "--whog FILE", "Path to the whog file."){ |v| o[:whog]=v }
+   opts.on("-i", "--blast FILE",
+      "Path to the Tabular BLAST file with COG IDs as subject."
+      ){ |v| o[:blast]=v }
+   opts.separator ""
+   opts.separator "Optional"
+   opts.on("-g", "--cog",
+      "If set, returns the COG ID, not the COG category."){ o[:cog]=true }
+   opts.on("-d", "--desc",
+      "Includes COG description (requires -g/--cog)."){ o[:desc]=true }
+   opts.on("-n", "--noverbose", "Run quietly, but show warnings."){ o[:q]=true }
+   opts.on("-q", "--quiet", "Run quietly."){ o[:q]=true; o[:w]=false }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+
+abort "-w/--whog is mandatory." if o[:whog].nil?
+abort "-i/--blast is mandatory." if o[:blast].nil?
+
+$stderr.puts "Parsing whog file." unless o[:q]
+cat = {}
+curCats = []
+fh = File.open o[:whog], "r"
+while ln=fh.gets
+   ln.chomp!
+   next if /^\s*$/.match ln
+   if m=/^\[([A-Z]+)\] (COG\d+) (.*)/.match(ln)
+      curCats = o[:cog] ? [ m[2]+(o[:desc]?" #{m[3]}":"") ] : m[1].split(//)
+   elsif /^_+$/.match ln
+      curCats = []
+   elsif m=/^\s+(?:.+?:\s+)?(.*)/.match(ln)
+      m[1].split(/\s+/).each do |g|
+         cat[g] ||= []
+	 curCats.each { |i| cat[g] << i }
+      end
+   else
+      abort "Impossible to parse line #{$.}: #{ln}"
+   end
+end
+fh.close
+
+$stderr.puts "Parsing BLAST." unless o[:q]
+fh = File.open(o[:blast], "r")
+while ln=fh.gets
+   row = ln.split(/\t/)
+   if cat[ row[1] ].nil?
+      $stderr.puts "Warning: line #{$.}: #{row[1]}: " +
+	 "Impossible to find category.\n" if o[:w]
+   else
+      cat[ row[1] ].each do |c|
+         row[1] = c
+	 puts row.join("\t")
+      end
+   end
+end
+fh.close
+

data/utils/enveomics/Scripts/BlastTab.filter.pl

@@ -0,0 +1,47 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+my %o;
+getopts('si', \%o);
+my($list, $blast) = @ARGV;
+
+($list and $blast) or die "
+.Description:
+   Extracts a subset of hits (queries or subjects) from a tabular BLAST.
+
+.Usage: $0 [options] list.txt blast.txt > subset.txt
+   
+   Options:
+      -s	If set, assumes that list.txt contains subject IDs.
+   		By default: assumes query IDs.
+      -i	If set, reports the inverse of the list (i.e., reports
+      		only hits absent in the list).
+
+   list.txt	List of IDs to extract.
+   blast.txt	Tabular BLAST file containing the superset of hits.
+   subset.txt	Tabulat BLAST file to be created.
+
+";
+
+open LI, "<", $list or die "Cannot read file: $list: $!\n";
+my %li = map { chomp; $_ => 1 } <LI>;
+close LI;
+
+open BLAST, "<", $blast or die "Cannot read file: $blast: $!\n";
+while(my $ln = <BLAST>){
+   chomp $ln;
+   my @ln = split("\t", $ln);
+   my $good = exists $li{$ln[ ($o{s} ? 1 : 0) ]};
+   $good = not $good if $o{i};
+   print "$ln\n" if $good;
+}
+close BLAST;
+

data/utils/enveomics/Scripts/BlastTab.kegg_pep2path_rest.pl

@@ -0,0 +1,194 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use LWP::Simple;
+use JSON;
+use File::Copy;
+
+my($blast, $cache_file, $max_cache) = @ARGV;
+($blast) or die "
+Description:
+   Takes a BLAST against KEGG_PEP (or KO) and retrieves the pathways in which the subject
+   peptides are involved.
+
+Usage:
+   $0 blast.txt[ cache_file] > output.txt
+
+   blast.txt	Input (filtered) BLAST file.
+   cache_file	(optional) File containing the saved cache.  If unset, the
+   		cache won't be recoverable across instances of this script.
+		It is strongly recommended to set a file.  Multiple
+		parallel instances of this script may use the same cache
+		file.
+   output.txt	Tab-delimited output file, with the columns:
+   		 o Query ID
+		 o Subject ID
+		 o Pathway ID
+		 o Pathway (reference) description
+		 o Organism
+
+";
+
+$max_cache ||= 0;
+$cache_file ||= "";
+
+sub read_cache($){
+   my ($cache_file) = @_;
+   my $cache = {};
+   my $n = 0;
+   if($cache_file and -s $cache_file){
+      local $/;
+      my $json = "";
+      while(-e "$cache_file.tmp"){
+         print STDERR "Locked cache (read), waiting 1 sec.\n";
+	 sleep 1;
+      }
+      open CACHE, "<", $cache_file or die "Cannot read file: $cache_file: $!\n";
+      while(<CACHE>){ $json.=$_ }
+      close CACHE;
+      $cache = decode_json($json);
+      $n = scalar keys %$cache;
+   }
+   return ($cache, $n);
+}
+
+sub write_cache($$){
+   my($cache, $cache_file) = @_;
+   if($cache_file){
+      # Get previously saved entries.
+      my($cache2, $cache_n2) = &read_cache($cache_file);
+      for my $k (keys %$cache2){
+	 $cache->{$k} ||= $cache2->{$k} unless $k eq "###:paths";
+      }
+      $cache->{'###:paths'} ||= {};
+      for my $p (keys %{$cache2->{'###:paths'}}){
+         $cache->{'###:paths'}->{$p} ||= $cache2->{'###:paths'}->{$p};
+      }
+      # Save merged cache.
+      if(-s $cache_file){ copy $cache_file, "$cache_file.pre" or die "Cannot create file: $cache_file.tmp: $!\n" }
+      my $json = encode_json($cache);
+      while(-e "$cache_file.tmp"){
+         print STDERR "Locked cache (write), waiting 1 sec.\n";
+	 sleep 1;
+      }
+      open CACHE, ">", "$cache_file.tmp" or die "Cannot create file: $cache_file.tmp: $!\n";
+      print CACHE $json;
+      close CACHE;
+      copy "$cache_file.tmp", $cache_file or die "Cannot create file: $cache_file: $!\n";
+      unlink "$cache_file.tmp" or die "Cannot unlink file: $cache_file.tmp: $!\n";
+   }
+}
+
+sub download_pathways($$){
+   my($cache, $ids) = @_;
+   my @todownload = ();
+   for my $id (@$ids){
+      push @todownload, $id unless exists $cache->{'###:paths'}->{$id};
+   }
+   while($#todownload>=0){
+      my @downloading = splice(@todownload, 0, 100);
+      my $path = get "http://rest.kegg.jp/list/".join("+", @downloading);
+      if($path){
+	 chomp $path;
+	 for my $p (split /\n/, $path){
+	    my @wl = split /\t/, $p;
+	    $wl[1] =~ s/ - /\t/;
+	    $cache->{'###:paths'}->{$wl[0]} = $wl[1];
+	 }
+      }
+   }
+   return $cache;
+}
+
+sub download($$){
+   my($cache, $todownload) = @_;
+   $cache->{'###:paths'} ||= {};
+   return $cache unless $#$todownload>=0;
+   $cache->{$_} = [] for @$todownload;
+   my $list = get "http://rest.kegg.jp/link/pathway/".join("+", @$todownload);
+   $list ||= "";
+   chomp $list;
+   my @pathids = ();
+   for my $res (split /\n/, $list){
+      my @rel = split /\t/, $res;
+      $#rel==1 or die "Unexpected number of columns:\n$res\n";
+      my $id = $rel[1];
+      push @pathids, $id;
+      unless(exists $cache->{$rel[0]}){
+	 #print STDERR "Request/response difference in ID: ".$rel[0].", searching match.\n";
+         for my $id (@$todownload){
+	    $rel[0] = $id if lc $id eq lc $rel[0];
+	 }
+	 die "Cannot find corresponding request.\n" unless exists $cache->{$rel[0]};
+      }
+      push @{ $cache->{$rel[0]} }, $id;
+   }
+   return &download_pathways($cache, \@pathids);
+}
+
+sub print_out($$){
+   my($cache, $hits) = @_;
+   for my $hit (@$hits){
+      die "Impossible to find gene in cache: ".$hit->[1]."\n" unless exists $cache->{$hit->[1]};
+      for my $path (@{$cache->{$hit->[1]}}){
+	 next if $path =~ /^path:ko\d/;
+	 unless(exists $cache->{'###:paths'}->{$path}){
+	    print STDERR "Cannot find pathway in cache: $path (from ".$hit->[1]."), emergency download\n";
+	    $cache = &download_pathways($cache, [$path]);
+	    die "Impossible to find pathway: $path.\n" unless exists $cache->{'###:paths'}->{$path};
+	 }
+	 print "", join("\t", $hit->[0], $hit->[1], $path, $cache->{'###:paths'}->{$path}), "\n";
+      }
+   }
+}
+
+print STDERR "Loading cache.\n";
+my ($cache, $n) = &read_cache($cache_file);
+print STDERR "   $n entries loaded.\n";
+my @nopath = ();
+for my $k (keys %$cache){
+   next if $k eq "###:paths";
+   for my $p (@{ $cache->{$k} }){
+      push @nopath, $p unless exists $cache->{'###:paths'}->{$p};
+   }
+}
+if($#nopath>=0){
+   print STDERR "   Sanitizing ".@nopath." pathways in cache.\n";
+   while($#nopath>=0){
+      my @paths = ();
+      for(1 .. 15){ push @paths, shift @nopath unless $#nopath==-1 }
+      $cache = &download_pathways($cache, \@paths);
+   }
+   &write_cache($cache, $cache_file);
+}
+
+my $lines=0;
+my $downs=0;
+my @buff = ();
+my @todownload = ();
+print STDERR "Mapping genes.\n";
+open BLAST, "<", $blast or die "Cannot read file: $blast: $!\n";
+while(<BLAST>){
+   chomp;
+   my @l = split /\t/;
+   print STDERR "   Mapping line ".(++$lines).". \r";
+   unless(($#todownload+2)%100){
+      print STDERR "+\r";
+      print STDERR " *\r" unless ++$downs%10;
+      $cache = &download($cache, \@todownload);
+      @todownload = ();
+      &print_out($cache, \@buff);
+      @buff = ();
+      &write_cache($cache, $cache_file) unless $downs%10;
+   }
+   push @buff, \@l;
+   push @todownload, $l[1] unless exists $cache->{$l[1]};
+}
+print STDERR "\nDone.\n";
+close BLAST;
+
+$cache = &download($cache, \@todownload);
+&print_out($cache, \@buff);
+&write_cache($cache, $cache_file);
+

data/utils/enveomics/Scripts/BlastTab.metaxaPrep.pl

@@ -0,0 +1,104 @@
+#!/usr/bin/env perl
+
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update Mar-23-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+
+sub HELP_MESSAGE { die "
+Usage:
+   $0 [options] genes.txt blast.txt ... > blast_metaxa.txt
+
+   genes.gff2		File containing the genes in any supported format
+   			(see option -f).
+   blast.txt ...	One or more tabular BLAST files.
+   blast_metaxa.txt	Input file for MeTaxa.
+
+   Options:
+   -l <float>		Minimum fraction of the gene aligned to consider a
+   			hit.  By default: 0.75.  Ignored if -f 'no'.
+   -f <str>		Format of the genes prediction.  Any of:
+   			 o gff2: GFF v2 as produced by MetaGeneMark.hmm.
+			 o gff3: GFF v3 with id field in the last column.
+			 o tab: Tabular file with columns gene, gene length,
+			   and contig.
+			 o no: Ignores genes file.
+			By default: gff2.
+   -q			Run quietly.
+   -h			Display this message and exit.
+
+";}
+
+my %o;
+getopts('l:f:qh',\%o);
+my($gff, @blasts) = @ARGV;
+($gff and $#blasts>=0) or &HELP_MESSAGE;
+$o{h} and &HELP_MESSAGE;
+$o{f} ||= "gff2";
+$o{f} = lc $o{f};
+$o{l} ||= 0.75;
+
+my %gene;
+if($o{f} ne 'no'){
+   print STDERR "Reading genes collection.\n" unless $o{q};
+   open GFF, "<", $gff or die "Cannot read file: $gff: $!\n";
+   while(<GFF>){
+      next if /^#/;
+      next if /^\s*$/;
+      chomp;
+      my @ln = split /\t/;
+      if($o{f} eq 'gff2'){
+	 exists $ln[8] or die "Cannot parse line $.: $_\n";
+	 my $id = $ln[8];
+	 $id =~ s/gene_id /gene_id_/;
+	 $ln[0] =~ s/ .*//;
+	 $gene{$id} = [$ln[0], (1+$ln[4]-$ln[3])/3];
+      }elsif($o{f} eq 'gff3'){
+	 exists $ln[8] or die "Cannot parse line $.: $_\n";
+	 $ln[8] =~ /id=([^;]+)/ or die "Cannot parse line $.: $_\n";
+	 my $id = $1;
+	 $ln[0] =~ s/ .*//;
+	 $gene{$id} = [$ln[0], (1+$ln[4]-$ln[3])/3];
+      }elsif($o{f} eq 'tab'){
+         exists $ln[2] or die "Cannot parse line $.: $_\n";
+	 $ln[1]+0 or die "$ln[0]: Length zero.\n";
+         $gene{$ln[0]} = [$ln[2], $ln[1]/3];
+      }else{
+         die "Unsupported format: ".$o{f}.".\n";
+      }
+   }
+   close GFF;
+}
+
+my $i=0;
+my $p=0;
+print STDERR "Generating MeTaxa input.\n" unless $o{q};
+for my $blast (@blasts){
+   print STDERR "  o $blast\n" unless $o{q};
+   open BLAST, "<", $blast or die "Cannot read file: $blast: $!\n";
+   while(<BLAST>){
+      chomp;
+      my @l = split /\t/;
+      $i++;
+      my $ctg;
+      if($o{f} eq 'no'){
+         $ctg = $l[0];
+      }else{
+	 exists $gene{$l[0]} or die "Cannot find contig for gene $l[0].\n";
+	 next unless $l[3] >= $o{l}*$gene{$l[0]}->[1];
+	 $ctg = $gene{$l[0]}->[0];
+      }
+      $l[1] =~ m/gi\|(\d+)\|/ or die "Cannot parse GI in $l[1].\n";
+      print "".join("\t", @l, $ctg, $l[0], $1)."\n";
+      $p++;
+   }
+   close BLAST;
+}
+print STDERR " Found $i results, reported $p.\n" unless $o{q};
+

data/utils/enveomics/Scripts/BlastTab.pairedHits.rb

@@ -0,0 +1,157 @@
+#!/usr/bin/env ruby
+
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Jul-29-2015
+# @license artistic license 2.0
+#
+
+require 'optparse'
+
+opts = {:minscore=>0, :besthits=>0, :orient=>0, :sisprefix=>"_"}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opt|
+   opt.separator "Identifies the best hits of paired-reads."
+   opt.separator ""
+   opt.on("-i", "--blast FILE", "Input BLAST file."){ |v| opts[:blast]=v }
+   opt.on("-s", "--minscore FLOAT", "Minimum (summed) Bit-Score to consider a pair-match."){ |v| opts[:minscore] = v.to_f }
+   opt.on("-b", "--besthits INT", "Outputs top best-hits only (use 0 to output all the paired hits)."){ |v| opts[:besthits]=v.to_i }
+   opt.on("-o", "--orient INT", "Checks the orientation of the hit.  Values are: 0, no checking; 1, same direction; 2,",
+   		"inwards; 3, outwards; 4, different direction (i.e., 2 or 3)."){ |v| opts[:orient]=v.to_i }
+   opt.on("-p", "--sisprefix STR", "Sister read number prefix in the name of the reads.  Escape characters as dots (\\.),",
+   		"parenthesis (\\(, \\), \\[, \\]), or other characters with special meaning in regular expressions",
+		"(\\*, \\+, \\^, \\$, \\|).  This prefix allows regular expressions (for example, use ':|\\.' to use any of",
+		"colon or dot).  Notice that the prefix will not be included in the base name reported in the output."){ |v| opts[:sisprefix]=v }
+   opt.on("-h","--help","Display this screen") do
+      puts opt
+      exit
+   end
+   opt.separator ""
+   opt.separator "Output:"
+   opt.separator "   Tab-delimited flat file, with the following columns:"
+   opt.separator "   1. Query ID (without the \"sister\" identifier)."
+   opt.separator "   2. Subject ID."
+   opt.separator "   3. Bit score (summed from both sister reads)."
+   opt.separator "   4/5. From/To (subject) coordinates for read 1."
+   opt.separator "   6/7. From/To (subject) coordinates for read 2."
+   opt.separator "   8. Reads orientation (1: same direction, 2: inwards, 3: outwards)."
+   opt.separator "   9. Estimated insert size."
+   opt.separator ""
+   opt.separator "Important note: This script assumes that paired hits are next to each other."
+   opt.separator "   If this is not the case (e.g., because the blast was concatenated),"
+   opt.separator "   you must sort the input before running this script."
+   opt.separator ""
+end.parse!
+abort "-i/--blast is mandatory." if opts[:blast].nil?
+abort "-i/--blast must exist." unless File.exists? opts[:blast]
+
+class SingleHit
+   attr_reader :sbj, :score, :orient, :sfrom, :sto, :qfrom, :qto
+   def initialize(blast_ln)
+      blast_ln.chomp!
+      ln = blast_ln.split("\t")
+      @sbj    = ln[1]
+      @score  = ln[11].to_f
+      @qfrom  = ln[6].to_i
+      @qto    = ln[7].to_i
+      @sfrom  = ln[8].to_i
+      @sto    = ln[9].to_i
+      @orient = @sfrom < @sto ? 1 : -1;
+   end
+end
+class DoubleHit
+   attr_reader :name, :sbj, :score, :orient, :hitA, :hitB
+   def initialize(name, hitA, hitB)
+      raise "Trying to set DoubleHit from hits with different subjects" unless hitA.sbj == hitB.sbj
+      @name = name
+      @hitA = hitA
+      @hitB = hitB
+      @sbj = hitA.sbj
+      @score = hitA.score + hitB.score
+      @orient = (hitA.orient == hitB.orient ? 1:
+      		((hitA.orient>0 and hitB.orient<0) ? 2: 3))
+   end
+   def to_s
+      coords = [@hitA.sfrom, @hitB.sfrom, @hitA.sto, @hitB.sto]
+      @name + "\t" + @sbj + "\t" + @score.to_s + "\t" +
+      	@hitA.sfrom.to_s + "\t" + @hitA.sto.to_s + "\t" +
+      	@hitB.sfrom.to_s + "\t" + @hitB.sto.to_s + "\t" +
+	@orient.to_s + "\t" + (coords.max-coords.min).to_s + "\n"
+   end
+end
+class PairedHits
+   attr_reader :name, :hitsA, :hitsB
+   @@minscore = 0
+   @@orient = 0
+   @@besthits = 0
+   def initialize(name)
+      @name = name
+      @hitsA = []
+      @hitsB = []
+      @hits  = []
+   end
+   def hits
+      @hits = []
+      # Search for paired hits
+      @hitsA.each do |hitA|
+         @hitsB.each do |hitB|
+	    if hitA.sbj == hitB.sbj
+	       hit = DoubleHit.new(@name, hitA, hitB)
+	       next if hit.score <= @@minscore # Minimum bit-score check
+	       next if ((1 .. 3).include?(@@orient) and @@orient != hit.orient) # "typical" orientation check
+	       next if (@@orient == 4 and not((2 .. 3).include?(hit.orient))) # "different-orientation" check
+	       @hits.push(hit)
+	    end
+	 end
+      end
+      # Sort the hits
+      @hits.sort! {|x,y| x.score <=> y.score }
+      if @@besthits==0
+         @hits
+      else
+         @hits.take(@@besthits)
+      end
+   end
+   def hitsX(x)
+      if x == 1
+         @hitsA
+      else
+         @hitsB
+      end
+   end
+   # Class methods
+   def PairedHits.minscore=(value)
+      @@minscore = value
+   end
+   def PairedHits.orient=(value)
+      @@orient = value
+   end
+   def PairedHits.besthits=(value)
+      @@besthits = value
+   end
+end
+
+PairedHits.minscore = opts[:minscore]
+PairedHits.orient   = opts[:orient]
+PairedHits.besthits = opts[:besthits]
+
+begin
+   f = File.open(opts[:blast], "r")
+   currPair = PairedHits.new("  ")
+   while(ln = f.gets)
+      m = /^([^\s]*)(?:#{opts[:sisprefix]})([12])/.match(ln)
+      raise "Impossible to parse read name in line #{$.} using sister prefix '#{opts[:sisprefix]}':\n#{ln}"  unless m
+      if m[1] != currPair.name
+	 currPair.hits.each { |hit| puts hit.to_s }
+	 currPair = PairedHits.new(m[1])
+      end
+      currPair.hitsX(m[2].to_i).push(SingleHit.new(ln));
+   end
+   currPair.hits.each { |hit| puts hit.to_s }
+   f.close
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+