miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Scripts/BlastTab.addlen.rb

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Feb-06-2015
+# @license: artistic license 2.0
+#
+
+require 'optparse'
+
+o = {:subject=>FALSE, :quiet=>FALSE}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "
+Appends an extra column to a BLAST with the length of the query or the subject sequence.
+You can pipe two instances to add both:
+   cat input.blast | #{$0} -f queries.fa | #{$0} -f subjects.fa -s > output.blast
+
+Usage: #{$0} [options] < input.blast > output.blast"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-f", "--fasta FILE", "Path to the FastA file"){ |v| o[:fasta] = v }
+   opts.separator ""
+   opts.separator "Options"
+   opts.on("-s", "--subject",
+   	"Use the subject column of the BLAST, by default the query column is used"){ o[:subject] = TRUE }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)"){ o[:quiet] = TRUE }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-f is mandatory" if o[:fasta].nil?
+
+len = {}
+id  = ""
+$stderr.puts "Reading FastA file: #{o[:fasta]}" unless o[:quiet]
+fh = File.open(o[:fasta], "r")
+fh.each_line do |ln|
+   defline = /^>(\S+)/.match(ln)
+   if defline.nil?
+      ln.gsub! /[^A-Za-z]/, ''
+      abort "Error: Unsupported format, expecting FastA" if len[id].nil?
+      len[id] = len[id] + ln.size
+   else
+      id = defline[1]
+      len[id] = 0
+   end
+end
+fh.close
+
+$stderr.puts "Appending #{o[:subject]?"subject":"query"} length column" unless o[:quiet]
+ARGF.each_line do |ln|
+   ln.chomp!
+   row = ln.split /\t/
+   id = o[:subject] ? row[1] : row[0];
+   abort "Impossible to find sequence of #{id}" if len[id].nil?
+   puts "#{ln}\t#{len[id]}"
+end
+

data/utils/enveomics/Scripts/BlastTab.advance.bash

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+#
+# @author  Luis M. Rodriguez-R
+# @update  Mar-23-2016
+# @license artistic license 2.0
+#
+
+if [[ ! $2 ]] ; then
+   echo "
+.DESCRIPTION
+   Calculates the percentage of a partial BLAST result. The
+   value produced slightly subestimates the actual advance,
+   due to un-flushed output and trailing queries that could
+   be processed but generate no results.
+
+.USAGE
+   $0 blast.txt qry.fasta
+
+   blast.txt	Incomplete Tabular BLAST output.
+   qry.fasta	FastA file with query sequences.
+";
+   exit 1;
+fi
+
+if [[ ! -r $1 ]]; then
+   echo "Cannot open file: $1";
+   exit 1;
+fi
+
+if [[ ! -r $2 ]]; then
+   echo "Cannot open file: $2";
+   exit 1;
+fi
+
+LAST_Q=`tail -n 2 $1 | head -n 1 | awk '{print $1}'`
+LAST_Q_NO=`grep -n "^>$LAST_Q\\( \\|$\\)" $2 | sed -e 's/:.*//'`
+if [[ ! $LAST_Q_NO ]]; then
+   echo "Cannot find sequence: $LAST_Q";
+   echo "Make sure you are providing the right query file.";
+   exit 1;
+fi
+TOTAL_Q_NO=`cat $2 | wc -l | sed -e 's/ *//'`
+let PERC=100*$LAST_Q_NO/$TOTAL_Q_NO
+
+echo "$PERC%: $LAST_Q_NO / $TOTAL_Q_NO"
+exit 0;
+

data/utils/enveomics/Scripts/BlastTab.best_hit_sorted.pl

@@ -0,0 +1,55 @@
+#!/usr/bin/env perl
+
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license: artistic license 2.0
+# @last_update: Mar-23-2015
+#
+
+use strict;
+use warnings;
+
+die "
+Usage:
+   sort blast.txt ... | $0 > blast.bh.txt
+   $0 blast_sorted.txt ... > blast.bh.txt
+   $0 -h|--help|-?
+
+   blast.txt ...	One or more files in Tabular BLAST format.
+   blast_sorted.txt ...	One or more files in Tabular BLAST format pre-sorted.
+   blast.bh.txt		Output file in BLAST format containing best-hits only.
+   -h|--help|-?		Any of these flags trigger this help message and exits.
+
+   NOTE: This script assumes that the BLAST is sorted.  Because it can read
+   from the STDIN, calling this script without arguments cause it to still until
+   killed or until an EOF (^D) is presented.
+
+" if exists $ARGV[0] and $ARGV[0] =~ /^\-?\-(h(elp)?|\?)/i;
+
+my $last_qry = '';
+my @best_res;
+
+sub best_result($$){
+   my($r1, $r2)=@_;
+   return $r1 unless $r2;
+   return $r1->[11] > $r2->[11] ? @$r1 : @$r2;
+}
+
+my $i=0;
+while(<>){
+   chomp;
+   #print STDERR " Reading entry $i...                             \r" unless $i%1000;
+   my @res = split /\t/;
+   die "\nCannot parse BLAST line $.: $_\n" unless exists $res[1];
+   if($last_qry eq $res[0]){
+      @best_res = &best_result(\@res, \@best_res);
+   }else{
+      print join("\t", @best_res), "\n" if $#best_res>0;
+      @best_res = @res;
+      $last_qry = $res[0];
+   }
+}
+print join("\t", @best_res), "\n" if @best_res;
+
+
+

data/utils/enveomics/Scripts/BlastTab.catsbj.pl

@@ -0,0 +1,106 @@
+#!/usr/bin/env perl
+# 
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @updated: Mar-23-2015
+# @license: artistic license 2.0
+# 
+
+use warnings;
+use strict;
+use List::Util qw/min max/;
+use Getopt::Std;
+
+sub HELP_MESSAGE { die "
+
+Description:
+   Generates a list of hits from a BLAST result concatenating the subject
+   sequences.  This can be used, e.g., to analyze BLAST results against
+   draft genomes.
+
+Usage:
+   $0 [options] seq.fa map.bls
+
+   seq.fa	Subject sequences (ref) in FastA format.
+   map.bls	Mapping of the reads to the reference in BLAST Tabular
+   		format.
+   
+   Options:
+   -i <float>	Minimum identity to report a result.  By default: 70.
+   -l <int>	Minimum alignment length to report a result.  By default: 60.
+   -s		The FastA provided is to be treated as a subset of the subject.
+   		By default, it expects all the subjects to be present in the
+		BLAST.
+   -q		Run quietly.
+   -h		Display this message and exit.
+   
+   This script creates two files using <map.bls> as prefix with extensions
+   .rec (for the recruitment plot) and .lim (for the limits of the different
+   sequences in <seq.fa>).
+
+";}
+
+my %o;
+getopts('i:l:sqh', \%o);
+my($fa, $map) = @ARGV;
+($fa and $map) or &HELP_MESSAGE;
+$o{h} and &HELP_MESSAGE;
+$o{i} ||= 70;
+$o{l} ||= 60;
+
+my %seq = ();
+my @seq = ();
+my $tot = 0;
+
+SEQ:{
+   print STDERR "== Reading reference sequences\n" unless $o{q};
+   open FA, "<", $fa or die "Cannot read the file: $fa: $!\n";
+   my $cur_seq = '';
+   while(<FA>){
+      chomp;
+      if(m/^>(\S+)/){
+         my $c = $1;
+	 $seq{$c} = exists $seq{$cur_seq} ? $seq{$cur_seq}+1 : 1;
+	 push @seq, $c;
+	 $cur_seq = $c;
+      }else{
+         s/[^A-Za-z]//g;
+	 $seq{$cur_seq} += length $_;
+      }
+   }
+   close FA;
+   print STDERR " Found ".(scalar @seq)." sequences.\n" unless $o{q};
+}
+
+open LIM, ">", "$map.lim" or die "Cannot create the file: $map.lim: $!\n";
+my $l = 0;
+for my $s (@seq){
+   print LIM "$s\t".(++$l)."\t$seq{$s}\n";
+   ($l, $seq{$s}) = ($seq{$s}, $l);
+}
+close LIM;
+
+MAP:{
+   print STDERR "== Reading mapping\n" unless $o{q};
+   open BLS, "<", $map or die "Cannot read the file: $map: $!\n";
+   open REC, ">", "$map.rec" or die "Cannot create the file: $map.rec: $!\n";
+   RESULT:while(<BLS>){
+      chomp;
+      my @ln = split /\t/;
+      $ln[11] or die "Cannot parse line $map:$.: $_\n";
+      next unless $ln[3]>=$o{l};
+      next unless $ln[2]>=$o{i};
+      unless(exists $seq{$ln[1]}){
+         die "Cannot find the subject sequence: $ln[1]\n" unless $o{s};
+	 next RESULT;
+      }
+      my $start = $seq{$ln[1]}+min($ln[8], $ln[9]);
+      my $end   = $seq{$ln[1]}+max($ln[8], $ln[9]);
+      print REC "$start\t$end\t$ln[2]\t$ln[11]\t$ln[0]",
+      		(exists($ln[13])?"\t".($ln[2]*$ln[3]/min($ln[12],$ln[13]))."\t":
+		exists($ln[12])?"\t".($ln[2]*$ln[3]/$ln[12])."\t":""),"\n";
+   }
+   close BLS;
+   close REC;
+   print STDERR " done.\n" unless $o{q};
+}
+

data/utils/enveomics/Scripts/BlastTab.cogCat.rb

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update  Mar-23-2016
+# @license artistic license 2.0
+#
+
+require "optparse"
+
+o = {:cog=>false, :desc=>false, :q=>false, :w=>true}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "Replaces the COG gene IDs in a BLAST for the COG category."
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-w", "--whog FILE", "Path to the whog file."){ |v| o[:whog]=v }
+   opts.on("-i", "--blast FILE",
+      "Path to the Tabular BLAST file with COG IDs as subject."
+      ){ |v| o[:blast]=v }
+   opts.separator ""
+   opts.separator "Optional"
+   opts.on("-g", "--cog",
+      "If set, returns the COG ID, not the COG category."){ o[:cog]=true }
+   opts.on("-d", "--desc",
+      "Includes COG description (requires -g/--cog)."){ o[:desc]=true }
+   opts.on("-n", "--noverbose", "Run quietly, but show warnings."){ o[:q]=true }
+   opts.on("-q", "--quiet", "Run quietly."){ o[:q]=true; o[:w]=false }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+
+abort "-w/--whog is mandatory." if o[:whog].nil?
+abort "-i/--blast is mandatory." if o[:blast].nil?
+
+$stderr.puts "Parsing whog file." unless o[:q]
+cat = {}
+curCats = []
+fh = File.open o[:whog], "r"
+while ln=fh.gets
+   ln.chomp!
+   next if /^\s*$/.match ln
+   if m=/^\[([A-Z]+)\] (COG\d+) (.*)/.match(ln)
+      curCats = o[:cog] ? [ m[2]+(o[:desc]?" #{m[3]}":"") ] : m[1].split(//)
+   elsif /^_+$/.match ln
+      curCats = []
+   elsif m=/^\s+(?:.+?:\s+)?(.*)/.match(ln)
+      m[1].split(/\s+/).each do |g|
+         cat[g] ||= []
+	 curCats.each { |i| cat[g] << i }
+      end
+   else
+      abort "Impossible to parse line #{$.}: #{ln}"
+   end
+end
+fh.close
+
+$stderr.puts "Parsing BLAST." unless o[:q]
+fh = File.open(o[:blast], "r")
+while ln=fh.gets
+   row = ln.split(/\t/)
+   if cat[ row[1] ].nil?
+      $stderr.puts "Warning: line #{$.}: #{row[1]}: " +
+	 "Impossible to find category.\n" if o[:w]
+   else
+      cat[ row[1] ].each do |c|
+         row[1] = c
+	 puts row.join("\t")
+      end
+   end
+end
+fh.close
+

data/utils/enveomics/Scripts/BlastTab.filter.pl

@@ -0,0 +1,47 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+my %o;
+getopts('si', \%o);
+my($list, $blast) = @ARGV;
+
+($list and $blast) or die "
+.Description:
+   Extracts a subset of hits (queries or subjects) from a tabular BLAST.
+
+.Usage: $0 [options] list.txt blast.txt > subset.txt
+   
+   Options:
+      -s	If set, assumes that list.txt contains subject IDs.
+   		By default: assumes query IDs.
+      -i	If set, reports the inverse of the list (i.e., reports
+      		only hits absent in the list).
+
+   list.txt	List of IDs to extract.
+   blast.txt	Tabular BLAST file containing the superset of hits.
+   subset.txt	Tabulat BLAST file to be created.
+
+";
+
+open LI, "<", $list or die "Cannot read file: $list: $!\n";
+my %li = map { chomp; $_ => 1 } <LI>;
+close LI;
+
+open BLAST, "<", $blast or die "Cannot read file: $blast: $!\n";
+while(my $ln = <BLAST>){
+   chomp $ln;
+   my @ln = split("\t", $ln);
+   my $good = exists $li{$ln[ ($o{s} ? 1 : 0) ]};
+   $good = not $good if $o{i};
+   print "$ln\n" if $good;
+}
+close BLAST;
+

data/utils/enveomics/Scripts/BlastTab.kegg_pep2path_rest.pl

@@ -0,0 +1,194 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use LWP::Simple;
+use JSON;
+use File::Copy;
+
+my($blast, $cache_file, $max_cache) = @ARGV;
+($blast) or die "
+Description:
+   Takes a BLAST against KEGG_PEP (or KO) and retrieves the pathways in which the subject
+   peptides are involved.
+
+Usage:
+   $0 blast.txt[ cache_file] > output.txt
+
+   blast.txt	Input (filtered) BLAST file.
+   cache_file	(optional) File containing the saved cache.  If unset, the
+   		cache won't be recoverable across instances of this script.
+		It is strongly recommended to set a file.  Multiple
+		parallel instances of this script may use the same cache
+		file.
+   output.txt	Tab-delimited output file, with the columns:
+   		 o Query ID
+		 o Subject ID
+		 o Pathway ID
+		 o Pathway (reference) description
+		 o Organism
+
+";
+
+$max_cache ||= 0;
+$cache_file ||= "";
+
+sub read_cache($){
+   my ($cache_file) = @_;
+   my $cache = {};
+   my $n = 0;
+   if($cache_file and -s $cache_file){
+      local $/;
+      my $json = "";
+      while(-e "$cache_file.tmp"){
+         print STDERR "Locked cache (read), waiting 1 sec.\n";
+	 sleep 1;
+      }
+      open CACHE, "<", $cache_file or die "Cannot read file: $cache_file: $!\n";
+      while(<CACHE>){ $json.=$_ }
+      close CACHE;
+      $cache = decode_json($json);
+      $n = scalar keys %$cache;
+   }
+   return ($cache, $n);
+}
+
+sub write_cache($$){
+   my($cache, $cache_file) = @_;
+   if($cache_file){
+      # Get previously saved entries.
+      my($cache2, $cache_n2) = &read_cache($cache_file);
+      for my $k (keys %$cache2){
+	 $cache->{$k} ||= $cache2->{$k} unless $k eq "###:paths";
+      }
+      $cache->{'###:paths'} ||= {};
+      for my $p (keys %{$cache2->{'###:paths'}}){
+         $cache->{'###:paths'}->{$p} ||= $cache2->{'###:paths'}->{$p};
+      }
+      # Save merged cache.
+      if(-s $cache_file){ copy $cache_file, "$cache_file.pre" or die "Cannot create file: $cache_file.tmp: $!\n" }
+      my $json = encode_json($cache);
+      while(-e "$cache_file.tmp"){
+         print STDERR "Locked cache (write), waiting 1 sec.\n";
+	 sleep 1;
+      }
+      open CACHE, ">", "$cache_file.tmp" or die "Cannot create file: $cache_file.tmp: $!\n";
+      print CACHE $json;
+      close CACHE;
+      copy "$cache_file.tmp", $cache_file or die "Cannot create file: $cache_file: $!\n";
+      unlink "$cache_file.tmp" or die "Cannot unlink file: $cache_file.tmp: $!\n";
+   }
+}
+
+sub download_pathways($$){
+   my($cache, $ids) = @_;
+   my @todownload = ();
+   for my $id (@$ids){
+      push @todownload, $id unless exists $cache->{'###:paths'}->{$id};
+   }
+   while($#todownload>=0){
+      my @downloading = splice(@todownload, 0, 100);
+      my $path = get "http://rest.kegg.jp/list/".join("+", @downloading);
+      if($path){
+	 chomp $path;
+	 for my $p (split /\n/, $path){
+	    my @wl = split /\t/, $p;
+	    $wl[1] =~ s/ - /\t/;
+	    $cache->{'###:paths'}->{$wl[0]} = $wl[1];
+	 }
+      }
+   }
+   return $cache;
+}
+
+sub download($$){
+   my($cache, $todownload) = @_;
+   $cache->{'###:paths'} ||= {};
+   return $cache unless $#$todownload>=0;
+   $cache->{$_} = [] for @$todownload;
+   my $list = get "http://rest.kegg.jp/link/pathway/".join("+", @$todownload);
+   $list ||= "";
+   chomp $list;
+   my @pathids = ();
+   for my $res (split /\n/, $list){
+      my @rel = split /\t/, $res;
+      $#rel==1 or die "Unexpected number of columns:\n$res\n";
+      my $id = $rel[1];
+      push @pathids, $id;
+      unless(exists $cache->{$rel[0]}){
+	 #print STDERR "Request/response difference in ID: ".$rel[0].", searching match.\n";
+         for my $id (@$todownload){
+	    $rel[0] = $id if lc $id eq lc $rel[0];
+	 }
+	 die "Cannot find corresponding request.\n" unless exists $cache->{$rel[0]};
+      }
+      push @{ $cache->{$rel[0]} }, $id;
+   }
+   return &download_pathways($cache, \@pathids);
+}
+
+sub print_out($$){
+   my($cache, $hits) = @_;
+   for my $hit (@$hits){
+      die "Impossible to find gene in cache: ".$hit->[1]."\n" unless exists $cache->{$hit->[1]};
+      for my $path (@{$cache->{$hit->[1]}}){
+	 next if $path =~ /^path:ko\d/;
+	 unless(exists $cache->{'###:paths'}->{$path}){
+	    print STDERR "Cannot find pathway in cache: $path (from ".$hit->[1]."), emergency download\n";
+	    $cache = &download_pathways($cache, [$path]);
+	    die "Impossible to find pathway: $path.\n" unless exists $cache->{'###:paths'}->{$path};
+	 }
+	 print "", join("\t", $hit->[0], $hit->[1], $path, $cache->{'###:paths'}->{$path}), "\n";
+      }
+   }
+}
+
+print STDERR "Loading cache.\n";
+my ($cache, $n) = &read_cache($cache_file);
+print STDERR "   $n entries loaded.\n";
+my @nopath = ();
+for my $k (keys %$cache){
+   next if $k eq "###:paths";
+   for my $p (@{ $cache->{$k} }){
+      push @nopath, $p unless exists $cache->{'###:paths'}->{$p};
+   }
+}
+if($#nopath>=0){
+   print STDERR "   Sanitizing ".@nopath." pathways in cache.\n";
+   while($#nopath>=0){
+      my @paths = ();
+      for(1 .. 15){ push @paths, shift @nopath unless $#nopath==-1 }
+      $cache = &download_pathways($cache, \@paths);
+   }
+   &write_cache($cache, $cache_file);
+}
+
+my $lines=0;
+my $downs=0;
+my @buff = ();
+my @todownload = ();
+print STDERR "Mapping genes.\n";
+open BLAST, "<", $blast or die "Cannot read file: $blast: $!\n";
+while(<BLAST>){
+   chomp;
+   my @l = split /\t/;
+   print STDERR "   Mapping line ".(++$lines).". \r";
+   unless(($#todownload+2)%100){
+      print STDERR "+\r";
+      print STDERR " *\r" unless ++$downs%10;
+      $cache = &download($cache, \@todownload);
+      @todownload = ();
+      &print_out($cache, \@buff);
+      @buff = ();
+      &write_cache($cache, $cache_file) unless $downs%10;
+   }
+   push @buff, \@l;
+   push @todownload, $l[1] unless exists $cache->{$l[1]};
+}
+print STDERR "\nDone.\n";
+close BLAST;
+
+$cache = &download($cache, \@todownload);
+&print_out($cache, \@buff);
+&write_cache($cache, $cache_file);
+