miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Scripts/EBIseq2tax.rb

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update  Oct-13-2015
+# @license Artistic License 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
+require "enveomics_rb/remote_data"
+use "nokogiri"
+
+#================================[ Options parsing ]
+$o = {
+   q: false, ids: [], dbfrom: "uniprotkb", header: true,
+   ret: "ScientificName",
+   ranks: %w(superkingdom phylum class order family genus species)}
+
+OptionParser.new do |opt|
+   opt.banner = "
+   Maps a list of EBI-supported IDs to their corresponding NCBI taxonomy using
+   EBI RESTful API. Avoid using this script on millions of entries at a time,
+   since each entry elicits requests to EBI and NCBI servers.
+
+   Usage: #{$0} [options]".gsub(/^ +/,"")
+   opt.separator ""
+   opt.on("-i", "--ids ID1,ID2,...", Array,
+      "Comma-separated list of EBI IDs. Required unless -I is passed."
+      ){ |v| $o[:ids]=v }
+   opt.on("-I", "--infile FILE",
+      "Raw text file containing the list of EBI IDs, one per line.",
+      "Required unless -i is passed."){ |v| $o[:infile]=v }
+   opt.on("-d", "--database DB",
+      "EBI database defining the EBI IDs. By default: " + $o[:dbfrom].to_s + "."
+      ){ |v| $o[:dbfrom]=v }
+   opt.on("-r", "--ranks RANK1,RANK2,...", Array,
+      "Taxonomic ranks to report. By default:",
+      $o[:ranks].join(",") + "."){ |v| $o[:ranks]=v }
+   opt.on("-n", "--noheader",
+      "Do not includ a header in the output."){ $o[:header]=false }
+   opt.on("-t", "--taxids",
+      "Return Taxonomy IDs instead of scientific names."){ $o[:ret]="TaxId" }
+   opt.on("-q", "--quiet", "Run quietly."){ |v| $o[:q]=true }
+   opt.on("-h", "--help","Display this screen") do
+      puts opt
+      exit
+   end
+   opt.separator ""
+end.parse!
+
+#================================[ Main ]
+begin
+   $o[:ids] += File.readlines($o[:infile]).map{ |l| l.chomp } unless
+      $o[:infile].nil?
+   $o[:ranks].map!{ |r| r.downcase }
+   puts (["ID", "TaxId"] + $o[:ranks].map{ |r| r.capitalize }).join("\t") if
+      $o[:header]
+   $o[:ids].each do |id|
+      id = $1 if id =~ /^[a-z]+\|\S+\|(\S+)/
+      taxid = RemoteData.ebiseq2taxid(id, $o[:dbfrom])
+      if taxid.nil?
+	 warn "Cannot find link to taxonomy: #{id}"
+	 next
+      end
+      taxonomy = {}
+      unless taxid.nil?
+	 doc = Nokogiri::XML( RemoteData.efetch({db: "taxonomy", id: taxid}) )
+	 taxonomy[ doc.at_xpath("/TaxaSet/Taxon/Rank").content ] =
+	    doc.at_xpath("/TaxaSet/Taxon/#{$o[:ret]}").content
+	 doc.xpath("/TaxaSet/Taxon/LineageEx/Taxon").each do |taxon|
+	    taxonomy[ taxon.at_xpath("./Rank").content ] =
+	       taxon.at_xpath("./#{$o[:ret]}").content
+	 end
+      end
+      puts ([id, taxid] +
+	 $o[:ranks].map{ |rank| taxonomy[ rank ] ||= "" }).join("\t")
+   end # $o[:ids].each
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+

data/utils/enveomics/Scripts/FastA.N50.pl

@@ -0,0 +1,60 @@
+#!/usr/bin/env perl
+
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license: Artistic-2.0
+
+use strict;
+use warnings;
+use List::Util qw/sum min max/;
+
+my ($seqs, $minlen, $n__) = @ARGV;
+$seqs or die "
+Description:
+  Calculates the N50 value of a set of sequences.  Alternatively, it
+  can calculate other N** values.  It also calculates the total number
+  of sequences, the total added length, and the longest sequence length.
+
+Usage:
+  $0 seqs.fa [minlen [**]]
+
+  seqs.fa	A FastA file containing the sequences
+  minlen	(optional) The minimum length to take into consideration
+  		By default: 0
+  **		(optional) Value N** to calculate.  By default: 50 (N50)
+
+";
+
+$minlen ||= 0;
+$n__    ||= 50;
+
+my @len = ();
+open SEQ, "<", $seqs or die "Cannot open file: $seqs: $!\n";
+while(<SEQ>){
+  if(/^>/){
+    push @len, 0;
+  }else{
+    next if /^;/;
+    chomp;
+    s/\W//g;
+    $len[-1] += length $_;
+  }
+}
+close SEQ;
+
+@len = sort { $a <=> $b } map { $_ >= $minlen ? $_ : () } @len;
+my $tot = (sum(@len) || 0);
+
+my $thr = $n__ * $tot / 100;
+my $pos = 0;
+for(@len){
+  $pos += $_;
+  if($pos >= $thr){
+    print "N$n__: $_\n";
+    last;
+  }
+}
+
+print "Sequences: " . scalar(@len) . "\n";
+print "Total length: $tot\n";
+print "Longest sequence: " . pop(@len) . "\n";
+

data/utils/enveomics/Scripts/FastA.extract.rb

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license Artistic-2.0
+#
+
+require 'optparse'
+
+o = {q: false}
+ARGV << '-h' if ARGV.size==0
+
+OptionParser.new do |opt|
+  opt.banner = "
+Extracts a list of sequences and/or coordinates from multi-FastA files.
+
+Usage: #{$0} [options]"
+  opt.separator ''
+  opt.separator 'Mandatory'
+  opt.on('-i', '--in PATH', 'Input FastA file.'){ |v| o[:i] = v }
+  opt.on('-o', '--out PATH', 'Output FastA file.'){ |v| o[:o] = v }
+  opt.on('-c', '--coords STRING',
+    'Comma-delimited list of coordinates (mandatory unless -C is passed).',
+    'The format of the coordinates is "SEQ:FROM..TO" or "SEQ:FROM~LEN":',
+    'SEQ: Sequence ID, or * (asterisk) to extract range from all sequences',
+    'FROM: Integer, position of the first base to include (can be negative)',
+    'TO: Integer, last base to include (can be negative)',
+    'LEN: Length of the range to extract'
+    ){ |v| o[:c] = v }
+  opt.separator ''
+  opt.separator 'Options'
+  opt.on('-C', '--coords-file PATH',
+    'File containing the coordinates, one per line.',
+    'Each line must follow the format described for -c.'){ |v| o[:C] = v }
+  opt.on('-q', '--quiet', 'Run quietly (no STDERR output).'){ o[:q] = true }
+  opt.on('-h', '--help', 'Display this screen.') do
+    puts opt
+    exit
+  end
+  opt.separator ''
+end.parse!
+abort '-i is mandatory.' if o[:i].nil?
+abort '-o is mandatory.' if o[:o].nil?
+abort '-c is mandatory.' if o[:c].nil? and o[:C].nil?
+
+# Classses to parse coordinates
+class SeqCoords
+  attr :id, :from, :to, :length, :str
+  def initialize(str)
+    @str = str
+    m = /(\S+):(-?\d+)(~|\.\.)(-?\d+)/.match str
+    raise "Cannot parse coordinates: #{str}" if m.nil?
+    @id = m[1]
+    @from = m[2].to_i
+    if m[3] == '~'
+      @length = m[4].to_i
+    else
+      @to = m[4].to_i
+    end
+  end
+
+  def extract(id, seq)
+    return nil unless concerns? id
+    from_i = from > 0 ? from : seq.length + 1 + from
+    if to.nil?
+      seq[from_i, length]
+    else
+      to_i = to > 0 ? to : seq.length + 1 + to
+      seq[from_i .. to_i]
+    end
+  end
+
+  def concerns?(seq_id)
+    return true if id == '*'
+    return id == seq_id
+  end
+end
+
+class SeqCoordsCollection
+  class << self
+    def from_str(str)
+      c = new
+      str.split(',').each { |i| c << SeqCoords.new(i) }
+      c
+    end
+    def from_file(path)
+      c = new
+      File.open(path, 'r') do |fh|
+        fh.each{ |i| c << SeqCoords.new(i.chomp) }
+      end
+      c
+    end
+  end
+
+  attr :collection
+
+  def initialize
+    @collection = []
+  end
+
+  def <<(coords)
+    @collection << coords
+  end
+
+  def extract(id, seq)
+    @collection.map{ |c| c.extract(id, seq) }.compact
+  end
+end
+
+# Functions to parse sequences
+def do_stuff(id, sq)
+  return if id.nil? or sq.empty?
+  @n_in += 1
+  sq.gsub!(/[^A-Za-z]/, '')
+  i = 0
+  @coll.extract(id, sq).each do |new_sq|
+    @ofh.puts ">#{id}:#{i += 1}"
+    @ofh.puts new_sq
+    @n_out += 1
+  end
+end
+
+# Parse coordinates
+$stderr.puts 'Parsing coordinates' unless o[:q]
+@coll = o[:c].nil? ? SeqCoordsCollection.from_file(o[:C]) :
+  SeqCoordsCollection.from_str(o[:c])
+$stderr.puts "  Coordinates found: #{@coll.collection.size}"
+
+# Parse sequences
+$stderr.puts 'Parsing sequences' unless o[:q]
+@n_in = 0
+@n_out = 0
+@ofh = File.open(o[:o], 'w')
+File.open(o[:i], 'r') do |fh|
+  id = nil
+  sq = ''
+  fh.each do |ln|
+    next if ln =~ /^;/
+    if ln =~ /^>(\S+)/
+      id = $1
+      do_stuff(id, sq)
+      sq = ''
+    else
+      sq << ln
+    end
+  end
+  do_stuff(id, sq)
+end
+@ofh.close
+$stderr.puts "  Input sequences: #{@n_in}"
+$stderr.puts "  Output fragments: #{@n_out}"
+

data/utils/enveomics/Scripts/FastA.filter.pl

@@ -0,0 +1,52 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update  Oct-07-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+sub HELP_MESSAGE { die "
+.Description:
+   Extracts a subset of sequences from a FastA file.
+
+.Usage: $0 [options] list.txt seqs.fa > subset.fa
+   
+   [options]
+   -r		Reverse list.  Extracts sequences NOT present in the list.
+   -q		Runs quietly.
+   -h		Prints this message and exits.
+
+   [mandatory]
+   list.txt	List of sequences to extract.
+   seqs.fa	FastA file containing the superset of sequences.
+   subset.fa	FastA file to be created.
+
+" }
+
+my %o=();
+getopts('rhq', \%o);
+my($list, $fa) = @ARGV;
+($list and $fa) or &HELP_MESSAGE;
+$o{h} and &HELP_MESSAGE;
+
+print STDERR "Reading list.\n" unless $o{q};
+open LI, "<", $list or die "Cannot read file: $list: $!\n";
+my %li = map { chomp; $_ => 1 } <LI>;
+close LI;
+
+print STDERR "Filtering FastA.\n" unless $o{q};
+open FA, "<", $fa or die "Cannot read file: $fa: $!\n";
+my $good = 0;
+while(my $ln = <FA>){
+   next if $ln =~ /^;/;
+   chomp $ln;
+   if($ln =~ m/^>((\S+).*)/){ $good = (exists $li{$1} or exists $li{">$1"} or exists $li{$2} or exists $li{$ln}) }
+   elsif($ln =~ m/^>/){ $good=$o{r}; print STDERR "Warning: Non-cannonical defline, line $.: $ln\n" }
+   print "$ln\n" if (($good and not $o{r}) or ($o{r} and not $good));
+}
+close FA;
+

data/utils/enveomics/Scripts/FastA.filterLen.pl

@@ -0,0 +1,28 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use Bio::SeqIO;
+
+my $file = $ARGV[0]; 
+my $min = $ARGV[1];
+($file and $min) or die <<HELP
+
+This script will filter a multi fastA file by length
+
+Usage "perl $0 fastafile minlenght "
+HELP
+;
+my $seq_in  = Bio::SeqIO->new( -format => 'fasta',-file => $file);
+
+while( my $seq1 = $seq_in->next_seq() ) {	
+	
+	my $id  = $seq1->primary_id;
+	chomp $id;
+	my $seq = $seq1->seq;
+	chomp $seq;
+	my $lseq = length($seq);
+	if($lseq>=$min){
+		print ">$id","\n",$seq,"\n";	
+	}
+}

data/utils/enveomics/Scripts/FastA.filterN.pl

@@ -0,0 +1,60 @@
+#!/usr/bin/env perl
+# 
+# @author  Luis M. Rodriguez-R
+# @update  Oct-07-2015
+# @license artistic license 2.0
+# 
+
+use warnings;
+use strict;
+
+my($file, $content, $stretch) = @ARGV;
+$file or die <<HELP
+
+Description:
+   Filter sequences by N-content and presence of long homopolymers.
+Usage:
+   $0 sequences.fa [content [stretch]] > filtered.fa
+Where:
+   sequences.fa	Input file in FastA format
+   content	A number between 0 and 1 indicating the maximum proportion of Ns
+   		(1 to turn off, 0.5 by default)
+   stretch	A number indicating the maximum number of consecutive identical
+   		nucleotides allowed (0 to turn off, 100 by default)
+   filtered.fa	Filtered set of sequences.
+
+HELP
+;
+($content ||= 0.5)+=0;
+($stretch ||= 100)+=0;
+
+my $good = 0;
+my $N = 0;
+
+FASTA: {
+   local $/ = "\n>";
+   open FILE, "<", $file or die "I can not open the file: $file: $!\n";
+   SEQ: while(<FILE>){
+      $N++;
+      s/^;.*//gm;
+      s/>//g;
+      my($n,$s) = split /\n/, $_, 2;
+      (my $clean = $s) =~ s/[^ACTGN]//g;
+      if($content < 1){
+         (my $Ns = $clean) =~ s/[^N]//g;
+	 next SEQ if length($Ns)>length($clean)*$content;
+      }
+      if($stretch > 0){
+         for my $nuc (qw(A C T G N)){
+	    next SEQ if $clean =~ m/[$nuc]{$stretch}/;
+	 }
+      }
+      print ">$n\n$s\n";
+      $good++;
+   }
+   close FILE;
+   print STDERR "Total sequences: $N\nAfter filtering: $good\n";
+}
+
+
+

data/utils/enveomics/Scripts/FastA.fragment.rb

@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+$:.push File.expand_path('../lib', __FILE__)
+require 'enveomics_rb/enveomics'
+require 'enveomics_rb/stats'
+$VERSION = 1.0
+
+o = { q: false, completeness: nil, minlen: 500, shuffle: true }
+OptionParser.new do |opts|
+  opts.version = $VERSION
+  Enveomics.opt_banner(
+    opts, 'Simulates incomplete (fragmented) drafts from complete genomes',
+    "#{File.basename($0)} -i in.fasta -o out.fasta -c 0.5 [options]"
+  )
+
+  opts.separator 'Mandatory'
+  opts.on(
+    '-i', '--in FILE',
+    'Path to the FastA file containing the complete sequences',
+    'Supports compression with .gz extension, use - for STDIN'
+  ) { |v| o[:in] = v }
+  opts.on(
+    '-o', '--out FILE', 'Path to the FastA to create',
+    'Supports compression with .gz extension, use - for STDOUT'
+  ) { |v| o[:out] = v }
+  opts.on(
+    '-c', '--completeness FLOAT',
+    'Fraction of genome completeness to simulate from 0 to 1'
+  ) { |v| o[:completeness] = v.to_f }
+
+  opts.separator ''
+  opts.separator 'Options'
+  opts.on(
+    '-m', '--minlen INT',
+    "Minimum fragment length to report. By default: #{o[:minlen]}"
+  ) { |v| o[:minlen] = v.to_i }
+  opts.on(
+    '-s', '--sorted', 'Keep fragments sorted as in the input file',
+    'By default, fragments are shuffled'
+  ) { |v| o[:shuffle] = !v }
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)') { o[:q] = true }
+  opts.on('-h', '--help', 'Display this screen') { puts opts ; exit }
+  opts.separator ''
+end.parse!
+
+raise Enveomics::OptionError.new('-i is mandatory') if o[:in].nil?
+raise Enveomics::OptionError.new('-o is mandatory') if o[:out].nil?
+raise Enveomics::OptionError.new('-c is mandatory') if o[:completeness].nil?
+
+begin
+  # Read input sequences
+  g_id  = []
+  g_seq = []
+  ifh = reader(o[:in])
+  id = ''
+  ifh.each_line do |ln|
+    if ln =~ /^>(\S*)/
+      g_id  << $1
+      g_seq << ''
+    else
+      g_seq[g_seq.size - 1] += ln.gsub(/[^A-Za-z]/, '')
+    end
+  end
+  ifh.close
+
+  # Fragment genomes
+  f = {}
+  binlen = [1, (o[:minlen].to_f/(1.5**2)).ceil].max
+  p = [0.001, [1.0, 1.0 - (o[:completeness]/1.25 + 0.1)].min].max
+  while !g_seq.empty?
+    id  = g_id.shift
+    seq = g_seq.shift
+    gL  = seq.length
+    while !seq.empty?
+      rand_x =
+        Enveomics::Stats.r_geom(p).to_f + Enveomics::Stats.r_unif(-0.5, 0.5)
+      fL = [0, (rand_x * binlen).round].max
+      f["#{f.size+1}_#{id}"] = seq[0, fL] if fL >= o[:minlen]
+      seq = seq[(fL + 1) .. -1]
+      seq = '' if seq.nil?
+    end
+  end
+
+  # Save output
+  k = f.keys
+  k.shuffle! if o[:shuffle]
+  ofh = writer(o[:out])
+  k.each do |id|
+    ofh.puts ">#{id}"
+    ofh.puts f[id].gsub(/(\S{50})/, "\\1\n")
+  end
+  ofh.close
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end
+