miga-base 1.2.17.0 → 1.2.17.2

data/utils/enveomics/Scripts/FastA.extract.rb

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license Artistic-2.0
+#
+
+require 'optparse'
+
+o = {q: false}
+ARGV << '-h' if ARGV.size==0
+
+OptionParser.new do |opt|
+  opt.banner = "
+Extracts a list of sequences and/or coordinates from multi-FastA files.
+
+Usage: #{$0} [options]"
+  opt.separator ''
+  opt.separator 'Mandatory'
+  opt.on('-i', '--in PATH', 'Input FastA file.'){ |v| o[:i] = v }
+  opt.on('-o', '--out PATH', 'Output FastA file.'){ |v| o[:o] = v }
+  opt.on('-c', '--coords STRING',
+    'Comma-delimited list of coordinates (mandatory unless -C is passed).',
+    'The format of the coordinates is "SEQ:FROM..TO" or "SEQ:FROM~LEN":',
+    'SEQ: Sequence ID, or * (asterisk) to extract range from all sequences',
+    'FROM: Integer, position of the first base to include (can be negative)',
+    'TO: Integer, last base to include (can be negative)',
+    'LEN: Length of the range to extract'
+    ){ |v| o[:c] = v }
+  opt.separator ''
+  opt.separator 'Options'
+  opt.on('-C', '--coords-file PATH',
+    'File containing the coordinates, one per line.',
+    'Each line must follow the format described for -c.'){ |v| o[:C] = v }
+  opt.on('-q', '--quiet', 'Run quietly (no STDERR output).'){ o[:q] = true }
+  opt.on('-h', '--help', 'Display this screen.') do
+    puts opt
+    exit
+  end
+  opt.separator ''
+end.parse!
+abort '-i is mandatory.' if o[:i].nil?
+abort '-o is mandatory.' if o[:o].nil?
+abort '-c is mandatory.' if o[:c].nil? and o[:C].nil?
+
+# Classses to parse coordinates
+class SeqCoords
+  attr :id, :from, :to, :length, :str
+  def initialize(str)
+    @str = str
+    m = /(\S+):(-?\d+)(~|\.\.)(-?\d+)/.match str
+    raise "Cannot parse coordinates: #{str}" if m.nil?
+    @id = m[1]
+    @from = m[2].to_i
+    if m[3] == '~'
+      @length = m[4].to_i
+    else
+      @to = m[4].to_i
+    end
+  end
+
+  def extract(id, seq)
+    return nil unless concerns? id
+    from_i = from > 0 ? from : seq.length + 1 + from
+    if to.nil?
+      seq[from_i, length]
+    else
+      to_i = to > 0 ? to : seq.length + 1 + to
+      seq[from_i .. to_i]
+    end
+  end
+
+  def concerns?(seq_id)
+    return true if id == '*'
+    return id == seq_id
+  end
+end
+
+class SeqCoordsCollection
+  class << self
+    def from_str(str)
+      c = new
+      str.split(',').each { |i| c << SeqCoords.new(i) }
+      c
+    end
+    def from_file(path)
+      c = new
+      File.open(path, 'r') do |fh|
+        fh.each{ |i| c << SeqCoords.new(i.chomp) }
+      end
+      c
+    end
+  end
+
+  attr :collection
+
+  def initialize
+    @collection = []
+  end
+
+  def <<(coords)
+    @collection << coords
+  end
+
+  def extract(id, seq)
+    @collection.map{ |c| c.extract(id, seq) }.compact
+  end
+end
+
+# Functions to parse sequences
+def do_stuff(id, sq)
+  return if id.nil? or sq.empty?
+  @n_in += 1
+  sq.gsub!(/[^A-Za-z]/, '')
+  i = 0
+  @coll.extract(id, sq).each do |new_sq|
+    @ofh.puts ">#{id}:#{i += 1}"
+    @ofh.puts new_sq
+    @n_out += 1
+  end
+end
+
+# Parse coordinates
+$stderr.puts 'Parsing coordinates' unless o[:q]
+@coll = o[:c].nil? ? SeqCoordsCollection.from_file(o[:C]) :
+  SeqCoordsCollection.from_str(o[:c])
+$stderr.puts "  Coordinates found: #{@coll.collection.size}"
+
+# Parse sequences
+$stderr.puts 'Parsing sequences' unless o[:q]
+@n_in = 0
+@n_out = 0
+@ofh = File.open(o[:o], 'w')
+File.open(o[:i], 'r') do |fh|
+  id = nil
+  sq = ''
+  fh.each do |ln|
+    next if ln =~ /^;/
+    if ln =~ /^>(\S+)/
+      id = $1
+      do_stuff(id, sq)
+      sq = ''
+    else
+      sq << ln
+    end
+  end
+  do_stuff(id, sq)
+end
+@ofh.close
+$stderr.puts "  Input sequences: #{@n_in}"
+$stderr.puts "  Output fragments: #{@n_out}"
+

data/utils/enveomics/Scripts/FastA.filter.pl

@@ -0,0 +1,52 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update  Oct-07-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+sub HELP_MESSAGE { die "
+.Description:
+   Extracts a subset of sequences from a FastA file.
+
+.Usage: $0 [options] list.txt seqs.fa > subset.fa
+   
+   [options]
+   -r		Reverse list.  Extracts sequences NOT present in the list.
+   -q		Runs quietly.
+   -h		Prints this message and exits.
+
+   [mandatory]
+   list.txt	List of sequences to extract.
+   seqs.fa	FastA file containing the superset of sequences.
+   subset.fa	FastA file to be created.
+
+" }
+
+my %o=();
+getopts('rhq', \%o);
+my($list, $fa) = @ARGV;
+($list and $fa) or &HELP_MESSAGE;
+$o{h} and &HELP_MESSAGE;
+
+print STDERR "Reading list.\n" unless $o{q};
+open LI, "<", $list or die "Cannot read file: $list: $!\n";
+my %li = map { chomp; $_ => 1 } <LI>;
+close LI;
+
+print STDERR "Filtering FastA.\n" unless $o{q};
+open FA, "<", $fa or die "Cannot read file: $fa: $!\n";
+my $good = 0;
+while(my $ln = <FA>){
+   next if $ln =~ /^;/;
+   chomp $ln;
+   if($ln =~ m/^>((\S+).*)/){ $good = (exists $li{$1} or exists $li{">$1"} or exists $li{$2} or exists $li{$ln}) }
+   elsif($ln =~ m/^>/){ $good=$o{r}; print STDERR "Warning: Non-cannonical defline, line $.: $ln\n" }
+   print "$ln\n" if (($good and not $o{r}) or ($o{r} and not $good));
+}
+close FA;
+

data/utils/enveomics/Scripts/FastA.filterLen.pl

@@ -0,0 +1,28 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use Bio::SeqIO;
+
+my $file = $ARGV[0]; 
+my $min = $ARGV[1];
+($file and $min) or die <<HELP
+
+This script will filter a multi fastA file by length
+
+Usage "perl $0 fastafile minlenght "
+HELP
+;
+my $seq_in  = Bio::SeqIO->new( -format => 'fasta',-file => $file);
+
+while( my $seq1 = $seq_in->next_seq() ) {	
+	
+	my $id  = $seq1->primary_id;
+	chomp $id;
+	my $seq = $seq1->seq;
+	chomp $seq;
+	my $lseq = length($seq);
+	if($lseq>=$min){
+		print ">$id","\n",$seq,"\n";	
+	}
+}

data/utils/enveomics/Scripts/FastA.filterN.pl

@@ -0,0 +1,60 @@
+#!/usr/bin/env perl
+# 
+# @author  Luis M. Rodriguez-R
+# @update  Oct-07-2015
+# @license artistic license 2.0
+# 
+
+use warnings;
+use strict;
+
+my($file, $content, $stretch) = @ARGV;
+$file or die <<HELP
+
+Description:
+   Filter sequences by N-content and presence of long homopolymers.
+Usage:
+   $0 sequences.fa [content [stretch]] > filtered.fa
+Where:
+   sequences.fa	Input file in FastA format
+   content	A number between 0 and 1 indicating the maximum proportion of Ns
+   		(1 to turn off, 0.5 by default)
+   stretch	A number indicating the maximum number of consecutive identical
+   		nucleotides allowed (0 to turn off, 100 by default)
+   filtered.fa	Filtered set of sequences.
+
+HELP
+;
+($content ||= 0.5)+=0;
+($stretch ||= 100)+=0;
+
+my $good = 0;
+my $N = 0;
+
+FASTA: {
+   local $/ = "\n>";
+   open FILE, "<", $file or die "I can not open the file: $file: $!\n";
+   SEQ: while(<FILE>){
+      $N++;
+      s/^;.*//gm;
+      s/>//g;
+      my($n,$s) = split /\n/, $_, 2;
+      (my $clean = $s) =~ s/[^ACTGN]//g;
+      if($content < 1){
+         (my $Ns = $clean) =~ s/[^N]//g;
+	 next SEQ if length($Ns)>length($clean)*$content;
+      }
+      if($stretch > 0){
+         for my $nuc (qw(A C T G N)){
+	    next SEQ if $clean =~ m/[$nuc]{$stretch}/;
+	 }
+      }
+      print ">$n\n$s\n";
+      $good++;
+   }
+   close FILE;
+   print STDERR "Total sequences: $N\nAfter filtering: $good\n";
+}
+
+
+

data/utils/enveomics/Scripts/FastA.fragment.rb

@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+$:.push File.expand_path('../lib', __FILE__)
+require 'enveomics_rb/enveomics'
+require 'enveomics_rb/stats'
+$VERSION = 1.0
+
+o = { q: false, completeness: nil, minlen: 500, shuffle: true }
+OptionParser.new do |opts|
+  opts.version = $VERSION
+  Enveomics.opt_banner(
+    opts, 'Simulates incomplete (fragmented) drafts from complete genomes',
+    "#{File.basename($0)} -i in.fasta -o out.fasta -c 0.5 [options]"
+  )
+
+  opts.separator 'Mandatory'
+  opts.on(
+    '-i', '--in FILE',
+    'Path to the FastA file containing the complete sequences',
+    'Supports compression with .gz extension, use - for STDIN'
+  ) { |v| o[:in] = v }
+  opts.on(
+    '-o', '--out FILE', 'Path to the FastA to create',
+    'Supports compression with .gz extension, use - for STDOUT'
+  ) { |v| o[:out] = v }
+  opts.on(
+    '-c', '--completeness FLOAT',
+    'Fraction of genome completeness to simulate from 0 to 1'
+  ) { |v| o[:completeness] = v.to_f }
+
+  opts.separator ''
+  opts.separator 'Options'
+  opts.on(
+    '-m', '--minlen INT',
+    "Minimum fragment length to report. By default: #{o[:minlen]}"
+  ) { |v| o[:minlen] = v.to_i }
+  opts.on(
+    '-s', '--sorted', 'Keep fragments sorted as in the input file',
+    'By default, fragments are shuffled'
+  ) { |v| o[:shuffle] = !v }
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)') { o[:q] = true }
+  opts.on('-h', '--help', 'Display this screen') { puts opts ; exit }
+  opts.separator ''
+end.parse!
+
+raise Enveomics::OptionError.new('-i is mandatory') if o[:in].nil?
+raise Enveomics::OptionError.new('-o is mandatory') if o[:out].nil?
+raise Enveomics::OptionError.new('-c is mandatory') if o[:completeness].nil?
+
+begin
+  # Read input sequences
+  g_id  = []
+  g_seq = []
+  ifh = reader(o[:in])
+  id = ''
+  ifh.each_line do |ln|
+    if ln =~ /^>(\S*)/
+      g_id  << $1
+      g_seq << ''
+    else
+      g_seq[g_seq.size - 1] += ln.gsub(/[^A-Za-z]/, '')
+    end
+  end
+  ifh.close
+
+  # Fragment genomes
+  f = {}
+  binlen = [1, (o[:minlen].to_f/(1.5**2)).ceil].max
+  p = [0.001, [1.0, 1.0 - (o[:completeness]/1.25 + 0.1)].min].max
+  while !g_seq.empty?
+    id  = g_id.shift
+    seq = g_seq.shift
+    gL  = seq.length
+    while !seq.empty?
+      rand_x =
+        Enveomics::Stats.r_geom(p).to_f + Enveomics::Stats.r_unif(-0.5, 0.5)
+      fL = [0, (rand_x * binlen).round].max
+      f["#{f.size+1}_#{id}"] = seq[0, fL] if fL >= o[:minlen]
+      seq = seq[(fL + 1) .. -1]
+      seq = '' if seq.nil?
+    end
+  end
+
+  # Save output
+  k = f.keys
+  k.shuffle! if o[:shuffle]
+  ofh = writer(o[:out])
+  k.each do |id|
+    ofh.puts ">#{id}"
+    ofh.puts f[id].gsub(/(\S{50})/, "\\1\n")
+  end
+  ofh.close
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end
+

data/utils/enveomics/Scripts/FastA.gc.pl

@@ -0,0 +1,42 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M Rodriguez-R
+# @update  Mar-23-2016
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+
+$#ARGV>=0 or die "
+Usage:
+   $0 seqs.fa... > gc.txt
+
+   seqs.fa	One or more FastA files.
+   gc.txt	A table with the G+C content of the sequences.
+
+";
+
+for my $fa (@ARGV){
+   open FA, "<", $fa or die "Cannot open file: $fa: $!\n";
+   my $def = "";
+   my $len = 0;
+   my $gc  = 0;
+   while(<FA>){
+      next if /^;/;
+      if(m/^>(\S*)/){
+         print "$def\t".($gc/$len)."\n" if $len;
+	 $def = $1;
+	 $len = 0;
+	 $gc  = 0;
+      }else{
+         s/[^ACTGactg]//g;
+	 $len += length $_;
+	 s/[^GC]//g;
+	 $gc  += length $_;
+      }
+   }
+   print "$def\t".($gc/$len)."\n" if $len;
+   close FA;
+}
+

data/utils/enveomics/Scripts/FastA.interpose.pl

@@ -0,0 +1,93 @@
+#!/usr/bin/env perl
+
+# @author Luis M. Rodriguez-R
+# @license artistic license 2.0
+
+use strict;
+use warnings;
+use Symbol;
+
+my $HELP = <<HELP
+
+  Description:
+    Interposes sequences in FastA format from two files into one output file.
+    If more than two files are provided, the script will interpose all the input
+    files.
+    Note that this script will check for the consistency of the names (assuming
+    a pair of related reads contains the same name varying only in a trailing
+    slash (/) followed by a digit.  If you want to turn this feature off just
+    set the -T option to zero.  If you want to decrease the sampling period (to
+    speed the script up) or increase it (to make it more sensitive to errors)
+    just change the -T option accordingly.
+
+  Usage:
+    $0 [-T <int> ]<output_fasta> <input_fasta_1> <input_fasta_2> [additional input files...]
+
+  Where,
+    -T <int>		: Optional.  Integer indicating the sampling period for
+    			  names evaluation (see Description above).
+			  By default: 1000.
+    output_fasta	: Output file
+    input_fasta_1	: First FastA file
+    input_fasta_2	: Second FastA file
+    ... 		: Any additional FastA files (or none)
+
+HELP
+;
+my $eval_T = 1000;
+if(exists $ARGV[0] and exists $ARGV[1] and $ARGV[0] eq '-T'){
+   $eval_T = $ARGV[1]+0;
+   shift @ARGV;
+   shift @ARGV;
+}
+my $out = shift @ARGV;
+my @in = @ARGV;
+$/ = "\n>";
+
+die $HELP unless $out and $#in >= 1;
+open OUT, ">", $out or die "Unable to write on $out: $!\n";
+print "Output file: $out\n";
+
+my @in_fh = ();
+
+for my $k (0 .. $#in) {
+   $in_fh[$k] = gensym;
+   open $in_fh[$k], "<", $in[$k] or die "Unable to read $in[$k]: $!\n";
+   print "Input file: $in[$k]\n";
+}
+
+my $i = 0;
+my $frl;
+LINE: while(1){
+   my $name = "";
+   print STDERR "\rEntry: $i    " unless $i % 1000;
+   FILE: for my $k (0 .. $#in_fh){
+      my $ln = readline($in_fh[$k]);
+      last LINE if $k==0 and not defined $ln;
+      defined $ln or die "Impossible to read next entry ($.) from $in[$k]: $!\n";
+      $ln =~ s/^\>?/>/;
+      $ln =~ s/\>$//;
+      $ln =~ s/^;.*//gm;
+      if($eval_T and not $i % $eval_T){
+	 unless($name){
+	    $ln =~ m/^>(.*?)[\/ \\_]\d+/ or die "Impossible to evaluate names!\n offending entry:\n$ln\n";
+	    $name = $1;
+	 }
+	 die "Inconsistent name!\n base name is $name\n offending entry is:\n$ln\n" unless $ln =~ /^>$name/;
+      }
+      unless($frl){
+         $ln =~ m/^>.*?\n(.*?)\n/ or die "Unexpected format!\n offending entry:\n$ln\n";
+	 my $i = $ln;
+	 $i =~ s/^>.*?\n//;
+	 $i =~ s/\n//g;
+	 $frl = length $i;
+      }
+      print OUT $ln;
+   }
+   $i++;
+}
+print "\rNumber of entries: $i     \nFirst read length: $frl\n";
+close OUT;
+
+for my $k(0..$#in_fh){print "ALERT: The file $in[$k] contains trailing entries\n" if defined readline($in_fh[$k])}
+

data/utils/enveomics/Scripts/FastA.length.pl

@@ -0,0 +1,38 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M Rodriguez-R
+# @update  Oct-07-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+
+$#ARGV>=0 or die "
+Usage:
+   $0 seqs.fa... > length.txt
+
+   seqs.fa	One or more FastA files.
+   length.txt	A table with the lengths of the sequences.
+
+";
+
+for my $fa (@ARGV){
+   open FA, "<", $fa or die "Cannot open file: $fa: $!\n";
+   my $def = '';
+   my $len = 0;
+   while(<FA>){
+      next if /^;/;
+      if(m/^>(\S+)\s?/){
+         print "$def\t$len\n" if $def;
+	 $def = $1;
+	 $len = 0;
+      }else{
+         s/[^A-Za-z]//g;
+	 $len+= length $_;
+      }
+   }
+   print "$def\t$len\n" if $def;
+   close FA;
+}
+

data/utils/enveomics/Scripts/FastA.mask.rb

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+o = {x: 'N', trim: false, wrap: 70}
+ARGV << '-h' if ARGV.empty?
+OptionParser.new do |opts|
+  opts.banner = "
+Mask sequence region(s) in a FastA file.
+
+Usage: #{$0} [options]"
+  opts.separator ''
+  opts.separator 'Mandatory'
+  opts.on('-i', '--in FILE', 'Input FastA file.'){ |v| o[:in] = v }
+  opts.on('-o', '--out FILE', 'Output FastA file.'){ |v| o[:out] = v }
+  opts.on('-r', '--regions REG1,REG2,...', Array,
+    'Regions to mask separated by commas.',
+    'Each region must be in the format "sequence_id:from..to"'
+    ){ |v| o[:reg] = v }
+  opts.separator ''
+  opts.separator 'Options'
+  opts.on('-x', '--symbol CHAR',
+    'Character used to mask the region(s)',
+    "By default: #{o[:x]}."){ |v| o[:x] = v }
+  opts.on('-t', '--trim',
+    'Trim masked regions extending to the edge of a sequence'
+    ){ |v| o[:trim] = v }
+  opts.on('-w', '--wrap INT',
+    'Line length to wrap sequences. Use 0 to generate 1-line sequences.',
+    "By default: #{o[:wrap]}."){ |v| o[:wrap] = v.to_i }
+  opts.on('-h', '--help', 'Display this screen.') do
+    puts opts
+    exit
+  end
+  opts.separator ''
+end.parse!
+abort '-i is mandatory' if o[:in].nil?
+abort '-o is mandatory' if o[:out].nil?
+abort '-r is mandatory' if o[:reg].nil?
+
+def wrap_width(txt, len)
+  return "" if txt.empty?
+  return "#{txt}\n" if len==0
+  txt.gsub(/(.{1,#{len}})/,"\\1\n")
+end
+
+# Read input sequences
+sq = {}
+File.open(o[:in], 'r') do |ifh|
+  bf = ''
+  ifh.each('>') do |i|
+    (dln, seq) = i.split(/[\n\r]+/, 2)
+    next if seq.nil?
+    id = dln.gsub(/\s.*/,  '')
+    seq.gsub!(/[\s>]/, '')
+    sq[id] = [dln, seq]
+  end
+end
+
+# Parse coordinates and mask regions
+last_id = nil
+o[:reg].each do |i|
+  m = i.match(/^(?:(.+):)?(\d+)\.\.(\d+)$/) or
+    abort "Unexpected region format: #{i}"
+  r = [m[1], m[2].to_i-1, m[3].to_i-1]
+  if r[0].nil?
+    abort "Region missing sequence ID: #{i}" if last_id.nil?
+    r[0] = last_id
+  end
+  last_id = r[0]
+  sq[r[0]] or abort "Cannot find sequence #{r[0]}"
+  r[1] <= r[2] or abort "Malformed range: #{i}"
+  if r[1] < 0 or r[2] > sq[r[0]][1].size
+    abort "Range extends beyond the edge of the sequence: #{i}"
+  end
+  sq[r[0]][1][r[1] .. r[2]] = o[:x]*(1+r[2]-r[1])
+end
+
+# Trim sequences and generate output
+ofh = File.open(o[:out], 'w')
+sq.each do |_k,v|
+  ofh.puts ">#{v[0]}"
+  if o[:trim]
+    v[1].gsub!(/^#{o[:x]}+/,'')
+    v[1].gsub!(/#{o[:x]}+$/,'')
+  end
+  ofh.print wrap_width(v[1], o[:wrap])
+end
+ofh.close
+

data/utils/enveomics/Scripts/FastA.per_file.pl

@@ -0,0 +1,36 @@
+#!/usr/bin/env perl
+#
+# @author  Luis M Rodriguez-R
+# @update  Mar-17-2016
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+
+$#ARGV>=1 or die "
+Usage:
+   $0 outdir seqs.fa...
+
+   outdir	Output directory for the individual files.
+   seqs.fa	One or more FastA files.
+
+";
+
+my $dir = shift @ARGV;
+
+for my $fa (@ARGV){
+   open FA, "<", $fa or die "Cannot open file: $fa: $!\n";
+   my $file = '';
+   while(<FA>){
+      next if /^;/;
+      if(m/^>(\S+)\s?/){
+	 close ONE if $file;
+         $file = $dir."/".$1.".fasta";
+	 open ONE, ">", $file or die "Cannot open file: $file: $!\n";
+      }
+      print ONE $_ if $file;
+   }
+   close ONE if $file;
+}
+