miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Scripts/FastA.tag.rb

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+
+# @author  Luis M. Rodriguez-R
+# @license artistic license 2.0
+
+require 'optparse'
+
+o = {q: false, p: '', s: '', d: false}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+  opts.banner = "
+Generates easy-to-parse tagged reads from FastA files.
+
+Usage: #{$0} [options]"
+  opts.separator ''
+  opts.separator 'Mandatory'
+  opts.on('-i', '--in FILE',
+    'Path to the FastA file containing the sequences.'){ |v| o[:in] = v }
+  opts.on('-o', '--out FILE',
+    'Path to the FastA to create.'){ |v| o[:out] = v }
+  opts.separator ''
+  opts.separator 'ID options'
+  opts.on('-p', '--prefix STR', 'Prefix to use in all IDs.'){ |v| o[:p] = v }
+  opts.on('-s', '--suffix STR', 'Suffix to use in all IDs.'){ |v| o[:s] = v }
+  opts.on('-d', '--defline',
+    'Keep the original defline after a space.'){ o[:d] = true }
+  opts.on('-l', '--list FILE',
+    'Reads a list of IDS.'){ |v| o[:l] = v }
+  opts.separator ''
+  opts.separator 'Other Options'
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)'){ o[:q] = true }
+  opts.on('-h', '--help', 'Display this screen') do
+    puts opts
+    exit
+  end
+  opts.separator ''
+end.parse!
+abort '-i is mandatory' if o[:in].nil?
+abort '-o is mandatory' if o[:out].nil?
+   
+begin
+  list = o[:l].nil? ? nil :
+    File.readlines(o[:l]).map{ |i| i.chomp.gsub(/^>/, '') }
+  ofh = File.open(o[:out], 'w')
+  i = 0
+  File.open(o[:in], 'r') do |ifh|
+    ifh.each do |ln|
+      ln.chomp!
+      next if ln =~ /^;/
+      unless /^>/.match(ln).nil?
+        i += 1
+        new_id = o[:l].nil? ? i : list.shift
+        ofh.puts ">#{o[:p]}#{new_id}#{o[:s]}#{o[:d]?" #{ln[1, ln.size-1]}":''}"
+      else
+        ofh.puts ln
+      end
+    end
+  end
+  ofh.close
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end
+

data/utils/enveomics/Scripts/FastA.toFastQ.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'zlib'
+
+o = { qual: 31, encoding: 33 }
+ARGV << '-h' if ARGV.empty?
+OptionParser.new do |opts|
+  opts.banner = "
+Creates a FastQ-compliant file from a FastA file.
+
+Usage: #{$0} [options]"
+  opts.separator ''
+  opts.separator 'Options'
+  opts.on(
+    '-i', '--in FILE', 'Input FastA file (supports .gz compression)'
+  ) { |v| o[:in] = v }
+  opts.on(
+    '-o', '--out FILE', 'Output FastQ file (supports .gz compression)'
+  ) { |v| o[:out] = v }
+  opts.on(
+    '-q', '--quality INT', Integer,
+    'PHRED quality score to use (fixed), in the range [-5, 41]',
+    "By default: #{o[:qual]}"
+  ) { |v| o[:qual] = v }
+  opts.on(
+    '--encoding INT', Integer,
+    "Base encoding (33 or 64). By default: #{o[:encoding]}"
+  ) { |v| o[:encoding] = v }
+  opts.on('-h', '--help', 'Display this screen.') do
+    puts opts
+    exit
+  end
+  opts.separator ''
+end.parse!
+abort '-i is mandatory' if o[:in].nil?
+abort '-o is mandatory' if o[:out].nil?
+abort '-q must be in the range -5 .. 41' if o[:qual] < -5 || o[:qual] > 41
+
+# Determine quality character
+$qchar = (o[:qual] + o[:encoding]).chr
+
+# Create file handlers
+ifh = o[:in] =~ /\.gz$/ ?
+  Zlib::GzipReader.open(o[:in]) : File.open(o[:in], 'r')
+ofh = o[:out] =~ /\.gz$/ ?
+  Zlib::GzipWriter.open(o[:out]) : File.open(o[:out], 'w')
+
+def print_seq(ofh, id, seq)
+  ofh.puts "@#{id}", seq, '+', $qchar * seq.length unless seq.empty?
+end
+
+# Generate FastQ
+id = ''
+seq = ''
+ifh.each_line do |ln|
+  next if ln =~ /^;/
+  if ln =~ /^>(.*)/
+    print_seq(ofh, id, seq)
+    seq = ''
+    id = $1
+  else
+    seq += ln.chomp.upcase.gsub(/[^A-Z]/,'')
+  end
+end
+print_seq(ofh, id, seq)
+ofh.close
+ifh.close
+

data/utils/enveomics/Scripts/FastA.wrap.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+o = {wrap:70}
+ARGV << "-h" if ARGV.empty?
+OptionParser.new do |opts|
+  opts.banner = "
+Wraps sequences in a FastA to a given line length.
+
+Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Options"
+  opts.on("-i", "--in FILE", "Input FastA file."){ |v| o[:in] = v }
+  opts.on("-o", "--out FILE", "Output FastA file."){ |v| o[:out] = v }
+  opts.on("-w", "--wrap INT",
+    "Line length to wrap sequences. Use 0 to generate 1-line sequences.",
+    "By default: #{o[:wrap]}."){ |v| o[:wrap] = v.to_i }
+  opts.on("-h", "--help", "Display this screen.") do
+    puts opts
+    exit
+  end
+  opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+
+def wrap_width(txt, len)
+  return "" if txt.empty?
+  return "#{txt}\n" if len==0
+  txt.gsub(/(.{1,#{len}})/,"\\1\n")
+end
+
+ofh = File.open(o[:out], "w")
+File.open(o[:in], "r") do |ifh|
+  bf = ""
+  ifh.each_line do |ln|
+    if ln =~ /^>/
+      ofh.print wrap_width(bf, o[:wrap])
+      ofh.puts ln
+      bf = ""
+    else
+      ln.chomp!
+      bf << ln
+    end
+  end
+  ofh.print wrap_width(bf, o[:wrap])
+end
+ofh.close

data/utils/enveomics/Scripts/FastQ.filter.pl

@@ -0,0 +1,54 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+sub HELP_MESSAGE { die "
+.Description:
+   Extracts a subset of sequences from a FastQ file.
+
+.Usage: $0 [options] list.txt seqs.fq > subset.fq
+   
+   [options]
+   -r		Reverse list.  Extracts sequences NOT present in the list.
+   -q		Runs quietly.
+   -h		Prints this message and exits.
+
+   [mandatory]
+   list.txt	List of sequences to extract.
+   seqs.fq	FastQ file containing the superset of sequences.
+   subset.fq	FastQ file to be created.
+
+" }
+
+my %o=();
+getopts('rhq', \%o);
+my($list, $fq) = @ARGV;
+($list and $fq) or &HELP_MESSAGE;
+$o{h} and &HELP_MESSAGE;
+
+print STDERR "Reading list.\n" unless $o{q};
+open LI, "<", $list or die "Cannot read file: $list: $!\n";
+my %li = map { chomp; $_ => 1 } <LI>;
+close LI;
+
+print STDERR "Filtering FastQ.\n" unless $o{q};
+open FQ, "<", $fq or die "Cannot read file: $fq: $!\n";
+my $good = 0;
+while(my $ln = <FQ>){
+   my @ln = ();
+   $ln[$_] = <FQ> for 0 .. 2;
+   chomp $ln;
+   if($ln =~ m/^@((\S+).*)/){ $good = (exists $li{$1} or exists $li{">$1"} or exists $li{"\@$1"} or exists $li{$2} or exists $li{$ln}) }
+   elsif($ln =~ m/^>/){ $good=0; print STDERR "Warning: Non-cannonical defline, line $.: $ln\n" }
+   else{ $good=$o{r}; print STDERR "Warning: Non-cannonical defline, line $.: $ln\n" }
+   print "".join("", "$ln\n", @ln) if (($good and not $o{r}) or ($o{r} and not $good));
+}
+close FQ;
+

data/utils/enveomics/Scripts/FastQ.interpose.pl

@@ -0,0 +1,90 @@
+#!/usr/bin/env perl
+
+# @author Luis M. Rodriguez-R
+# @license artistic license 2.0
+
+use strict;
+use warnings;
+use Symbol;
+
+my $HELP = <<HELP
+
+  Description:
+    Interposes sequences in FastQ format from two files into one output file.
+    If more than two files are provided, the script will interpose all the input
+    files.
+    Note that this script will check for the consistency of the names (assuming
+    a pair of related reads contains the same name varying only in a trailing
+    slash (/) followed by a digit.  If you want to turn this feature off just
+    set the -T option to zero.  If you want to decrease the sampling period (to
+    speed the script up) or increase it (to make it more sensitive to errors)
+    just change the -T option accordingly.
+
+  Usage:
+    $0 [-T <int> ]<output_fastq> <input_fastq_1> <input_fastq_2> [additional input files...]
+
+  Where,
+    -T <int>		: Optional.  Integer indicating the sampling period for
+    			  names evaluation (see Description above).
+			  By default: 1000.
+    output_fastq	: Output file
+    input_fastq_1	: First FastQ file
+    input_fastq_2	: Second FastQ file
+    ... 		: Any additional FastQ files (or none)
+
+HELP
+;
+my $eval_T = 1000;
+if(exists $ARGV[0] and exists $ARGV[1] and $ARGV[0] eq '-T'){
+   $eval_T = $ARGV[1]+0;
+   shift @ARGV;
+   shift @ARGV;
+}
+my $out = shift @ARGV;
+my @in = @ARGV;
+
+
+die $HELP unless $out and $#in >= 1;
+open OUT, ">", $out or die "Unable to write on $out: $!\n";
+print "Output file: $out\n";
+
+my @in_fh = ();
+
+for my $k (0 .. $#in) {
+   $in_fh[$k] = gensym;
+   open $in_fh[$k], "<", $in[$k] or die "Unable to read $in[$k]: $!\n";
+   print "Input file: $in[$k]\n";
+}
+
+my $i = 0;
+my $frl;
+LINE: while(1){
+   my $name = "";
+   print STDERR "\rEntry: $i    " unless $i % 1000;
+   FILE: for my $k (0 .. $#in_fh){
+      my @ln = ();
+      for my $l (0 .. 3){
+	 $ln[$l] = readline($in_fh[$k]);
+	 last LINE if $k==0 and $l==0 and (not defined $ln[$l]);
+	 defined $ln[$l] or die "Impossible to read next entry (line $.) from $in[$k]: $!\n";
+	 chomp $ln[$l];
+      }
+      if($eval_T and not $i % $eval_T){
+	 $ln[0] =~ m/^\@(.*?)\/\d+\s*$/ or die "Impossible to evaluate names!\n offending entry:\n$ln[0]\n";
+	 $name ||= $1;
+	 die "Inconsistent name!\n base name is $name\n offending entry is:\n$ln[0]\n" unless $1 eq $name;
+      }
+      unless($frl){
+	 $ln[0] =~ /^@/ or die "Unexpected format! (missing @)\n offending entry: $ln[0].\n";
+         $ln[2] =~ /^\+/ or die "Unexpected format! (missing +)\n offending entry: $ln[0].\n";
+	 $frl = length $ln[1];
+      }
+      print OUT "".join("\n", @ln, "");
+   }
+   $i++;
+}
+print "\rNumber of entries: $i     \nFirst read length: $frl\n";
+close OUT;
+
+for my $k(0..$#in_fh){print "ALERT: The file $in[$k] contains trailing entries\n" if defined readline($in_fh[$k])}
+

data/utils/enveomics/Scripts/FastQ.maskQual.rb

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+
+$VERSION = 1.2
+$:.push File.expand_path('../lib', __FILE__)
+require 'enveomics_rb/enveomics'
+
+o = { q: false, offset: 33, qual: 15, fasta: false }
+OptionParser.new do |opts|
+  opts.version = $VERSION
+  Enveomics.opt_banner(
+    opts, 'Masks low-quality bases in a FastQ file',
+    "#{File.basename($0)} -i in.fastq -o out.fastq [options]"
+  )
+
+  opts.separator 'Mandatory'
+  opts.on(
+    '-i', '--input FILE',
+    'Path to the FastQ file containing the sequences',
+    'Supports compression with .gz extension, use - for STDIN'
+  ) { |v| o[:in] = v }
+  opts.on(
+    '-o', '--out FILE',
+    'Path to the output FastQ file',
+    'Supports compression with .gz extension, use - for STDOUT'
+  ) { |v| o[:out] = v }
+
+  opts.separator ''
+  opts.separator 'Quality Options'
+  opts.on(
+    '-q', '--qual INT', Integer,
+    "Minimum quality score to allow a base, by default: #{o[:qual]}"
+  ) { |v| o[:qual] = v }
+  opts.on(
+    '--offset INT', Integer,
+    "Q-score offset, by default: #{o[:offset]}"
+  ) { |v| o[:offset] = v }
+
+  opts.separator ''
+  opts.separator 'Other Options'
+  opts.on(
+    '-a', '--fasta', 'Output sequences in FastA format'
+  ) { |v| o[:fasta] = v }
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)') { o[:q] = true }
+  opts.on('-h', '--help', 'Display this screen') do
+    puts opts
+    exit
+  end
+  opts.separator ''
+end.parse!
+
+raise Enveomics::OptionError.new('-i is mandatory') if o[:in].nil?
+raise Enveomics::OptionError.new('-o is mandatory') if o[:out].nil?
+$QUIET = o[:q]
+
+# Open in/out files
+say 'Reading FastQ file'
+ifh = reader(o[:in])
+ofh = writer(o[:out])
+
+# Parse and mask
+entry = []
+lno = 0
+ifh.each_line do |ln|
+  lno += 1 # <- Gzip doesn't support $.
+  case lno % 4
+  when 1
+    ln =~ /^@(\S+)/ or
+      raise Enveomics::ParseError.new("Unexpected defline format: #{ln}")
+    entry << ln
+  when 2, 3
+    entry << ln
+  when 0
+    entry << ln
+    q = entry[3].chomp.split('').map { |i| (i.ord - o[:offset]) }
+    q.map { |i| i < o[:qual] }.each_with_index { |i, k| entry[1][k] = 'N' if i }
+    ofh.puts(o[:fasta] ? [entry[0].gsub(/^@/, '>'), entry[1]] : entry)
+    entry = []
+  end
+end
+
+# Finalize
+say "  Lines: #{lno}"
+unless entry.empty?
+  raise Enveomics::ParseError.new('Unexpected trailing lines in FastQ')
+end
+say "  Sequences: #{lno / 4}"
+ifh.close
+ofh.close
+

data/utils/enveomics/Scripts/FastQ.offset.pl

@@ -0,0 +1,90 @@
+#!/usr/bin/env perl
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update Mar-23-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+
+my ($in, $off, $force) = @ARGV;
+$in or die "
+.Description:
+   There are several FastQ formats (see http://en.wikipedia.org/wiki/FASTQ_format).
+   This script takes a FastQ in any of them, identifies the type of FastQ (this is,
+   the offset), and generates a FastQ with the given offset.  Note that Solexa+64
+   FastQ can cause problematic values when using the offset 33, since there is no
+   equivalent in Phred+33 for negative values (the range of Solexa+64 is -5 to 40).
+
+.Usage:
+   $0 in.fastq[ offset[ force]] > out.fastq
+
+   in.fastq	Input file in FastQ format (range is automatically detected).
+   offset	(optional) Offset to use for the output.  Use 0 (zero) to detect
+   		the input format and exit.  By default: 33.
+   force	(optional) If true, turns errors into warnings and continues.
+   		Out-of-range values are set to the closest range limit.
+   out.fastq	Output file in FastQ format with the specified offset.
+
+";
+
+$off = 33 unless defined $off;
+
+my $in_off = 0;
+open IN, "<", $in or die "Cannot read file: $in: $!\n";
+GUESS_FORMAT: while(<IN>){
+   unless($.%4){
+      chomp;
+      for my $chr (split //){
+         my $o = ord $chr;
+	 if($o < 55){
+	    $in_off = 33;
+	    last GUESS_FORMAT;
+	 }elsif($o > 80){
+	    $in_off = 64;
+	    last GUESS_FORMAT;
+	 }
+      }
+   }
+}
+close IN;
+print STDERR "Detected input offset: Phred+$in_off\n";
+exit unless $off;
+
+my $Solexa64=0;
+die "Couldn't guess input format.\n" unless $in_off;
+open IN, "<", $in or die "Cannot read file: $in: $!\n";
+while(<IN>){
+   if($in_off==$off or $.%4){
+      print $_;
+   }else{
+      chomp;
+      for my $chr (split //){
+         my $score = ord($chr) - $in_off;
+	 my $err = '';
+	 if($score < -5){
+	    $err = "Out-of-range value $chr ($score) in line $..\n";
+	    $score = $off==64 ? -5 : 0;
+	 }elsif(!$Solexa64 and $score < 0){
+	    if($in_off==64){
+	       print STDERR "Format variant: Solexa+64\n";
+	       $Solexa64 = 1;
+	    }else{
+	       $err = "Out-of-range value $chr ($score) in line $..\n";
+	       $score = 0;
+	    }
+	 }elsif($score>41){
+	    $err = "Out-of-range value $chr ($score) in line $..\n";
+	    $score = 41;
+	 }
+	 if($err){
+	    if($force){ warn $err } else { die $err }
+	 }
+	 print chr( $score + $off );
+      }
+      print "\n";
+   }
+}
+close IN;
+