miga-base 0.7.26.0 → 0.7.26.1

data/utils/enveomics/Scripts/FastA.wrap.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+o = {wrap:70}
+ARGV << "-h" if ARGV.empty?
+OptionParser.new do |opts|
+  opts.banner = "
+Wraps sequences in a FastA to a given line length.
+
+Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Options"
+  opts.on("-i", "--in FILE", "Input FastA file."){ |v| o[:in] = v }
+  opts.on("-o", "--out FILE", "Output FastA file."){ |v| o[:out] = v }
+  opts.on("-w", "--wrap INT",
+    "Line length to wrap sequences. Use 0 to generate 1-line sequences.",
+    "By default: #{o[:wrap]}."){ |v| o[:wrap] = v.to_i }
+  opts.on("-h", "--help", "Display this screen.") do
+    puts opts
+    exit
+  end
+  opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+
+def wrap_width(txt, len)
+  return "" if txt.empty?
+  return "#{txt}\n" if len==0
+  txt.gsub(/(.{1,#{len}})/,"\\1\n")
+end
+
+ofh = File.open(o[:out], "w")
+File.open(o[:in], "r") do |ifh|
+  bf = ""
+  ifh.each_line do |ln|
+    if ln =~ /^>/
+      ofh.print wrap_width(bf, o[:wrap])
+      ofh.puts ln
+      bf = ""
+    else
+      ln.chomp!
+      bf << ln
+    end
+  end
+  ofh.print wrap_width(bf, o[:wrap])
+end
+ofh.close

data/utils/enveomics/Scripts/FastQ.filter.pl

@@ -0,0 +1,54 @@
+#!/usr/bin/env perl
+#
+# @author: Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+use warnings;
+use strict;
+use Getopt::Std;
+
+sub HELP_MESSAGE { die "
+.Description:
+   Extracts a subset of sequences from a FastQ file.
+
+.Usage: $0 [options] list.txt seqs.fq > subset.fq
+   
+   [options]
+   -r		Reverse list.  Extracts sequences NOT present in the list.
+   -q		Runs quietly.
+   -h		Prints this message and exits.
+
+   [mandatory]
+   list.txt	List of sequences to extract.
+   seqs.fq	FastQ file containing the superset of sequences.
+   subset.fq	FastQ file to be created.
+
+" }
+
+my %o=();
+getopts('rhq', \%o);
+my($list, $fq) = @ARGV;
+($list and $fq) or &HELP_MESSAGE;
+$o{h} and &HELP_MESSAGE;
+
+print STDERR "Reading list.\n" unless $o{q};
+open LI, "<", $list or die "Cannot read file: $list: $!\n";
+my %li = map { chomp; $_ => 1 } <LI>;
+close LI;
+
+print STDERR "Filtering FastQ.\n" unless $o{q};
+open FQ, "<", $fq or die "Cannot read file: $fq: $!\n";
+my $good = 0;
+while(my $ln = <FQ>){
+   my @ln = ();
+   $ln[$_] = <FQ> for 0 .. 2;
+   chomp $ln;
+   if($ln =~ m/^@((\S+).*)/){ $good = (exists $li{$1} or exists $li{">$1"} or exists $li{"\@$1"} or exists $li{$2} or exists $li{$ln}) }
+   elsif($ln =~ m/^>/){ $good=0; print STDERR "Warning: Non-cannonical defline, line $.: $ln\n" }
+   else{ $good=$o{r}; print STDERR "Warning: Non-cannonical defline, line $.: $ln\n" }
+   print "".join("", "$ln\n", @ln) if (($good and not $o{r}) or ($o{r} and not $good));
+}
+close FQ;
+

data/utils/enveomics/Scripts/FastQ.interpose.pl

@@ -0,0 +1,90 @@
+#!/usr/bin/env perl
+
+# @author Luis M. Rodriguez-R
+# @license artistic license 2.0
+
+use strict;
+use warnings;
+use Symbol;
+
+my $HELP = <<HELP
+
+  Description:
+    Interposes sequences in FastQ format from two files into one output file.
+    If more than two files are provided, the script will interpose all the input
+    files.
+    Note that this script will check for the consistency of the names (assuming
+    a pair of related reads contains the same name varying only in a trailing
+    slash (/) followed by a digit.  If you want to turn this feature off just
+    set the -T option to zero.  If you want to decrease the sampling period (to
+    speed the script up) or increase it (to make it more sensitive to errors)
+    just change the -T option accordingly.
+
+  Usage:
+    $0 [-T <int> ]<output_fastq> <input_fastq_1> <input_fastq_2> [additional input files...]
+
+  Where,
+    -T <int>		: Optional.  Integer indicating the sampling period for
+    			  names evaluation (see Description above).
+			  By default: 1000.
+    output_fastq	: Output file
+    input_fastq_1	: First FastQ file
+    input_fastq_2	: Second FastQ file
+    ... 		: Any additional FastQ files (or none)
+
+HELP
+;
+my $eval_T = 1000;
+if(exists $ARGV[0] and exists $ARGV[1] and $ARGV[0] eq '-T'){
+   $eval_T = $ARGV[1]+0;
+   shift @ARGV;
+   shift @ARGV;
+}
+my $out = shift @ARGV;
+my @in = @ARGV;
+
+
+die $HELP unless $out and $#in >= 1;
+open OUT, ">", $out or die "Unable to write on $out: $!\n";
+print "Output file: $out\n";
+
+my @in_fh = ();
+
+for my $k (0 .. $#in) {
+   $in_fh[$k] = gensym;
+   open $in_fh[$k], "<", $in[$k] or die "Unable to read $in[$k]: $!\n";
+   print "Input file: $in[$k]\n";
+}
+
+my $i = 0;
+my $frl;
+LINE: while(1){
+   my $name = "";
+   print STDERR "\rEntry: $i    " unless $i % 1000;
+   FILE: for my $k (0 .. $#in_fh){
+      my @ln = ();
+      for my $l (0 .. 3){
+	 $ln[$l] = readline($in_fh[$k]);
+	 last LINE if $k==0 and $l==0 and (not defined $ln[$l]);
+	 defined $ln[$l] or die "Impossible to read next entry (line $.) from $in[$k]: $!\n";
+	 chomp $ln[$l];
+      }
+      if($eval_T and not $i % $eval_T){
+	 $ln[0] =~ m/^\@(.*?)\/\d+\s*$/ or die "Impossible to evaluate names!\n offending entry:\n$ln[0]\n";
+	 $name ||= $1;
+	 die "Inconsistent name!\n base name is $name\n offending entry is:\n$ln[0]\n" unless $1 eq $name;
+      }
+      unless($frl){
+	 $ln[0] =~ /^@/ or die "Unexpected format! (missing @)\n offending entry: $ln[0].\n";
+         $ln[2] =~ /^\+/ or die "Unexpected format! (missing +)\n offending entry: $ln[0].\n";
+	 $frl = length $ln[1];
+      }
+      print OUT "".join("\n", @ln, "");
+   }
+   $i++;
+}
+print "\rNumber of entries: $i     \nFirst read length: $frl\n";
+close OUT;
+
+for my $k(0..$#in_fh){print "ALERT: The file $in[$k] contains trailing entries\n" if defined readline($in_fh[$k])}
+

data/utils/enveomics/Scripts/FastQ.offset.pl

@@ -0,0 +1,90 @@
+#!/usr/bin/env perl
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @update Mar-23-2015
+# @license artistic license 2.0
+#
+
+use warnings;
+use strict;
+
+my ($in, $off, $force) = @ARGV;
+$in or die "
+.Description:
+   There are several FastQ formats (see http://en.wikipedia.org/wiki/FASTQ_format).
+   This script takes a FastQ in any of them, identifies the type of FastQ (this is,
+   the offset), and generates a FastQ with the given offset.  Note that Solexa+64
+   FastQ can cause problematic values when using the offset 33, since there is no
+   equivalent in Phred+33 for negative values (the range of Solexa+64 is -5 to 40).
+
+.Usage:
+   $0 in.fastq[ offset[ force]] > out.fastq
+
+   in.fastq	Input file in FastQ format (range is automatically detected).
+   offset	(optional) Offset to use for the output.  Use 0 (zero) to detect
+   		the input format and exit.  By default: 33.
+   force	(optional) If true, turns errors into warnings and continues.
+   		Out-of-range values are set to the closest range limit.
+   out.fastq	Output file in FastQ format with the specified offset.
+
+";
+
+$off = 33 unless defined $off;
+
+my $in_off = 0;
+open IN, "<", $in or die "Cannot read file: $in: $!\n";
+GUESS_FORMAT: while(<IN>){
+   unless($.%4){
+      chomp;
+      for my $chr (split //){
+         my $o = ord $chr;
+	 if($o < 55){
+	    $in_off = 33;
+	    last GUESS_FORMAT;
+	 }elsif($o > 80){
+	    $in_off = 64;
+	    last GUESS_FORMAT;
+	 }
+      }
+   }
+}
+close IN;
+print STDERR "Detected input offset: Phred+$in_off\n";
+exit unless $off;
+
+my $Solexa64=0;
+die "Couldn't guess input format.\n" unless $in_off;
+open IN, "<", $in or die "Cannot read file: $in: $!\n";
+while(<IN>){
+   if($in_off==$off or $.%4){
+      print $_;
+   }else{
+      chomp;
+      for my $chr (split //){
+         my $score = ord($chr) - $in_off;
+	 my $err = '';
+	 if($score < -5){
+	    $err = "Out-of-range value $chr ($score) in line $..\n";
+	    $score = $off==64 ? -5 : 0;
+	 }elsif(!$Solexa64 and $score < 0){
+	    if($in_off==64){
+	       print STDERR "Format variant: Solexa+64\n";
+	       $Solexa64 = 1;
+	    }else{
+	       $err = "Out-of-range value $chr ($score) in line $..\n";
+	       $score = 0;
+	    }
+	 }elsif($score>41){
+	    $err = "Out-of-range value $chr ($score) in line $..\n";
+	    $score = 41;
+	 }
+	 if($err){
+	    if($force){ warn $err } else { die $err }
+	 }
+	 print chr( $score + $off );
+      }
+      print "\n";
+   }
+}
+close IN;
+

data/utils/enveomics/Scripts/FastQ.split.pl

@@ -0,0 +1,53 @@
+#!/usr/bin/env perl
+#
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update Jul-05-2015
+#
+
+use warnings;
+use strict;
+use Symbol;
+
+my ($file, $base, $outN) = @ARGV;
+
+$outN ||= 2;
+($file and $base) or die "
+Usage
+   $0 in_file.fq out_base[ no_files]
+   
+   in_file.fq	Input file in FastA format.
+   out_base	Prefix for the name of the output files.  It will
+   		be appended with .<i>.fastq, where <i> is a consecutive
+		number starting in 1.
+   no_files	Number of files to generate.  By default: 2.
+
+";
+
+
+my @outSym = ();
+for my $i (1 .. $outN){
+   $outSym[$i-1] = gensym;
+   open $outSym[$i-1], ">", "$base.$i.fastq" or die "I can not create the file: $base.$i.fa: $!\n";
+}
+
+
+my($i, $seq) = (-1, '');
+open FILE, "<", $file or die "I can not read the file: $file: $!\n";
+while(my $ln=<FILE>){
+   if($.%4 == 1){
+      print { $outSym[$i % $outN] } $seq if $seq;
+      $i++;
+      $seq = '';
+   }
+   $seq.=$ln;
+}
+print { $outSym[$i % $outN] } $seq if $seq;
+close FILE;
+
+for(my $j=0; $j<$outN; $j++){
+   close $outSym[$j];
+}
+
+print STDERR "Sequences: ".($i+1)."\nFiles: $outN\n";
+

data/utils/enveomics/Scripts/FastQ.tag.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Feb-06-2015
+# @license: artistic license 2.0
+#
+
+require 'optparse'
+
+o = {:q=>FALSE, :p=>"", :s=>""}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "
+Generates easy-to-parse tagged reads from FastQ files.
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-i", "--in FILE", "Path to the FastQ file containing the sequences."){ |v| o[:in] = v }
+   opts.on("-o", "--out FILE", "Path to the FastQ to create."){ |v| o[:out] = v }
+   opts.separator ""
+   opts.separator "ID options"
+   opts.on("-p", "--prefix STR", "Prefix to use in all IDs."){ |v| o[:p] = v }
+   opts.on("-s", "--suffix STR", "Suffix to use in all IDs."){ |v| o[:s] = v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)"){ o[:q] = TRUE }
+   opts.on("-h", "--help", "Display this screen") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+   
+begin
+   ifh = File.open(o[:in], 'r');
+   ofh = File.open(o[:out], 'w');
+   i=0
+   while ln=ifh.gets
+      ln.chomp!
+      if $.%4==1 and not /^@/.match(ln).nil?
+	 i+=1
+	 ofh.puts "@#{o[:p]}#{i}#{o[:s]}"
+      elsif $.%4==2 or $.%4==0
+         ofh.puts ln
+      elsif $.%4==3 and not /^\+/.match(ln).nil?
+         ofh.puts "+"
+      else
+         abort "Impossible to parse line #{$.}: #{ln}.\n"
+      end
+   end
+   ifh.close
+   ofh.close
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/FastQ.test-error.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+o = {q:false, key:2}
+ARGV << '-h' if ARGV.empty?
+OptionParser.new do |opts|
+   opts.banner = "
+Compares the estimated error of sequencing reads (Q-score) with
+observed mismatches (identity against a know reference sequence).
+
+Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Mandatory"
+  opts.on("-f", "--fastq FILE",
+       "Path to the FastQ file containing the sequences."){ |v| o[:fastq] = v }
+  opts.on("-b", "--blast FILE",
+       "Path to the tabular BLAST file mapping reads to reference sequences."
+       ){ |v| o[:blast] = v }
+  opts.on("-o", "--out FILE",
+      "Path to the output tab-delimited file to create."){ |v| o[:out] = v }
+  opts.separator ""
+  opts.separator "Other Options"
+  opts.on("-q", "--quiet", "Run quietly (no STDERR output)"){ o[:q] = TRUE }
+  opts.on("-h", "--help", "Display this screen") do
+    puts opts
+    exit
+  end
+  opts.separator ""
+end.parse!
+abort "-f is mandatory" if o[:fastq].nil?
+abort "-b is mandatory" if o[:blast].nil?
+abort "-o is mandatory" if o[:out].nil?
+
+# Read the Q scores and estimate expected mismatches
+mm = {} # <- Hash with read IDs as key, and arrays as values:
+        #    [ expected mismatches, variance of mismatches, length ]
+$stderr.puts "Reading FastQ file" unless o[:q]
+File.open(o[:fastq], "r") do |fh|
+  id = nil
+  fh.each_line do |ln|
+    case $.%4
+    when 1
+      ln =~ /^@(\S+)/ or raise "Unexpected defline format: #{ln}"
+      id = $1
+      $stderr.print " #{mm.size} reads...\r" unless o[:q]
+    when 0
+      ln.chomp!
+      # I'm assuming ALWAYS Phred+33!!!
+      p = ln.split('').map{ |i| (i.ord - 33).to_f }.map{ |q| 10.0**(-q/10.0) }
+      mu = p.inject(:+)
+      var = p.map{ |i| i*(1.0-i) }.inject(:+)
+      mm[id] = [mu, var, p.size]
+    end
+  end
+  $stderr.puts " Found: #{mm.size} reads." unless o[:q]
+end
+
+ofh = File.open(o[:out], "w")
+ofh.puts %w[id obs_subs obs_id aln_len obs_ins obs_del obs_gap mu var len].join("\t")
+
+# Read Identities and compare against expectation
+$stderr.puts "Reading Tabular BLAST file" unless o[:q]
+File.open(o[:blast], "r") do |fh|
+  k = 0
+  fh.each_line do |ln|
+    r = ln.chomp.split("\t")
+    id = r[0]
+    next if mm[id].nil?
+    k += 1
+    $stderr.print " #{k} alignments...\r" unless o[:q]
+    obs_m = r[4].to_i + (r[6].to_i - 1) + (mm[id][2] - r[7].to_i)
+    obs_del = r[3].to_i - (r[7].to_i - r[6].to_i).abs
+    obs_ins = r[3].to_i - (r[9].to_i - r[8].to_i).abs
+    ofh.puts ([id, obs_m, r[2], r[7].to_i - r[6].to_i + 1,
+          obs_ins, obs_del, r[5]] + mm[id]).join("\t")
+  end
+  $stderr.puts " Found #{k} alignments." unless o[:q]
+end
+
+ofh.close