bio-faster 0.2.2 → 0.4.2

data/lib/bio-faster.rb

@@ -1,6 +1,7 @@
-#
-# Copyright(C) 2011 Francesco Strozzi <francesco.strozzi@gmail.com>
-#
 
-require File.expand_path(File.join(File.dirname(__FILE__),'..','ext','faster'))
+require 'ffi'
+require 'bio/faster/library'
+require 'bio/faster'
+
+
 

data/lib/bio/faster.rb

@@ -0,0 +1,57 @@
+
+#
+# Copyright(C) 2012 Francesco Strozzi <francesco.strozzi@gmail.com>
+#
+
+module Bio
+
+  class Faster
+
+    extend FFI::Library
+
+    ffi_lib Bio::Faster::Library.load
+
+    attr_accessor :file
+    attr_accessor :encoding
+    def initialize(file, encoding = :sanger)
+        self.file = file
+        self.encoding = encoding
+    end
+
+    class FastQRecord < FFI::Struct
+      layout :id, :pointer,
+             :seq, :pointer,
+             :quality, :pointer,
+             :raw_quality, :pointer,
+             :filename, :pointer,
+             :stream, :pointer,
+             :line, :pointer,
+             :bad_chars, :pointer
+
+    end
+
+    attach_function :fastQ_iterator, [FastQRecord, :int], :int
+
+    def each_record
+        raise ArgumentError, "File #{self.file} does not exist" unless File.exists? self.file
+        record = FastQRecord.new
+        scale_factor = nil
+        case self.encoding
+          when :sanger then scale_factor = 33
+          when :solexa then scale_factor = 64
+        end
+        record[:filename] = FFI::MemoryPointer.from_string self.file
+        while (result = Bio::Faster.fastQ_iterator(record,scale_factor)) == 1
+          yield [record[:id].read_string,record[:seq].read_string,record[:quality].read_array_of_int(record[:raw_quality].read_string.length)]
+        end
+        case result
+          when -1 then raise RuntimeError, "Bad formatted FastQ file!"
+          when -2 then raise RuntimeError, "Sequence or quality is truncated!"
+        end
+
+
+    end
+
+
+  end
+end

data/lib/bio/faster/library.rb

@@ -0,0 +1,26 @@
+module Bio
+  class Faster
+    class Library
+
+      require 'rbconfig'
+
+      def self.lib_extension
+        case RbConfig::CONFIG['host_os']
+          when /linux/ then return 'so'
+          when /darwin/ then return 'dylib'
+          else raise NotImplementedError, "Native library is not available for Windows platform"
+        end
+      end
+
+      # Load the correct library for the OS system in use
+      # @return [String] the absolute path for the filename of the shared library
+      # @note this method is called automatically when the module is loaded
+      def self.load
+        path = File.expand_path File.dirname(__FILE__)
+        path.gsub!(/lib\/bio\/faster/,'ext')
+        File.join(path,"libfaster.#{self.lib_extension}")
+      end
+
+    end
+  end
+end

data/spec/fastq_error_spec.rb

@@ -0,0 +1,55 @@
+require 'helper'
+
+describe Bio::Faster do
+
+  describe "#each_record error handling" do
+
+    it "should raise an error if the header is wrong" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_header.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+    it "should raise and error if there is a space in the quality string" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_qual_space.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+    it "should raise and error if there is a tab in the quality string" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_qual_tab.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+    it "should raise and error if there is a v-tab in the quality string" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_qual_vtab.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+    it "should raise and error if there is a space in the sequence string" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_spaces.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+    it "should raise and error if there is a tab in the sequence string" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_tabs.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+    it "should raise and error if there is a delete char in the quality string" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_qual_del.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+    it "should raise and error if there is an escape in the quality string" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_qual_escape.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+    it "should raise and error if there is a unit separator char in the quality string" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_qual_unit_sep.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+    it "should raise an error if sequence and quality are truncated or different in length" do
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_trunc_at_qual.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_trunc_at_seq.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_qual_null.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_long_qual.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_trunc_in_seq.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+      expect {Bio::Faster.new(File.join(TEST_DATA+"/errors/","error_trunc_in_qual.fastq")).each_record {|seq|}}.to raise_error(RuntimeError)
+    end
+
+
+  end
+
+end

data/spec/parser_spec.rb

@@ -2,136 +2,48 @@ require 'helper'
 
 describe Bio::Faster do
   
-  describe "#parser" do
-  
-    it "reads both Fasta and FastQ format files" do
-      res = Bio::Faster.parse(File.join(TEST_DATA,"sample.fastq")) {|seq|}
-      res.should == true
-      res = Bio::Faster.parse(File.join(TEST_DATA,"sample.fasta")) {|seq|}
-      res.should == true
-    end
-    
-    it "needs a valid block to parse files" do
-      expect { Bio::Faster.parse(File.join(TEST_DATA,"sample.fastq")) }.to raise_error(ArgumentError)
-    end
-  
-    it "throws an error if file does not exists" do
-      expect { Bio::Faster.parse(File.join(TEST_DATA,"dummy.fastq")) }.to raise_error(ArgumentError)
-    end
-    
-    it "returns an array with id, comment, sequence and an array with qualities (FastQ only)" do
-      Bio::Faster.parse(File.join(TEST_DATA,"sample.fastq")) do |seq| 
-        seq.class.should == Array
-        seq.size.should == 4
-        seq[-1].class.should == Array
-      end
-      
-      Bio::Faster.parse(File.join(TEST_DATA,"sample.fasta")) do |seq| 
+  describe "#each_record" do
+
+    it "should read a FastQ file returning an array with sequence data" do
+      Bio::Faster.new(TEST_DATA+"/formats/illumina_full_range_as_illumina.fastq").each_record do |seq|
         seq.class.should == Array
-        seq.size.should == 3
       end
-      
-    end
-    
-    it "reads gzipped files" do
-      res = Bio::Faster.parse(File.join(TEST_DATA,"sample.fastq")) {|seq|}
-      res.should == true
-    end
-    
-    it "parse correctly FastQ files" do
-      faster_res = []
-      Bio::Faster.parse(File.join(TEST_DATA,"sample.fastq")) {|seq| faster_res << seq}
-      faster_res[0][0].should == "HISEQ1:86:D0306ACXX:2:1101:20970:17588"
-      faster_res[0][1].should == "1:N:0:CTTGTA"
-      faster_res[0][2].should == "CGGTGCTGTTGTTATGCTGATGCTTATTAGTGCAAGTGTAGCTCCTCCGATTAGATGAATTAACAGGTGTCCTGCAGTAATGTTGGCTGTTAGTCGTAC"
-      
-      faster_res[-1][0].should == "HISEQ1:86:D0306ACXX:2:1101:1411:17830"
-      faster_res[-1][1].should == "1:Y:0:CTTGTA"
-      faster_res[-1][2].should == "CGGCGGGCGTGGGGAGAGAGCTATGAAGGCCTCAGGGAAGCTTCGAGAGTATAAGGTGTTGGGGTGCTGCCTGCCAACCCCCAAATTCCACACACCACC"
     end
-    
-    it "parse correctly compressed FastQ files" do
-      faster_res = []
-      Bio::Faster.parse(File.join(TEST_DATA,"sample.fastq.gz")) {|seq| faster_res << seq}
-      faster_res[0][0].should == "HISEQ1:86:D0306ACXX:2:1101:20970:17588"
-      faster_res[0][1].should == "1:N:0:CTTGTA"
-      faster_res[0][2].should == "CGGTGCTGTTGTTATGCTGATGCTTATTAGTGCAAGTGTAGCTCCTCCGATTAGATGAATTAACAGGTGTCCTGCAGTAATGTTGGCTGTTAGTCGTAC"
-      
-      faster_res[-1][0].should == "HISEQ1:86:D0306ACXX:2:1101:1411:17830"
-      faster_res[-1][1].should == "1:Y:0:CTTGTA"
-      faster_res[-1][2].should == "CGGCGGGCGTGGGGAGAGAGCTATGAAGGCCTCAGGGAAGCTTCGAGAGTATAAGGTGTTGGGGTGCTGCCTGCCAACCCCCAAATTCCACACACCACC"
-    end
-    
 
-    it "parse correctly Fasta files" do
-      faster_res = []
-      Bio::Faster.parse(File.join(TEST_DATA,"sample.fasta")) {|seq| faster_res << seq}
-      faster_res[0][0].should == "seq1"
-      faster_res[0][1].should == "comment1"
-      faster_res[0][2].should == "AGCAATTTCCCTTTTCCTGTCCTTTTTATAACATTGTGGAGGAAGACGGCAGCATAAAAAGGACAGTATTTGATTAAAAAATGATAAAAATTTTCAAAC"
-      
-      faster_res[-1][0].should == "seq4"
-      faster_res[-1][1].should == nil
-      faster_res[-1][2].should == "mgltrrealssiaavggekalkdalavlggps"
-    end
-  
-    it "can return the sequence data in a more friendly way" do
-      Bio::Faster.parse(File.join(TEST_DATA,"sample.fastq")) do |sequence_id, comment, sequence, quality|
-        sequence_id.should == "HISEQ1:86:D0306ACXX:2:1101:20970:17588"
-        comment.should == "1:N:0:CTTGTA"
-        sequence.should == "CGGTGCTGTTGTTATGCTGATGCTTATTAGTGCAAGTGTAGCTCCTCCGATTAGATGAATTAACAGGTGTCCTGCAGTAATGTTGGCTGTTAGTCGTAC"
-        quality.class.should == Array 
-        break
+    it "should handle correctly Phred64 qualities (Solexa)" do
+      file = TEST_DATA+"/formats/misc_rna_as_solexa.fastq"
+      bioruby_data = []
+      Bio::FlatFile.open(File.open(file)).each_entry do |seq|
+        seq.format = "fastq-solexa"
+        bioruby_data << [seq.entry_id,seq.seq,seq.qualities]
       end
-    end
-    
-    it "can return the sequence data in a more friendly way (also for FastA)" do
-      Bio::Faster.parse(File.join(TEST_DATA,"sample.fasta")) do |sequence_id, comment, sequence, quality|
-        quality.should == nil # it is a Fasta file, so no quality
-        sequence_id.should == "seq1"
-        comment.should == "comment1"
-        sequence.should == "AGCAATTTCCCTTTTCCTGTCCTTTTTATAACATTGTGGAGGAAGACGGCAGCATAAAAAGGACAGTATTTGATTAAAAAATGATAAAAATTTTCAAAC"
-        break
+      faster_data = []
+      Bio::Faster.new(file, :solexa).each_record do |seq|
+        seq[0] = seq[0].split(" ").first
+        faster_data << seq
       end
+      faster_data.should == bioruby_data
     end
-  
-  
-    describe "quality conversion for FastQ files (Sanger/Phred only)" do
     
-      it "converts directly quality scores for Illumina 1.8+ FastQ files" do
-
-        bioruby_quals = []
-        # standard Quality conversion as done in BioRuby Bio::FastQ
-        Bio::FlatFile.open(Bio::Fastq,File.open(File.join(TEST_DATA,"sample.fastq"))).each_entry do |seq|
-          bioruby_quals << seq.qualities
+    it "should read different FastQ formats" do
+      files = Dir.glob(TEST_DATA+"/formats/*.fastq")
+      files.each do |file|
+        bioruby_data = []
+        Bio::FlatFile.open(Bio::Fastq,File.open(file)).each_entry do |seq|
+           bioruby_data << [seq.entry_id,seq.seq,seq.qualities]
         end
-
-        faster_quals = []
-        Bio::Faster.parse(File.join(TEST_DATA,"sample.fastq")) do |sequence_id, comment, sequence, quality|
-          faster_quals << quality
+        faster_data = []
+        Bio::Faster.new(file).each_record do |seq|
+           seq[0] = seq[0].split(" ").first
+           faster_data << seq
         end
-        faster_quals.should == bioruby_quals
-
+        faster_data.should == bioruby_data
       end
 
-      it "converts directly quality scores for SFF 454 FastQ files" do
-
-        bioruby_quals = []
-        # standard Quality conversion as done in BioRuby Bio::FastQ
-        Bio::FlatFile.open(Bio::Fastq,File.open(File.join(TEST_DATA,"sff_sample.fastq"))).each_entry do |seq|
-          bioruby_quals << seq.qualities
-        end
+    end
 
-        faster_quals = []
-        Bio::Faster.parse(File.join(TEST_DATA,"sff_sample.fastq")) do |sequence_id, comment, sequence, quality|
-          faster_quals << quality
-        end
-        faster_quals.should == bioruby_quals
 
-      end
-    
-    end
-  
   end
+
   
 end

data/test/data/errors/error_header.fastq

@@ -0,0 +1,20 @@
+SLXA-B3_649_FC8437_R1_1_1_610_79
+GATGTGCAATACCTTTGTAGAGGAA
++SLXA-B3_649_FC8437_R1_1_1_610_79
+YYYYYYYYYYYYYYYYYYWYWYYSU
+@SLXA-B3_649_FC8437_R1_1_1_397_389
+GGTTTGAGAAAGAGAAATGAGATAA
++SLXA-B3_649_FC8437_R1_1_1_397_389
+YYYYYYYYYWYYYYWWYYYWYWYWW
+@SLXA-B3_649_FC8437_R1_1_1_850_123
+GAGGGTGTTGATCATGATGATGGCG
++SLXA-B3_649_FC8437_R1_1_1_850_123
+YYYYYYYYYYYYYWYYWYYSYYYSY
+@SLXA-B3_649_FC8437_R1_1_1_362_549
+GGAAACAAAGTTTTTCTCAACATAG
++SLXA-B3_649_FC8437_R1_1_1_362_549
+YYYYYYYYYYYYYYYYYYWWWWYWYY
+@SLXA-B3_649_FC8437_R1_1_1_183_714
+GTATTATTTAATGGCATACACTCAA
++SLXA-B3_649_FC8437_R1_1_1_183_714
+YYYYYYYYYYWYYYYWYWWUWWWQQ

data/test/data/errors/error_long_qual.fastq

@@ -0,0 +1,20 @@
+@SLXA-B3_649_FC8437_R1_1_1_610_79
+GATGTGCAATACCTTTGTAGAGGAA
++SLXA-B3_649_FC8437_R1_1_1_610_79
+YYYYYYYYYYYYYYYYYYWYWYYSU
+@SLXA-B3_649_FC8437_R1_1_1_397_389
+GGTTTGAGAAAGAGAAATGAGATAA
++SLXA-B3_649_FC8437_R1_1_1_397_389
+YYYYYYYYYWYYYYWWYYYWYWYWW
+@SLXA-B3_649_FC8437_R1_1_1_850_123
+GAGGGTGTTGATCATGATGATGGCG
++SLXA-B3_649_FC8437_R1_1_1_850_123
+YYYYYYYYYYYYYWYYWYYSYYYSY
+@SLXA-B3_649_FC8437_R1_1_1_362_549
+GGAAACAAAGTTTTTCTCAACATAG
++SLXA-B3_649_FC8437_R1_1_1_362_549
+YYYYYYYYYYYYYYYYYYWWWWYWYY
+@SLXA-B3_649_FC8437_R1_1_1_183_714
+GTATTATTTAATGGCATACACTCAA
++SLXA-B3_649_FC8437_R1_1_1_183_714
+YYYYYYYYYYWYYYYWYWWUWWWQQ

data/test/data/errors/error_qual_del.fastq

@@ -0,0 +1,20 @@
+@SLXA-B3_649_FC8437_R1_1_1_610_79
+GATGTGCAATACCTTTGTAGAGGAA
++SLXA-B3_649_FC8437_R1_1_1_610_79
+YYYYYYYYYYYYYYYYYYWYWYYSU
+@SLXA-B3_649_FC8437_R1_1_1_397_389
+GGTTTGAGAAAGAGAAATGAGATAA
++SLXA-B3_649_FC8437_R1_1_1_397_389
+YYYYYYYYYWYYYYWWYYYWYWYWW
+@SLXA-B3_649_FC8437_R1_1_1_850_123
+GAGGGTGTTGATCATGATGATGGCG
++SLXA-B3_649_FC8437_R1_1_1_850_123
+YYYYYYYYYYYYYWYYWYYSYYYSY
+@SLXA-B3_649_FC8437_R1_1_1_362_549
+GGAAACAAAGTTTTTCTCAACATAG
++SLXA-B3_649_FC8437_R1_1_1_362_549
+YYYYYYYYYYYYYYYYYWWWWYWY
+@SLXA-B3_649_FC8437_R1_1_1_183_714
+GTATTATTTAATGGCATACACTCAA
++SLXA-B3_649_FC8437_R1_1_1_183_714
+YYYYYYYYYYWYYYYWYWWUWWWQQ

data/test/data/errors/error_qual_escape.fastq

@@ -0,0 +1,20 @@
+@SLXA-B3_649_FC8437_R1_1_1_610_79
+GATGTGCAATACCTTTGTAGAGGAA
++SLXA-B3_649_FC8437_R1_1_1_610_79
+YYYYYYYYYYYYYYYYYYWYWYYSU
+@SLXA-B3_649_FC8437_R1_1_1_397_389
+GGTTTGAGAAAGAGAAATGAGATAA
++SLXA-B3_649_FC8437_R1_1_1_397_389
+YYYYYYYYYWYYYYWWYYYWYWYWW
+@SLXA-B3_649_FC8437_R1_1_1_850_123
+GAGGGTGTTGATCATGATGATGGCG
++SLXA-B3_649_FC8437_R1_1_1_850_123
+YYYYYYYYYYYYYWYYWYYSYYYSY
+@SLXA-B3_649_FC8437_R1_1_1_362_549
+GGAAACAAAGTTTTTCTCAACATAG
++SLXA-B3_649_FC8437_R1_1_1_362_549
+YYYYYYYYYYYYYYYYYYWWWWYWY
+@SLXA-B3_649_FC8437_R1_1_1_183_714
+GTATTATTTAATGGCATACACTCAA
++SLXA-B3_649_FC8437_R1_1_1_183_714
+YYYYYYYYYWYYYYWYWWUWWWQQ

data/test/data/errors/error_qual_space.fastq

@@ -0,0 +1,20 @@
+@SLXA-B3_649_FC8437_R1_1_1_610_79
+GATGTGCAATACCTTTGTAGAGGAA
++SLXA-B3_649_FC8437_R1_1_1_610_79
+YYYYYYYYYYYYYYYYYYWYWYYSU
+@SLXA-B3_649_FC8437_R1_1_1_397_389
+GGTTTGAGAAAGAGAAATGAGATAA
++SLXA-B3_649_FC8437_R1_1_1_397_389
+YYYYYYYYYWYYYYWWYYYWYWYWW
+@SLXA-B3_649_FC8437_R1_1_1_850_123
+GAGGGTGTTGATCATGATGATGGCG
++SLXA-B3_649_FC8437_R1_1_1_850_123
+YYYYYYYYYYYYYWYYWYYSYYYSY
+@SLXA-B3_649_FC8437_R1_1_1_362_549
+GGAAACAAAGTTTTTCTCAACATAG
++SLXA-B3_649_FC8437_R1_1_1_362_549
+YYYYYYYYYYYYYYYYYY WWWYWY
+@SLXA-B3_649_FC8437_R1_1_1_183_714
+GTATTATTTAATGGCATACACTCAA
++SLXA-B3_649_FC8437_R1_1_1_183_714
+YYYYYYYYYYWYYYYWYWWUWWWQQ

data/test/data/errors/error_qual_tab.fastq

@@ -0,0 +1,20 @@
+@SLXA-B3_649_FC8437_R1_1_1_610_79
+GATGTGCAATACCTTTGTAGAGGAA
++SLXA-B3_649_FC8437_R1_1_1_610_79
+YYYYYYYYYYYYYYYYYYWYWYYSU
+@SLXA-B3_649_FC8437_R1_1_1_397_389
+GGTTTGAGAAAGAGAAATGAGATAA
++SLXA-B3_649_FC8437_R1_1_1_397_389
+YYYYYYYYYWYYYYWWYYYWYWYWW
+@SLXA-B3_649_FC8437_R1_1_1_850_123
+GAGGGTGTTGATCATGATGATGGCG
++SLXA-B3_649_FC8437_R1_1_1_850_123
+YYYYYYYYYYYYYWYYWYYSYYYSY
+@SLXA-B3_649_FC8437_R1_1_1_362_549
+GGAAACAAAGTTTTTCTCAACATAG
++SLXA-B3_649_FC8437_R1_1_1_362_549
+YYYYYYYYYYYYYYYYYYWWWWYWY
+@SLXA-B3_649_FC8437_R1_1_1_183_714
+GTATTATTTAATGGCATACACTCAA
++SLXA-B3_649_FC8437_R1_1_1_183_714
+YYYYYYYYYY      YYYYWYWWUWWWQQ