bio-gff3 0.8.5 → 0.8.6

data/lib/bio/db/gff/digest/gffnocache.rb

@@ -21,19 +21,24 @@ module Bio
         # record
         module SeekRec
           # Fetch a record using fh and file seek position
-          def SeekRec::fetch(fh,fpos)
+          def SeekRec::fetch(fh,fpos,parser)
             return nil if fh==nil or fpos==nil
             fh.seek(fpos)
-            GFF::GFF3::FileRecord.new(fpos, fh.gets)
+            if parser == :bioruby 
+              GFF::GFF3::BioRubyFileRecord.new(fpos, fh.gets)
+            else
+              GFF::GFF3::FastParserFileRecord.new(fpos, fh.gets)
+            end
           end
         end
 
         # Helper class which gives Hash-like access to the 
         # no-cache GFF3 file
         class SeekRecList 
-          def initialize fh
+          def initialize fh, parser
             @fh = fh
             @h = {}
+            @parser = parser
           end
 
           def []= id, rec
@@ -44,7 +49,7 @@ module Bio
 
           def [](id)
             fpos = @h[id]
-            SeekRec::fetch(@fh,fpos)
+            SeekRec::fetch(@fh,fpos,@parser)
           end
 
           def each 
@@ -56,7 +61,7 @@ module Bio
 
         # List of ids
         class SeekLinkedRecs < Hash
-          include Helpers::Error
+          include Helpers::Logger
           def add id, rec
             info "Adding #{rec.feature_type} <#{id}>"
             self[id] = [] if self[id] == nil
@@ -91,14 +96,22 @@ module Bio
           info "---- Digest DB and store data in mRNA Hash (NoCache)"
           @count_ids          = Counter.new   # Count ids
           @count_seqnames     = Counter.new   # Count seqnames
-          @componentlist      = SeekRecList.new(@iter.fh) # Store containers, like genes, contigs
+          @componentlist      = SeekRecList.new(@iter.fh,@options[:parser]) # Store containers, like genes, contigs
           @orflist            = SeekLinkedRecs.new   # Store linked gene records
           @mrnalist           = SeekLinkedRecs.new   # Store linked mRNA records
           @cdslist            = SeekLinkedRecs.new
           @exonlist           = SeekLinkedRecs.new
           @sequencelist       = {}
           @unrecognized_features = {}
-          @iter.each_rec do | id, rec |
+          @iter.each_rec do |fpos, line|
+            rec = case @options[:parser]
+              when :bioruby
+                Bio::GFF::GFF3::BioRubyFileRecord.new(fpos, line)
+              when :line
+                Bio::GFF::GFF3::FastParserFileRecord.new(fpos, line)
+              else
+                raise 'Unknown parser'
+            end
             store_record(rec)
           end
           @iter.each_sequence do | id, bioseq |
@@ -117,11 +130,17 @@ module Bio
           list.each do | id, io_seeklist |
             recs = []
             io_seeklist.each do | fpos |
-              recs << SeekRec::fetch(fh,fpos)
+              recs << SeekRec::fetch(fh,fpos,@options[:parser])
             end
             seqid = recs[0].seqname
             component = find_component(recs[0])
-            yield id, recs, component
+            if @options[:no_assemble]
+              recs.each do | rec |
+                yield id, [rec], component
+              end
+            else
+              yield id, recs, component
+            end
           end
         end
         

data/lib/bio/db/gff/digest/gffparser.rb

@@ -19,7 +19,7 @@ module Bio
 
         include Bio::GFFbrowser::Helpers
         include Bio::GFFbrowser::Helpers::Validate
-        include Bio::GFFbrowser::Helpers::Error
+        include Bio::GFFbrowser::Helpers::Logger
         include Gff3Component
         include Gff3Features
 

data/lib/bio/db/gff/file/gfffileiterator.rb

@@ -13,7 +13,7 @@ module Bio
 
       # FileRecord inherits from the BioRuby Record, but
       # adds the file seek position.
-      class FileRecord < Record
+      class BioRubyFileRecord < Record
         attr_accessor :io_seek
         def initialize io_seek, buf
           @io_seek = io_seek
@@ -21,6 +21,16 @@ module Bio
         end
       end
 
+      class FastParserFileRecord < GFFbrowser::FastLineRecord
+        attr_accessor :io_seek
+
+        include Bio::GFFbrowser::FastLineParser 
+        def initialize io_seek, buf
+          @io_seek = io_seek
+          super(parse_line_fast(buf))
+        end
+      end
+
       # GFF3::FileIterator takes a file and yields GFF3 records with their
       # seek position included in the record.
       class FileIterator
@@ -31,9 +41,10 @@ module Bio
           @fh = File.open(filename)
         end
 
-        # Iterate over every record in the file, yielding the record ID and
-        # (File)Record, which includes the io_seek position in the file
-        def each_rec()
+        # Iterate over every record in the file, yielding the seekpos
+        # and line containing the record
+        def each_rec
+          @fh.seek(0)
           fpos = 0
           @fh.each_line do | line |
             line = line.strip
@@ -42,10 +53,8 @@ module Bio
               break
             end
             if line.size != 0 and line !~ /^#/
-              rec = FileRecord.new(fpos, line)
               lastpos = @fh.tell
-              id = rec.id
-              yield id, rec
+              yield fpos, line
               @fh.seek(lastpos) # reset filepos, just in case it changed
             end
             fpos = @fh.tell

data/lib/bio/db/gff/gff3.rb

@@ -9,6 +9,8 @@
 
 require 'bio/db/gff/digest/gffinmemory'
 require 'bio/db/gff/digest/gffnocache'
+require 'bio/db/gff/digest/gfflrucache'
+require 'bio/db/gff/block/gffblockparser'
 
 module Bio
   module GFFbrowser
@@ -16,16 +18,24 @@ module Bio
       attr_reader :assembler
 
       include Digest
+      include Block
 
       # Initialize a GFF parser
       def initialize filename, options = {}
+        options[:parser] = :line if options[:parser] == nil
         cache_recs    = options[:cache_records]
         @assembler = 
-          case cache_recs
-            when :cache_none 
-              NoCache.new(filename, options)
-            else
-              InMemory.new(filename, options)  # default 
+          if options[:block]
+            GffBlockParser.new(filename, options)
+          else 
+            case cache_recs
+              when :cache_none 
+                NoCache.new(filename, options)
+              when :cache_lru
+                LruCache.new(filename, options)
+              else
+                InMemory.new(filename, options)  # default 
+            end
           end
       end
 

data/lib/bio/db/gff/gff3parserec.rb

@@ -24,7 +24,7 @@ module Bio
     #
     module FastLineParser
 
-      include Helpers::Error
+      include Helpers::Logger
 
       # Returns a (partial) record, assuming it is a valid GFF3
       # format, no validation takes place, other than field counting (!)
@@ -44,11 +44,6 @@ module Bio
           return nil
         end
 
-        fs[GFF3_START]      = fs[GFF3_START].to_i
-        fs[GFF3_END]        = fs[GFF3_END].to_i
-        fs[GFF3_SCORE]      = fs[GFF3_SCORE].to_f 
-        fs[GFF3_PHASE]      = fs[GFF3_PHASE].to_i
-        fs[GFF3_ATTRIBUTES] = parse_attributes_fast(fs[GFF3_ATTRIBUTES],options)
         fs
       end
 

data/lib/bio/db/gff/gffcomponent.rb

@@ -7,13 +7,15 @@
 #
 # Fetch information from a GFF file
 
+require 'set'
+
 module Bio
   module GFFbrowser
 
     module Helpers
 
       module Record
-        include Error
+        include Logger
         # Format a record ID by, first, getting the ID attribute. If that fails
         # the seqname is used with the start/stop positions.
         def Record::formatID rec  
@@ -33,11 +35,11 @@ module Bio
 
       module Gff3Component
 
-        include Error
+        include Logger
 
-        COMPONENT_TYPES = %w{
+        COMPONENT_TYPES = Set.new(%w{
           gene SO:0000704 contig transcript Component region
-        }
+        })
  
         # Walk the component list to find a matching component/container for a
         # record. First use the parent ID. If that is missing go by sequence
@@ -80,13 +82,13 @@ module Bio
       module Gff3Features
 
         # Ignore the following features (case sensitive?)
-        IGNORE_FEATURES = Gff3Component::COMPONENT_TYPES + %w{
+        IGNORE_FEATURES = Gff3Component::COMPONENT_TYPES + Set.new(%w{
           transposon Match similarity UTR
           TF_binding_site intronSO:0000188 polyA_sequence SO:0000610
           polyA_site SO:0000553
           five_prime_UTR SO:0000204 three_prime_UTR SO:0000205
           exon SO:0000147
-        }
+        })
       end
 
     end

data/lib/bio/db/gff/gffrecord.rb

@@ -1,3 +1,5 @@
+require 'bio/db/gff/gff3parserec'
+
 module Bio
   module GFFbrowser
     
@@ -7,6 +9,9 @@ module Bio
 
     # Using the fast line parser
     class FastLineRecord < Record
+
+      include FastLineParser
+
       def initialize fields
         @fields = fields
       end
@@ -16,27 +21,27 @@ module Bio
       end
 
       def seqid
-        @fields[GFF3_SEQID]
+        @seqid_ ||= @fields[GFF3_SEQID]
       end
 
       alias seqname :seqid
 
       def phase
-        @fields[GFF3_PHASE]
+        @phase_ ||= @fields[GFF3_PHASE].to_i
       end
 
       alias frame :phase
 
       def start
-        @fields[GFF3_START]
+        @start_ ||= @fields[GFF3_START].to_i
       end
 
       def end
-        @fields[GFF3_END]
+        @end_ ||= @fields[GFF3_END].to_i
       end
 
       def score
-        @fields[GFF3_SCORE]
+        @score_ ||= @fields[GFF3_SCORE].to_f
       end
 
       def strand
@@ -44,7 +49,7 @@ module Bio
       end
 
       def feature
-        @fields[GFF3_TYPE]
+        @feature_ ||= @fields[GFF3_TYPE]
       end
 
       alias feature_type :feature
@@ -53,7 +58,7 @@ module Bio
       end
 
       def attributes 
-        @fields[GFF3_ATTRIBUTES]
+        @attributes_ ||= parse_attributes_fast(@fields[GFF3_ATTRIBUTES])
       end
 
       def get_attribute name
@@ -61,7 +66,7 @@ module Bio
       end 
 
       def id
-        attributes['ID']
+        @id_ ||= attributes['ID']
       end
 
       alias entry_id :id

data/lib/bio/db/gff/gffsection.rb

@@ -1,4 +1,3 @@
-
 module Bio
   module GFFbrowser
 

data/lib/bio/db/gff/gffsequence.rb

@@ -15,7 +15,7 @@ module Bio
 
       module Gff3Sequence
 
-        include Bio::GFFbrowser::Helpers::Error
+        include Bio::GFFbrowser::Helpers::Logger
 
 
         # Patch a sequence together from a Sequence string and an array
@@ -61,13 +61,13 @@ module Bio
           orf_frame = startpos - 1
           orf_frameshift = orf_frame % 3
           sectionlist = sectionlist.reverse if orf_reverse
-          # if do_debug
+          if do_debug
             debug options.to_s
             debug [:reverse,do_reverse].to_s
             debug [:complement,do_complement].to_s
             debug [:trim,do_trim].to_s
             debug [:orf_reverse, orf_reverse, rec0.strand].to_s
-          # end
+          end
 
           if sequence.kind_of?(Bio::FastaFormat)
             # BioRuby conversion
@@ -80,18 +80,12 @@ module Bio
             if do_reverse and orf_reverse
               s = s.reverse 
             end
-            # Correct for phase. Unfortunately the use of phase is ambiguous.
-            # Here we check whether rec.start is in line with orf_frame. If it
-            # is, we correct for phase. Otherwise it is ignored.
             if do_phase and rec.phase
               phase = rec.phase.to_i
-              # if ((rec.start-startpos) % 3 == 0) 
               s = s[phase..-1]
-              # end
             end
             s
           }
-          # p seq
           seq = seq.join
           if do_complement and do_reverse and orf_reverse
             ntseq = Bio::Sequence::NA.new(seq)

data/lib/bio/db/gff/gffvalidate.rb

@@ -33,7 +33,7 @@ module Bio
 
       # Helper class for storing linked records based on a shared ID
       class LinkedRecs < Hash
-        include Error
+        include Logger
         def add id, rec
           info "Adding #{rec.feature_type} <#{id}>"
           self[id] = [] if self[id] == nil

data/lib/bio/output/gfflogger.rb

@@ -4,7 +4,7 @@ module Bio
 
     module Helpers
 
-      module Error
+      module Logger
         include Bio::Log
 
         def debug str, id=''
@@ -27,6 +27,15 @@ module Bio
           log.error_(str+" <#{id}>",:act => FailOnError.new)
 
         end
+
+        def log_sys_info msg
+          log = LoggerPlus['bio-gff3']
+          rmem = `ps -o rss= -p #{Process.pid}`.to_i
+          vmem = `ps -o vsz= -p #{Process.pid}`.to_i
+          if rmem or vmem
+            log.info7 "Memory used #{msg} RAM #{rmem/1024}M, VMEM #{vmem/1024}M"
+          end
+        end
       end
     end
   end

data/spec/gff3_fileiterator_spec.rb

@@ -13,10 +13,11 @@ TEST2='test/data/gff/standard.gff3'
 
 describe Bio::GFF::GFF3::FileIterator, "iterates a GFF3 file" do
 
+
   it "should parse a file and yield records" do 
     iter = Bio::GFF::GFF3::FileIterator.new(TEST1)
-    iter.each_rec do | id, rec |
-      # p [id, rec, rec.io_seek]
+    iter.each_rec do | fpos, line |
+      rec = Bio::GFF::GFF3::FastParserFileRecord.new(fpos, line)
       rec.io_seek.should == 51
       break
     end
@@ -25,8 +26,8 @@ describe Bio::GFF::GFF3::FileIterator, "iterates a GFF3 file" do
   it "should handle embedded FASTA records" do
     iter = Bio::GFF::GFF3::FileIterator.new(TEST1)
     last = nil
-    iter.each_rec do | id, rec |
-      # p [id, rec]
+    iter.each_rec do | fpos, line |
+      rec = Bio::GFF::GFF3::FastParserFileRecord.new(fpos, line)
       last = rec
     end
     last.io_seek.should == 3342

data/spec/gffdb_spec.rb

@@ -54,7 +54,13 @@ describe GFF3, "GFF3 API (InMemory) with everything in memory" do
 end
 
 describe GFF3, "GFF3 API with :cache_components => 1000, :cache_records => :cache_none" do
-  # iterators_should_be_implemented
+  before :all do 
+    # initialize
+    gff3 = Bio::GFFbrowser::GFF3.new(TESTGFF1, :cache_components => :cache_none, :cache_records => :cache_lru)
+    @gff = gff3.assembler
+  end
+
+  iterators_should_be_implemented
 end
 
 describe GFF3, "GFF3 API with :cache_components => 1000, :cache_records => 1000" do