bio-gff3 0.6.0

data/lib/bio/db/gff/gffdb.rb

@@ -0,0 +1,40 @@
+#
+# = bio/db/gff/gffdb.rb - GFF database class
+#
+# Copyright::  Copyright (C) 2010
+#              Pjotr Prins <pjotr.prins@thebird.nl>
+# License::    The Ruby License
+
+# Create db from a GFF file
+
+require 'bio'
+require 'bio/db/gff/gfffileiterator'
+require 'bio/db/gff/gfffasta'
+require 'bio/db/gff/gffassemble'
+require 'bio/db/gff/gffparser'
+require 'bio/db/gff/gffinmemory'
+require 'bio/db/gff/gffnocache'
+
+module Bio
+  module GFFbrowser
+    class GFFdb
+      attr_reader :assembler
+
+      include Digest
+
+      # Initialize a GFF parser
+      def initialize filename, options = {}
+        cache_recs    = options[:cache_records]
+        @assembler = 
+          case cache_recs
+            when :cache_none 
+              NoCache.new(filename, options)
+            else
+              InMemory.new(filename, options)  # default 
+          end
+      end
+
+    end # GFFdb
+  end # GFFbrowser
+end # Bio
+

data/lib/bio/db/gff/gfffasta.rb

@@ -0,0 +1,68 @@
+# = bio/db/gff/gfffasta.rb - Fetch records from a file in FASTA format
+#
+# Copyright::  Copyright (C) 2010
+#              Pjotr Prins <pjotr.prins@thebird.nl>
+# License::    The Ruby License
+#
+# This requires a special implementation as it uses an open file and we 
+# retain file seek positions.
+
+module Bio
+
+  class GFF
+
+    # Read FASTA records from file and store seek positions, which are
+    # used to retrieve the records. Note, this implementation merely retains
+    # records in memory (FIXME)
+    class FastaReader
+      def initialize fh, io_seek=nil
+        @fh = fh
+        @h = {}
+        parse
+      end
+
+      def parse
+        # read FASTA records
+        header = nil
+        seqs   = []
+        @fh.each_line do | line |
+          line = line.strip
+          next if line =~ /^#/
+          if line =~ /^>/  # FASTA record header
+            add(header,seqs)
+            header = line
+            seqs   = []
+          else
+            seqs << line
+          end
+        end
+        add(header,seqs)
+      end
+
+      def [] index
+        @h[index]
+      end
+
+      def each 
+        @h.each do | k,v |
+          yield k, v
+        end
+      end
+
+    private
+       def add header, seqs
+         if header
+           id, fastarec = fasta_rec(header, seqs)
+           @h[id] = fastarec.data.strip
+         end
+       end
+
+      def fasta_rec header, buf
+        fst = Bio::FastaFormat.new(header+"\n"+buf.to_s)
+        return fst.definition, fst
+      end
+
+    end
+
+  end # GFF
+end # Bio

data/lib/bio/db/gff/gfffileiterator.rb

@@ -0,0 +1,77 @@
+#
+# = bio/db/gff/gfffileiterator.rb - Fetch records from a file
+#
+# Copyright::  Copyright (C) 2010
+#              Pjotr Prins <pjotr.prins@thebird.nl>
+# License::    The Ruby License
+
+module Bio
+
+  class GFF
+
+    class GFF3
+
+      class FileRecord < Record
+        attr_accessor :io_seek
+        def initialize io_seek, buf
+          @io_seek = io_seek
+          super(buf)
+        end
+      end
+
+      # GFF3::FileIterator takes a file and yields GFF3 records with their
+      # seek position included in the record.
+      class FileIterator
+        attr_accessor :fh
+        attr_reader :fasta_io_seek
+
+        def initialize filename
+          @fh = File.open(filename)
+        end
+
+        # Iterate over every record in the file, yielding the record ID and
+        # (File)Record, which includes the io_seek position in the file
+        def each_rec()
+          fpos = 0
+          @fh.each_line do | line |
+            line = line.strip
+            if line == "##FASTA"
+              @fasta_io_seek = fpos
+              break
+            end
+            if line.size != 0 and line !~ /^#/
+              rec = FileRecord.new(fpos, line)
+              lastpos = @fh.tell
+              id = rec.id
+              yield id, rec
+              @fh.seek(lastpos) # reset filepos, just in case it changed
+            end
+            fpos = @fh.tell
+          end
+        end
+
+        # Iterate over all contained FASTA sequences, yielding the ID
+        # and sequence as a FASTA record. Normally call each_rec first and 
+        # you can test for existing FASTA records if fasta_io_seek != nil
+        def each_sequence
+          if @fasta_io_seek == nil
+            # Find the FASTA location first
+            @fh.each_line do | line |
+              break if line.strip == "##FASTA"
+            end
+          else
+            @fh.seek(@fasta_io_seek)
+          end
+          fasta = Bio::GFF::FastaReader.new(@fh)
+          fasta.each do | id, fastarec |
+            yield id, fastarec
+          end
+        end
+      end
+    end # GFF3
+  end # GFF
+end # Bio
+
+    
+
+

data/lib/bio/db/gff/gffinmemory.rb

@@ -0,0 +1,63 @@
+#
+# = bio/db/gff/gffinmemory.rb - Assemble mRNA and CDS from GFF in RAM
+#
+# Copyright::  Copyright (C) 2010
+#              Pjotr Prins <pjotr.prins@thebird.nl>
+# License::    The Ruby License
+#
+# Fetch information from a GFF file
+
+module Bio
+  module GFFbrowser
+
+    module Digest
+
+      class InMemory
+        include Parser
+        include Gff3Sequence
+        attr_reader :sequencelist
+
+        def initialize filename, options
+          @options = options
+          # Invoke the BioRuby in memory parser
+          @gff = Bio::GFF::GFF3.new(File.read(filename))
+        end
+
+        # Digest mRNA from the GFFdb and store in Hash
+        # Next yield(id, seq) from Hash
+        def parse 
+          info "---- Digest DB and store data in mRNA Hash"
+          @count_ids          = Counter.new   # Count ids
+          @count_seqnames     = Counter.new   # Count seqnames
+          @componentlist      = {} # Store containers, like genes, contigs
+          @mrnalist           = LinkedRecs.new   # Store linked mRNA records
+          @cdslist            = LinkedRecs.new
+          @exonlist           = LinkedRecs.new
+          @sequencelist       = {}
+          @unrecognized_features = {}
+          @gff.records.each do | rec |
+            store_record(rec)
+          end
+          @gff.sequences.each do | bioseq |
+            id = bioseq.entry_id
+            @sequencelist[id] = bioseq.to_s # in Bio::Sequence with contained Bio::FastaFormat
+          end
+          validate_mrnas
+          validate_cdss 
+          show_unrecognized_features 
+          @genelist      = @count_ids.keys 
+          read_fasta
+        end
+
+        def each_item list
+          list.each do | id, recs |
+            seqid = recs[0].seqname
+            component = find_component(recs[0])
+            yield id, recs, component
+          end
+        end
+        
+      end
+    end
+  end
+end

data/lib/bio/db/gff/gffnocache.rb

@@ -0,0 +1,124 @@
+#
+# = bio/db/gff/gffnocache.rb - Assemble mRNA and CDS from GFF by fseek
+#
+# Copyright::  Copyright (C) 2010
+#              Pjotr Prins <pjotr.prins@thebird.nl>
+# License::    The Ruby License
+#
+# Fetch information from a GFF file without using RAM - also check
+# out the caching edition, which uses limited amounts of RAM
+
+module Bio
+  module GFFbrowser
+
+    module Digest
+
+      module NoCacheHelpers 
+
+        module SeekRec
+          # Fetch a record using fh and file seek position
+          def SeekRec::fetch(fh,fpos)
+            return nil if fh==nil or fpos==nil
+            fh.seek(fpos)
+            GFF::GFF3::FileRecord.new(fpos, fh.gets)
+          end
+        end
+
+        # The hardwired to file RecList
+        class SeekRecList 
+          def initialize fh
+            @fh = fh
+            @h = {}
+          end
+
+          def []= id, rec
+            raise "id #{id} occurs twice!" if @h[id]
+            fpos = rec.io_seek
+            @h[id] = fpos
+          end
+
+          def [](id)
+            fpos = @h[id]
+            SeekRec::fetch(@fh,fpos)
+          end
+
+          def each 
+            @h.each do | id,fpos |
+              yield id, self[id]
+            end
+          end
+        end
+
+        class SeekLinkedRecs < Hash
+          include Helpers::Error
+          def add id, rec
+            info "Adding #{rec.feature_type} <#{id}>"
+            self[id] = [] if self[id] == nil
+            self[id] << rec.io_seek
+          end
+          # validation is switched off for NoCache
+          def validate_seqname
+          end
+          # validation is switched off for NoCache
+          def validate_nonoverlapping
+          end
+          # validation is switched off for NoCache
+          def validate_shared_parent
+          end
+        end
+      end
+
+      class NoCache
+        include Parser
+        include NoCacheHelpers
+        include Gff3Sequence
+
+        def initialize filename, options
+          @filename = filename
+          @options = options
+          @iter = Bio::GFF::GFF3::FileIterator.new(@filename)
+        end
+
+        # parse the whole file once and store all seek locations, 
+        # rather than the records themselves
+        def parse
+          info "---- Digest DB and store data in mRNA Hash (NoCache)"
+          @count_ids          = Counter.new   # Count ids
+          @count_seqnames     = Counter.new   # Count seqnames
+          @componentlist      = SeekRecList.new(@iter.fh) # Store containers, like genes, contigs
+          @mrnalist           = SeekLinkedRecs.new   # Store linked mRNA records
+          @cdslist            = SeekLinkedRecs.new
+          @exonlist           = SeekLinkedRecs.new
+          @sequencelist       = {}
+          @unrecognized_features = {}
+          @iter.each_rec do | id, rec |
+            store_record(rec)
+          end
+          @iter.each_sequence do | id, bioseq |
+            @sequencelist[id] = bioseq.to_s
+          end
+          validate_mrnas 
+          validate_cdss
+          show_unrecognized_features
+          @genelist      = @count_ids.keys 
+          read_fasta
+        end
+
+        def each_item list
+          # p list.class
+          fh = @iter.fh
+          list.each do | id, io_seeklist |
+            recs = []
+            io_seeklist.each do | fpos |
+              recs << SeekRec::fetch(fh,fpos)
+            end
+            seqid = recs[0].seqname
+            component = find_component(recs[0])
+            yield id, recs, component
+          end
+        end
+        
+      end
+    end
+  end
+end

data/lib/bio/db/gff/gffparser.rb

@@ -0,0 +1,154 @@
+#
+# = bio/db/gff/gffparser.rb - Parsing logic for GFF3 file
+#
+# Copyright::  Copyright (C) 2010
+#              Pjotr Prins <pjotr.prins@thebird.nl>
+# License::    The Ruby License
+#
+
+module Bio
+  module GFFbrowser
+    module Digest
+
+      module Parser
+
+        include Bio::GFFbrowser::Helpers
+        include Bio::GFFbrowser::Helpers::Error
+        include Gff3Component
+        include Gff3Features
+
+        def store_record rec
+            return if rec.comment # skip GFF comments
+            id = Record::formatID(rec)
+            @count_ids.add(id)
+            @count_seqnames.add(rec.seqname)
+
+            if COMPONENT_TYPES.include?(rec.feature_type)
+              # check for container ID
+              warn("Container <#{rec.feature_type}> has no ID, so using sequence name instead",id) if rec.id == nil
+              @componentlist[id] = rec
+              info "Added #{rec.feature_type} with component ID #{id}"
+            else
+              case rec.feature_type
+                when 'mRNA' || 'SO:0000234'
+                  @mrnalist.add(id,rec)
+                when 'CDS'  || 'SO:0000316'
+                  @cdslist.add(id,rec)
+                when 'exon' || 'SO:0000147'
+                  @exonlist.add(id,rec)
+                else
+                  if !IGNORE_FEATURES.include?(rec.feature_type)
+                    @unrecognized_features[rec.feature_type] = true
+                  end
+              end
+            end
+        end
+
+        def validate_mrnas 
+          return if not @options[:validate]
+          # validate gene/container/component seqname is shared
+          @mrnalist.validate_seqname
+          @mrnalist.validate_shared_parent
+        end
+
+        def validate_cdss 
+          return if not @options[:validate]
+          @cdslist.validate_seqname
+          # validate CDS sections do not overlap
+          @cdslist.validate_nonoverlapping
+          # validate sections share the parent
+          @cdslist.validate_shared_parent
+          # display unhandled features
+        end
+
+        def show_unrecognized_features 
+          @unrecognized_features.keys.each do | k |
+            warn "Feature has no match",k if k
+          end
+        end
+
+        def read_fasta
+          if @options[:fasta_filename]
+            File.open(@options[:fasta_filename]) do | f |
+              fasta = Bio::GFF::FastaReader.new(f)
+              fasta.each do | id, fastarec |
+                # p fastarec
+                @sequencelist[id] = fastarec
+              end
+            end
+          end
+          # p :inmemory, @sequencelist
+        end
+
+        # Yield the id, recs, containing component and sequence of mRNAs
+        def each_mRNA
+          parse if !@mrnalist
+          each_item(@mrnalist) { |id, recs, component | yield id, recs, component }
+        end
+
+        # Yield the id, recs, and containing component
+        def each_CDS
+          parse if !@cdslist
+          each_item(@cdslist) { |id, recs, component | yield id, recs, component }
+        end
+
+        # Yield the id, recs, and containing component
+        def each_exon
+          parse if !@exonlist
+          each_item(@exonlist) { |id, recs, component | yield id, recs, component }
+        end
+
+        # Yield a unique description and the sequence
+        def each_mRNA_seq
+          each_mRNA do | id, reclist, component |
+            if component
+              sequence = @sequencelist[component.seqname]
+              # p sequence
+              if sequence
+                yield description(id,component,reclist), assemble(sequence,component.start,reclist)
+              else 
+                warn "No sequence information for",id
+              end
+            end
+          end
+        end
+
+        # Yield a unique description and the sequence
+        def each_CDS_seq
+          each_CDS do | id, reclist, component |
+            if component
+              sequence = @sequencelist[component.seqname]
+              # p sequence
+              if sequence
+                seq = assemble(sequence,component.start,reclist,:codonize=>true)
+                if seq.size % 3 != 0
+                  p reclist # leave this in
+                  # raise "CDS size #{seq.size} is not a multiple of 3! <#{seq}>"
+                  warn "CDS size is not a multiple of 3",id
+                end
+                yield description(id,component,reclist), seq
+              else 
+                warn "No sequence information for",id
+              end
+            end
+          end
+        end
+
+        # Yield a unique description and the sequence
+        def each_exon_seq
+          each_exon do | id, reclist, component |
+            if component
+              sequence = @sequencelist[component.seqname]
+              if sequence
+                seq = assemble(sequence,component.start,reclist)
+                yield description(id,component,reclist), seq
+              else 
+                warn "No sequence information for",id
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end