bio-gff3 0.8.5 → 0.8.6

data/Gemfile

@@ -2,7 +2,8 @@ source "http://rubygems.org"
 
 # Runtime dependencies
 gem "bio", ">= 1.3.1"
-gem "bio-logger", "> 0.5.0"
+gem "log4r", "> 1.1.6"
+gem "bio-logger", "> 0.8.0"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
@@ -11,5 +12,5 @@ group :development do
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.5.2"
   gem "rcov", ">= 0"
-  gem "rspec", ">= 2.0.0"
+  gem "rspec", ">= 2.3.0"
 end

data/Rakefile

@@ -24,10 +24,9 @@ Support for external FASTA files.
   gem.authors = ["Pjotr Prins"]
   # Include your dependencies below. Runtime dependencies are required when using your gem,
   # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
-  gem.add_runtime_dependency 'bio', '>= 1.4.1'
-  gem.add_runtime_dependency 'log4r', '> 1.1.6'
-  gem.add_runtime_dependency 'bio-logger', '>= 0.6.1'
-  gem.add_development_dependency 'rspec', '> 2.0'
+  # gem.add_runtime_dependency 'bio', '>= 1.4.1'
+  # gem.add_runtime_dependency 'log4r', '> 1.1.6'
+  # gem.add_runtime_dependency 'bio-logger', '>= 0.8.0'
 end
 Jeweler::RubygemsDotOrgTasks.new
 
@@ -36,8 +35,18 @@ Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
+  Kernel.system('rspec spec/*.rb')
 end
 
+#require 'spec/rake/spectask'
+#Spec::Rake::SpecTask.new(:spec) do |t|
+#  t.spec_files = Dir.glob('spec/**/*_spec.rb')
+#  t.spec_opts << '--format specdoc'
+#  t.warning = true
+#  t.rcov = true
+#end
+
+
 require 'rcov/rcovtask'
 Rcov::RcovTask.new do |test|
   test.libs << 'test'

data/VERSION

@@ -1 +1 @@
-0.8.5
+0.8.6

data/bin/gff3-fetch

@@ -25,16 +25,16 @@ USAGE = <<EOM
     CDS              : assemble CDS 
     exon             : list all exons
     gene|ORF         : list gene ORFs 
-    other            : use any type from GFF3 definition, e.g. 'Terminate' -- NYI
+    other            : use any type from GFF3 definition, e.g. 'Terminate'
 
   and the following performance options:
 
 		--parser bioruby : use BioRuby GFF3 parser (slow)
-    --parser line    : use GFF3 line parser -- in preparation
-    --parser block   : use GFF3 block parser (optimistic) -- NYI
+    --parser line    : use GFF3 line parser (faster, default)
+    --block          : parse GFF3 by block (optimistic) -- NYI
     --cache full     : load all in RAM (fast, default)
     --cache none     : do not load anything in memory (slow)
-    --cache lru      : use LRU cache (limit RAM use, fast) -- NYI
+    --cache lru      : use least recently used cache (limit RAM use, fast) -- NYI
     --max-cpus num   : use num threads -- NYI
     --emboss         : use EMBOSS translation (fast) -- NYI
 
@@ -74,21 +74,45 @@ USAGE = <<EOM
 
       gff3-fetch mRNA test/data/gff/test.gff3 --trace ERROR
 
+    Fine tuning outpt - show messages matching regex 
+
+      gff3-fetch mRNA test/data/gff/test.gff3 --trace '=msg =~ /component/'
+
     Fine tuning output - write log messages to file.log
 
       gff3-fetch mRNA test/data/gff/test.gff3 --trace ERROR --logger file.log
 
+For more information on output, see the bioruby-logger plugin.
+
 == Performance
 
 time gff3-fetch cds m_hapla.WS217.dna.fa m_hapla.WS217.gff3 2> /dev/null > test.fa 
 
-  Cache              real     user     sys
+  Digesting parser:
+
+  Cache              real     user     sys  version     RAM
   ------------------------------------------------------------
   full,bioruby       12m41    12m28    0m09 (0.8.0)
   full,line          12m13    12m06    0m07 (0.8.5)
-  none,bioruby      504m39   477m49   26m50 (0.8.0)
-  lru,bioruby        ?
-  ----------------------------------------------------
+  full,line,lazy     11m51    11m43    0m07 (0.8.6)     6,600M
+
+  none,bioruby      504m     477m     26m50 (0.8.0)
+  none,line         297m     267m     28m36 (0.8.5)       
+  none,line,lazy    132m     106m     26m01 (0.8.6)       650M
+
+  lru,bioruby       533m     510m     22m47 (0.8.5)
+  lru,line          353m     326m     26m44 (0.8.5)  1K
+  lru,line          305m     281m     22m30 (0.8.5) 10K
+  lru,line,lazy     182m     161m     21m10 (0.8.6) 10K
+  lru,line,lazy      75m      75m      0m17 (0.8.6) 50K   730M
+  ------------------------------------------------------------
+
+  Block parser:
+
+  Cache              real     user     sys  gff3 version
+  ------------------------------------------------------------
+  in preparation
+  ------------------------------------------------------------
 
 where
 
@@ -132,7 +156,7 @@ Bio::Log::CLI.trace('info')
 options = OpenStruct.new()
 
 # ---- Default options
-options.parser = :bioruby
+options.parser = :line
 
 opts = OptionParser.new() { |opts|
   opts.on_tail("-h", "--help", "Show help and examples") {
@@ -141,7 +165,7 @@ opts = OptionParser.new() { |opts|
     exit()
   }
 
-  opts.on("--parser [bioruby,line]", String, "Parser (default bioruby)") do |p|
+  opts.on("--parser [bioruby,line]", String, "Parser (default line)") do |p|
     case p.downcase
       when 'bioruby'
         options.parser = :bioruby
@@ -152,12 +176,12 @@ opts = OptionParser.new() { |opts|
     end
   end
 
-  opts.on("--cache [none,full]", String, "Caching (default full)") do |cache|
+  opts.on("--cache [full,lru,none]", String, "Caching (default full)") do |cache|
     case cache.downcase
       when 'none'
         options.cache = :cache_none
-      # when 'lru'
-      #   options.cache = :cache_lru
+      when 'lru'
+        options.cache = :cache_lru
       when 'full'
         options.cache = :cache_full
       else
@@ -165,6 +189,10 @@ opts = OptionParser.new() { |opts|
     end
   end
 
+  opts.on("--block", "Parse by block") do |b|
+    options.block = true
+  end
+
   opts.on("--no-assemble", "output sequences without assembling") do |b|
     options.no_assemble = true
   end
@@ -229,6 +257,7 @@ ARGV.each do | fn |
   opts = {}
   opts[:validate]         = options.validate
   opts[:parser]           = options.parser
+  opts[:block]            = options.block
   opts[:cache_components] = options.cache
   opts[:cache_records]    = options.cache
   opts[:fasta_filename]   = fastafn if fastafn
@@ -238,6 +267,10 @@ ARGV.each do | fn |
   opts[:phase]            = options.phase
   opts[:debug]            = options.debug
 
+  include Bio::GFFbrowser::Helpers::Logger
+  debug $:.to_s
+  debug opts.to_s
+  log_sys_info("BaseLine")
   gff3 = Bio::GFFbrowser::GFF3.new(fn,opts)
 
   gff = gff3.assembler
@@ -265,8 +298,11 @@ ARGV.each do | fn |
             writer.put(id,seq)
           end
     else
-      raise "Unknown action on type <#{gfftype}>"
+          gff.each_seq(gfftype.downcase) do | id, seq |
+            writer.put(id,seq)
+          end
   end
+  log_sys_info("Done")
   fastafn = nil
 end
 

data/bio-gff3.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bio-gff3}
-  s.version = "0.8.5"
+  s.version = "0.8.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Pjotr Prins"]
-  s.date = %q{2011-01-13}
+  s.date = %q{2011-01-17}
   s.default_executable = %q{gff3-fetch}
   s.description = %q{GFF3 (genome browser) information and digest mRNA and CDS sequences.
 Options for low memory use and caching of records.
@@ -31,7 +31,9 @@ Support for external FASTA files.
     "bin/gff3-fetch",
     "bio-gff3.gemspec",
     "lib/bio-gff3.rb",
+    "lib/bio/db/gff/block/gffblockparser.rb",
     "lib/bio/db/gff/digest/gffinmemory.rb",
+    "lib/bio/db/gff/digest/gfflrucache.rb",
     "lib/bio/db/gff/digest/gffnocache.rb",
     "lib/bio/db/gff/digest/gffparser.rb",
     "lib/bio/db/gff/file/gfffasta.rb",
@@ -65,6 +67,8 @@ Support for external FASTA files.
     "test/data/gff/test.gff3",
     "test/data/regression/test_ext_gff3.rtest",
     "test/data/regression/test_gff3.rtest",
+    "test/data/regression/test_lrucache_ext_gff3.rtest",
+    "test/data/regression/test_lrucache_gff3.rtest",
     "test/data/regression/test_nocache_ext_gff3.rtest",
     "test/data/regression/test_nocache_gff3.rtest",
     "test/helper.rb",
@@ -94,41 +98,32 @@ Support for external FASTA files.
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<bio>, [">= 1.3.1"])
-      s.add_runtime_dependency(%q<bio-logger>, ["> 0.5.0"])
+      s.add_runtime_dependency(%q<log4r>, ["> 1.1.6"])
+      s.add_runtime_dependency(%q<bio-logger>, ["> 0.8.0"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
-      s.add_development_dependency(%q<rspec>, [">= 2.0.0"])
-      s.add_runtime_dependency(%q<bio>, [">= 1.4.1"])
-      s.add_runtime_dependency(%q<log4r>, ["> 1.1.6"])
-      s.add_runtime_dependency(%q<bio-logger>, [">= 0.6.1"])
-      s.add_development_dependency(%q<rspec>, ["> 2.0"])
+      s.add_development_dependency(%q<rspec>, [">= 2.3.0"])
     else
       s.add_dependency(%q<bio>, [">= 1.3.1"])
-      s.add_dependency(%q<bio-logger>, ["> 0.5.0"])
+      s.add_dependency(%q<log4r>, ["> 1.1.6"])
+      s.add_dependency(%q<bio-logger>, ["> 0.8.0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_dependency(%q<rcov>, [">= 0"])
-      s.add_dependency(%q<rspec>, [">= 2.0.0"])
-      s.add_dependency(%q<bio>, [">= 1.4.1"])
-      s.add_dependency(%q<log4r>, ["> 1.1.6"])
-      s.add_dependency(%q<bio-logger>, [">= 0.6.1"])
-      s.add_dependency(%q<rspec>, ["> 2.0"])
+      s.add_dependency(%q<rspec>, [">= 2.3.0"])
     end
   else
     s.add_dependency(%q<bio>, [">= 1.3.1"])
-    s.add_dependency(%q<bio-logger>, ["> 0.5.0"])
+    s.add_dependency(%q<log4r>, ["> 1.1.6"])
+    s.add_dependency(%q<bio-logger>, ["> 0.8.0"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
     s.add_dependency(%q<rcov>, [">= 0"])
-    s.add_dependency(%q<rspec>, [">= 2.0.0"])
-    s.add_dependency(%q<bio>, [">= 1.4.1"])
-    s.add_dependency(%q<log4r>, ["> 1.1.6"])
-    s.add_dependency(%q<bio-logger>, [">= 0.6.1"])
-    s.add_dependency(%q<rspec>, ["> 2.0"])
+    s.add_dependency(%q<rspec>, [">= 2.3.0"])
   end
 end
 

data/lib/bio/db/gff/block/gffblockparser.rb

@@ -0,0 +1,93 @@
+
+module Bio
+  module GFFbrowser
+
+    module Block
+
+      # The block parser simplifies parsing, by assuming GFF3 is 
+      # organised into blocks. All relevant information is 
+      # resolved a block at a time.
+      class GffBlockParser
+        include FastLineParser 
+
+        def initialize filename, options
+          info "Starting block parser"
+          @filename = filename
+          @options = options
+          @iter = Bio::GFF::GFF3::FileIterator.new(@filename)
+        end
+
+        def parse(gfftype)
+          @inseqidlist = {}
+          # Fetch FASTA first
+          @sequencelist = {}
+          if @options[:fasta_filename]
+            File.open(@options[:fasta_filename]) do | f |
+              fasta = Bio::GFF::FastaReader.new(f)
+              fasta.each do | id, fastarec |
+                # p fastarec
+                @sequencelist[id] = fastarec
+              end
+            end
+          else
+            # Embedded FASTA
+            @iter.each_sequence do | id, bioseq |
+              @sequencelist[id] = bioseq.to_s
+            end
+          end
+          seqid = nil
+          recs = []
+          @iter.each_rec do | fpos, line |
+            rec = FastLineRecord.new(parse_line_fast(line))
+            if seqid != rec.seqid 
+              # starting a new block
+              if @inseqidlist[rec.seqid]
+                # not a well formed GFF3 file, we need
+                # to drop
+                error "GFF3 file not sorted, falling back to line parser"
+                raise "ERROR, bailing out"
+              end
+              parse_block(gfftype,recs,@sequencelist[seqid])  { | id, seq | yield id,seq } if seqid
+              recs = []
+              seqid = rec.seqid
+              @inseqidlist[seqid] = true
+            end
+            recs.push rec
+          end
+          parse_block(gfftype,recs,@sequencelist[seqid])  { | id, seq | yield id,seq } if seqid
+        end
+
+        # Parse sequence objects sharing the same seqid 
+        # and yield each +gfftype+ as an iq,seq
+        def parse_block gfftype, recs, sequence
+          recs.each do | rec |
+            if rec.feature_type.downcase == gfftype
+              yield rec.id, sequence[rec.start-1..rec.end-1]
+            end
+          end
+        end
+   
+        def each_seq(gfftype) 
+          parse(gfftype) { | id, seq | yield id,seq }
+
+        end
+
+        def each_gene_seq
+          each_seq('gene') { | id, seq | yield id,seq }
+        end
+        def each_mRNA_seq
+          each_seq('mrna') { | id, seq | yield id,seq }
+
+        end
+        def each_exon_seq
+          each_seq('exon') { | id, seq | yield id,seq }
+
+        end
+        def each_CDS_seq
+          each_seq('cds') { | id, seq | yield id,seq }
+
+        end
+      end
+    end
+  end
+end

data/lib/bio/db/gff/digest/gffinmemory.rb

@@ -54,7 +54,9 @@ module Bio
           validate_cdss 
           show_unrecognized_features 
           @genelist      = @count_ids.keys 
+          log_sys_info("After reading GFF")
           read_fasta
+          log_sys_info("After reading FASTA")
         end
 
         def each_item list

data/lib/bio/db/gff/digest/gfflrucache.rb

@@ -0,0 +1,208 @@
+#
+# = bio/db/gff/gfflrucache.rb - Assemble mRNA and CDS from GFF by LRU cache
+#
+# Copyright::  Copyright (C) 2010
+#              Pjotr Prins <pjotr.prins@thebird.nl>
+# License::    The Ruby License
+#
+# Fetch information from a GFF file without using RAM - using a least 
+# recently used cache 'LRU' - also check out the caching edition, 
+# which uses limited amounts of RAM.
+#
+# In effect it is NoCache with parser recs cached in the LRU hash
+
+require 'bio/db/gff/digest/gffparser'
+require 'bio/system/lruhash'
+
+module Bio
+  module GFFbrowser
+
+    module Digest
+
+      module LruCacheHelpers 
+
+        # Module to fetch a line from GFF3 file and returns a parsed 
+        # record
+        module SeekRec
+          # Fetch a record using fh and file seek position
+          def SeekRec::fetch(fh,fpos,parser)
+            return nil if fh==nil or fpos==nil
+            fh.seek(fpos)
+            if parser == :bioruby 
+              GFF::GFF3::BioRubyFileRecord.new(fpos, fh.gets)
+            else
+              GFF::GFF3::FastParserFileRecord.new(fpos, fh.gets)
+            end
+          end
+        end
+
+        module LruRec
+          # Fetch a record using fh and file seek position,
+          # utilising the LRU cache
+          def fetch(fh,fpos,parser)
+            return nil if fh==nil or fpos==nil
+            rec = @lru[fpos]
+            if rec==nil
+              rec = SeekRec::fetch(fh,fpos,parser)
+              @lru[fpos] = rec
+            end
+            rec
+          end
+        end
+
+        # Helper class which gives Hash-like access to the 
+        # no-cache GFF3 file
+        class SeekRecList 
+          include LruRec
+
+          def initialize fh, parser, lru
+            @fh = fh
+            @h = {}
+            @parser = parser
+            @lru = lru
+          end
+
+          def []= id, rec
+            raise "id #{id} occurs twice!" if @h[id]
+            fpos = rec.io_seek
+            @h[id] = fpos
+          end
+
+          def [](id)
+            fpos = @h[id]
+            fetch(@fh,fpos,@parser)
+          end
+
+          def each 
+            @h.each do | id,fpos |
+              yield id, self[id]
+            end
+          end
+        end
+
+        # List of ids
+        class SeekLinkedRecs < Hash
+          include Helpers::Logger
+          def add id, rec
+            info "Adding #{rec.feature_type} <#{id}>"
+            self[id] = [] if self[id] == nil
+            self[id] << rec.io_seek
+          end
+          # validation is switched off for LruCache
+          def validate_seqname
+          end
+          # validation is switched off for LruCache
+          def validate_nonoverlapping
+          end
+          # validation is switched off for LruCache
+          def validate_shared_parent
+          end
+        end
+      end
+
+      class LruTracker
+        include Helpers::Logger
+        attr_accessor :hits, :misses, :calls
+        attr_reader :cache
+
+        def initialize 
+          @cache = LRUHash.new 50000
+          @hits = 0
+          @misses = 0
+          @calls = 0
+        end
+ 
+        def [](name)
+          @calls += 1
+          item = @cache[name]
+          if @cache[name] == nil
+            @misses += 1
+          else
+            @hits += 1
+          end
+          item
+        end
+
+        def []=(name,item)
+          @cache[name] = item
+        end
+        def display msg
+          info "Cache calls #{msg}  = #{@calls}" 
+          info "Cache hits #{msg}   = #{@hits}" 
+          info "Cache misses #{msg} = #{@misses}" 
+        end
+      end
+
+      class LruCache
+        include Parser
+        include LruCacheHelpers
+        include Gff3Sequence
+        include LruRec
+
+        def initialize filename, options
+          @filename = filename
+          @options = options
+          @iter = Bio::GFF::GFF3::FileIterator.new(@filename)
+          @lru = LruTracker.new
+        end
+
+        # parse the whole file once and store all seek locations, 
+        # rather than the records themselves
+        def parse
+          info "---- Digest DB and store data in mRNA Hash (LruCache)"
+          @count_ids          = Counter.new   # Count ids
+          @count_seqnames     = Counter.new   # Count seqnames
+          @componentlist      = SeekRecList.new(@iter.fh,@options[:parser],@lru) # Store containers, like genes, contigs
+          @orflist            = SeekLinkedRecs.new   # Store linked gene records
+          @mrnalist           = SeekLinkedRecs.new   # Store linked mRNA records
+          @cdslist            = SeekLinkedRecs.new
+          @exonlist           = SeekLinkedRecs.new
+          @sequencelist       = {}
+          @unrecognized_features = {}
+          @iter.each_rec do |fpos, line|
+            rec = case @options[:parser]
+              when :bioruby
+                Bio::GFF::GFF3::BioRubyFileRecord.new(fpos, line)
+              when :line
+                Bio::GFF::GFF3::FastParserFileRecord.new(fpos, line)
+              else
+                raise 'Unknown parser'
+            end
+            store_record(rec)
+          end
+          @iter.each_sequence do | id, bioseq |
+            @sequencelist[id] = bioseq.to_s
+          end
+          validate_mrnas 
+          validate_cdss
+          show_unrecognized_features
+          @genelist      = @count_ids.keys 
+          read_fasta
+          @lru.display('After reading files')
+        end
+
+        def each_item list
+          # p list.class
+          fh = @iter.fh
+          list.each do | id, io_seeklist |
+            recs = []
+            io_seeklist.each do | fpos |
+              recs << fetch(fh,fpos,@options[:parser])
+            end
+            seqid = recs[0].seqname
+            component = find_component(recs[0])
+            if @options[:no_assemble]
+              recs.each do | rec |
+                yield id, [rec], component
+              end
+            else
+              yield id, recs, component
+            end
+          end
+          @lru.display('After iterating')
+        end
+        
+      end
+    end
+  end
+end