bio-blastxmlparser 0.6.0

data/bio-blastxmlparser.gemspec

@@ -0,0 +1,88 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{bio-blastxmlparser}
+  s.version = "0.6.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Pjotr Prins"]
+  s.date = %q{2011-02-14}
+  s.default_executable = %q{blastxmlparser}
+  s.description = %q{Fast big data XML parser and library, written in Ruby}
+  s.email = %q{pjotr.public01@thebird.nl}
+  s.executables = ["blastxmlparser"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bin/blastxmlparser",
+    "bio-blastxmlparser.gemspec",
+    "lib/bio-blastxmlparser.rb",
+    "lib/bio/db/blast/parser/nokogiri.rb",
+    "lib/bio/db/blast/xmliterator.rb",
+    "lib/bio/db/blast/xmlsplitter.rb",
+    "sample/bioruby.rb",
+    "sample/blastxmlparserdemo.rb",
+    "sample/libxml_sax.rb",
+    "sample/nokogiri_dom.rb",
+    "sample/nokogiri_sax.rb",
+    "sample/nokogiri_split_dom.rb",
+    "spec/bio-blastxmlparser_spec.rb",
+    "spec/spec_helper.rb",
+    "test/data/aa_example.fasta",
+    "test/data/aa_example_blastp.m7",
+    "test/data/nt_example.fasta",
+    "test/data/nt_example_blastn.m7",
+    "timings.sh"
+  ]
+  s.homepage = %q{http://github.com/pjotrp/bioruby-blastxmlparser}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{BLAST XML parser}
+  s.test_files = [
+    "spec/bio-blastxmlparser_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<bio-logger>, ["> 0.8.0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.4"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<bio-logger>, ["> 0.8.0"])
+      s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
+      s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<bio-logger>, ["> 0.8.0"])
+    s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
+    s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/lib/bio-blastxmlparser.rb

@@ -0,0 +1,17 @@
+# find local plugin installation, and use it when there
+rootpath = File.dirname(File.dirname(__FILE__))
+bio_logger_path = File.join(rootpath,'..','bioruby-logger','lib')
+if File.directory? bio_logger_path
+  $: << bio_logger_path
+  $stderr.print "bio-logger loaded directly\n"
+else
+  require "rubygems"
+  gem "bio-logger"
+end
+require 'bio-logger'
+
+Bio::Log::LoggerPlus.new('bio-blastxmlparser')
+
+require 'bio/db/blast/parser/nokogiri'
+require 'bio/db/blast/xmlsplitter'
+require 'bio/db/blast/xmliterator'

data/lib/bio/db/blast/parser/nokogiri.rb

@@ -0,0 +1,203 @@
+
+require 'nokogiri'
+require 'enumerator'
+
+module Bio
+  module Blast
+
+    module XPath
+      def field name
+        res = if @prefix
+          @xml.xpath(@prefix+name+'/text()')
+        else
+          @xml.xpath(name+'/text()')
+        end
+        if res == nil
+          logger = Bio::Log::LoggerPlus['bio-blastxmlparser']
+          logger.warn("XML elemement <#{name}> has no content")
+        end
+        res.to_s
+      end
+
+    end
+
+    # Some magic to create XML -> method mappers, on the fly
+    module MapXPath
+      include XPath
+      def MapXPath.define_s map
+        map.each { |k,v| 
+          define_method(v) {
+            field(k)
+          }
+        }
+      end
+      def MapXPath.define_i map
+        map.each { |k,v| 
+          define_method(v) {
+            field(k).to_i
+          }
+        }
+      end
+      def MapXPath.define_f map
+        map.each { |k,v| 
+          define_method(v) {
+            field(k).to_f
+          }
+        }
+      end
+      def [] name
+        field(name)
+      end
+
+    end
+
+    class NokogiriBlastHsp
+      include MapXPath
+      attr_reader :parent
+      MapXPath.define_s 'Hsp_id'          => :hsp_id,
+                        'Hsp_qseq'        => :qseq,
+                        'Hsp_hseq'        => :hseq,
+                        'Hsp_midline'     => :midline
+      MapXPath.define_i 'Hsp_num'         => :hsp_num,
+                        'Hsp_score'       => :score,
+                        'Hsp_query-from'  => :query_from,
+                        'Hsp_query-to'    => :query_to,
+                        'Hsp_hit-from'    => :hit_from,
+                        'Hsp_hit-to'      => :hit_to,
+                        'Hsp_query-frame' => :query_frame,
+                        'Hsp_hit-frame'   => :hit_frame,
+                        'Hsp_identity'    => :identity,
+                        'Hsp_positive'    => :positive,
+                        'Hsp_align-len'   => :align_len
+      MapXPath.define_f 'Hsp_bit-score'   => :bit_score,
+                        'Hsp_evalue'      => :evalue
+                         
+      def initialize xml, parent
+        @xml = xml
+        @parent = parent
+      end
+
+      def to_s
+        s = <<EOM
+  hit_num=#{parent.hit_num}, hsp_num=#{hsp_num}, score=#{score}, bit_score=#{bit_score}
+EOM
+      end
+      
+    end
+
+    class NokogiriBlastHit
+      include MapXPath
+      attr_reader :parent
+      MapXPath.define_s 'Hit_id' => :hit_id,
+                        'Hit_def' => :hit_def,
+                        'Hit_accession' => :accession
+      MapXPath.define_i 'Hit_num' => :hit_num, 
+                        'Hit_len' => :len
+
+      def initialize hit, parent
+        @xml = hit
+        @parent = parent
+      end
+
+      def hsps
+        Enumerator.new { |yielder|
+          @xml.children.each do | hit_field |
+            if hit_field.name == 'Hit_hsps'
+              hit_field.children.each do | hsp |
+                if hsp.name == 'Hsp'
+                  yielder.yield NokogiriBlastHsp.new(hsp,self)
+                end
+              end
+            end
+          end
+        }
+      end
+
+      def each
+        hsps.each { | h | yield h }
+      end
+
+
+      def to_s
+        s = <<EOM
+iter_num=#{parent.iter_num}, hit_id=#{hit_id}, hit_def=#{hit_def}, hit_num=#{hit_num}
+EOM
+      end
+      
+    end
+
+    class NokogiriBlastIterator
+      include MapXPath
+      attr_reader :parent
+      MapXPath.define_s 'Iteration_query-ID'  => :query_id,  
+                        'Iteration_query-def' => :query_def 
+
+      MapXPath.define_i 'Iteration_iter-num'  => :iter_num,
+                        'Iteration_query-len' => :query_len
+
+
+      def initialize iterator, parent, opts = { :prefix => nil }
+        @parent = parent
+        @prefix = opts[:prefix]
+        @xml = if iterator.name == 'document'
+                 iterator.children.first
+               else
+                 iterator
+               end
+        name2 = @xml.name
+        raise "Error in BLAST XML, expected Iteration node, but got #{name2}" if name2 != 'Iteration'
+        # p [:iter,@prefix,'@@Iteratition_iter-num',field('Iteration_iter-num')]
+        # print @xml.to_s
+      end
+
+      def hits
+        Enumerator.new { |yielder|
+          @xml.children.each do | iter_field |
+            if iter_field.name == 'Iteration_hits'
+              iter_field.children.each do | hit |
+                if hit.name == 'Hit'
+                  yielder.yield NokogiriBlastHit.new(hit,self)
+                end
+              end
+            end
+          end
+        }
+      end
+
+      def each
+        hits.each { | h | yield h }
+      end
+
+      def to_s
+        s = <<EOM
+iter_num=#{iter_num}, query_id=#{query_id}
+EOM
+      end
+    end
+
+    class NokogiriBlastXml 
+      def initialize document
+        @xml = document
+      end
+
+      def to_enum
+        Enumerator.new { |yielder| 
+          each { | iterator | yielder.yield(iterator) }
+        }
+      end
+
+      def each &block
+        doc = Nokogiri::XML(@xml) { | cfg | cfg.noblanks }
+        doc.root.children.each do |blastnode|
+          if blastnode.name == 'BlastOutput_iterations'
+            blastnode.children.each do | iteration |
+              if iteration.name == 'Iteration'
+                block.call(NokogiriBlastIterator.new(iteration,self))
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bio/db/blast/xmliterator.rb

@@ -0,0 +1,19 @@
+
+
+module Bio
+  module Blast
+
+    # Iterate a BLAST file yielding (lazy) results
+    class XmlIterator
+      def initialize blastfilename
+        @fn = blastfilename
+      end
+      
+      def to_enum
+        logger = Bio::Log::LoggerPlus['bio-blastxmlparser']
+        logger.info("parsing (full) #{@fn}")
+        NokogiriBlastXml.new(File.new(@fn)).to_enum
+      end
+    end
+  end
+end

data/lib/bio/db/blast/xmlsplitter.rb

@@ -0,0 +1,43 @@
+module Bio
+  module Blast
+    # Reads a full XML result and splits it out into a buffer for each
+    # Iteration (query result).
+    class XmlSplitterIterator
+      def initialize fn
+        @fn = fn
+      end
+
+      def to_enum 
+        Enumerator.new do | yielder | 
+          logger = Bio::Log::LoggerPlus['bio-blastxmlparser']
+          logger.info("split file parsing #{@fn}")
+          f = File.open(@fn)
+          # Skip BLAST header
+          f.each_line do | line |
+            break if line.strip == "<Iteration>"
+          end
+          # Return each Iteration as an XML DOM
+          each_iteration(f) do | buf |
+            iteration = Nokogiri::XML.parse(buf.join) { | cfg | cfg.noblanks }
+            yielder.yield NokogiriBlastIterator.new(iteration,self,:prefix=>nil)
+          end
+        end
+      end
+
+    private
+
+      def each_iteration f
+        # b = ["<?xml version=\"1.0\"?>\n","<Iteration>\n"]
+        # b = []
+        b = ["<Iteration>\n"]
+        f.each_line do | line |
+          b << line
+          if line.strip == "</Iteration>"
+            yield b
+            b = []
+          end
+        end
+      end
+    end
+  end
+end

data/sample/bioruby.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require 'bio'
+
+fn = 'test/data/nt_example_blastn.m7'
+
+# Iterates over each XML result.
+# The variable "report" is a Bio::Blast::Report object.
+# Bio::Blast.reports(ARGF) do |report| 
+Bio::Blast.reports(File.new(fn)) do |report| 
+  puts "Hits for " + report.query_def + " against " + report.db
+  report.each do |hit|
+    print hit.target_id, "\t", hit.evalue, "\n" if hit.evalue < 0.001
+  end
+end

data/sample/blastxmlparserdemo.rb

@@ -0,0 +1,17 @@
+#! /usr/bin/ruby
+
+rootpath = File.dirname(File.dirname(__FILE__))
+$: << File.join(rootpath,'lib')
+
+require 'bio-blastxmlparser'
+fn = 'test/data/nt_example_blastn.m7'
+n = Bio::Blast::XmlIterator.new(fn).to_enum
+n.each do | iter |
+  puts "Hits for " + iter.query_id
+  iter.each do | hit |
+    hit.each do | hsp |
+      print hit.hit_id, "\t", hsp.evalue, "\n" if hsp.evalue < 0.001
+    end
+  end
+end
+

data/sample/libxml_sax.rb

@@ -0,0 +1,25 @@
+#! /usr/bin/ruby
+
+require 'rubygems'
+require 'libxml'
+
+include LibXML
+
+class PostCallbacks
+  include XML::SaxParser::Callbacks
+
+  def on_start_element(element, attributes)
+    if element == 'Iteration_iter-num'
+      # Process row of data here
+      print "---- ",element
+    end
+    # if element == 'Hsp_score'
+    #   print "---- ",element
+    # end
+  end
+end
+
+parser = XML::SaxParser.file("test/data/nt_example_blastn.m7")
+parser.callbacks = PostCallbacks.new
+parser.parse
+

data/sample/nokogiri_dom.rb

@@ -0,0 +1,17 @@
+#! /usr/bin/ruby
+
+require 'rubygems'
+require 'nokogiri'
+
+include Nokogiri
+
+input = Nokogiri::XML(File.new("test/data/nt_example_blastn.m7"))
+
+input.root.xpath("//Iteration").each do | e |
+  print "---- "
+  print e.xpath("Iteration_iter-num/text()"),"\n"
+  print e.xpath("Iteration_hits/Hit/Hit_hsps/Hsp/Hsp_score/text()").map {|n| n.to_s}, "\n"
+
+end
+
+