bio-blastxmlparser 0.6.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,18 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Runtime dependencies
+# gem "bio", ">= 1.3.1"
+gem "bio-logger", "> 0.8.0"
+gem "nokogiri", ">= 1.4.4"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2.3.0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.2"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,34 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    bio-logger (0.9.0)
+      log4r (>= 1.1.9)
+    diff-lcs (1.1.2)
+    git (1.2.5)
+    jeweler (1.5.2)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    log4r (1.1.9)
+    nokogiri (1.4.4)
+    rake (0.8.7)
+    rcov (0.9.9)
+    rspec (2.3.0)
+      rspec-core (~> 2.3.0)
+      rspec-expectations (~> 2.3.0)
+      rspec-mocks (~> 2.3.0)
+    rspec-core (2.3.1)
+    rspec-expectations (2.3.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bio-logger (> 0.8.0)
+  bundler (~> 1.0.0)
+  jeweler (~> 1.5.2)
+  nokogiri (>= 1.4.4)
+  rcov
+  rspec (~> 2.3.0)

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Pjotr Prins
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,211 @@
+= bio-blastxmlparser
+
+blastxmlparser is a fast big-data BLAST XML file parser. Rather than
+loading everything in memory, XML is parsed by BLAST query
+(Iteration). Not only has this the advantage of low memory use, it may
+also be faster when IO continues in parallel (disks read ahead).
+
+Next to the API, blastxmlparser comes as a command line utility, which
+can be used to filter results and requires no understanding of Ruby.
+
+== Performance
+
+XML parsing is expensive. blastxmlparser uses the Nokogiri C, or Java, XML
+parser, based on libxml2. Basically a DOM parser is used for subsections of a
+document, tests show this is faster than a SAX parser with Ruby callbacks.  To
+see why libxml2 based Nokogiri is fast, see
+http://www.rubyinside.com/ruby-xml-performance-benchmarks-1641.html and
+http://www.xml.com/lpt/a/1703. 
+
+The parser is also designed with other optimizations, such as lazy evaluation,
+only creating objects when required, and (future) parallelization. When parsing
+a full BLAST result usually only a few fields are used. By using XPath queries
+only the relevant fields are queried.
+
+Timings for parsing test/data/nt_example_blastn.m7 (file size 3.4Mb) 
+
+Nokogiri DOM (default)
+
+real    0m1.259s
+user    0m1.052s
+sys     0m0.144s
+
+Nokogiri split DOM
+
+real    0m1.713s
+user    0m1.444s
+sys     0m0.160s
+
+BioRuby ReXML DOM parser
+
+real    1m14.548s
+user    1m13.065s
+sys     0m0.472s
+
+== Install
+
+  gem install bio-blastxmlparser
+
+Nokogiri XML parser is required. To install it,
+the libxml2 libraries and headers need to be installed first, for
+example on Debian:
+
+  apt-get install libxslt-dev libxml2-dev
+  gem install bio-blastxmlparser
+
+for more installation on other platforms see
+http://nokogiri.org/tutorials/installing_nokogiri.html. 
+
+== API
+
+To loop through a BLAST result:
+
+    >> require 'bio-blastxmlparser'
+    >> fn = 'test/data/nt_example_blastn.m7'
+    >>   n = Bio::Blast::XmlIterator.new(fn).to_enum
+    >>   n.each do | iter |
+    >>     puts "Hits for " + iter.query_id
+    >>     iter.each do | hit |
+    >>       hit.each do | hsp |
+    >>         print hit.hit_id, "\t", hsp.evalue, "\n" if hsp.evalue < 0.001
+    >>       end
+    >>     end
+    >>   end
+
+The next example parses XML using less memory
+
+    >> blast = XmlSplitterIterator.new(fn).to_enum
+    >> iter = blast.next
+    >> iter.iter_num
+    >> 1
+    >> iter.query_id
+    => "lcl|1_0"
+
+Get the first hit
+
+    >> hit = iter.hits.first
+    >> hit.hit_num
+    => 1
+    >> hit.hit_id
+    => "lcl|I_74685"
+    >> hit.hit_def
+    => "[57809 - 57666] (REVERSE SENSE) "
+    >> hit.accession
+    => "I_74685"
+    >> hit.len
+    => 144
+
+Get the parent info
+
+    >> hit.parent.query_id
+    => "lcl|1_0"
+ 
+Get the first Hsp
+
+    >> hsp = hit.hsps.first
+    >> hsp.hsp_num
+    => 1
+    >> hsp.bit_score
+    => 145.205
+    >> hsp.score
+    => 73
+    >> hsp.evalue
+    => 5.82208e-34
+    >> hsp.query_from
+    => 28
+    >> hsp.query_to
+    => 100
+    >> hsp.query_frame
+    => 1
+    >> hsp.hit_frame
+    => 1
+    >> hsp.identity
+    => 73
+    >> hsp.positive
+    => 73
+    >> hsp.align_len
+    => 73
+    >> hsp.qseq
+    => "AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCTGCCTGCCAACCTATATGCTCCTGTGTTTAG"
+    >> hsp.hseq
+    => "AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCTGCCTGCCAACCTATATGCTCCTGTGTTTAG"
+    >> hsp.midline
+    => "|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||"
+
+It is possible to use the XML element names, over methods. E.g.
+
+    >> hsp.field("Hsp_bit-score")
+    => "145.205"
+    >> hsp["Hsp_bit-score"]
+    => "145.205"
+
+Note that these are always String values.
+
+Fetch the next result (Iteration)
+
+    >> iter2 = blast.next
+    >> iter2.iter_num
+    >> 2 
+    >> iter2.query_id
+    => "lcl|2_0"
+
+etc. etc.
+
+For more examples see the files in ./spec
+
+== Usage
+
+  blastxmlparser [options] file(s)
+
+    -p, --parser name                Use full|split parser (default full)
+    -n, --named fields               Set named fields
+    -e, --exec filter                Execute filter
+
+        --logger filename            Log to file (default stderr)
+        --trace options              Set log level (default INFO, see bio-logger)
+    -q, --quiet                      Run quietly
+    -v, --verbose                    Run verbosely
+        --debug                      Show debug messages
+    -h, --help                       Show help and examples
+
+  bioblastxmlparser filename(s)
+
+    Use --help switch for more information
+
+== Examples
+
+Print result fields of iterations containing 'lcl', using a regex
+
+  blastxmlparser -e 'iter.query_id=~/lcl/' test/data/nt_example_blastn.m7
+
+Print fields where bit_score > 145
+
+  blastxmlparser -e 'hsp.bit_score>145' test/data/nt_example_blastn.m7
+
+It is also possible to use the XML element names directly
+
+  blastxmlparser -e 'hsp["Hsp_bit-score"].to_i>145' test/data/nt_example_blastn.m7
+
+Print named fields where E-value < 0.001 and hit length > 100
+
+  blastxmlparser -n 'hsp.evalue,hsp.qseq' -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+
+  1       5.82208e-34     AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCT...
+  2       5.82208e-34     AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCT...
+  3       2.76378e-11     AATATGGTAGCTACAGAAACGGTAGTACACTCTTC     
+  4       1.13373e-13     CTAAACACAGGAGCATATAGGTTGGCAGGCAGGCAAAAT 
+  5       2.76378e-11     GAAGAGTGTACTACCGTTTCTGTAGCTACCATATT     
+  etc. etc.
+
+To use the low-mem version use
+
+  blastxmlparser --parser split -n 'hsp.evalue,hsp.qseq' -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+
+== URL
+
+The project lives at http://github.com/pjotrp/blastxmlparser. If you use this software, please cite http://dx.doi.org/10.1093/bioinformatics/btq475
+
+== Copyright
+
+Copyright (c) 2011 Pjotr Prins under the MIT licence.  See LICENSE.txt and http://www.opensource.org/licenses/mit-license.html for further details.
+

data/Rakefile

@@ -0,0 +1,50 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "bio-blastxmlparser"
+  gem.homepage = "http://github.com/pjotrp/bioruby-blastxmlparser"
+  gem.license = "MIT"
+  gem.summary = %Q{BLAST XML parser}
+  gem.description = %Q{Fast big data XML parser and library, written in Ruby}
+  gem.email = "pjotr.public01@thebird.nl"
+  gem.authors = ["Pjotr Prins"]
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "bio-blastxmlparser #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.6.0

data/bin/blastxmlparser

@@ -0,0 +1,165 @@
+#!/usr/bin/env ruby
+#
+# BioRuby bio-blastxmlparser Plugin
+# Author:: Pjotr Prins
+# Copyright:: 2011
+# License:: MIT License
+#
+# Copyright (C) 2010,2011 Pjotr Prins <pjotr.prins@thebird.nl>
+
+rootpath = File.dirname(File.dirname(__FILE__))
+$: << File.join(rootpath,'lib')
+
+BLASTXML_VERSION = File.new(File.join(rootpath,'VERSION')).read.chomp
+
+$stderr.print "BioRuby BLAST XML Parser "+BLASTXML_VERSION+" Copyright (C) 2011 Pjotr Prins <pjotr.prins@thebird.nl>\n\n"
+
+USAGE = <<EOM
+
+  bioblastxmlparser filename(s)
+
+    Use --help switch for more information
+
+== Examples
+
+Print result fields of iterations containing 'lcl', using a regex
+
+  blastxmlparser -e 'iter.query_id=~/lcl/' test/data/nt_example_blastn.m7
+
+Print fields where bit_score > 145
+
+  blastxmlparser -e 'hsp.bit_score>145' test/data/nt_example_blastn.m7
+
+It is also possible to use the XML element names directly
+
+  blastxmlparser -e 'hsp["Hsp_bit-score"].to_i>145' test/data/nt_example_blastn.m7
+
+Print named fields where E-value < 0.001 and hit length > 100
+
+  blastxmlparser -n 'hsp.evalue,hsp.qseq' -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+
+  1       5.82208e-34     AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCT...
+  2       5.82208e-34     AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCT...
+  3       2.76378e-11     AATATGGTAGCTACAGAAACGGTAGTACACTCTTC     
+  4       1.13373e-13     CTAAACACAGGAGCATATAGGTTGGCAGGCAGGCAAAAT 
+  5       2.76378e-11     GAAGAGTGTACTACCGTTTCTGTAGCTACCATATT     
+  etc. etc.
+
+To use the low-mem version use
+
+  blastxmlparser --parser split -n 'hsp.evalue,hsp.qseq' -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+
+== URL
+
+The project lives at http://github.com/pjotrp/blastxmlparser. If you use this software, please cite http://dx.doi.org/10.1093/bioinformatics/btq475
+
+== Copyright
+
+Copyright (c) 2011 Pjotr Prins under the MIT licence.  See LICENSE.txt and http://www.opensource.org/licenses/mit-license.html for further details.
+
+EOM
+
+if ARGV.size == 0
+  print USAGE
+  exit 1
+end
+
+require 'bio-blastxmlparser'
+require 'optparse'
+require 'ostruct'
+
+require 'bio-logger'
+Bio::Log::CLI.logger('stderr')
+Bio::Log::CLI.trace('info')
+
+options = OpenStruct.new()
+
+opts = OptionParser.new do |o|
+
+  o.on_tail("-h", "--help", "Show help and examples") {
+    print(opts)
+    print USAGE
+    exit()
+  }
+
+  o.banner = "== Usage\n  #{File.basename($0)} [options] file(s)"
+
+  o.separator ""
+
+  o.on("-p name", "--parser name", "Use full|split parser (default full)") do |p|
+    options.parser = p.to_sym
+  end
+  
+  o.on("-n fields","--named fields",String, "Set named fields") do |s|
+    options.fields = s.split(/,/)
+  end
+  
+  o.on("-e filter","--exec filter",String, "Execute filter") do |s|
+    options.exec = s
+  end
+  
+  o.separator ""
+
+  o.on("--logger filename",String,"Log to file (default stderr)") do | name |
+    Bio::Log::CLI.logger(name)
+  end
+ 
+  o.on("--trace options",String,"Set log level (default INFO, see bio-logger)") do | s |
+    Bio::Log::CLI.trace(s)
+  end
+  
+  o.on("-q", "--quiet", "Run quietly") do |q|
+    Bio::Log::CLI.trace('error')
+  end
+  
+  o.on("-v", "--verbose", "Run verbosely") do |v|
+    Bio::Log::CLI.trace('info')
+  end
+  
+  o.on("--debug", "Show debug messages") do |v|
+    Bio::Log::CLI.trace('debug')
+  end
+
+end
+
+begin
+  opts.parse!(ARGV)
+
+  Bio::Log::CLI.configure('bio-blastxmlparser')
+  logger = Bio::Log::LoggerPlus['bio-blastxmlparser']
+
+  ARGV.each do | fn |
+    logger.info("XML parsing #{fn}")
+    n = if options.parser == :split
+      Bio::Blast::XmlSplitterIterator.new(fn).to_enum
+    else
+      Bio::Blast::XmlIterator.new(fn).to_enum
+    end
+    i = 1
+    n.each do | iter |
+      iter.each do | hit |
+        hit.each do | hsp |
+          do_print = if options.exec
+                       eval(options.exec)
+                     else
+                       true
+                     end
+          if do_print
+            if options.fields
+              print i,"\t"
+              options.fields.each do | f |
+                print eval(f),"\t"
+              end
+              print "\n"
+            else
+              print [i,iter.iter_num,iter.query_id,hit.hit_id,hsp.hsp_num,hsp.evalue].join("\t"),"\n" 
+            end
+            i += 1
+          end
+        end
+      end
+    end
+  end
+rescue OptionParser::InvalidOption => e
+  opts[:invalid_argument] = e.message
+end