bio-blastxmlparser 1.1.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31b42217bb809cde8d5ef3c06d11c6c9123c6413
-  data.tar.gz: 5d23e19fb8c774f7edaffd03bbcb156800679f7f
+  metadata.gz: b96fa7141abe77c13e1e34eb1d920a624c22f83d
+  data.tar.gz: 1b038243195b478d18b2a2c72b3fb0b4538a6701
 SHA512:
-  metadata.gz: de99019d564d5ea759f6e3ef330b8e9e68f6a7bbdb0578c34699ad7f716da16562d702da935bcfc5e3baa9a9e673b2ad99a62ae07210c4feea144134ad822e94
-  data.tar.gz: 24bb61197ff82129b404dcaf928b38ac9f7b9d5c10b90a630032253e468adc558c1675639b11c4bc60f7da8082626b3b288c2f0b9e6328e0ef2525b3a79453a8
+  metadata.gz: 87ea99f1ac87b528e8a08c5490cb500038fdd06ea18def8cda1276cf5e7ba7976ca4f0a49934023be16ea122ae1d91e514ad5a5fbdf282245a105ce22131dc6f
+  data.tar.gz: 2237ce97c067f42123c9aeba9ae8cb77cf97d11ef0ad7211ceb8a7e6110f6cc6c71d2a7a2e1e1dcf31b5f6c67d55af8a992b82eb43e377caf858fff6cac3e4d9

data/Gemfile

@@ -1,7 +1,7 @@
 source "http://rubygems.org"
 # Runtime dependencies
 gem "bio-logger"
-gem "nokogiri", "~>1.6.0"
+gem "nokogiri", "~>1.6.3"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.

data/Gemfile.lock

@@ -66,7 +66,7 @@ DEPENDENCIES
   bio-logger
   bundler
   jeweler (~> 2.0.1)
-  nokogiri (~> 1.6.0)
+  nokogiri (~> 1.6.3)
   rake
   rdoc
   rspec

data/README.md

@@ -2,8 +2,9 @@
 
 # bio-blastxmlparser
 
-blastxmlparser is a very fast big-data BLAST XML file parser, which can be used
-as command line utility. Use blastxmlparser to:
+blastxmlparser is a very fast parallelised big-data BLAST XML file
+parser, which can be used as command line utility. Use blastxmlparser
+to:
 
 * Parse BLAST XML
 * Filter output
@@ -24,12 +25,10 @@ can be used to filter results and requires no understanding of Ruby.
   blastxmlparser --help
 ```
 
-(see Installation, below, if it does not work)
-
 ## Performance
 
 XML parsing is expensive. blastxmlparser can use the fast Nokogiri C, or
-Java XML parsers, based on libxml2. Basically, a DOM parser is used
+Java XML parsers, based on libxml2 in parallel. A DOM parser is used
 after splitting the BLAST XML document into subsections.
 Tests show this is faster than a SAX
 parser with Ruby callbacks.  To see why libxml2 based Nokogiri is
@@ -38,33 +37,21 @@ fast, see this
 and [xml.com](http://www.xml.com/lpt/a/1703). 
 
 Blastxmlparser is designed with other optimizations, such as lazy
-evaluation, i.e., only creating objects when required, and (in a
-future version) parallelization. When parsing a full BLAST result
-usually only a few fields are used. By using XPath queries the parser
-makes sure only the relevant fields are queried.
+evaluation, i.e., only creating objects when required, and
+parallelism. When parsing a full BLAST result usually only a few
+fields are used. By using XPath queries the parser makes sure only the
+relevant fields are queried.
 
-Timings for parsing test/data/nt_example_blastn.m7 (file size 3.4Mb) 
+Timings for parsing a 128 Mb BLAST XML file on 4x1.2GHz laptop 
 
 ```
-  bio-blastxmlparser + Nokogiri DOM (default)
-
-  real    0m1.259s
-  user    0m1.052s
-  sys     0m0.144s
-
-  bio-blastxmlparser + Nokogiri split DOM
-
-  real    0m1.713s
-  user    0m1.444s
-  sys     0m0.160s
-
-  BioRuby ReXML DOM parser (old style)
-
-  real    1m14.548s
-  user    1m13.065s
-  sys     0m0.472s
+  real    0m13.985s
+  user    0m44.951s
+  sys     0m3.676s
 ```
 
+which makes for pretty good core utilisation.
+
 ## Install
 
 ```sh
@@ -99,9 +86,11 @@ provide build paths, as described [here](http://nokogiri.org/tutorials/installin
   blastxmlparser [options] file(s)
 
     -p, --parser name                Use full|split parser (default full)
+    -e, --exec filter                Evaluate filter
+
+    -n, --named fields               Print named fields
         --output-fasta               Output FASTA
-    -n, --named fields               Set named fields
-    -e, --exec filter                Execute filter
+    -t, --template erb               Use ERB template for output
 
         --logger filename            Log to file (default stderr)
         --trace options              Set log level (default INFO, see bio-logger)
@@ -109,10 +98,6 @@ provide build paths, as described [here](http://nokogiri.org/tutorials/installin
     -v, --verbose                    Run verbosely
         --debug                      Show debug messages
     -h, --help                       Show help and examples
-
-  bioblastxmlparser filename(s)
-
-    Use --help switch for more information
 ```
 
 ### Examples
@@ -204,7 +189,7 @@ template could be
 To get JSON, run it with
 
 ```sh
-  blastxmlparser --template template/json.erb -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+  blastxmlparser --template template/blast2json.erb -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
 ```
 
 ```Javascript
@@ -223,7 +208,7 @@ To get JSON, run it with
 Likewise, using the RDF template
 
 ```sh
-  blastxmlparser --template template/rdf.erb -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+  blastxmlparser --template template/blast2rdf.erb -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
 ```
 
 ```ruby
@@ -235,14 +220,13 @@ Likewise, using the RDF template
   :accession   "Minc02032",
   :id          "lcl|Minc02032",
   :len         147,
-  :E-value     8.1089e-12,
   :identity    60,
   :align_len   69,
   :bitscore    69.8753,
   :qseq        "ATGGGAGATGGAATTGAACCGTCATGGAAAGGGCCCAAACCGAAGCACAACCGACTGTGCCACCATCCA",
   :midline     "|||||||||||||||||||| |||||||| |       |||||||||||||||||||||||||||||||", 
   :hseq        "ATGGGAGATGGAATTGAACCATCATGGAATG-------ACCGAAGCACAACCGACTGTGCCACCATCCA",
-  :evalue   8.1089e-12 .
+  :evalue      8.1089e-12 .
 ```
 
 ## Additional options

data/Rakefile

@@ -15,7 +15,7 @@ Jeweler::Tasks.new do |gem|
   gem.name = "bio-blastxmlparser"
   gem.homepage = "http://github.com/pjotrp/blastxmlparser"
   gem.license = "MIT"
-  gem.summary = %Q{Very fast BLAST XML to RDF/HTML/JSON/YAML/csv transformer}
+  gem.summary = %Q{Very fast parallel BLAST XML to RDF/HTML/JSON/YAML/csv transformer}
   gem.description = %Q{Fast big data BLAST XML parser and library; this libxml2 based version is 50x faster than BioRuby and comes with a nice CLI}
   gem.email = "pjotr.public01@thebird.nl"
   gem.authors = ["Pjotr Prins"]

data/VERSION

@@ -1 +1 @@
-1.1.2
+2.0.0

data/bin/blastxmlparser

@@ -52,8 +52,17 @@ opts = OptionParser.new do |o|
     options.parser = p.to_sym
   end
   
-  o.on("-e filter","--exec filter",String, "Evaluate filter") do |s|
-    options.exec = s
+  o.on("--filter filter",String, "Filtering expression") do |s|
+    options.filter = s
+  end
+
+  o.on("-t num", "--threads num",String, "Use parallel threads") do |num|
+    options.threads = num.to_i
+  end
+
+  o.on("-e filter","--exec filter",String, "Evaluate filter (deprecated)") do |s|
+    $stderr.print "WARNING: -e,--exec switch is deprecated, use --filter instead!\n"
+    options.filter = s
   end
 
   o.separator ""
@@ -61,7 +70,7 @@ opts = OptionParser.new do |o|
   o.on("-n fields","--named fields",String, "Print named fields") do |s|
     options.fields = s.split(/,/)
   end
-   o.on("--output-fasta","Output FASTA") do |b|
+  o.on("--output-fasta","Output FASTA") do |b|
     options.output_fasta = true
   end
   
@@ -100,7 +109,16 @@ begin
   Bio::Log::CLI.configure('bio-blastxmlparser')
   logger = Bio::Log::LoggerPlus['bio-blastxmlparser']
 
-  if options[:template]
+  if options.threads != 1
+    begin
+      require 'parallel'
+    rescue LoadError
+      $stderr.print "Error: Missing 'parallel' module. Install with command 'gem install parallel' if you want multiple threads\n"
+      options.threads = 1
+    end
+  end
+
+  if options.template
     include BioRdf
     fn = options.template
     raise "No template #{fn}!" if not File.exist?(fn)
@@ -114,39 +132,73 @@ begin
     else
       Bio::BlastXMLParser::XmlIterator.new(fn).to_enum
     end
-    i = 1
-    n.each do | iter |
+    chunks = []
+    chunks_count = 0
+    NUM_CHUNKS=10_000
+
+    process = lambda { |iter,i|  # Process one BLAST iter block
+      res = []
+      line_count = 0
+      hit_count = 0
       iter.each do | hit |
+        hit_count += 1
         hit.each do | hsp |
-          do_print = if options.exec
-                       eval(options.exec)
+          do_print = if options.filter
+                       eval(options.filter)
                      else
                        true
                      end
           if do_print
+            line_count += 1
             if template
-              print template.result(binding)
+              res << template.result(binding)
             elsif options.output_fasta
-              print ">"+hit.accession+' '+iter.iter_num.to_s+'|'+iter.query_id+' '+hit.hit_id+' '+hit.hit_def+"\n"
-              print hsp.qseq+"\n"
+              res << ">"+hit.accession+' '+iter.iter_num.to_s+'|'+iter.query_id+' '+hit.hit_id+' '+hit.hit_def+"\n"
+              res << hsp.qseq+"\n"
             else
               # Default output
               if options.fields
-                print i,"\t"
+                out = [iter.iter_num,hit_count,hsp.hsp_num]
                 options.fields.each do | f |
-                  print eval(f),"\t"
+                  out << eval(f)
                 end
-                print "\n"
-                else
-                  print [i,iter.iter_num,iter.query_id,hit.hit_id,hsp.hsp_num,hsp.evalue].join("\t"),"\n" 
+                res << out.join("\t")+"\n"
+              else
+                res << [hit_count,iter.iter_num,iter.query_id,hit.hit_id,hsp.hsp_num,hsp.evalue].join("\t")+"\n"
               end
             end
-            i += 1
           end
         end
       end
+      res
+    } # end process 
+
+    output = lambda { |collection| 
+      collection.each do | result |
+        result.each { |line| print line }
+      end
+    } # end output
+
+    if options.threads == 1
+      n.each do | iter |
+        process.call(iter,0).each { | line | print line }
+      end
+    else
+      n.each do | iter |
+        chunks << iter
+        chunks_count += 1
+        if chunks.size > NUM_CHUNKS
+          output.call Parallel.map_with_index(chunks, :in_processes => options.threads) { | iter,i |
+            process.call(iter,i)
+          }
+          chunks = []
+        end
+      end
+      output.call Parallel.map_with_index(chunks, :in_processes => options.threads) { | iter,i |
+            process.call(iter,i)
+          }
     end
   end
 rescue OptionParser::InvalidOption => e
-  opts[:invalid_argument] = e.message
+  $stderr.print e.message
 end

data/bio-blastxmlparser.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "bio-blastxmlparser"
-  s.version = "1.1.2"
+  s.version = "2.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Pjotr Prins"]
-  s.date = "2014-09-02"
+  s.date = "2014-09-06"
   s.description = "Fast big data BLAST XML parser and library; this libxml2 based version is 50x faster than BioRuby and comes with a nice CLI"
   s.email = "pjotr.public01@thebird.nl"
   s.executables = ["blastxmlparser"]
@@ -42,8 +42,9 @@ Gem::Specification.new do |s|
     "sample/nokogiri_split_dom.rb",
     "spec/bio-blastxmlparser_spec.rb",
     "spec/spec_helper.rb",
-    "template/json.erb",
-    "template/rdf.erb",
+    "template/blast2json.erb",
+    "template/blast2rdf-minimal.erb",
+    "template/blast2rdf.erb",
     "test/data/aa_example.fasta",
     "test/data/aa_example_blastp.m7",
     "test/data/nt_example.fasta",
@@ -54,14 +55,14 @@ Gem::Specification.new do |s|
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
   s.rubygems_version = "2.0.3"
-  s.summary = "Very fast BLAST XML to RDF/HTML/JSON/YAML/csv transformer"
+  s.summary = "Very fast parallel BLAST XML to RDF/HTML/JSON/YAML/csv transformer"
 
   if s.respond_to? :specification_version then
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<bio-logger>, [">= 0"])
-      s.add_runtime_dependency(%q<nokogiri>, ["~> 1.6.0"])
+      s.add_runtime_dependency(%q<nokogiri>, ["~> 1.6.3"])
       s.add_development_dependency(%q<rake>, [">= 0"])
       s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
@@ -69,7 +70,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<rdoc>, [">= 0"])
     else
       s.add_dependency(%q<bio-logger>, [">= 0"])
-      s.add_dependency(%q<nokogiri>, ["~> 1.6.0"])
+      s.add_dependency(%q<nokogiri>, ["~> 1.6.3"])
       s.add_dependency(%q<rake>, [">= 0"])
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
@@ -78,7 +79,7 @@ Gem::Specification.new do |s|
     end
   else
     s.add_dependency(%q<bio-logger>, [">= 0"])
-    s.add_dependency(%q<nokogiri>, ["~> 1.6.0"])
+    s.add_dependency(%q<nokogiri>, ["~> 1.6.3"])
     s.add_dependency(%q<rake>, [">= 0"])
     s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, ["~> 2.0.1"])

data/lib/bio/writers/rdf.rb

@@ -63,7 +63,7 @@ module BioRdf
         # Don't want Bio depency in templates!
         # logger = Bio::Log::LoggerPlus.new 'bio-rdf'
         # logger.warn "\nWARNING: Changed identifier <#{s}> to <#{id}>"
-        $stderr.print "\nWARNING: Changed identifier <#{s}> to <#{id}>"
+        # $stderr.print "\nWARNING: Changed identifier <#{s}> to <#{id}>"
       end
       if not RDF::valid_uri?(id)
         raise "Invalid URI after mangling <#{s}> to <#{id}>!"

data/template/blast2rdf-minimal.erb

@@ -0,0 +1,14 @@
+<%
+  blastid = Turtle::mangle_identifier(hit.parent.query_def) 
+  id = blastid+'_'+hit.hit_num.to_s
+%>
+:<%= blastid %>  :query  :<%= id %>
+:<%= id %>
+  :query_def   "<%= hit.parent.query_def %>",
+  :num         <%= hit.hit_num %>,
+  :accession   "<%= hit.accession %>",
+  :len         <%= hit.len %>,
+  :identity    <%= hsp.identity %>,
+  :align_len   <%= hsp.align_len %>,
+  :bitscore    <%= hsp.bit_score %>,
+  :evalue   <%= hsp.evalue %> .

data/template/{rdf.erb → blast2rdf.erb}

@@ -10,12 +10,11 @@
   :accession   "<%= hit.accession %>",
   :id          "<%= hit.hit_id %>",
   :len         <%= hit.len %>,
-  :E-value     <%= hsp.evalue %>,
   :identity    <%= hsp.identity %>,
   :align_len   <%= hsp.align_len %>,
   :bitscore    <%= hsp.bit_score %>,
   :qseq        "<%= hsp.qseq %>",
   :midline     "<%= hsp.midline %>", 
   :hseq        "<%= hsp.hseq %>",
-  :evalue   <%= hsp.evalue %> .
+  :evalue      <%= hsp.evalue %> .
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bio-blastxmlparser
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Pjotr Prins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-02 00:00:00.000000000 Z
+date: 2014-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bio-logger
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.6.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.6.3
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,8 +142,9 @@ files:
 - sample/nokogiri_split_dom.rb
 - spec/bio-blastxmlparser_spec.rb
 - spec/spec_helper.rb
-- template/json.erb
-- template/rdf.erb
+- template/blast2json.erb
+- template/blast2rdf-minimal.erb
+- template/blast2rdf.erb
 - test/data/aa_example.fasta
 - test/data/aa_example_blastp.m7
 - test/data/nt_example.fasta
@@ -172,5 +173,5 @@ rubyforge_project:
 rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
-summary: Very fast BLAST XML to RDF/HTML/JSON/YAML/csv transformer
+summary: Very fast parallel BLAST XML to RDF/HTML/JSON/YAML/csv transformer
 test_files: []