bio-blastxmlparser 1.1.1 → 1.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 31b42217bb809cde8d5ef3c06d11c6c9123c6413
+  data.tar.gz: 5d23e19fb8c774f7edaffd03bbcb156800679f7f
+SHA512:
+  metadata.gz: de99019d564d5ea759f6e3ef330b8e9e68f6a7bbdb0578c34699ad7f716da16562d702da935bcfc5e3baa9a9e673b2ad99a62ae07210c4feea144134ad822e94
+  data.tar.gz: 24bb61197ff82129b404dcaf928b38ac9f7b9d5c10b90a630032253e468adc558c1675639b11c4bc60f7da8082626b3b288c2f0b9e6328e0ef2525b3a79453a8

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 rvm:
-  - 1.9.2
   - 1.9.3
   - jruby-19mode # JRuby in 1.9 mode
   - rbx-19mode
+  - 2.1.0
 #  - 1.8.7
 #  - jruby-18mode # JRuby in 1.8 mode
 #  - rbx-18mode

data/Gemfile

@@ -1,21 +1,14 @@
 source "http://rubygems.org"
-# Add dependencies required to use your gem here.
-# Example:
-#   gem "activesupport", ">= 2.3.5"
-
 # Runtime dependencies
-# gem "bio", ">= 1.3.1"
-gem "bio-logger", ">= 1.0.0"
-gem "nokogiri", ">= 1.5.0"
+gem "bio-logger"
+gem "nokogiri", "~>1.6.0"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
-  # gem "rspec", "~> 2.3.0"
-  gem "rake", ">= 0.9.2.2"
+  gem "rake"
   gem "bundler"
-  gem "jeweler", "~> 1.8.4"
-  gem "rspec", ">= 2.3.0"
-  gem "rdoc", ">= 2.4.2"
-  # gem "rcov", ">= 0"
+  gem "jeweler", "~> 2.0.1"
+  gem "rspec"
+  gem "rdoc"
 end

data/Gemfile.lock

@@ -1,18 +1,51 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    addressable (2.3.6)
     bio-logger (1.0.0)
       log4r (>= 1.1.9)
+    builder (3.2.2)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.1.3)
-    git (1.2.5)
-    jeweler (1.8.4)
-      bundler (~> 1.0)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    git (1.2.8)
+    github_api (0.12.1)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.2)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.3)
+      oauth2
+    hashie (3.3.1)
+    highline (1.6.21)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
       git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
       rake
       rdoc
     json (1.6.5)
+    jwt (1.0.0)
     log4r (1.1.9)
-    nokogiri (1.5.0)
+    mini_portile (0.6.0)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
     rake (0.9.2.2)
     rdoc (3.12)
       json (~> 1.4)
@@ -24,15 +57,16 @@ GEM
     rspec-expectations (2.8.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.8.0)
+    thread_safe (0.3.4)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bio-logger (>= 1.0.0)
+  bio-logger
   bundler
-  jeweler (~> 1.8.4)
-  nokogiri (>= 1.5.0)
-  rake (>= 0.9.2.2)
-  rdoc (>= 2.4.2)
-  rspec (>= 2.3.0)
+  jeweler (~> 2.0.1)
+  nokogiri (~> 1.6.0)
+  rake
+  rdoc
+  rspec

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2011 Pjotr Prins
+Copyright (c) 2011-2014 Pjotr Prins
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/{README.rdoc → README.md}

@@ -1,41 +1,51 @@
-blastxmlparser is listed at http://biogems.info
+[![Build Status](https://travis-ci.org/pjotrp/blastxmlparser.svg?branch=master)](https://travis-ci.org/pjotrp/blastxmlparser)
+
+# bio-blastxmlparser
 
-= bio-blastxmlparser
+blastxmlparser is a very fast big-data BLAST XML file parser, which can be used
+as command line utility. Use blastxmlparser to:
 
-blastxmlparser is a very fast big-data BLAST XML file parser, which can be
-used as command line utility, or as a Ruby library. Rather than
-loading everything in memory, XML is parsed by BLAST query
-(Iteration). Not only has this the advantage of low memory use, it
-also shows results early, and it may be faster when IO continues in
-parallel (disk read-ahead).
+* Parse BLAST XML
+* Filter output
+* Generate FASTA, JSON, YAML, RDF, HTML, tabular output etc.
+
+Rather than loading everything in memory, XML is parsed by BLAST query
+(Iteration). Not only has this the advantage of low memory use, it also shows
+results early, and it may be faster when IO continues in parallel (disk
+read-ahead).
 
 Next to the API, blastxmlparser comes as a command line utility, which
 can be used to filter results and requires no understanding of Ruby.
 
-= Quick start
+# Quick start
 
+```sh
   gem install bio-blastxmlparser
   blastxmlparser --help
+```
 
 (see Installation, below, if it does not work)
 
-== Performance
+## Performance
 
-XML parsing is expensive. blastxmlparser uses the fast Nokogiri C, or Java, XML
-parsers, based on libxml2. Basically, a DOM parser is used for subsections of a
-document. Tests show this is faster than a SAX parser with Ruby callbacks.  To
-see why libxml2 based Nokogiri is fast, see
-http://www.rubyinside.com/ruby-xml-performance-benchmarks-1641.html and
-http://www.xml.com/lpt/a/1703. 
+XML parsing is expensive. blastxmlparser can use the fast Nokogiri C, or
+Java XML parsers, based on libxml2. Basically, a DOM parser is used
+after splitting the BLAST XML document into subsections.
+Tests show this is faster than a SAX
+parser with Ruby callbacks.  To see why libxml2 based Nokogiri is
+fast, see this
+[benchmark](http://www.rubyinside.com/ruby-xml-performance-benchmarks-1641.html)
+and [xml.com](http://www.xml.com/lpt/a/1703). 
 
-The parser is also designed with other optimizations, such as lazy
-evaluation, i.e. only creating objects when required, and (in a future
-version) parallelization. When parsing a full BLAST result usually
-only a few fields are used. By using XPath queries only the relevant
-fields are queried.
+Blastxmlparser is designed with other optimizations, such as lazy
+evaluation, i.e., only creating objects when required, and (in a
+future version) parallelization. When parsing a full BLAST result
+usually only a few fields are used. By using XPath queries the parser
+makes sure only the relevant fields are queried.
 
 Timings for parsing test/data/nt_example_blastn.m7 (file size 3.4Mb) 
 
+```
   bio-blastxmlparser + Nokogiri DOM (default)
 
   real    0m1.259s
@@ -53,29 +63,39 @@ Timings for parsing test/data/nt_example_blastn.m7 (file size 3.4Mb)
   real    1m14.548s
   user    1m13.065s
   sys     0m0.472s
+```
 
-== Install
+## Install
 
+```sh
   gem install bio-blastxmlparser
+```
 
-Important: the parser is written for Ruby >= 1.9. You can check with
+Important: the parser is written for Ruby >= 1.9. Check with
 
+```sh
   ruby -v
   gem env
+```
 
 Nokogiri XML parser is required. To install it,
 the libxml2 libraries and headers need to be installed first, for
 example on Debian:
 
+```sh
   apt-get install libxslt-dev libxml2-dev
   gem install bio-blastxmlparser
+```
+
+Nokogiri balks when libxml2 or libxslt is missing on your system (or
+may install something automatically). In the worst case you'll have to
+provide build paths, as described [here](http://nokogiri.org/tutorials/installing_nokogiri.html).
 
-for more installation on other platforms see
-http://nokogiri.org/tutorials/installing_nokogiri.html. 
+## Command line usage
 
-== Command line usage
+### Usage
 
-=== Usage
+```
   blastxmlparser [options] file(s)
 
     -p, --parser name                Use full|split parser (default full)
@@ -93,23 +113,24 @@ http://nokogiri.org/tutorials/installing_nokogiri.html.
   bioblastxmlparser filename(s)
 
     Use --help switch for more information
+```
 
-=== Examples
+### Examples
 
 Print result fields of iterations containing 'lcl', using a regex
 
+```sh
   blastxmlparser -e 'iter.query_id=~/lcl/' test/data/nt_example_blastn.m7
-
-Print fields where bit_score > 145
-
-  blastxmlparser -e 'hsp.bit_score>145' test/data/nt_example_blastn.m7
+```
 
 prints a tab delimited
 
+```sh
   1       1       lcl|1_0 lcl|I_74685     1       5.82208e-34
   2       1       lcl|1_0 lcl|I_1 1       5.82208e-34
   3       2       lcl|2_0 lcl|I_2 1       6.05436e-59
   4       3       lcl|3_0 lcl|I_3 1       2.03876e-56
+```
 
 The second and third column show the BLAST iteration, and the others
 relate to the hits.
@@ -117,11 +138,20 @@ relate to the hits.
 As this is evaluated Ruby, it is also possible to use the XML element
 names directly
 
+```sh
   blastxmlparser -e 'hsp["Hsp_bit-score"].to_i>145' test/data/nt_example_blastn.m7
+```
+
+Or the shorter 
+
+```sh
+  blastxmlparser -e 'hsp.bit_score>145' test/data/nt_example_blastn.m7
+```
 
 And it is possible to print (non default) named fields where E-value < 0.001 
 and hit length > 100. E.g.
 
+```sh
   blastxmlparser -n 'hsp.evalue,hsp.qseq' -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
 
   1       5.82208e-34     AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCT...
@@ -130,28 +160,104 @@ and hit length > 100. E.g.
   4       1.13373e-13     CTAAACACAGGAGCATATAGGTTGGCAGGCAGGCAAAAT 
   5       2.76378e-11     GAAGAGTGTACTACCGTTTCTGTAGCTACCATATT     
   etc. etc.
+```
 
 prints the evalue and qseq columns. To output FASTA use --output-fasta
 
+```sh
   blastxmlparser --output-fasta -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+```
 
 which prints matching sequences, where the first field is the accession, followed
 by query iteration id, and hit_id. E.g.
 
+```sh
   >I_74685 1|lcl|1_0 lcl|I_74685 [57809 - 57666] (REVERSE SENSE) 
   AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCTGCCTGCCAACCTATATGCTCCTGTGTTTAG
   >I_1 1|lcl|1_0 lcl|I_1 [477 - 884] 
   AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCTGCCTGCCAACCTATATGCTCCTGTGTTTAG
   etc. etc.
+```
+
+## Modify output
+
+To have more output options blastxmlparser can use an [ERB
+template](http://www.stuartellis.eu/articles/erb/) for every match. This is a
+very flexible option that can output textual formats such as JSON, YAML, HTML
+and RDF. Examples are provided in
+[./templates](https://github.com/pjotrp/bioruby-vcf/templates/). A JSON
+template could be
+
+```Javascript
+{ "<%= hit.parent.query_def %>": {
+  "num":      <%= hit.hit_num %>,
+  "id":       "<%= hit.hit_id %>",
+  "len":      <%= hit.len %>,
+  "E-value":  <%= hsp.evalue %>,
+  "bitscore": <%= hsp.bit_score %>,
+  "qseq":     "<%= hsp.qseq %>",
+  "midline":  "<%= hsp.midline %>", 
+  "hseq":     "<%= hsp.hseq %>",
+};
+```
+
+To get JSON, run it with
+
+```sh
+  blastxmlparser --template template/json.erb -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+```
+
+```Javascript
+{ "B0511.9d gene=WBGene00015235": {
+  "num":      5,
+  "id":       "gi|268566471|ref|XP_002639731.1|",
+  "len":      199,
+  "E-value":  1.72502e-22,
+  "bitscore": 96.6709,
+  "qseq":     "MSMLRRPLTQLELSVI------------------VPKCXXXXXXXXXXXXQSEPPRGITRRNLRSADRKNRDVPGPSTGECTRTSIAPNRCEMSFTEVQ-TLTSARTPVAAPTLTLSTPVNPVSSAEMLX----XXXXXXXXXXXASRSGDNDSPLLFNAYDTPQQ--GINXXXXXXXXXXXXXNAHLYAXXXXXXXXXXXXXXXXRSHRH",
+  "midline":  "MSMLRRPLTQLEL                       K             QSEP  GI++RNLRSADR+ +DVPG ++GE  +           FT+   +++SARTPV+  ++ LSTPVNP SS EM+                 SR  + D PL+FNAYDTPQQ  G +             NAHLY+                RS RH", 
+  "hseq":     "MSMLRRPLTQLELCEDDIQWLSEQLAKKETGFEDEVKYEVMDVDEDEPMDQSEPTGGISKRNLRSADRRKKDVPG-TSGEGAQ-----------FTDQGLSISSARTPVSGASVNLSTPVNPSSSNEMMALPPPVRLARAGRRQRDSRVVNGDVPLMFNAYDTPQQPAGGSNGSPTPSDSPESPNAHLYSTPINPTSSSGGPSSNTRSQRH",
+};
+```
+
+Likewise, using the RDF template
+
+```sh
+  blastxmlparser --template template/rdf.erb -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+```
+
+```ruby
+:Minc_Contig50_77_42056___42484_1_64492  :query  :Minc_Contig50_77_42056___42484_1_64492_23
+:Minc_Contig50_77_42056___42484_1_64492_23
+  :query_id    "lcl|30_0",
+  :query_def   "Minc_Contig50_77 [42056 - 42484] 1 64492",
+  :num         23,
+  :accession   "Minc02032",
+  :id          "lcl|Minc02032",
+  :len         147,
+  :E-value     8.1089e-12,
+  :identity    60,
+  :align_len   69,
+  :bitscore    69.8753,
+  :qseq        "ATGGGAGATGGAATTGAACCGTCATGGAAAGGGCCCAAACCGAAGCACAACCGACTGTGCCACCATCCA",
+  :midline     "|||||||||||||||||||| |||||||| |       |||||||||||||||||||||||||||||||", 
+  :hseq        "ATGGGAGATGGAATTGAACCATCATGGAATG-------ACCGAAGCACAACCGACTGTGCCACCATCCA",
+  :evalue   8.1089e-12 .
+```
+
+## Additional options
 
 To use the low-mem (iterated slower) version of the parser use
 
+```sh
   blastxmlparser --parser split -n 'hsp.evalue,hsp.qseq' -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
+```
 
-== API (Ruby library)
+## API (Ruby library)
 
 To loop through a BLAST result:
 
+```ruby
     >> require 'bio-blastxmlparser'
     >> fn = 'test/data/nt_example_blastn.m7'
     >>   n = Bio::BlastXMLParser::XmlIterator.new(fn).to_enum
@@ -163,19 +269,23 @@ To loop through a BLAST result:
     >>       end
     >>     end
     >>   end
+```
 
 The next example parses XML using less memory by using a Ruby
 Iterator
 
+```ruby
     >> blast = Bio::BlastXMLParser::XmlSplitterIterator.new(fn).to_enum
     >> iter = blast.next
     >> iter.iter_num
     => 1
     >> iter.query_id
     => "lcl|1_0"
+```
 
 Get the first hit
 
+```ruby
     >> hit = iter.hits.first
     >> hit.hit_num
     => 1
@@ -187,14 +297,18 @@ Get the first hit
     => "I_74685"
     >> hit.len
     => 144
+```
 
 Get the parent info
 
+```ruby
     >> hit.parent.query_id
     => "lcl|1_0"
- 
+```
+
 Get the first Hsp
 
+```ruby
     >> hsp = hit.hsps.first
     >> hsp.hsp_num
     => 1
@@ -224,6 +338,7 @@ Get the first Hsp
     => "AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCTGCCTGCCAACCTATATGCTCCTGTGTTTAG"
     >> hsp.midline
     => "|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||"
+```
 
 Unlike BioRuby, this module uses the actual element names in the XML
 definition, to avoid confusion (if anyone wants a translation,
@@ -232,30 +347,36 @@ feel free to contribute an adaptor).
 It is also possible to use the XML element names as Strings, rather
 than methods. E.g.
 
+```ruby
     >> hsp.field("Hsp_bit-score")
     => "145.205"
     >> hsp["Hsp_bit-score"]
     => "145.205"
+```
 
 Note that, when using the element names, the results are always String values.
 
 Fetch the next result (Iteration)
 
+```ruby
     >> iter2 = blast.next
     >> iter2.iter_num
     >> 2 
     >> iter2.query_id
     => "lcl|2_0"
+```
 
 etc. etc.
 
 For more examples see the files in ./spec
 
-== URL
+## URL
 
 The project lives at http://github.com/pjotrp/blastxmlparser. If you use this software, please cite http://dx.doi.org/10.1093/bioinformatics/btq475
 
-== Copyright
+blastxmlparser is listed at http://biogems.info
+
+## Copyright
 
-Copyright (c) 2011,2012 Pjotr Prins under the MIT licence.  See LICENSE.txt and http://www.opensource.org/licenses/mit-license.html for further details.
+Copyright (c) 2011-2014 Pjotr Prins under the MIT licence.  See LICENSE.txt and http://www.opensource.org/licenses/mit-license.html for further details.
 

data/Rakefile

@@ -15,14 +15,10 @@ Jeweler::Tasks.new do |gem|
   gem.name = "bio-blastxmlparser"
   gem.homepage = "http://github.com/pjotrp/blastxmlparser"
   gem.license = "MIT"
-  gem.summary = %Q{Very fast BLAST XML parser and library for big data}
-  gem.description = %Q{Fast big data BLAST XML parser and library; this libxml2 based version is 50x faster than BioRuby}
+  gem.summary = %Q{Very fast BLAST XML to RDF/HTML/JSON/YAML/csv transformer}
+  gem.description = %Q{Fast big data BLAST XML parser and library; this libxml2 based version is 50x faster than BioRuby and comes with a nice CLI}
   gem.email = "pjotr.public01@thebird.nl"
   gem.authors = ["Pjotr Prins"]
-  # Include your dependencies below. Runtime dependencies are required when using your gem,
-  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
-  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
-  #  gem.add_development_dependency 'rspec', '> 1.2.3'
 end
 Jeweler::RubygemsDotOrgTasks.new
 
@@ -32,17 +28,11 @@ RSpec::Core::RakeTask.new(:spec) do |spec|
   spec.pattern = FileList['spec/**/*_spec.rb']
 end
 
-# RSpec::Core::RakeTask.new(:rcov) do |spec|
-#   spec.pattern = 'spec/**/*_spec.rb'
-#   spec.rcov = true
-# end
-
 task :default => [ :test, :spec ]
 
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  # test.pattern = 'test/**/test_*.rb'  # breaks in 1.9.3
   test.test_files = Dir.glob("test/**/test_*.rb")
   test.verbose = true
 end

data/VERSION

@@ -1 +1 @@
-1.1.1
+1.1.2

data/bin/blastxmlparser

@@ -4,83 +4,21 @@
 # Author:: Pjotr Prins
 # License:: MIT License
 #
-# Copyright (C) 2010-2013 Pjotr Prins <pjotr.prins@thebird.nl>
+# Copyright (C) 2010-2014 Pjotr Prins <pjotr.prins@thebird.nl>
 
 rootpath = File.dirname(File.dirname(__FILE__))
 $: << File.join(rootpath,'lib')
 
 BLASTXML_VERSION = File.new(File.join(rootpath,'VERSION')).read.chomp
 
-$stderr.print "BioRuby BLAST XML Parser "+BLASTXML_VERSION+" Copyright (C) 2011 Pjotr Prins <pjotr.prins@thebird.nl>\n\n"
+$stderr.print "BLAST XML Parser "+BLASTXML_VERSION+" Copyright (C) 2014 Pjotr Prins <pjotr.prins@thebird.nl>\n\n"
 
 USAGE = <<EOM
 
-  bioblastxmlparser filename(s)
+  blastxmlparser filename(s)
 
     Use --help switch for more information
 
-== Examples
-
-Print result fields of iterations containing 'lcl', using a regex
-
-  blastxmlparser -e 'iter.query_id=~/lcl/' test/data/nt_example_blastn.m7
-
-Print fields where bit_score > 145
-
-  blastxmlparser -e 'hsp.bit_score>145' test/data/nt_example_blastn.m7
-
-prints a tab delimited
-
-  1       1       lcl|1_0 lcl|I_74685     1       5.82208e-34
-  2       1       lcl|1_0 lcl|I_1 1       5.82208e-34
-  3       2       lcl|2_0 lcl|I_2 1       6.05436e-59
-  4       3       lcl|3_0 lcl|I_3 1       2.03876e-56
-
-The second and third column show the BLAST iteration, and the others
-relate to the hits.
-
-As this is evaluated Ruby, it is also possible to use the XML element
-names directly
-
-  blastxmlparser -e 'hsp["Hsp_bit-score"].to_i>145' test/data/nt_example_blastn.m7
-
-And it is possible to print (non default) named fields where E-value < 0.001 
-and hit length > 100. E.g.
-
-  blastxmlparser -n 'hsp.evalue,hsp.qseq' -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
-
-  1       5.82208e-34     AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCT...
-  2       5.82208e-34     AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCT...
-  3       2.76378e-11     AATATGGTAGCTACAGAAACGGTAGTACACTCTTC     
-  4       1.13373e-13     CTAAACACAGGAGCATATAGGTTGGCAGGCAGGCAAAAT 
-  5       2.76378e-11     GAAGAGTGTACTACCGTTTCTGTAGCTACCATATT     
-  etc. etc.
-
-prints the evalue and qseq columns. To output FASTA use --output-fasta
-
-  blastxmlparser --output-fasta -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
-
-which prints matching sequences, where the first field is the accession, followed
-by query iteration id, and hit_id. E.g.
-
-  >I_74685 1|lcl|1_0 lcl|I_74685 [57809 - 57666] (REVERSE SENSE) 
-  AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCTGCCTGCCAACCTATATGCTCCTGTGTTTAG
-  >I_1 1|lcl|1_0 lcl|I_1 [477 - 884] 
-  AGTGAAGCTTCTAGATATTTGGCGGGTACCTCTAATTTTGCCTGCCTGCCAACCTATATGCTCCTGTGTTTAG
-  etc. etc.
-
-To use the low-mem (iterated slower) version of the parser use
-
-  blastxmlparser --parser split -n 'hsp.evalue,hsp.qseq' -e 'hsp.evalue<0.01 and hit.len>100' test/data/nt_example_blastn.m7
-
-== URL
-
-The project lives at http://github.com/pjotrp/blastxmlparser. If you use this software, please cite http://dx.doi.org/10.1093/bioinformatics/btq475
-
-== Copyright
-
-Copyright (c) 2011 Pjotr Prins under the MIT licence.  See LICENSE.txt and http://www.opensource.org/licenses/mit-license.html for further details.
-
 EOM
 
 if ARGV.size == 0
@@ -114,16 +52,22 @@ opts = OptionParser.new do |o|
     options.parser = p.to_sym
   end
   
-  o.on("--output-fasta","Output FASTA") do |b|
-    options.output_fasta = true
+  o.on("-e filter","--exec filter",String, "Evaluate filter") do |s|
+    options.exec = s
   end
-  
-  o.on("-n fields","--named fields",String, "Set named fields") do |s|
+
+  o.separator ""
+
+  o.on("-n fields","--named fields",String, "Print named fields") do |s|
     options.fields = s.split(/,/)
   end
+   o.on("--output-fasta","Output FASTA") do |b|
+    options.output_fasta = true
+  end
   
-  o.on("-e filter","--exec filter",String, "Execute filter") do |s|
-    options.exec = s
+  o.on("-t erb","--template erb",String, "Use ERB template for output") do |s|
+    require 'erb'
+    options.template = s
   end
   
   o.separator ""
@@ -156,6 +100,13 @@ begin
   Bio::Log::CLI.configure('bio-blastxmlparser')
   logger = Bio::Log::LoggerPlus['bio-blastxmlparser']
 
+  if options[:template]
+    include BioRdf
+    fn = options.template
+    raise "No template #{fn}!" if not File.exist?(fn)
+    template = ERB.new(File.read(fn))
+  end
+
   ARGV.each do | fn |
     logger.info("XML parsing #{fn}")
     n = if options.parser == :split
@@ -173,10 +124,13 @@ begin
                        true
                      end
           if do_print
-            if options.output_fasta
+            if template
+              print template.result(binding)
+            elsif options.output_fasta
               print ">"+hit.accession+' '+iter.iter_num.to_s+'|'+iter.query_id+' '+hit.hit_id+' '+hit.hit_def+"\n"
               print hsp.qseq+"\n"
             else
+              # Default output
               if options.fields
                 print i,"\t"
                 options.fields.each do | f |

data/bio-blastxmlparser.gemspec

@@ -5,17 +5,17 @@
 
 Gem::Specification.new do |s|
   s.name = "bio-blastxmlparser"
-  s.version = "1.1.1"
+  s.version = "1.1.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Pjotr Prins"]
-  s.date = "2013-02-07"
-  s.description = "Fast big data BLAST XML parser and library; this libxml2 based version is 50x faster than BioRuby"
+  s.date = "2014-09-02"
+  s.description = "Fast big data BLAST XML parser and library; this libxml2 based version is 50x faster than BioRuby and comes with a nice CLI"
   s.email = "pjotr.public01@thebird.nl"
   s.executables = ["blastxmlparser"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
     "Gemfile",
     "Gemfile.lock",
     "LICENSE.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
     "bin/blastxmlparser",
@@ -33,6 +33,7 @@ Gem::Specification.new do |s|
     "lib/bio/db/blast/parser/nokogiri.rb",
     "lib/bio/db/blast/xmliterator.rb",
     "lib/bio/db/blast/xmlsplitter.rb",
+    "lib/bio/writers/rdf.rb",
     "sample/bioruby.rb",
     "sample/blastxmlparserdemo.rb",
     "sample/libxml_sax.rb",
@@ -41,6 +42,8 @@ Gem::Specification.new do |s|
     "sample/nokogiri_split_dom.rb",
     "spec/bio-blastxmlparser_spec.rb",
     "spec/spec_helper.rb",
+    "template/json.erb",
+    "template/rdf.erb",
     "test/data/aa_example.fasta",
     "test/data/aa_example_blastp.m7",
     "test/data/nt_example.fasta",
@@ -50,37 +53,37 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/pjotrp/blastxmlparser"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
-  s.summary = "Very fast BLAST XML parser and library for big data"
+  s.rubygems_version = "2.0.3"
+  s.summary = "Very fast BLAST XML to RDF/HTML/JSON/YAML/csv transformer"
 
   if s.respond_to? :specification_version then
-    s.specification_version = 3
+    s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<bio-logger>, [">= 1.0.0"])
-      s.add_runtime_dependency(%q<nokogiri>, [">= 1.5.0"])
-      s.add_development_dependency(%q<rake>, [">= 0.9.2.2"])
+      s.add_runtime_dependency(%q<bio-logger>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, ["~> 1.6.0"])
+      s.add_development_dependency(%q<rake>, [">= 0"])
       s.add_development_dependency(%q<bundler>, [">= 0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
-      s.add_development_dependency(%q<rspec>, [">= 2.3.0"])
-      s.add_development_dependency(%q<rdoc>, [">= 2.4.2"])
+      s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<rdoc>, [">= 0"])
     else
-      s.add_dependency(%q<bio-logger>, [">= 1.0.0"])
-      s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
-      s.add_dependency(%q<rake>, [">= 0.9.2.2"])
+      s.add_dependency(%q<bio-logger>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, ["~> 1.6.0"])
+      s.add_dependency(%q<rake>, [">= 0"])
       s.add_dependency(%q<bundler>, [">= 0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
-      s.add_dependency(%q<rspec>, [">= 2.3.0"])
-      s.add_dependency(%q<rdoc>, [">= 2.4.2"])
+      s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+      s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<rdoc>, [">= 0"])
     end
   else
-    s.add_dependency(%q<bio-logger>, [">= 1.0.0"])
-    s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
-    s.add_dependency(%q<rake>, [">= 0.9.2.2"])
+    s.add_dependency(%q<bio-logger>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, ["~> 1.6.0"])
+    s.add_dependency(%q<rake>, [">= 0"])
     s.add_dependency(%q<bundler>, [">= 0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
-    s.add_dependency(%q<rspec>, [">= 2.3.0"])
-    s.add_dependency(%q<rdoc>, [">= 2.4.2"])
+    s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+    s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<rdoc>, [">= 0"])
   end
 end
 

data/lib/bio-blastxmlparser.rb

@@ -17,3 +17,4 @@ Bio::Log::LoggerPlus.new('bio-blastxmlparser')
 require 'bio/db/blast/parser/nokogiri'
 require 'bio/db/blast/xmlsplitter'
 require 'bio/db/blast/xmliterator'
+require 'bio/writers/rdf'

data/lib/bio/writers/rdf.rb

@@ -0,0 +1,79 @@
+# RDF support module. Original is part of bioruby-rdf by Pjotr Prins
+#
+module BioRdf
+
+  module RDF
+
+    def RDF::valid_uri? uri
+      uri =~ /^([!#$&-;=?_a-z~]|%[0-9a-f]{2})+$/i
+    end
+
+    def RDF::escape_string_literal(literal)
+      s = literal.to_s
+      # Put a slash before every double quote if there is no such slash already
+      s = s.gsub(/(?<!\\)"/,'\"')
+      # Put a slash before a single slash if it is not \["utnr>\]
+      if s =~ /[^\\]\\[^\\]/
+        s2 = []
+        s.each_char.with_index { |c,i| 
+          res = c
+          if i>0 and c == '\\' and s[i-1] != '\\' and s[i+1] !~ /^[uUtnr\\"]/
+            res = '\\' + c
+          end
+          # p [i,c,s[i+1],res]
+          s2 << res
+        }
+        s = s2.join('')
+      end
+      s
+    end
+
+    def RDF::stringify_literal(literal)
+      RDF::escape_string_literal(literal.to_s)
+    end
+
+    def RDF::quoted_stringify_literal(literal)
+      '"' + stringify_literal(literal) + '"'
+    end
+  end
+
+  module Turtle
+
+    def Turtle::stringify_literal(literal)
+      RDF::stringify_literal(literal)
+    end
+
+    def Turtle::identifier(id)
+      raise "Illegal identifier #{id}" if id != Turtle::mangle_identifier(id)
+    end
+
+    # Replace letters/symbols that are not allowed in a Turtle identifier
+    # (short hand URI). This should be the definite mangler and replace the 
+    # ones in bioruby-table and bio-exominer. Manglers are useful when using
+    # data from other sources and trying to transform them into simple RDF 
+    # identifiers.
+
+    def Turtle::mangle_identifier(s)
+      id = s.strip.gsub(/[^[:print:]]/, '').gsub(/[#)(,]/,"").gsub(/[%]/,"perc").gsub(/(\s|\.|\$|\/|\\|\>)+/,"_")
+      id = id.gsub(/\[|\]/,'')
+      # id = URI::escape(id)
+      id = id.gsub(/\|/,'_')
+      id = id.gsub(/\-|:/,'_')
+      if id != s 
+        # Don't want Bio depency in templates!
+        # logger = Bio::Log::LoggerPlus.new 'bio-rdf'
+        # logger.warn "\nWARNING: Changed identifier <#{s}> to <#{id}>"
+        $stderr.print "\nWARNING: Changed identifier <#{s}> to <#{id}>"
+      end
+      if not RDF::valid_uri?(id)
+        raise "Invalid URI after mangling <#{s}> to <#{id}>!"
+      end
+      valid_id = if id =~ /^\d/
+                   'r' + id
+                 else
+                   id
+                 end
+      valid_id  # we certainly hope so!
+    end
+  end
+end

data/template/json.erb

@@ -0,0 +1,15 @@
+{ "<%= hit.parent.query_def %>": {
+  "query_id":  "<%= hit.parent.query_id %>",
+  "num":       <%= hit.hit_num %>,
+  "accession": "<%= hit.accession %>",
+  "id":        "<%= hit.hit_id %>",
+  "len":       <%= hit.len %>,
+  "E-value":   <%= hsp.evalue %>,
+  "identity":  <%= hsp.identity %>,
+  "align_len": <%= hsp.align_len %>,
+  "bitscore":  <%= hsp.bit_score %>,
+  "qseq":      "<%= hsp.qseq %>",
+  "midline":   "<%= hsp.midline %>", 
+  "hseq":      "<%= hsp.hseq %>",
+};
+

data/template/rdf.erb

@@ -0,0 +1,21 @@
+<%
+  blastid = Turtle::mangle_identifier(hit.parent.query_def) 
+  id = blastid+'_'+hit.hit_num.to_s
+%>
+:<%= blastid %>  :query  :<%= id %>
+:<%= id %>
+  :query_id    "<%= hit.parent.query_id %>",
+  :query_def   "<%= hit.parent.query_def %>",
+  :num         <%= hit.hit_num %>,
+  :accession   "<%= hit.accession %>",
+  :id          "<%= hit.hit_id %>",
+  :len         <%= hit.len %>,
+  :E-value     <%= hsp.evalue %>,
+  :identity    <%= hsp.identity %>,
+  :align_len   <%= hsp.align_len %>,
+  :bitscore    <%= hsp.bit_score %>,
+  :qseq        "<%= hsp.qseq %>",
+  :midline     "<%= hsp.midline %>", 
+  :hseq        "<%= hsp.hseq %>",
+  :evalue   <%= hsp.evalue %> .
+

metadata

@@ -1,110 +1,130 @@
 --- !ruby/object:Gem::Specification
 name: bio-blastxmlparser
 version: !ruby/object:Gem::Version
-  version: 1.1.1
-  prerelease: 
+  version: 1.1.2
 platform: ruby
 authors:
 - Pjotr Prins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-07 00:00:00.000000000Z
+date: 2014-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bio-logger
-  requirement: &24214160 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *24214160
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &24213120 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *24213120
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &24212220 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.2.2
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *24212220
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &24211440 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *24211440
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &24174660 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.4
+        version: 2.0.1
   type: :development
   prerelease: false
-  version_requirements: *24174660
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.1
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &24173840 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *24173840
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &24173100 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.2
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *24173100
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fast big data BLAST XML parser and library; this libxml2 based version
-  is 50x faster than BioRuby
+  is 50x faster than BioRuby and comes with a nice CLI
 email: pjotr.public01@thebird.nl
 executables:
 - blastxmlparser
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
-- README.rdoc
+- README.md
 files:
-- .document
-- .rspec
-- .travis.yml
+- ".document"
+- ".rspec"
+- ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
 - bin/blastxmlparser
@@ -113,6 +133,7 @@ files:
 - lib/bio/db/blast/parser/nokogiri.rb
 - lib/bio/db/blast/xmliterator.rb
 - lib/bio/db/blast/xmlsplitter.rb
+- lib/bio/writers/rdf.rb
 - sample/bioruby.rb
 - sample/blastxmlparserdemo.rb
 - sample/libxml_sax.rb
@@ -121,6 +142,8 @@ files:
 - sample/nokogiri_split_dom.rb
 - spec/bio-blastxmlparser_spec.rb
 - spec/spec_helper.rb
+- template/json.erb
+- template/rdf.erb
 - test/data/aa_example.fasta
 - test/data/aa_example_blastp.m7
 - test/data/nt_example.fasta
@@ -129,29 +152,25 @@ files:
 homepage: http://github.com/pjotrp/blastxmlparser
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3287387609254152406
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
-summary: Very fast BLAST XML parser and library for big data
+specification_version: 4
+summary: Very fast BLAST XML to RDF/HTML/JSON/YAML/csv transformer
 test_files: []