scbi_blast 0.0.33 → 0.0.34

data/History.txt

@@ -1,3 +1,7 @@
+=== 0.0.34 2012-04-30
+
+Added streamed XML parser to reduce memory usage with xml tree
+
 === 0.0.33 2011-07-06
 
 Added do_blast_cmd to launch any blast command

data/Manifest.txt

@@ -14,8 +14,10 @@ test/empty_blast.xml
 test/blast.txt
 lib/scbi_blast/blast_hit.rb
 lib/scbi_blast/blast_query.rb
+lib/scbi_blast/blast_result.rb
 lib/scbi_blast/blast_table_result.rb
 lib/scbi_blast/blast_xml_result.rb
 lib/scbi_blast/blast_simplexml_result.rb
+lib/scbi_blast/blast_streamxml_result.rb
 lib/scbi_blast/batch_blast.rb
 lib/scbi_blast/dust_masker.rb

data/Rakefile

@@ -14,7 +14,7 @@ $hoe = Hoe.spec 'scbi_blast' do
   self.developer 'Dario Guerrero & Almudena Bocinos', 'dariogf@gmail.com, alkoke@gmail.com'
   # self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
   self.rubyforge_name       = self.name # TODO this is default value
-  # self.extra_deps         = [['activesupport','>= 2.0.2']]
+  self.extra_deps         = [['xml-simple','>= 1.0.12']]
 
 end
 

data/lib/scbi_blast/batch_blast.rb

@@ -68,7 +68,7 @@ class BatchBlast
       if fmt == :table
         res = BlastTableResult.new(res)
       elsif fmt == :xml
-        res = BlastSimplexmlResult.new(res)
+        res = BlastStreamxmlResult.new(res)
         # elsif fmt ==:xml2
         # res = BlastXmlResult.new(res)
       end

data/lib/scbi_blast/blast_hit.rb

@@ -25,11 +25,16 @@ class BlastHit
 
   # initializes a new hit
   def initialize(q_beg,q_end,s_beg,s_end)
-
+    set_limits(q_beg,q_end,s_beg,s_end)
+  end
+  
+  def set_limits(q_beg,q_end,s_beg,s_end)
+    # puts "Set limits #{[q_beg,q_end,s_beg,s_end].join(',')}"
     @q_beg = q_beg.to_i-1  #blast indexes are 1 based
     @q_end = q_end.to_i-1
     @s_beg = s_beg.to_i-1
     @s_end = s_end.to_i-1
+    # puts "Set limits2 #{[@q_beg,@q_end,@s_beg,@s_end].join(',')}"
 
     @reversed = false
 
@@ -41,7 +46,9 @@ class BlastHit
       @s_end = s_beg.to_i-1
       @reversed = true
     end
-
+    # puts "Set limits3 #{[@q_beg,@q_end,@s_beg,@s_end].join(',')}"
+    # puts "Set limits4 #{[q_beg,q_end,s_beg,s_end].join(',')}"
+    
   end
 
   # some accessors
@@ -125,6 +132,47 @@ class BlastHit
   def size
     return (@q_end-@q_beg+1)
   end
+  
+  def compare?(hit)
+    res=true
+    
+    res &&=( @q_beg==hit.q_beg)
+    res &&=( @q_end==hit.q_end)
+    res &&=( @s_beg==hit.s_beg)
+    res &&=( @s_end==hit.s_end)
+    
+    res &&=( @subject_id==hit.subject_id)
+    res &&=( @align_len==hit.align_len)
+    res &&=( @gaps==hit.gaps)
+    res &&=( @mismatches==hit.mismatches)
+    
+    
+    res &&=( @reversed==hit.reversed)
+    res &&=( @score==hit.score)
+    res &&=( @acc==hit.acc)
+    res &&=( @definition==hit.definition)
+    
+    
+    res &&=( @q_frame==hit.q_frame)
+    res &&=( @s_frame==hit.s_frame)
+    res &&=( @full_subject_length==hit.full_subject_length)
+    res &&=( @ident==hit.ident)
+    
+    
+    res &&=( @e_val==hit.e_val)
+    res &&=( @bit_score==hit.bit_score)
+    res &&=( @q_seq==hit.q_seq)
+    res &&=( @s_seq==hit.s_seq)
+    
+    if !res 
+      puts "Hits not equal:"
+      puts inspect
+      puts "="*20
+      puts hit.inspect
+    end
+    
+    return res
+  end
 
   # readers and accessor for properties
   attr_accessor :q_beg, :q_end, :s_beg, :s_end

data/lib/scbi_blast/blast_query.rb

@@ -42,9 +42,9 @@ class BlastQuery
 
   # inspect query values with all hits
   def inspect
-    res = "\n * Query  #{@query_id} :"
+    res = "\n * Query  #{@query_id}, #{@query_def}, #{@full_query_length} :"
     res += "subject_id ident align_len mismatches gaps q_beg q_end s_beg s_end e_val bit_score reversed\n\n"
-    @hits.each{ |h| res+= h.inspect+"\n" }
+    @hits.each{ |h| res+= "=="+h.inspect+"\n" }
 
     return res
   end
@@ -72,6 +72,41 @@ class BlastQuery
     end until (res2.count == res.count)
 
 
+    return res
+  end
+  
+  def compare?(query)
+    res=true
+    
+
+    # same hits
+    res &&=( @hits.count==query.hits.count)
+    
+    # if !res 
+    #    puts "Queries not equal:"
+    #    puts inspect
+    #    puts "="*20
+    #    puts query.inspect
+    #  end
+    
+    if res
+      @hits.each_with_index do |h,i|
+        res &&= h.compare?(query.hits[i])
+      end
+    end
+    
+    # if !res 
+    #    puts "Queries hits not equal:"
+    #    puts inspect
+    #    puts "="*20
+    #    puts query.inspect
+    #  end
+    
+    res &&=( @query_id==query.query_id)
+    res &&=( @query_def==query.query_def)
+    res &&=( @full_query_length==query.full_query_length)
+    
+    
     return res
   end
 
@@ -86,7 +121,7 @@ private
       merged_ids.push hit.definition if !merged_ids.nil? && (!merged_ids.include?(hit.definition))
 
       # find overlapping hits
-      c=merged_hits.find{|c| hit.query_overlaps?(c)}
+      c=merged_hits.find{|c2| hit.query_overlaps?(c2)}
 
       if (c.nil?)
         # add new hit

data/lib/scbi_blast/blast_result.rb

@@ -0,0 +1,89 @@
+# Copyright (c) 2010 Dario Guerrero & Almudena Bocinos
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+require "blast_query.rb"
+require "blast_hit.rb"
+
+
+
+# Extracts results from blast table's file and uses it to create instances of "BlastQuery" and "BlastHit"
+class BlastResult
+  
+  attr_accessor :querys
+  
+  # Parser initialization
+  def initialize(input)
+
+    @querys = []
+
+    # if input.is_a?(Array)
+    #   lines=input
+    # else
+    #   fich = File.open(input,'r')
+    #   lines = fich.readlines
+    #   fich.close
+    # end
+  end
+
+
+  # inspect results
+  def inspect
+    res = "Blast results:\n"
+    res+= '-'*20
+    res+= "\nQuerys: #{@querys.count}\n"
+    @querys.each{|q| res+=q.inspect+"\n"}
+    return res
+  end
+
+  # find query by name
+  def find_query(querys,name_q)
+    #  newq = querys.find{|q| ( q.find{|h| (h.subject_id)})}
+    new_q=nil
+
+    if !querys.empty?
+      new_q=querys.find{|q| (q.query_id==name_q)}
+    end
+
+    return new_q
+  end
+
+  # check if there are querys
+  def empty?
+    return @querys.empty?
+  end
+  
+  # get query count
+  def size
+    @querys.size
+  end
+  
+  def compare?(results)
+    res = true
+    
+    @querys.each_with_index  do |q,i|
+      res &&= q.compare?(results.querys[i])
+    end
+    
+    return res
+  end
+  
+end

data/lib/scbi_blast/blast_simplexml_result.rb

@@ -27,12 +27,12 @@ require 'xmlsimple'
 
 # Extracts results from a blast results in XML
 # format and uses it to create instances of "BlastQuery" and "BlastHit"
-class BlastSimplexmlResult
+class BlastSimplexmlResult < BlastResult
 
   # Parser initialization
   def initialize(input)
+    super(input)
 
-    @querys = []
     lines=[]
     if input.is_a?(Array)
       lines=input
@@ -44,8 +44,11 @@ class BlastSimplexmlResult
       else
         raise "File #{input} doesn't exists"
       end
-
     end
+    parse(lines)
+  end
+  
+  def parse(lines)
 
     # puts "lines length #{lines.length}"
     if !lines.empty?
@@ -59,12 +62,13 @@ class BlastSimplexmlResult
         # puts JSON::pretty_generate(iteration)
 
         query_id = iteration['Iteration_query-ID'][0]
-        full_query_length = iteration['Iteration_query-len'][0]
+        full_query_length = iteration['Iteration_query-len'][0].to_i
         query_def = iteration['Iteration_query-def'][0]
 
         if query_def =~ /^([^\s]+)/
           query_def=$1
         end
+        
 
         #@query_def = iteration['Iteration_query-def'][0]
 
@@ -135,38 +139,4 @@ class BlastSimplexmlResult
     #inspect
   end
 
-
-
-  def inspect
-
-    res = "Blast results:\n"
-    res+= '-'*20
-    res+= "\nQuerys: #{@querys.count}\n"
-    @querys.each{|q| res+=q.inspect+"\n"}
-    return res
-  end
-
-  # finds a query by name
-  def find_query(querys,name_q)
-    #  newq = querys.find{|q| ( q.find{|h| (h.subject_id)})}
-    new_q=nil
-
-    if !querys.empty?
-      new_q=querys.find{|q| (q.query_id==name_q)}
-    end
-
-    return new_q
-  end
-
-  # check if there are querys
-  def empty?
-    return @querys.empty?
-  end
-
-  # get num of querys
-  def size
-    @querys.size
-  end
-
-  attr_accessor :querys
 end

data/lib/scbi_blast/blast_streamxml_result.rb

@@ -0,0 +1,181 @@
+require "blast_query.rb"
+require "blast_hit.rb"
+
+require 'rexml/document'
+require 'rexml/streamlistener'
+
+include REXML
+include REXML::StreamListener
+
+# Extracts results from a blast results in XML
+# format and uses it to create instances of "BlastQuery" and "BlastHit"
+class BlastStreamxmlResult < BlastResult
+  
+  # Parser initialization
+  def initialize(input)
+    super(input)
+
+    lines=[]
+    
+    # some variables for tracking the state of the parse
+    @current_query = nil
+    @current_hit = nil
+    @current_element = nil
+    @current_hit_subject_id=''
+    @current_hit_acc=''
+    @current_hit_full_subject_length=0
+    @current_hit_hit_def=''
+    
+    # @state = nil # (values :in_query,:in_hit)
+    
+    
+    if input.is_a?(Array)
+      lines=input.join("\n")
+      do_parse(lines)
+      
+    elsif !input.strip.empty?
+      
+      if File.exists?(input)
+        lines= File.open(input,'r')
+        # @lines = fich.readlines
+        # fich.close
+        do_parse(lines)
+        lines.close
+      else
+        raise "File #{input} doesn't exists"
+      end
+    end
+    
+  end
+  
+
+  def tag_start(name, attributes)
+    # puts "+#{name}: #{attributes}"
+    case name
+      when 'Iteration'
+        @current_query= BlastQuery.new(-1)
+        
+      when 'Hit'
+        @current_hit_subject_id=''
+        @current_hit_acc=''
+        @current_hit_full_subject_length=0
+        @current_hit_hit_def=''
+        
+      when 'Hsp'
+        @current_hit = BlastHit.new(0,0,0,0)
+        
+        # populate results from hit
+        @current_hit.subject_id=@current_hit_subject_id
+        @current_hit.full_subject_length=@current_hit_full_subject_length
+        @current_hit.definition=@current_hit_hit_def
+        @current_hit.acc=@current_hit_acc
+      else
+        @current_element = name
+    end
+    
+  end
+
+  def tag_end(name)
+    
+    case name
+      when 'Iteration'
+        @querys.push @current_query
+        @current_query=nil
+      when 'Hit'
+        @current_hit_subject_id=''
+        @current_hit_acc=''
+        @current_hit_full_subject_length=0
+        @current_hit_hit_def=''
+        @current_hit=nil
+        
+      when 'Hsp'
+        @current_hit.set_limits(@current_hit.q_beg,@current_hit.q_end,@current_hit.s_beg,@current_hit.s_end)
+        
+        @current_query.hits.push @current_hit
+        
+    end
+    
+    @current_element=nil
+    # puts "-#{name}"
+
+  end
+
+  def text(text)
+    
+    case @current_element
+      
+      # values for querys 
+      when 'Iteration_query-ID'
+        @current_query.query_id=text
+      when 'Iteration_query-len'
+        @current_query.full_query_length=text.to_i
+      when 'Iteration_query-def'
+        @current_query.query_def=text
+        
+      # values for hits
+      when 'Hit_id'
+        @current_hit_subject_id=text
+      when 'Hit_accession'
+        @current_hit_acc=text
+      when 'Hit_len'
+        @current_hit_full_subject_length=text.to_i
+      when 'Hit_def'
+        @current_hit_hit_def=text
+        if @current_hit_hit_def=='No definition line'
+          @current_hit_hit_def =@current_hit_subject_id
+        end
+
+      # values for HSPs
+      
+
+        when 'Hsp_query-from'
+          # puts "QBEG1:#{text.to_i}"
+          @current_hit.q_beg=text.to_i
+          # puts "QBEG2:#{@current_hit.q_beg},#{text.to_i}"
+        when 'Hsp_query-to'
+          @current_hit.q_end=text.to_i
+        when 'Hsp_hit-from'
+          @current_hit.s_beg=text.to_i
+        when 'Hsp_hit-to'
+          @current_hit.s_end=text.to_i
+        when 'Hsp_align-len'
+          @current_hit.align_len=text.to_i
+          @current_hit.ident=(@current_hit.ident/@current_hit.align_len)*100
+        when 'Hsp_identity'
+          
+          @current_hit.ident=(text.to_f)
+          # @current_hit.ident=(text.to_f/@current_hit.align_len)*100
+          # percent calculation now goes to align-len
+        when 'Hsp_gaps'
+          @current_hit.gaps=text.to_i
+        when 'Hsp_midline'
+          @current_hit.mismatches= text.count(' ').to_i - @current_hit.gaps
+        when 'Hsp_evalue'
+          @current_hit.e_val=text.to_f
+          @current_hit.e_val = (@current_hit.e_val*1000).round/1000.0
+        when 'Hsp_bit-score'
+          @current_hit.bit_score=text.to_f
+          @current_hit.bit_score = (@current_hit.bit_score*100).round/100.0
+        when 'Hsp_score'
+          @current_hit.score =text.to_f
+        when 'Hsp_query-frame'
+          @current_hit.q_frame = text.to_i
+        when 'Hsp_hit-frame'
+          @current_hit.s_frame =text.to_i
+
+        when 'Hsp_qseq'
+          @current_hit.q_seq = text
+        when 'Hsp_hseq'
+          @current_hit.s_seq = text
+      
+    end
+
+    # reset the current element so we don't pick up empty text
+    @current_element = nil
+  end
+  
+  def do_parse(lines)
+    Document.parse_stream(lines, self)
+  end
+
+end

data/lib/scbi_blast/blast_table_result.rb

@@ -24,14 +24,13 @@ require "blast_query.rb"
 require "blast_hit.rb"
 
 
-
 # Extracts results from blast table's file and uses it to create instances of "BlastQuery" and "BlastHit"
-class BlastTableResult
+class BlastTableResult < BlastResult
 
   # Parser initialization
   def initialize(input)
 
-    @querys = []
+    super(input)
 
 
     if input.is_a?(Array)
@@ -44,7 +43,10 @@ class BlastTableResult
       fich.close
 
     end
-
+    parse(lines)
+  end
+  
+  def parse(lines)
     query_name=''
 
     lines.each do |line|
@@ -57,9 +59,9 @@ class BlastTableResult
         elsif line =~ /^#\s0\shits\sfound$/
           @querys.push BlastQuery.new(query_name)
         end
+        
         # 0 hits found
 
-
       else
         params = line.split(/\t+/)
 
@@ -168,38 +170,4 @@ class BlastTableResult
 
   end
 
-
-  # inspect results
-  def inspect
-    res = "Blast results:\n"
-    res+= '-'*20
-    res+= "\nQuerys: #{@querys.count}\n"
-    @querys.each{|q| res+=q.inspect+"\n"}
-    return res
-  end
-
-  # find query by name
-  def find_query(querys,name_q)
-    #  newq = querys.find{|q| ( q.find{|h| (h.subject_id)})}
-    new_q=nil
-
-    if !querys.empty?
-      new_q=querys.find{|q| (q.query_id==name_q)}
-    end
-
-    return new_q
-  end
-
-  # check if there are querys
-  def empty?
-
-    return @querys.empty?
-  end
-
-  # get query count
-  def size
-    @querys.size
-  end
-
-  attr_accessor :querys
 end

data/lib/scbi_blast.rb

@@ -26,12 +26,14 @@ require 'batch_blast'
 require 'dust_masker'
 require 'blast_query'
 # require 'blast_xml_result'
+require 'blast_result'
 require 'blast_simplexml_result'
+require 'blast_streamxml_result'
 require 'blast_hit'
 require 'blast_table_result'
 
 module ScbiBlast
-   VERSION = '0.0.33'
+   VERSION = '0.0.34'
 end
 
 

data/test/test_scbi_blast.rb

@@ -17,15 +17,48 @@ class TestScbiBlast < Test::Unit::TestCase
     puts res.inspect
   end
   
-	def test_querys
-	  res = BlastSimplexmlResult.new(File.join(File.dirname(__FILE__),'blast.xml'))  	
+  def test_querys
+    res = BlastSimplexmlResult.new(File.join(File.dirname(__FILE__),'blast.xml'))   
     # puts "BLAST.XML"
     # puts res.inspect
-  	assert_equal(2,res.querys.count)
-  	assert_equal(2,res.querys[0].hits.count)
-  	assert_equal(2,res.querys[1].hits.count)
-  	
-	end
+    assert_equal(2,res.querys.count)
+    assert_equal(2,res.querys[0].hits.count)
+    assert_equal(2,res.querys[1].hits.count)
+    
+  end
+
+  def test_streamxml
+    file='blast.xml'
+    
+    resStream = BlastStreamxmlResult.new(File.join(File.dirname(__FILE__),file))
+    resSimple = BlastSimplexmlResult.new(File.join(File.dirname(__FILE__),file))
+    
+    # res = BlastStreamxmlResult.new(File.join(File.dirname(__FILE__),'db_1.xml'))
+    # puts "BLAST.XML"
+    # puts res.inspect
+    assert_equal(2,resStream.querys.count)
+    assert_equal(2,resStream.querys[0].hits.count)
+    assert_equal(2,resStream.querys[1].hits.count)
+    
+    assert(resStream.compare?(resSimple))
+    
+    resStream.querys.each_with_index do |q,i|
+      q2=resSimple.querys[i]
+      
+      q.hits.each_with_index do |h,i2|
+        h2=q2.hits[i2]
+        assert(h.compare?(h2),"Hits not equal: #{h.inspect},#{h2.inspect}")
+      end
+      
+      assert_equal(q.query_id,q2.query_id)
+      assert_equal(q.query_def,q2.query_def)
+      assert_equal(q.full_query_length , q2.full_query_length)
+      
+      assert(q.compare?(q2),"Query not equal: #{q.inspect},#{q2.inspect}")
+      
+    end
+    
+  end
   
   def test_hits
     assert true

metadata

@@ -2,7 +2,7 @@
 name: scbi_blast
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.33
+  version: 0.0.34
 platform: ruby
 authors: 
 - Dario Guerrero & Almudena Bocinos
@@ -10,19 +10,30 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-06 00:00:00 Z
+date: 2012-04-30 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: hoe
+  name: xml-simple
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.12
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
         version: 2.8.0
   type: :development
-  version_requirements: *id001
+  version_requirements: *id002
 description: |-
   scbi_blast is a ruby gem to handle blast+ executions without the need of temporary files, 
   it has been developed at [SCBI](http://www.scbi.uma.es) by Almudena Bocinos & Dario Guerrero.
@@ -53,9 +64,11 @@ files:
 - test/blast.txt
 - lib/scbi_blast/blast_hit.rb
 - lib/scbi_blast/blast_query.rb
+- lib/scbi_blast/blast_result.rb
 - lib/scbi_blast/blast_table_result.rb
 - lib/scbi_blast/blast_xml_result.rb
 - lib/scbi_blast/blast_simplexml_result.rb
+- lib/scbi_blast/blast_streamxml_result.rb
 - lib/scbi_blast/batch_blast.rb
 - lib/scbi_blast/dust_masker.rb
 homepage: http://www.scbi.uma.es/downloads