rubyful_soup 1.0.3 → 1.0.4

data/CHANGELOG

@@ -1,5 +1,15 @@
 Rubyful Soup Changelog
 
+1.0.4
+
+Major performance improvements: the code is now over three times
+faster. It's still relatively slower than Beautiful Soup, but it's no
+longer unusably slow on large documents.
+
+You can now tell the parser to only parse certain tags (and their
+recursive contents). This can mean a much smaller memory footprint if
+all you care about are the A tags or whatever.
+
 1.0.3
 
 Minor bugfixes to handle more types of data. Still more changes to use

data/lib/rubyful_soup.rb

@@ -1,7 +1,7 @@
 #Rubyful Soup
 #Elixir and Tonic
 #"The Screen-Scraper's Friend"
-#v1.0.3
+#v1.0.4
 #http://www.crummy.com/software/RubyfulSoup/
 #
 #Rubyful Soup is a port to the Ruby language and idiom of the Python
@@ -12,6 +12,7 @@
 #Maebashi. The easiest way to get it is to install the "htmltools"
 #gem.
 require 'html/sgml-parser'
+require 'set'
 
 #UTF-8 voodoo--does this really work?
 $KCODE = 'u'
@@ -213,16 +214,33 @@ module TagModule
   #  return methods('find_all').to_proc
   #end
 
-  def initialize(parser, name, attrs=nil, parent=nil, previous=nil)
+  def initialize(parser, name, attr_list=[], parent=nil, previous=nil)
     @hidden = false
     @parser = parser
     @name = name
-    attrs ||= {}
-    @attrs = attrs
+    @attr_list = attr_list
+    @attrs = nil
     @contents = []
     setup(parent, previous)
   end
 
+  # Turn the list of attributes into a hash on demand, so we don't have
+  # to do it for every tag while parsing.
+
+  def attrs
+    unless @attrs
+      @attrs = @attr_list.inject({}) do |m,v|
+        if v[1][0] == ?" and v[1][-1] == ?":
+            v[1] = v[1][1..-2]
+        end
+        m[v[0]] = v[1]
+        m
+      end
+      @attr_list = nil
+    end
+    return @attrs
+  end
+
   #soup.title_tag, or soup.title, is the same as soup.find('title')
   def method_missing(name, *args)
     #puts "Missing method #{name} for #{self.class.name}"
@@ -233,25 +251,22 @@ module TagModule
     return find(name, *args)
   end
 
-  #TODO: is there a mixin for Hash?
   def [](k)
-    return @attrs[k]
+    attrs[k]
   end
 
   def []=(k, v)
-    @attrs[k] = v
+    attrs[k] = v
   end
 
   def delete(k)
-    @attrs.delete(k)
+    attrs.delete(k)
   end
 
   def has_key?(k)
-    return @attrs.has_key(k)
+    attrs.has_key(k)
   end
 
-  #End things that would go away if there was a mixin for Hash.
-
   def each
     @contents.each { |x| yield x }
   end
@@ -287,8 +302,8 @@ module TagModule
   #consumes whitespace, this method is not certain to reproduce the
   #whitespace present in the original string.
   def to_s(show_structure_indent=nil)
-    attrs = []
-    @attrs.each { |k,v| attrs.push("#{k}=\"#{v}\"") if v }
+    attr_strings = []
+    attrs.each { |k,v| attr_strings << %{#{k}="#{v}"} if v }
     if self_closing?
       close = ' /'
       closeTag = nil
@@ -307,8 +322,8 @@ module TagModule
     else
       s = []
       attribute_string = ''
-      unless attrs.empty?
-        attribute_string = ' ' + attrs.join(' ')
+      unless attr_strings.empty?
+        attribute_string = ' ' + attr_strings.join(' ')
       end
       s.push(space) if show_structure_indent
       s.push("<#{@name}#{attribute_string}#{close}>")
@@ -489,11 +504,22 @@ class BeautifulStoneSoup < HTML::SGMLParser
     @@self_closing_tags.has_key?(tag)
   end
 
-  #Args: :initial_text_is_everything, :avoid_parser_problems
+  #Args: :initial_text_is_everything, :avoid_parser_problems, :parse_only_these
   def initialize(text, args={})
     super(self, @@rootTagName)
     @quote_stack = []
     @hidden = 1
+    if args[:parse_only_these]
+      @parse_only_these = Set.new
+      p = args[:parse_only_these]
+      if p.respond_to? :each
+        p.each { |x| @parse_only_these << x }
+      else
+        @parse_only_these << p
+      end      
+    else
+      @parse_only_these = nil
+    end
     reset
 
     @avoid_parser_problems = args[:avoid_parser_problems] || true
@@ -522,7 +548,7 @@ class BeautifulStoneSoup < HTML::SGMLParser
   end
 
   def ==(anObject)
-    return anObject.to_s == to_s
+    return anObject != nil && anObject.to_s == to_s
   end
 
   def done
@@ -564,13 +590,7 @@ class BeautifulStoneSoup < HTML::SGMLParser
   
   def unknown_starttag(name, attrs)
     #puts "Starting tag #{name} #{attrs.inspect}"
-    attrs = attrs.inject({}) do |m,v|
-      if v[1][0] == ?" and v[1][-1] == ?":
-        v[1] = v[1][1..-2]
-      end
-      m[v[0]] = v[1]
-      m
-    end
+
     unless @quote_stack.empty?
       #This is not a real tag.
       #puts "<#{name}> is not real!"
@@ -582,6 +602,8 @@ class BeautifulStoneSoup < HTML::SGMLParser
     end
 
     end_text
+
+    return unless !@parse_only_these or @tag_stack.size > 1 or @parse_only_these.member?(name)
     self_closing = @@self_closing_tags.has_key?(name)
     smart_pop(name) unless self_closing
     tag = Tag.new(self, name, attrs, @currentTag, @previous_parsed)
@@ -608,12 +630,16 @@ class BeautifulStoneSoup < HTML::SGMLParser
       handle_data('</#{name}>')
       return
     end   
+
+    return unless !@parse_only_these or @tag_stack.size > 1 or @parse_only_these.member?(name)
+
     end_text
     pop_to_tag(name)
     @quote_stack.pop if not @quote_stack.empty? and @quote_stack[-1] == name
   end
 
   def handle_data(data)
+    return unless !@parse_only_these or @tag_stack.size > 1
     @currentText.push(data)
   end
 
@@ -712,7 +738,7 @@ class BeautifulStoneSoup < HTML::SGMLParser
     #puts "Pop to tag #{ name }. Inclusive? #{inclusive_pop}"
     num_pops = 0
     mostRecentTag = nil
-    (0...@tag_stack.length).to_a.reverse.each do |i|
+    (@tag_stack.length-1).downto(0) do |i|
       if name == @tag_stack[i].name
         #puts "Found at #{i}, #{@tag_stack.length-i}"
         num_pops = @tag_stack.length-i
@@ -748,7 +774,7 @@ class BeautifulStoneSoup < HTML::SGMLParser
     is_reset_nesting = @@reset_nesting_tags.has_key?(name)
     popTo = nil
     inclusive = true
-    for p in @tag_stack.reverse
+    @tag_stack.reverse_each do |p|
         if (p == nil or p.name == name) and not is_nestable
             #Non-nestable tags get popped to the top or to their
             #last occurance.

data/tests/rubyful_soup_tests.rb

@@ -391,6 +391,16 @@ class SOAPMeUp < SoupTest
   end
 end
 
+# Verifies that you can decide not to parse certain tags.
+class OnlyTheLonely < SoupTest
+  def test_parse_only_these
+    html = "<a>1<b>2</b>3</a><b>4<a>5</a>6</b>"
+    soup = BeautifulStoneSoup.new(html, :parse_only_these=>'b')
+    puts soup
+    assert_equal(soup.to_s, "<b>2</b><b>4<a>5</a>6</b>")
+  end
+end
+
 #The Unicode test suite has not yet been ported because I haven't
 #figured out how Ruby does Unicode.
 

metadata

@@ -3,50 +3,51 @@ rubygems_version: 0.8.4
 specification_version: 1
 name: rubyful_soup
 version: !ruby/object:Gem::Version 
-  version: 1.0.3
-date: 2005-11-04
+  version: 1.0.4
+date: 2006-03-01
 summary: An HTML/XML parser that handles bad markup and provides tree traversal methods.
 require_paths: 
-  - lib
+- lib
 email: leonardr@segfault.org
 homepage: http://www.crummy.com/software/RubyfulSoup/
 rubyforge_project: 
-description: "Rubyful Soup is a *ML parser that makes screen-scraping easy. It won't choke on
-  bad markup, and it's easy to locate the part of a document you want."
+description: Rubyful Soup is a *ML parser that makes screen-scraping easy. It won't choke on bad markup, and it's easy to locate the part of a document you want.
 autorequire: 
 default_executable: 
 bindir: bin
 has_rdoc: true
 required_ruby_version: !ruby/object:Gem::Version::Requirement 
   requirements: 
-    - 
-      - ">"
-      - !ruby/object:Gem::Version 
-        version: 0.0.0
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
   version: 
 platform: ruby
 authors: 
-  - Leonard Richardson
+- Leonard Richardson
 files: 
-  - lib/rubyful_soup.rb
-  - tests/rubyful_soup_tests.rb
-  - CHANGELOG
+- lib/rubyful_soup.rb
+- tests/rubyful_soup_tests.rb
+- CHANGELOG
 test_files: 
-  - tests/rubyful_soup_tests.rb
+- tests/rubyful_soup_tests.rb
 rdoc_options: []
+
 extra_rdoc_files: 
-  - CHANGELOG
+- CHANGELOG
 executables: []
+
 extensions: []
+
 requirements: []
+
 dependencies: 
-  - !ruby/object:Gem::Dependency 
-    name: htmltools
-    version_requirement: 
-    version_requirements: !ruby/object:Gem::Version::Requirement 
-      requirements: 
-        - 
-          - ">"
-          - !ruby/object:Gem::Version 
-            version: 0.0.0
-      version: 
+- !ruby/object:Gem::Dependency 
+  name: htmltools
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: