hpricot_scrub 0.2.3 → 0.3.0

data/CHANGELOG.txt

@@ -1,3 +1,12 @@
+2008-01-11 Mina Naguib <mina.hpricotscrub@naguib.ca>
+	Release 0.3.0
+	Large overhaul of the module's logic to mimic most of perl's HTML::Scrubber
+	functionality:
+  	- Deprecate config keys :allow_tags, :remove_tags and :allow_attributes
+		- Introduce config keys :elem_rules, :default_elem_rule,
+		  :default_comment_rule and :default_attribute_rule
+		- Document the above (inline - visible in rdoc & the likes)
+
 2007-04-05 Michael <michael@underpantsgnome.com>
 	Release 0.2.3
 	Add patches from Eric Wong

data/Manifest.txt

@@ -9,4 +9,5 @@ lib/hpricot_scrub.rb
 test/test_helper.rb
 test/scrubber_data.rb
 test/hpricot_scrub_test.rb
-examples/config.yml
+test/old_hpricot_scrub_test.rb
+examples/old_config.yml

data/lib/hpricot_scrub/hpricot_scrub.rb

@@ -9,14 +9,120 @@ end
 require 'hpricot'
 
 module Hpricot
-  module Scrubable
-    # TODO: figure out how to handle comments
-    def scrubable?
-      ! [ Hpricot::Text, 
-          Hpricot::BogusETag, 
-          Hpricot::Comment
-        ].include?(self.class) && self.respond_to?(:scrub)
+
+  class Scrub
+
+    def self.normalize_config(config) #:nodoc:#
+      config = {} unless config.is_a?(Hash)
+
+      return config if config[:normalized]
+
+      config = {
+
+        # Legacy config keys:
+        :remove_tags => [],
+        :allow_tags => [],
+        :allow_attributes => [],
+
+        # New fine-grained hotness:
+        :elem_rules => {
+          "script"  =>  false,
+          "style"  =>  false
+        },
+        :default_elem_rule => :strip,
+        :default_comment_rule => false,
+        :default_attribute_rule => false
+
+      }.merge(config)
+
+      #
+      # Merge+delete legacy config keys
+      #
+      # :remove_tags
+      (config.delete(:remove_tags) || []).each do |tag|
+        config[:elem_rules][tag] = false unless config[:elem_rules].has_key?(tag)
+      end
+      # :allow_tags
+      (config.delete(:allow_tags) || []).each do |tag|
+        config[:elem_rules][tag] = true unless config[:elem_rules].has_key?(tag)
+      end
+      # :allow_attributes
+      (config.delete(:allow_attributes) || []).each do |attribute|
+        #
+        # Add it to the default attribute rule
+        #
+        old_rule = config[:default_attribute_rule]
+        config[:default_attribute_rule] = Proc.new do |parent_element, key, value|
+          if key == attribute
+            true
+          else
+            Scrub::keep_attribute?(parent_element, key, value, old_rule)
+          end
+        end
+      end
+
+      config[:normalized] = true
+      return config
+    end
+
+    #
+    # Takes:
+    #
+    #  An element
+    #  An attribute key found in that element
+    #  The attribute value attached to the key
+    #  An attribute rule
+    #
+    # Checks the rule aginst the attribute and returns:
+    #
+    #  true = the attribute should be kept
+    #  false = the attribute should NOT be kept
+    #
+    # Acceptable attribute rules are:
+    #
+    #  true: Keep the attribute without inspection
+    #  a String: Attribute value must be the same as the string
+    #  an Array: Attribute key must exist in the array
+    #  a Regexp: Attribute value must match the regexp
+    #  a Hash: The attribute key is found in the hash, and the value is considered a new rule and follows these same rules via recursion
+    #  a Proc: The Proc is called with arguments (parent_element, key, value), the returned value is considered a new rule and follows these same rules via recursion
+    #  otherwise: Remove the attribute
+    #
+    def self.keep_attribute?(parent_element, key, value, attribute_rule)
+
+      if attribute_rule == true
+        keep = true
+      elsif attribute_rule.is_a?(String)
+        keep = (attribute_rule == value)
+      elsif attribute_rule.is_a?(Array)
+        keep = attribute_rule.include?(key)
+      elsif attribute_rule.is_a?(Regexp)
+        keep = attribute_rule.match(value)
+      elsif attribute_rule.is_a?(Hash)
+        # Allow hash value to be new rule via recursion
+        new_rule = attribute_rule[key]
+        keep = keep_attribute?(parent_element, key, value, new_rule)
+      elsif attribute_rule.is_a?(Proc)
+        # Allow the proc to return a new rule - recurse:
+        new_rule = attribute_rule.call(parent_element, key, value)
+        keep = keep_attribute?(parent_element, key, value, new_rule)
+      else
+        # Err on the side of caution
+        keep = false
+      end
+
+      return keep
+
     end
+
+    module Scrubbable
+      def scrubbable?
+        ! [ Hpricot::Text, 
+            Hpricot::BogusETag,
+          ].include?(self.class) && self.respond_to?(:scrub)
+      end
+    end
+
   end
 
   class Elements
@@ -25,73 +131,124 @@ module Hpricot
     end
     
     def strip_attributes(safe=[])
-      each { |x| x.strip_attributes(safe) }
+      each { |x| x.scrub_attributes(safe) }
     end
   end
 
   class BaseEle
-    include Scrubable
+    include Scrub::Scrubbable
   end
 
-  class Elem
-    include Scrubable
+  class Comment
+    include Scrub::Scrubbable
 
-    def scrub(config)
-      children.reverse.each { |c| c.scrub(config) if c.scrubable? }
-      strip unless config[:allow_tags].include?(name)
+    def remove
+      parent.children.delete(self)
     end
 
+    #
+    # Scrubs this comment according to the given config
+    # If the config key :default_comment_rule is true, the comment is kept. Otherwise it's removed.
+    #
+    def scrub(config = nil)
+      config = Scrub::normalize_config(config)
+      rule = config[:default_comment_rule]
+      remove unless rule
+      return true
+    end
+
+  end
+
+  class Elem
+    include Scrub::Scrubbable
+
     def remove
       parent.children.delete(self)
     end
 
     def strip
-      children.each { |c| c.strip if c.scrubable? }
+      swap(inner_html)
+    end
+
+    #
+    # Scrubs the element according to the given config
+    # The relevant config key is :elem_rules.  It is expected to be a Hash having String HTML tag names as keys, and a rule as values
+    # The rule value dictates what happens to the element.  The following logic is used:
+    #   If the rules is false, the element is removed
+    #   If the rule is :strip, the element is stripped (the element itself is deleted and its children are promoted upwards to where it was)
+    #   Otherwise the element is kept
+    #
+    #  If the element name (HTML tag) was not found in :elem_rules, the default rule in config key :default_elem_rule is used
+    #
+    # After the above is done, scrub_attributes is called if the element was kept.  The rule is passed to it as it's assumed to be the attribute rules (see Hpricot::Scrub.keep_attribute?) to apply to the attributes, UNLESS the rule was explicitly "true", in which case the config key :default_attribute_rule is passed.
+    #
+    # This is recursive and will do all the above to all the children of the element as well.
+    #
+    def scrub(config = nil)
 
-      if strip_removes?
+      config = Scrub::normalize_config(config)
+
+      children.reverse.each do |child|
+        child.scrub(config) if child.scrubbable?
+      end
+
+      rule = config[:elem_rules].has_key?(name) ? config[:elem_rules][name] : config[:default_elem_rule]
+
+      if !rule
         remove
+      elsif rule == :strip
+        strip
       else
-        parent.replace_child self, Hpricot.make(inner_html) unless parent.nil?
+        # Positive rule
+        # Keep the element
+        # On to attributes
+        scrub_attributes(rule == true ? config[:default_attribute_rule] : rule)
       end
+
+      return self
     end
-    
-    def strip_attributes(safe=[])
-      attributes.each {|atr|
-        remove_attribute(atr[0]) unless safe.include?(atr[0])
-      } unless attributes.nil?
+
+    #
+    # Loops over all the attributes on this element, and removes any which Hpricot::Scrub.keep_attribute? returns false for
+    #
+    def scrub_attributes(attribute_rule = nil)
+      if attributes
+        attributes.each do |key, value|
+          remove_attribute(key) unless Scrub.keep_attribute?(self, key, value, attribute_rule)
+        end
+      end
+      return true
     end
     
-    def strip_removes?
-      # TODO: find other elements that should be removed instead of stripped
-      attributes && attributes['type'] =~ /script|css/
-    end
-  end
+  end #class Elem
 
   class Doc
-    def scrub(config={})
-      config = {
-        :remove_tags => [],
-        :allow_tags => [],
-        :allow_attributes => []
-      }.merge(config)
-      
-      config[:remove_tags].each { |tag| (self/tag).remove }
-      config[:allow_tags].each { |tag|
-        (self/tag).strip_attributes(config[:allow_attributes])
-      }
-      children.reverse.each {|c| c.scrub(config) if c.scrubable? }
-      self
+
+    #
+    # Scrubs the Hpricot document by removing certain elements and attributes
+    # according to the passed-in config
+    # WARNING: This is destructive.  If you want to keep your document untouched use a duplicate copy
+    # See the documentation on Hpricot::Elem#scrub for documentation of config
+    #
+    def scrub(config=nil)
+      config = Scrub::normalize_config(config)
+      children.reverse.each do |child|
+        child.scrub(config) if child.scrubbable?
+      end
+      return self
     end
-  end
+
+  end #class Doc
+
 end
 
 class String
-  def scrub!
-    self.gsub!(/^(\n|.)*$/, Hpricot(self).scrub.inner_html)
+  def scrub!(config=nil)
+    self.gsub!(/^(\n|.)*$/, Hpricot(self).scrub(config).inner_html)
   end
 
-  def scrub
-    dup.scrub!
+  def scrub(config=nil)
+    dup.scrub!(config)
   end
 end
 
@@ -116,4 +273,5 @@ begin
       dup.decode!
     end
   end
+
 rescue LoadError; end

data/lib/hpricot_scrub/version.rb

@@ -1,8 +1,8 @@
 module HpricotScrub #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 2
-    TINY  = 3
+    MINOR = 3
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/test/hpricot_scrub_test.rb

@@ -1,84 +1,128 @@
+#
+# This test case tests HpricotScrub features that were introduced in version 0.3.0
+# introduction of more fine-grained filtering
+#
 require File.dirname(__FILE__) + '/test_helper.rb'
 require File.dirname(__FILE__) + '/scrubber_data.rb'
+require "uri"
 
 class HpricotScrubTest < Test::Unit::TestCase
 
   def setup
-    @clean = Hpricot(MARKUP).scrub.inner_html
-    @config = YAML.load_file('examples/config.yml')
 
-    # add some tags that most users will probably want
-    @config_full = @config.dup
-    %w(body head html).each { |x| @config_full[:allow_tags].push(x) }
+    config = {
+      :elem_rules => {
+        "a" =>  {
+          "href"  =>  %r|^https?://|i
+        },
+        "b" => true,
+        "body"  =>  {
+          "lang" => %w(en es fr)
+        },
+        "br"  =>  true,
+        "div" =>  %w(id class style),
+        "hr"  =>  true,
+        "html"  => true,
+        "img" =>  {
+          "src" =>  Proc.new do |parent_element, attribute_key, attribute_value|
+            begin
+              uri = URI.parse(attribute_value)
+              uri.is_a?(URI::HTTP) && uri.host != /imageshack/i
+            rescue
+              false
+            end
+          end,
+          "align" =>  "middle",
+          "alt" =>  true
+        },
+        "marquee" =>  :strip,
+        "p"  =>  true,
+        "script"  => false,
+        "span" =>  :strip,
+        "strong" => true,
+        "style"  => false
+      },
+      :default_elem_rule => :strip,
+      :default_comment_rule => false,
+      :default_attribute_rule => false
+    }
+
+    @docs = [
+      Hpricot(MARKUP),
+      Hpricot(GOOGLE)
+    ]
+    @scrubbed_docs = [
+      Hpricot(MARKUP).scrub(config),
+      Hpricot(GOOGLE).scrub(config)
+    ]
+
   end
 
-  def test_full_markup_partial_scrub
-    full = Hpricot(MARKUP)
-    full_markup = '<html><head></head><body>' + MARKUP + '</body></html>'
-    doc = Hpricot(full_markup).scrub(@config_full)
-    partial_scrub_common(doc, full)
+  def test_elem_default_rule_strips
+     @scrubbed_docs.each do |doc|
+       assert_equal 0, doc.search("//span").length
+     end
   end
 
-  def test_full_scrub
-    doc = Hpricot(MARKUP).scrub
-    # using the divisor search throws warnings in test
-    assert_tag_count(doc, 'a', 0)
-    assert_tag_count(doc, 'p', 0)
-    assert_tag_count(doc, 'img', 0)
-    assert_tag_count(doc, 'br', 0)
-    assert_tag_count(doc, 'div', 0)
-    assert_tag_count(doc, 'script', 0)
+  def test_elem_rule_keep
+     @scrubbed_docs.each_with_index do |doc, i|
+       assert_equal @docs[i].search("//a").length, doc.search("//a").length
+       assert_equal @docs[i].search("//b").length, doc.search("//b").length
+       assert_equal @docs[i].search("//img").length,  doc.search("//img").length
+     end
   end
 
-  def test_partial_scrub
-    full = Hpricot(MARKUP)
-    doc = Hpricot(MARKUP).scrub(@config)
-    partial_scrub_common(doc, full)
+  def test_elem_rule_remove
+     @scrubbed_docs.each do |doc|
+       assert_equal 0, doc.search("//script").length
+       assert_equal 0, doc.search("//style").length
+     end
   end
 
-  def test_full_doc
-    doc = Hpricot(GOOGLE).scrub
-    assert_tag_count(doc, 'a', 0)
-    assert_tag_count(doc, 'p', 0)
-    assert_tag_count(doc, 'img', 0)
-    assert_tag_count(doc, 'br', 0)
-    assert_tag_count(doc, 'div', 0)
-    assert_tag_count(doc, 'script', 0)
+  def test_elem_rule_strip
+     @scrubbed_docs.each do |doc|
+       assert_equal 0, doc.search("//marquee").length
+       assert_equal 0, doc.search("//span").length
+     end
   end
 
-  def test_string_scrub
-    formatted = MARKUP
-    assert formatted.scrub == @clean
-    assert formatted == MARKUP
+  def test_attr_default_rule_removes
+     @scrubbed_docs.each do |doc|
+       assert_equal 0, doc.search("*[@mce_src]").length
+       assert_equal 0, doc.search("*[@target]").length
+     end
   end
 
-  def test_string_scrub!
-    formatted = MARKUP
-    assert formatted.scrub! == @clean
-    assert formatted == @clean
+  def test_attr_rule_true
+     @scrubbed_docs.each_with_index do |doc, i|
+       assert_equal @docs[i].search("*[@alt]").length, doc.search("*[@alt]").length
+     end
   end
 
-  def test_decoder
-    str = 'some <a href="http://example.com/">example&nbsp;link</a> to nowhere'
-    scrubbed_str = str.scrub
-    assert scrubbed_str.include?('&nbsp;')
+  def test_attr_rule_string
+     @scrubbed_docs.each_with_index do |doc, i|
+       assert_equal @docs[i].search("img[@align='middle']").length, doc.search("img[@align]").length
+     end
+  end
 
-    if defined?(HTMLEntities)
-      assert ! scrubbed_str.decode.include?('&nbsp;')
+  def test_attr_rule_regexp
+     @scrubbed_docs.each_with_index do |doc, i|
+       assert_equal @docs[i].search("a[@href^='http']").length, doc.search("a[@href]").length
+     end
+  end
 
-      scrubbed_str.decode!
-      assert ! scrubbed_str.include?('&nbsp;')
-    end
+  def test_attr_rule_array
+     @scrubbed_docs.each_with_index do |doc, i|
+       assert_equal @docs[i].search("div[@id]").length, doc.search("div[@id]").length
+       assert_equal @docs[i].search("div[@class]").length, doc.search("div[@class]").length
+       assert_equal @docs[i].search("div[@style]").length, doc.search("div[@style]").length
+     end
   end
 
-private
-  def partial_scrub_common(doc, full)
-    # using the divisor search throws warnings in test
-    assert_tag_count(doc, 'a', 0)
-    assert_tag_count(doc, 'p', full.search('//p').size)
-    assert_tag_count(doc, 'div', full.search('//div').size)
-    assert_tag_count(doc, 'img', full.search('//img').size)
-    assert_tag_count(doc, 'br', full.search('//br').size)
-    assert_tag_count(doc, 'script', 0)
+  def test_attr_rule_proc
+     @scrubbed_docs.each_with_index do |doc, i|
+       assert_equal @docs[i].search("img[@src^='http']").length, doc.search("img[@src]").length
+     end
   end
+
 end

data/test/old_hpricot_scrub_test.rb

@@ -0,0 +1,88 @@
+#
+# This test case tests HpricotScrub features that were present in version 0.2.3 before
+# introduction of more fine-grained filtering
+#
+require File.dirname(__FILE__) + '/test_helper.rb'
+require File.dirname(__FILE__) + '/scrubber_data.rb'
+
+class OldHpricotScrubTest < Test::Unit::TestCase
+
+  def setup
+    @clean = Hpricot(MARKUP).scrub.inner_html
+    @config = YAML.load_file('examples/old_config.yml')
+
+    # add some tags that most users will probably want
+    @config_full = @config.dup
+    %w(body head html).each { |x| @config_full[:allow_tags].push(x) }
+  end
+
+  def test_full_markup_partial_scrub
+    full = Hpricot(MARKUP)
+    full_markup = '<html><head></head><body>' + MARKUP + '</body></html>'
+    doc = Hpricot(full_markup).scrub(@config_full)
+    partial_scrub_common(doc, full)
+  end
+
+  def test_full_scrub
+    doc = Hpricot(MARKUP).scrub
+    # using the divisor search throws warnings in test
+    assert_tag_count(doc, 'a', 0)
+    assert_tag_count(doc, 'p', 0)
+    assert_tag_count(doc, 'img', 0)
+    assert_tag_count(doc, 'br', 0)
+    assert_tag_count(doc, 'div', 0)
+    assert_tag_count(doc, 'script', 0)
+  end
+
+  def test_partial_scrub
+    full = Hpricot(MARKUP)
+    doc = Hpricot(MARKUP).scrub(@config)
+    partial_scrub_common(doc, full)
+  end
+
+  def test_full_doc
+    doc = Hpricot(GOOGLE).scrub
+    assert_tag_count(doc, 'a', 0)
+    assert_tag_count(doc, 'p', 0)
+    assert_tag_count(doc, 'img', 0)
+    assert_tag_count(doc, 'br', 0)
+    assert_tag_count(doc, 'div', 0)
+    assert_tag_count(doc, 'script', 0)
+  end
+
+  def test_string_scrub
+    formatted = MARKUP
+    assert formatted.scrub == @clean
+    assert formatted == MARKUP
+  end
+
+  def test_string_scrub!
+    formatted = MARKUP
+    assert formatted.scrub! == @clean
+    assert formatted == @clean
+  end
+
+  def test_decoder
+    str = 'some <a href="http://example.com/">example&nbsp;link</a> to nowhere'
+    scrubbed_str = str.scrub
+    assert scrubbed_str.include?('&nbsp;')
+
+    if defined?(HTMLEntities)
+      assert ! scrubbed_str.decode.include?('&nbsp;')
+
+      scrubbed_str.decode!
+      assert ! scrubbed_str.include?('&nbsp;')
+    end
+  end
+
+private
+  def partial_scrub_common(doc, full)
+    # using the divisor search throws warnings in test
+    assert_tag_count(doc, 'a', 0)
+    assert_tag_count(doc, 'p', full.search('//p').size)
+    assert_tag_count(doc, 'div', full.search('//div').size)
+    assert_tag_count(doc, 'img', full.search('//img').size)
+    assert_tag_count(doc, 'br', full.search('//br').size)
+    assert_tag_count(doc, 'script', 0)
+  end
+end

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.2.1
+rubygems_version: 0.9.4
 specification_version: 1
 name: hpricot_scrub
 version: !ruby/object:Gem::Version 
-  version: 0.2.3
-date: 2007-04-05 00:00:00 -07:00
+  version: 0.3.0
+date: 2008-04-22 00:00:00 -05:00
 summary: Scrub HTML with Hpricot
 require_paths: 
 - lib
@@ -40,13 +40,18 @@ files:
 - test/test_helper.rb
 - test/scrubber_data.rb
 - test/hpricot_scrub_test.rb
-- examples/config.yml
+- test/old_hpricot_scrub_test.rb
+- examples/old_config.yml
 test_files: 
 - test/hpricot_scrub_test.rb
-rdoc_options: []
-
-extra_rdoc_files: []
-
+- test/old_hpricot_scrub_test.rb
+rdoc_options: 
+- --main
+- README.txt
+extra_rdoc_files: 
+- README.txt
+- CHANGELOG.txt
+- Manifest.txt
 executables: []
 
 extensions: []