hpricot_scrub 0.2.2 → 0.2.3

data/CHANGELOG.txt

@@ -1,3 +1,9 @@
+2007-04-05 Michael <michael@underpantsgnome.com>
+	Release 0.2.3
+	Add patches from Eric Wong
+		- Recursive scrubbing wasn't scrubbing if parent was in allow
+		- Add optional use of HTMLEntities
+
 2007-03-04 Michael <michael@underpantsgnome.com>
 	Release 0.2.2
 	Add patches from Eric Wong 
@@ -6,10 +12,10 @@
 	
 2007-03-04 Michael <michael@underpantsgnome.com>
 	Release 0.2.0
-	Add String methods for scrub and scrub!
+		- Add String methods for scrub and scrub!
 	
-	Fixed a bug where nested elements were not being scrubbed when using a 
-	config hash
+		- Fixed a bug where nested elements were not being scrubbed when using a 
+			config hash
 
 2007-03-03 Michael <michael@underpantsgnome.com>
-	Release 0.1.0, Initial Gem version
+		- Release 0.1.0, Initial Gem version

data/examples/config.yml

@@ -1,47 +1,49 @@
-
 ---
-    :allow_tags: # let these tags stay, but will strip attributes
-        - 'b'
-        - 'blockquote'
-        - 'br'
-        - 'div'
-        - 'h1'
-        - 'h2'
-        - 'h3'
-        - 'h4'
-        - 'h5'
-        - 'h6'
-        - 'hr'
-        - 'i'
-        - 'em'
-        - 'img'
-        - 'li'
-        - 'ol'
-        - 'p'
-        - 'pre'
-        - 'small'
-        - 'span'
-        - 'span'
-        - 'strike'
-        - 'strong'
-        - 'sub'
-        - 'sup'
-        - 'table'
-        - 'tbody'
-        - 'td'
-        - 'tfoot'
-        - 'thead'
-        - 'tr'
-        - 'u'
-        - 'ul'
+  :allow_tags: # let these tags stay, but will strip attributes
+    - 'html'
+    - 'head'
+    - 'body'
+    - 'b'
+    - 'blockquote'
+    - 'br'
+    - 'div'
+    - 'h1'
+    - 'h2'
+    - 'h3'
+    - 'h4'
+    - 'h5'
+    - 'h6'
+    - 'hr'
+    - 'i'
+    - 'em'
+    - 'img'
+    - 'li'
+    - 'ol'
+    - 'p'
+    - 'pre'
+    - 'small'
+    - 'span'
+    - 'span'
+    - 'strike'
+    - 'strong'
+    - 'sub'
+    - 'sup'
+    - 'table'
+    - 'tbody'
+    - 'td'
+    - 'tfoot'
+    - 'thead'
+    - 'tr'
+    - 'u'
+    - 'ul'
 
-    :remove_tags: # completely removes everything between open and close tag
-        - 'form'
-        - 'script'
-        
-    :allow_attributes: # let these attributes stay, strip all others
-        - 'src'
-        - 'font'
-        - 'alt'
-        - 'style'
-        - 'align'
+  :remove_tags: # completely removes everything between open and close tag
+    - 'form'
+    - 'script'
+    
+  :allow_attributes: # let these attributes stay, strip all others
+    - 'src'
+    - 'font'
+    - 'alt'
+    - 'style'
+    - 'align'

data/lib/hpricot_scrub/hpricot_scrub.rb

@@ -10,9 +10,12 @@ require 'hpricot'
 
 module Hpricot
   module Scrubable
+    # TODO: figure out how to handle comments
     def scrubable?
-      ! [Hpricot::Text, Hpricot::BogusETag].include?(self.class) &&
-        self.respond_to?(:scrub)
+      ! [ Hpricot::Text, 
+          Hpricot::BogusETag, 
+          Hpricot::Comment
+        ].include?(self.class) && self.respond_to?(:scrub)
     end
   end
 
@@ -34,9 +37,7 @@ module Hpricot
     include Scrubable
 
     def scrub(config)
-      children.reverse.each { |c| 
-        c.scrub(config) if c.scrubable? && ! config[:allow_tags].include?(c.name)
-      }
+      children.reverse.each { |c| c.scrub(config) if c.scrubable? }
       strip unless config[:allow_tags].include?(name)
     end
 
@@ -61,7 +62,7 @@ module Hpricot
     end
     
     def strip_removes?
-      # I'm sure there are others that shuould be ripped instead of stripped
+      # TODO: find other elements that should be removed instead of stripped
       attributes && attributes['type'] =~ /script|css/
     end
   end
@@ -93,3 +94,26 @@ class String
     dup.scrub!
   end
 end
+
+begin
+  require 'htmlentities'
+  
+  module Hpricot
+    class Scrub
+      @coder = HTMLEntities.new
+      class << self
+        def entifier; @coder end
+      end
+    end
+  end
+  
+  class String
+    def decode!
+      self.gsub!(/^(\n|.)*$/, Hpricot::Scrub.entifier.decode(self))
+    end
+
+    def decode
+      dup.decode!
+    end
+  end
+rescue LoadError; end

data/lib/hpricot_scrub/version.rb

@@ -2,7 +2,7 @@ module HpricotScrub #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 2
-    TINY  = 2
+    TINY  = 3
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/test/hpricot_scrub_test.rb

@@ -6,6 +6,17 @@ class HpricotScrubTest < Test::Unit::TestCase
   def setup
     @clean = Hpricot(MARKUP).scrub.inner_html
     @config = YAML.load_file('examples/config.yml')
+
+    # add some tags that most users will probably want
+    @config_full = @config.dup
+    %w(body head html).each { |x| @config_full[:allow_tags].push(x) }
+  end
+
+  def test_full_markup_partial_scrub
+    full = Hpricot(MARKUP)
+    full_markup = '<html><head></head><body>' + MARKUP + '</body></html>'
+    doc = Hpricot(full_markup).scrub(@config_full)
+    partial_scrub_common(doc, full)
   end
 
   def test_full_scrub
@@ -22,12 +33,16 @@ class HpricotScrubTest < Test::Unit::TestCase
   def test_partial_scrub
     full = Hpricot(MARKUP)
     doc = Hpricot(MARKUP).scrub(@config)
-    # using the divisor search throws warnings in test
+    partial_scrub_common(doc, full)
+  end
+
+  def test_full_doc
+    doc = Hpricot(GOOGLE).scrub
     assert_tag_count(doc, 'a', 0)
-    assert_tag_count(doc, 'p', full.search('//p').size)
-    assert_tag_count(doc, 'div', full.search('//div').size)
-    assert_tag_count(doc, 'img', full.search('//img').size)
-    assert_tag_count(doc, 'br', full.search('//br').size)
+    assert_tag_count(doc, 'p', 0)
+    assert_tag_count(doc, 'img', 0)
+    assert_tag_count(doc, 'br', 0)
+    assert_tag_count(doc, 'div', 0)
     assert_tag_count(doc, 'script', 0)
   end
 
@@ -36,10 +51,34 @@ class HpricotScrubTest < Test::Unit::TestCase
     assert formatted.scrub == @clean
     assert formatted == MARKUP
   end
-  
+
   def test_string_scrub!
     formatted = MARKUP
     assert formatted.scrub! == @clean
     assert formatted == @clean
   end
+
+  def test_decoder
+    str = 'some <a href="http://example.com/">example&nbsp;link</a> to nowhere'
+    scrubbed_str = str.scrub
+    assert scrubbed_str.include?('&nbsp;')
+
+    if defined?(HTMLEntities)
+      assert ! scrubbed_str.decode.include?('&nbsp;')
+
+      scrubbed_str.decode!
+      assert ! scrubbed_str.include?('&nbsp;')
+    end
+  end
+
+private
+  def partial_scrub_common(doc, full)
+    # using the divisor search throws warnings in test
+    assert_tag_count(doc, 'a', 0)
+    assert_tag_count(doc, 'p', full.search('//p').size)
+    assert_tag_count(doc, 'div', full.search('//div').size)
+    assert_tag_count(doc, 'img', full.search('//img').size)
+    assert_tag_count(doc, 'br', full.search('//br').size)
+    assert_tag_count(doc, 'script', 0)
+  end
 end

data/test/scrubber_data.rb

@@ -12,3 +12,23 @@ alert("gotcha");</script><img src="http://content.example.com/content/3587a2f6ee
 <span>some random unclosed span
 <style type="text/css">.foo {color:blue}</style>
 EOS
+
+GOOGLE = <<-EOS
+<html><head><meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"><title>Google</title><style><!--
+body,td,a,p,.h{font-family:arial,sans-serif}
+.h{font-size:20px}
+.h{color:#3366cc}
+.q{color:#00c}
+--></style>
+<script>
+<!--
+function sf(){document.f.q.focus();}
+// -->
+</script>
+</head><body bgcolor=#ffffff text=#000000 link=#0000cc vlink=#551a8b alink=#ff0000 onload="sf();if(document.images){new Image().src='/images/nav_logo2.png'}" topmargin=3 marginheight=3><center><div align=right nowrap style="padding-bottom:4px" width=100%><font size=-1><a href="/url?sa=p&pref=ig&pval=3&q=http://www.google.com/ig%3Fhl%3Den&usg=__yvmOvIrk79QYmDkrJAeuYO8jTmo=">Personalize this page</a>&nbsp;|&nbsp;<a href="https://www.google.com/accounts/Login?continue=http://www.google.com/&hl=en">Sign in</a></font></div><img alt="Google" height=110 src="/intl/en_ALL/images/logo.gif" width=276><br><br><form action="/search" name=f><script defer><!--
+function togDisp(e){stopB(e);var elems=document.getElementsByName('more');for(var i=0;i<elems.length;i++){var obj=elems[i],dp="";if(obj.style.display==""){dp="none";}obj.style.display=dp;}return false;}
+function stopB(e){if(!e)e=window.event;e.cancelBubble=true;}
+document.onclick=function(event){var elems=document.getElementsByName('more');if(elems[0].style.display==""){togDisp(event);}}
+//-->
+</script><table border=0 cellspacing=0 cellpadding=4><tr><td nowrap><font size=-1><b>Web</b>&nbsp;&nbsp;&nbsp;&nbsp;<a class=q href="http://images.google.com/imghp?ie=ISO-8859-1&oe=ISO-8859-1&hl=en&tab=wi">Images</a>&nbsp;&nbsp;&nbsp;&nbsp;<a class=q href="http://video.google.com/?ie=ISO-8859-1&oe=ISO-8859-1&hl=en&tab=wv">Video</a>&nbsp;&nbsp;&nbsp;&nbsp;<a class=q href="http://news.google.com/nwshp?ie=ISO-8859-1&oe=ISO-8859-1&hl=en&tab=wn">News</a>&nbsp;&nbsp;&nbsp;&nbsp;<a class=q href="http://maps.google.com/maps?ie=ISO-8859-1&oe=ISO-8859-1&hl=en&tab=wl">Maps</a>&nbsp;&nbsp;&nbsp;&nbsp;<b><a href="/intl/en/options/" class=q onclick="this.blur();return togDisp(event)">more&nbsp;&raquo;</a></b><span name=more id=more style="display:none;position:absolute;background:#fff;border:1px solid #369;margin:-.5ex 2ex;padding:0 0 .5ex .8ex;width:16ex;line-height:1.9;z-index:1000" onclick="stopB(event)"><a href=# onclick="return togDisp(event)"><img border=0 src=/images/x2.gif width=12 height=12 alt="Close menu" align=right hspace=4 vspace=4></a><a class=q href="http://blogsearch.google.com/?ie=ISO-8859-1&oe=ISO-8859-1&hl=en&tab=wb">Blogs</a><br><a class=q href="http://books.google.com/bkshp?ie=ISO-8859-1&oe=ISO-8859-1&hl=en&tab=wp">Books</a><br><a class=q href="http://froogle.google.com/frghp?ie=ISO-8859-1&oe=ISO-8859-1&hl=en&tab=wf">Froogle</a><br><a class=q href="http://groups.google.com/grphp?ie=ISO-8859-1&oe=ISO-8859-1&hl=en&tab=wg">Groups</a><br><a class=q href="http://www.google.com/ptshp?ie=ISO-8859-1&oe=ISO-8859-1&hl=en&tab=wt">Patents</a><br><a href="/intl/en/options/" class=q><b>even more &raquo;</b></a></span></font></td></tr></table><table cellpadding=0 cellspacing=0><tr valign=top><td width=25%>&nbsp;</td><td align=center nowrap><input name=hl type=hidden value=en><input type=hidden name=ie value="ISO-8859-1"><input maxlength=2048 name=q size=55 title="Google Search" value=""><br><input name=btnG type=submit value="Google Search"><input name=btnI type=submit value="I'm Feeling Lucky"></td><td nowrap width=25%><font size=-2>&nbsp;&nbsp;<a href=/advanced_search?hl=en>Advanced Search</a><br>&nbsp;&nbsp;<a href=/preferences?hl=en>Preferences</a><br>&nbsp;&nbsp;<a href=/language_tools?hl=en>Language Tools</a></font></td></tr></table></form><br><br><font size=-1><a href="/intl/en/ads/">Advertising&nbsp;Programs</a> - <a href="/services/">Business Solutions</a> - <a href=/intl/en/about.html>About Google</a></font><p><font size=-2>&copy;2007 Google</font></p></center></body></html>
+EOS

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.1
+rubygems_version: 0.9.2.1
 specification_version: 1
 name: hpricot_scrub
 version: !ruby/object:Gem::Version 
-  version: 0.2.2
-date: 2007-03-13 00:00:00 -07:00
+  version: 0.2.3
+date: 2007-04-05 00:00:00 -07:00
 summary: Scrub HTML with Hpricot
 require_paths: 
 - lib