loofah 1.0.0 → 1.1.0

data/CHANGELOG.rdoc

@@ -1,5 +1,14 @@
 = Changelog
 
+== 1.1.0 (2011-08-08)
+
+Enhancements:
+
+* Additional HTML5lib whitelist elements (from html5lib 1524:80b5efe26230).
+  Up to date with HTML5lib ruby code as of 1723:7ee6a0331856.
+* Whitelists (which are not part of the public API) are now Sets (were previously Arrays).
+* Don't explode when encountering UTF-8 URIs. (#25, #29)
+
 == 1.0.0 (2010-10-26)
 
 Notes:

data/Gemfile

@@ -1 +1,19 @@
-gemspec
+# -*- ruby -*-
+
+# DO NOT EDIT THIS FILE. Instead, edit Rakefile, and run `rake bundler:gemfile`.
+
+source :gemcutter
+
+gem "nokogiri", ">=1.4.4"
+
+gem "rake", ">=0.8", :group => [:development, :test]
+gem "minitest", "~>2.2", :group => [:development, :test]
+gem "rr", "~>1.0", :group => [:development, :test]
+gem "json", ">=0", :group => [:development, :test]
+gem "hoe-gemspec", ">=0", :group => [:development, :test]
+gem "hoe-debugging", ">=0", :group => [:development, :test]
+gem "hoe-bundler", ">=0", :group => [:development, :test]
+gem "hoe-git", ">=0", :group => [:development, :test]
+gem "hoe", ">=2.9.4", :group => [:development, :test]
+
+# vim: syntax=ruby

data/Manifest.txt

@@ -21,6 +21,7 @@ lib/loofah/scrubber.rb
 lib/loofah/scrubbers.rb
 lib/loofah/xml/document.rb
 lib/loofah/xml/document_fragment.rb
+test/assets/testdata_sanitizer_tests1.dat
 test/helper.rb
 test/html5/test_sanitizer.rb
 test/integration/test_ad_hoc.rb
@@ -29,6 +30,7 @@ test/integration/test_html.rb
 test/integration/test_scrubbers.rb
 test/integration/test_xml.rb
 test/unit/test_api.rb
+test/unit/test_encoding.rb
 test/unit/test_helpers.rb
 test/unit/test_scrubber.rb
 test/unit/test_scrubbers.rb

data/README.rdoc

@@ -291,7 +291,7 @@ name that nobody could spell properly.
 
 The MIT License
 
-Copyright (c) 2009, 2010 by Mike Dalessio, Bryan Helmkamp
+Copyright (c) 2009, 2010, 2011 by Mike Dalessio, Bryan Helmkamp
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/Rakefile

@@ -4,6 +4,8 @@ require 'hoe'
 
 Hoe.plugin :git
 Hoe.plugin :gemspec
+Hoe.plugin :bundler
+Hoe.plugin :debugging
 
 Hoe.spec "loofah" do
   developer "Mike Dalessio", "mike.dalessio@gmail.com"
@@ -13,10 +15,16 @@ Hoe.spec "loofah" do
   self.history_file     = "CHANGELOG.rdoc"
   self.readme_file      = "README.rdoc"
 
-  extra_deps     << ["nokogiri", ">=1.3.3"]
-  extra_dev_deps << ["mocha", ">=0.9"]
-  extra_dev_deps << ["shoulda", ">=2.10"]
+  extra_deps     << ["nokogiri", ">=1.4.4"]
+
   extra_dev_deps << ["rake", ">=0.8"]
+  extra_dev_deps << ["minitest", "~>2.2"]
+  extra_dev_deps << ["rr", "~>1.0"]
+  extra_dev_deps << ["json", ">=0"]
+  extra_dev_deps << ["hoe-gemspec", ">=0"]
+  extra_dev_deps << ["hoe-debugging", ">=0"]
+  extra_dev_deps << ["hoe-bundler", ">=0"]
+  extra_dev_deps << ["hoe-git", ">=0"]
 end
 
 task :gemspec do

data/lib/loofah.rb

@@ -29,10 +29,7 @@ require 'loofah/helpers'
 #
 module Loofah
   # The version of Loofah you are using
-  VERSION = '1.0.0'
-
-  # The minimum required version of Nokogiri
-  REQUIRED_NOKOGIRI_VERSION = '1.3.3'
+  VERSION = '1.1.0'
 
   class << self
     # Shortcut for Loofah::HTML::Document.parse
@@ -81,7 +78,3 @@ module Loofah
 
   end
 end
-
-if Nokogiri::VERSION < Loofah::REQUIRED_NOKOGIRI_VERSION
-  raise RuntimeError, "Loofah requires Nokogiri #{Loofah::REQUIRED_NOKOGIRI_VERSION} or later (currently #{Nokogiri::VERSION})"
-end

data/lib/loofah/elements.rb

@@ -1,19 +1,17 @@
+require 'set'
+
 module Loofah
   module Elements
     # Block elements in HTML4
-    STRICT_BLOCK_LEVEL = %w[address blockquote center dir div dl
+    STRICT_BLOCK_LEVEL = Set.new %w[address blockquote center dir div dl
       fieldset form h1 h2 h3 h4 h5 h6 hr isindex menu noframes
       noscript ol p pre table ul]
 
     # The following elements may also be considered block-level elements since they may contain block-level elements
-    LOOSE_BLOCK_LEVEL = %w[dd dt frameset li tbody td tfoot th thead tr]
+    LOOSE_BLOCK_LEVEL = Set.new %w[dd dt frameset li tbody td tfoot th thead tr]
 
     BLOCK_LEVEL = STRICT_BLOCK_LEVEL + LOOSE_BLOCK_LEVEL
   end
 
-  module HashedElements
-    include Loofah::MetaHelpers::HashifiedConstants(Elements)
-  end
+  ::Loofah::MetaHelpers.add_downcased_set_members_to_all_set_constants ::Loofah::Elements
 end
-
-

data/lib/loofah/html/document_fragment.rb

@@ -15,7 +15,9 @@ module Loofah
         #  parse a fragment.
         #
         def parse tags
-          self.new(Loofah::HTML::Document.new, tags)
+          doc = Loofah::HTML::Document.new
+          doc.encoding = tags.encoding.name if tags.respond_to?(:encoding)
+          self.new(doc, tags)
         end
       end
 

data/lib/loofah/html5/scrub.rb

@@ -4,38 +4,53 @@ module Loofah
   module HTML5 # :nodoc:
     module Scrub
 
+      CONTROL_CHARACTERS = if RUBY_VERSION =~ /^1\.8/
+                             /`|[\000-\040\177\s]+|\302[\200-\240]/
+                           else
+                             /[`\u0000-\u0020\u007F\s\u0080-\u0101]/
+                           end
+
       class << self
 
+        def allowed_element? element_name
+          ::Loofah::HTML5::WhiteList::ALLOWED_ELEMENTS_WITH_LIBXML2.include? element_name
+        end
+
         #  alternative implementation of the html5lib attribute scrubbing algorithm
-        def scrub_attributes(node)
+        def scrub_attributes node
           node.attribute_nodes.each do |attr_node|
             attr_name = if attr_node.namespace
                           "#{attr_node.namespace.prefix}:#{attr_node.node_name}"
                         else
                           attr_node.node_name
                         end
-            attr_node.remove unless HashedWhiteList::ALLOWED_ATTRIBUTES[attr_name]
-            if HashedWhiteList::ATTR_VAL_IS_URI[attr_name]
+            unless WhiteList::ALLOWED_ATTRIBUTES.include?(attr_name)
+              attr_node.remove
+              next
+            end
+            if WhiteList::ATTR_VAL_IS_URI.include?(attr_name)
               # this block lifted nearly verbatim from HTML5 sanitization
-              val_unescaped = CGI.unescapeHTML(attr_node.value).gsub(/`|[\000-\040\177\s]+|\302[\200-\240]/,'').downcase
-              if val_unescaped =~ /^[a-z0-9][-+.a-z0-9]*:/ and HashedWhiteList::ALLOWED_PROTOCOLS[val_unescaped.split(':')[0]].nil?
+              val_unescaped = CGI.unescapeHTML(attr_node.value).gsub(CONTROL_CHARACTERS,'').downcase
+              if val_unescaped =~ /^[a-z0-9][-+.a-z0-9]*:/ && ! WhiteList::ALLOWED_PROTOCOLS.include?(val_unescaped.split(':')[0])
                 attr_node.remove
+                next
               end
             end
-            if HashedWhiteList::SVG_ATTR_VAL_ALLOWS_REF[attr_name]
+            if WhiteList::SVG_ATTR_VAL_ALLOWS_REF.include?(attr_name)
               attr_node.value = attr_node.value.gsub(/url\s*\(\s*[^#\s][^)]+?\)/m, ' ') if attr_node.value
             end
-            if HashedWhiteList::SVG_ALLOW_LOCAL_HREF[node.name] && attr_name == 'xlink:href' && attr_node.value =~ /^\s*[^#\s].*/m
+            if WhiteList::SVG_ALLOW_LOCAL_HREF.include?(node.name) && attr_name == 'xlink:href' && attr_node.value =~ /^\s*[^#\s].*/m
               attr_node.remove
+              next
             end
           end
           if node.attributes['style']
-            node['style'] = scrub_css(node.attributes['style'])
+            node['style'] = scrub_css node.attributes['style']
           end
         end
 
         #  lifted nearly verbatim from html5lib
-        def scrub_css(style)
+        def scrub_css style
           # disallow urls
           style = style.to_s.gsub(/url\s*\(\s*[^\s)]+?\s*\)\s*/, ' ')
 
@@ -47,14 +62,14 @@ module Loofah
           style.scan(/([-\w]+)\s*:\s*([^:;]*)/) do |prop, val|
             next if val.empty?
             prop.downcase!
-            if HashedWhiteList::ALLOWED_CSS_PROPERTIES[prop]
+            if WhiteList::ALLOWED_CSS_PROPERTIES.include?(prop)
               clean << "#{prop}: #{val};"
             elsif %w[background border margin padding].include?(prop.split('-')[0])
               clean << "#{prop}: #{val};" unless val.split().any? do |keyword|
-                HashedWhiteList::ALLOWED_CSS_KEYWORDS[keyword].nil? and
+                WhiteList::ALLOWED_CSS_KEYWORDS.include?(keyword) &&
                   keyword !~ /^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$/
               end
-            elsif HashedWhiteList::ALLOWED_SVG_PROPERTIES[prop]
+            elsif WhiteList::ALLOWED_SVG_PROPERTIES.include?(prop)
               clean << "#{prop}: #{val};"
             end
           end
@@ -67,4 +82,3 @@ module Loofah
     end
   end
 end
-

data/lib/loofah/html5/whitelist.rb

@@ -1,3 +1,5 @@
+require 'set'
+
 module Loofah
   module HTML5 # :nodoc:
     #
@@ -43,53 +45,59 @@ module Loofah
     #
     # </html5_license>
     module WhiteList
-      ACCEPTABLE_ELEMENTS = %w[a abbr acronym address area b big blockquote br
+      ACCEPTABLE_ELEMENTS = Set.new %w[a abbr acronym address area audio b big blockquote br
       button caption center cite code col colgroup dd del dfn dir div dl dt
       em fieldset font form h1 h2 h3 h4 h5 h6 hr i img input ins kbd label
       legend li map menu ol optgroup option p pre q s samp select small span
       strike strong sub sup table tbody td textarea tfoot th thead tr tt u
-      ul var]
+      ul var video]
 
-      MATHML_ELEMENTS = %w[annotation annotation-xml maction math merror mfrac
+      MATHML_ELEMENTS = Set.new %w[annotation annotation-xml maction math merror mfrac
       mfenced mi mmultiscripts mn mo mover mpadded mphantom mprescripts mroot mrow
       mspace msqrt mstyle msub msubsup msup mtable mtd mtext mtr munder
       munderover none semantics]
 
-      SVG_ELEMENTS = %w[a animate animateColor animateMotion animateTransform
-      circle defs desc ellipse font-face font-face-name font-face-src foreignObject
-      g glyph hkern linearGradient line marker metadata missing-glyph
+      SVG_ELEMENTS = Set.new %w[a animate animateColor animateMotion animateTransform
+      circle clipPath defs desc ellipse feGaussianBlur filter font-face
+      font-face-name font-face-src foreignObject
+      g glyph hkern linearGradient line marker mask metadata missing-glyph
       mpath path polygon polyline radialGradient rect set stop svg switch
-      text title tspan use]
+      text textPath title tspan use]
 
-      ACCEPTABLE_ATTRIBUTES = %w[abbr accept accept-charset accesskey action
+      ACCEPTABLE_ATTRIBUTES = Set.new %w[abbr accept accept-charset accesskey action
       align alt axis border cellpadding cellspacing char charoff charset
       checked cite class clear cols colspan color compact coords datetime
       dir disabled enctype for frame headers height href hreflang hspace id
-      ismap label lang longdesc maxlength media method multiple name nohref
-      noshade nowrap prompt readonly rel rev rows rowspan rules scope
+      ismap label lang longdesc loop loopcount loopend loopstart
+      maxlength media method multiple name nohref
+      noshade nowrap poster prompt readonly rel rev rows rowspan rules scope
       selected shape size span src start style summary tabindex target title
       type usemap valign value vspace width xml:lang]
 
-      MATHML_ATTRIBUTES = %w[actiontype align close columnalign columnalign
+      MATHML_ATTRIBUTES = Set.new %w[actiontype align close
       columnalign columnlines columnspacing columnspan depth display
       displaystyle encoding equalcolumns equalrows fence fontstyle fontweight
       frame height linethickness lspace mathbackground mathcolor mathvariant
-      mathvariant maxsize minsize open other rowalign rowalign rowalign rowlines
+      maxsize minsize open other rowalign rowlines
       rowspacing rowspan rspace scriptlevel selection separator separators
-      stretchy width width xlink:href xlink:show xlink:type xmlns xmlns:xlink]
+      stretchy width xlink:href xlink:show xlink:type xmlns xmlns:xlink]
 
-      SVG_ATTRIBUTES = %w[accent-height accumulate additive alphabetic
+      SVG_ATTRIBUTES = Set.new %w[accent-height accumulate additive alphabetic
        arabic-form ascent attributeName attributeType baseProfile bbox begin
-       by calcMode cap-height class color color-rendering content cx cy d dx
-       dy descent display dur end fill fill-opacity fill-rule font-family
+       by calcMode cap-height class clip-path clip-rule color
+       color-interpolation-filters color-rendering content cx cy d dx
+       dy descent display dur end fill fill-opacity fill-rule
+       filterRes filterUnits font-family
        font-size font-stretch font-style font-variant font-weight from fx fy g1
        g2 glyph-name gradientUnits hanging height horiz-adv-x horiz-origin-x id
        ideographic k keyPoints keySplines keyTimes lang marker-end
        marker-mid marker-start markerHeight markerUnits markerWidth
-       mathematical max min name offset opacity orient origin
-       overline-position overline-thickness panose-1 path pathLength points
-       preserveAspectRatio r refX refY repeatCount repeatDur
-       requiredExtensions requiredFeatures restart rotate rx ry slope stemh
+       maskContentUnits maskUnits mathematical max method min name offset opacity orient origin
+       overline-position overline-thickness panose-1 path pathLength
+       patternContentUnits patternTransform patternUnits  points
+       preserveAspectRatio primitiveUnits r refX refY repeatCount repeatDur
+       requiredExtensions requiredFeatures restart rotate rx ry slope spacing
+       startOffset stdDeviation stemh
        stemv stop-color stop-opacity strikethrough-position
        strikethrough-thickness stroke stroke-dasharray stroke-dashoffset
        stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity
@@ -100,16 +108,16 @@ module Loofah
        xlink:show xlink:title xlink:type xml:base xml:lang xml:space xmlns
        xmlns:xlink y y1 y2 zoomAndPan]
 
-      ATTR_VAL_IS_URI = %w[href src cite action longdesc xlink:href xml:base]
+      ATTR_VAL_IS_URI = Set.new %w[href src cite action longdesc xlink:href xml:base]
 
-      SVG_ATTR_VAL_ALLOWS_REF = %w[clip-path color-profile cursor fill
+      SVG_ATTR_VAL_ALLOWS_REF = Set.new %w[clip-path color-profile cursor fill
       filter marker marker-start marker-mid marker-end mask stroke]
 
-      SVG_ALLOW_LOCAL_HREF = %w[altGlyph animate animateColor animateMotion
+      SVG_ALLOW_LOCAL_HREF = Set.new %w[altGlyph animate animateColor animateMotion
       animateTransform cursor feImage filter linearGradient pattern
       radialGradient textpath tref set use]
 
-      ACCEPTABLE_CSS_PROPERTIES = %w[azimuth background-color
+      ACCEPTABLE_CSS_PROPERTIES = Set.new %w[azimuth background-color
       border-bottom-color border-collapse border-color border-left-color
       border-right-color border-top-color clear color cursor direction
       display elevation float font font-family font-size font-style
@@ -119,16 +127,16 @@ module Loofah
       text-align text-decoration text-indent unicode-bidi vertical-align
       voice-family volume white-space width]
 
-      ACCEPTABLE_CSS_KEYWORDS = %w[auto aqua black block blue bold both bottom
+      ACCEPTABLE_CSS_KEYWORDS = Set.new %w[auto aqua black block blue bold both bottom
       brown center collapse dashed dotted fuchsia gray green !important
       italic left lime maroon medium none navy normal nowrap olive pointer
       purple red right solid silver teal top transparent underline white
       yellow]
 
-      ACCEPTABLE_SVG_PROPERTIES = %w[fill fill-opacity fill-rule stroke
+      ACCEPTABLE_SVG_PROPERTIES = Set.new %w[fill fill-opacity fill-rule stroke
       stroke-width stroke-linecap stroke-linejoin stroke-opacity]
 
-      ACCEPTABLE_PROTOCOLS = %w[ed2k ftp http https irc mailto news gopher nntp
+      ACCEPTABLE_PROTOCOLS = Set.new %w[ed2k ftp http https irc mailto news gopher nntp
       telnet webcal xmpp callto feed urn aim rsync tag ssh sftp rtsp afs]
 
       # subclasses may define their own versions of these constants
@@ -139,7 +147,7 @@ module Loofah
       ALLOWED_SVG_PROPERTIES = ACCEPTABLE_SVG_PROPERTIES
       ALLOWED_PROTOCOLS = ACCEPTABLE_PROTOCOLS
 
-      VOID_ELEMENTS = %w[
+      VOID_ELEMENTS = Set.new %w[
         base
         link
         meta
@@ -154,15 +162,10 @@ module Loofah
       ]
 
       # additional tags we should consider safe since we have libxml2 fixing up our documents.
-      TAGS_SAFE_WITH_LIBXML2 = %w[html head body]
+      TAGS_SAFE_WITH_LIBXML2 = Set.new %w[html head body]
       ALLOWED_ELEMENTS_WITH_LIBXML2 = ALLOWED_ELEMENTS + TAGS_SAFE_WITH_LIBXML2
     end      
 
-    #
-    #  The HTML5lib whitelist arrays, transformed into hashes for faster lookup.
-    #
-    module HashedWhiteList
-      include Loofah::MetaHelpers::HashifiedConstants(WhiteList)
-    end
+    ::Loofah::MetaHelpers.add_downcased_set_members_to_all_set_constants ::Loofah::HTML5::WhiteList
   end
 end

data/lib/loofah/instance_methods.rb

@@ -41,7 +41,7 @@ module Loofah
         when Nokogiri::XML::Document
           scrubber.traverse(root) if root
         when Nokogiri::XML::DocumentFragment
-          children.each { |node| node.scrub!(scrubber) } # TODO: children.scrub! once Nokogiri 1.4.2 is out
+          children.scrub! scrubber
         else
           scrubber.traverse(self)
         end

data/lib/loofah/metahelpers.rb

@@ -1,15 +1,13 @@
 module Loofah
   module MetaHelpers
-    def self.HashifiedConstants(orig_module)
-      hashed_module = Module.new
-      orig_module.constants.each do |constant|
-        next unless orig_module.module_eval("#{constant}").is_a?(Array)
-        hashed_module.module_eval <<-CODE
-          #{constant} = {}
-          #{orig_module.name}::#{constant}.each { |c| #{constant}[c] = true ; #{constant}[c.downcase] = true }
-        CODE
+    def self.add_downcased_set_members_to_all_set_constants mojule
+      mojule.constants.each do |constant_sym|
+        constant = mojule.const_get constant_sym
+        next unless Set === constant
+        constant.dup.each do |member|
+          constant.add member.downcase
+        end
       end
-      hashed_module
     end
   end
 end

data/lib/loofah/scrubber.rb

@@ -91,7 +91,7 @@ module Loofah
     def html5lib_sanitize(node)
       case node.type
       when Nokogiri::XML::Node::ELEMENT_NODE
-        if HTML5::HashedWhiteList::ALLOWED_ELEMENTS_WITH_LIBXML2[node.name]
+        if HTML5::Scrub.allowed_element? node.name
           HTML5::Scrub.scrub_attributes node
           return Scrubber::CONTINUE
         end

data/lib/loofah/scrubbers.rb

@@ -75,7 +75,7 @@ module Loofah
 
       def scrub(node)
         return CONTINUE if html5lib_sanitize(node) == CONTINUE
-        node.before node.inner_html
+        node.before node.children
         node.remove
       end
     end
@@ -117,8 +117,7 @@ module Loofah
 
       def scrub(node)
         return CONTINUE if html5lib_sanitize(node) == CONTINUE
-        replacement_killer = Nokogiri::XML::Text.new(node.to_s, node.document)
-        node.add_next_sibling replacement_killer
+        node.add_next_sibling Nokogiri::XML::Text.new(node.to_s, node.document)
         node.remove
         return STOP
       end
@@ -150,7 +149,7 @@ module Loofah
       def scrub(node)
         case node.type
         when Nokogiri::XML::Node::ELEMENT_NODE
-          if HTML5::HashedWhiteList::ALLOWED_ELEMENTS_WITH_LIBXML2[node.name]
+          if HTML5::Scrub.allowed_element? node.name
             node.attributes.each { |attr| node.remove_attribute(attr.first) }
             return CONTINUE if node.namespaces.empty?
           end
@@ -190,9 +189,8 @@ module Loofah
       end
 
       def scrub(node)
-        return CONTINUE unless Loofah::HashedElements::BLOCK_LEVEL[node.name]
-        replacement_killer = Nokogiri::XML::Text.new("\n#{node.content}\n", node.document)
-        node.add_next_sibling replacement_killer
+        return CONTINUE unless Loofah::Elements::BLOCK_LEVEL.include?(node.name)
+        node.add_next_sibling Nokogiri::XML::Text.new("\n#{node.content}\n", node.document)
         node.remove
       end
     end