sanitize 2.1.1 → 6.0.0

data/lib/sanitize.rb

@@ -1,94 +1,87 @@
 # encoding: utf-8
-#--
-# Copyright (c) 2013 Ryan Grove <ryan@wonko.com>
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the 'Software'), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#++
 
+require 'nokogiri'
 require 'set'
 
-require 'nokogiri'
-require 'sanitize/version'
-require 'sanitize/config'
-require 'sanitize/config/restricted'
-require 'sanitize/config/basic'
-require 'sanitize/config/relaxed'
-require 'sanitize/transformers/clean_cdata'
-require 'sanitize/transformers/clean_comment'
-require 'sanitize/transformers/clean_element'
+require_relative 'sanitize/version'
+require_relative 'sanitize/config'
+require_relative 'sanitize/config/default'
+require_relative 'sanitize/config/restricted'
+require_relative 'sanitize/config/basic'
+require_relative 'sanitize/config/relaxed'
+require_relative 'sanitize/css'
+require_relative 'sanitize/transformers/clean_cdata'
+require_relative 'sanitize/transformers/clean_comment'
+require_relative 'sanitize/transformers/clean_css'
+require_relative 'sanitize/transformers/clean_doctype'
+require_relative 'sanitize/transformers/clean_element'
 
 class Sanitize
   attr_reader :config
 
-  # Matches a valid HTML5 data attribute name. The unicode ranges included here
-  # are a conservative subset of the full range of characters that are
-  # technically allowed, with the intent of matching the most common characters
-  # used in data attribute names while excluding uncommon or potentially
-  # misleading characters, or characters with the potential to be normalized
-  # into unsafe or confusing forms.
+  # Matches one or more control characters that should be removed from HTML
+  # before parsing, as defined by the HTML living standard.
   #
-  # If you need data attr names with characters that aren't included here (such
-  # as combining marks, full-width characters, or CJK), please consider creating
-  # a custom transformer to validate attributes according to your needs.
+  # -   https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream
+  # -   https://infra.spec.whatwg.org/#control
+  REGEX_HTML_CONTROL_CHARACTERS = /[\u0001-\u0008\u000b\u000e-\u001f\u007f-\u009f]+/u
+
+  # Matches one or more non-characters that should be removed from HTML before
+  # parsing, as defined by the HTML living standard.
   #
-  # http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#embedding-custom-non-visible-data-with-the-data-*-attributes
-  REGEX_DATA_ATTR = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u
+  # -   https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream
+  # -   https://infra.spec.whatwg.org/#noncharacter
+  REGEX_HTML_NON_CHARACTERS = /[\ufdd0-\ufdef\ufffe\uffff\u{1fffe}\u{1ffff}\u{2fffe}\u{2ffff}\u{3fffe}\u{3ffff}\u{4fffe}\u{4ffff}\u{5fffe}\u{5ffff}\u{6fffe}\u{6ffff}\u{7fffe}\u{7ffff}\u{8fffe}\u{8ffff}\u{9fffe}\u{9ffff}\u{afffe}\u{affff}\u{bfffe}\u{bffff}\u{cfffe}\u{cffff}\u{dfffe}\u{dffff}\u{efffe}\u{effff}\u{ffffe}\u{fffff}\u{10fffe}\u{10ffff}]+/u
 
   # Matches an attribute value that could be treated by a browser as a URL
   # with a protocol prefix, such as "http:" or "javascript:". Any string of zero
   # or more characters followed by a colon is considered a match, even if the
   # colon is encoded as an entity and even if it's an incomplete entity (which
   # IE6 and Opera will still parse).
-  REGEX_PROTOCOL = /\A([^\/#]*?)(?:\:|&#0*58|&#x0*3a)/i
+  REGEX_PROTOCOL = /\A\s*([^\/#]*?)(?:\:|&#0*58|&#x0*3a)/i
+
+  # Matches one or more characters that should be stripped from HTML before
+  # parsing. This is a combination of `REGEX_HTML_CONTROL_CHARACTERS` and
+  # `REGEX_HTML_NON_CHARACTERS`.
+  #
+  # https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream
+  REGEX_UNSUITABLE_CHARS = /(?:#{REGEX_HTML_CONTROL_CHARACTERS}|#{REGEX_HTML_NON_CHARACTERS})/u
 
   #--
   # Class Methods
   #++
 
-  # Returns a sanitized copy of _html_, using the settings in _config_ if
-  # specified.
-  def self.clean(html, config = {})
-    Sanitize.new(config).clean(html)
+  # Returns a sanitized copy of the given full _html_ document, using the
+  # settings in _config_ if specified.
+  #
+  # When sanitizing a document, the `<html>` element must be allowlisted or an
+  # error will be raised. If this is undesirable, you should probably use
+  # {#fragment} instead.
+  def self.document(html, config = {})
+    Sanitize.new(config).document(html)
   end
 
-  # Performs Sanitize#clean in place, returning _html_, or +nil+ if no changes
-  # were made.
-  def self.clean!(html, config = {})
-    Sanitize.new(config).clean!(html)
+  # Returns a sanitized copy of the given _html_ fragment, using the settings in
+  # _config_ if specified.
+  def self.fragment(html, config = {})
+    Sanitize.new(config).fragment(html)
   end
 
-  # Performs a Sanitize#clean using a full-document HTML parser instead of
-  # the default fragment parser. This will add a DOCTYPE and html tag
-  # unless they are already present
-  def self.clean_document(html, config = {})
-    Sanitize.new(config).clean_document(html)
+  # Sanitizes the given `Nokogiri::XML::Node` instance and all its children.
+  def self.node!(node, config = {})
+    Sanitize.new(config).node!(node)
   end
 
-  # Performs Sanitize#clean_document in place, returning _html_, or +nil+ if no
-  # changes were made.
-  def self.clean_document!(html, config = {})
-    Sanitize.new(config).clean_document!(html)
-  end
+  # Aliases for pre-3.0.0 backcompat.
+  class << Sanitize
+    # @deprecated Use {.document} instead.
+    alias_method :clean_document, :document
 
-  # Sanitizes the specified Nokogiri::XML::Node and all its children.
-  def self.clean_node!(node, config = {})
-    Sanitize.new(config).clean_node!(node)
+    # @deprecated Use {.fragment} instead.
+    alias_method :clean, :fragment
+
+    # @deprecated Use {.node!} instead.
+    alias_method :clean_node!, :node!
   end
 
   #--
@@ -97,118 +90,154 @@ class Sanitize
 
   # Returns a new Sanitize object initialized with the settings in _config_.
   def initialize(config = {})
-    @config = Config::DEFAULT.merge(config)
-
-    @transformers = {
-      :breadth => Array(@config[:transformers_breadth].dup),
-      :depth   => Array(@config[:transformers]) + Array(@config[:transformers_depth])
-    }
+    @config = Config.merge(Config::DEFAULT, config)
 
-    # Default depth transformers. These always run at the end of the chain,
-    # after any custom transformers.
-    @transformers[:depth] << Transformers::CleanComment unless @config[:allow_comments]
+    @transformers = Array(@config[:transformers]).dup
 
-    @transformers[:depth] <<
-        Transformers::CleanCDATA <<
-        Transformers::CleanElement.new(@config)
-  end
+    # Default transformers always run at the end of the chain, after any custom
+    # transformers.
+    @transformers << Transformers::CleanElement.new(@config)
+    @transformers << Transformers::CleanComment unless @config[:allow_comments]
 
-  # Returns a sanitized copy of the given _html_ fragment.
-  def clean(html)
-    if html
-      dupe = html.dup
-      clean!(dupe) || dupe
+    if @config[:elements].include?('style')
+      scss = Sanitize::CSS.new(config)
+      @transformers << Transformers::CSS::CleanElement.new(scss)
     end
-  end
 
-  # Performs clean in place, returning _html_, or +nil+ if no changes were
-  # made.
-  def clean!(html, parser = Nokogiri::HTML::DocumentFragment)
-    fragment = parser.parse(html)
-    clean_node!(fragment)
-
-    output_method_params = {:encoding => @config[:output_encoding], :indent => 0}
-
-    if @config[:output] == :xhtml
-      output_method = fragment.method(:to_xhtml)
-      output_method_params[:save_with] = Nokogiri::XML::Node::SaveOptions::AS_XHTML
-    elsif @config[:output] == :html
-      output_method = fragment.method(:to_html)
-    else
-      raise Error, "unsupported output format: #{@config[:output]}"
+    if @config[:attributes].values.any? {|attr| attr.include?('style') }
+      scss ||= Sanitize::CSS.new(config)
+      @transformers << Transformers::CSS::CleanAttribute.new(scss)
     end
 
-    result = output_method.call(output_method_params)
+    @transformers << Transformers::CleanDoctype
+    @transformers << Transformers::CleanCDATA
 
-    return result == html ? nil : html[0, html.length] = result
+    @transformer_config = { config: @config }
   end
 
-  # Returns a sanitized copy of the given full _html_ document.
-  def clean_document(html)
-    unless html.nil?
-      clean_document!(html.dup) || html
-    end
+  # Returns a sanitized copy of the given _html_ document.
+  #
+  # When sanitizing a document, the `<html>` element must be allowlisted or an
+  # error will be raised. If this is undesirable, you should probably use
+  # {#fragment} instead.
+  def document(html)
+    return '' unless html
+
+    doc = Nokogiri::HTML5.parse(preprocess(html), **@config[:parser_options])
+    node!(doc)
+    to_html(doc)
   end
 
-  # Performs clean_document in place, returning _html_, or +nil+ if no changes
-  # were made.
-  def clean_document!(html)
-    if !@config[:elements].include?('html') && !@config[:remove_contents]
-      raise 'You must have the HTML element whitelisted to call #clean_document unless remove_contents is set to true'
-      # otherwise Nokogiri will raise for having multiple root nodes when
-      # it moves its children to the root document context
-    end
+  # @deprecated Use {#document} instead.
+  alias_method :clean_document, :document
+
+  # Returns a sanitized copy of the given _html_ fragment.
+  def fragment(html)
+    return '' unless html
 
-    clean!(html, Nokogiri::HTML::Document)
+    frag = Nokogiri::HTML5.fragment(preprocess(html), **@config[:parser_options])
+    node!(frag)
+    to_html(frag)
   end
 
-  # Sanitizes the specified Nokogiri::XML::Node and all its children.
-  def clean_node!(node)
+  # @deprecated Use {#fragment} instead.
+  alias_method :clean, :fragment
+
+  # Sanitizes the given `Nokogiri::XML::Node` and all its children, modifying it
+  # in place.
+  #
+  # If _node_ is a `Nokogiri::XML::Document`, the `<html>` element must be
+  # allowlisted or an error will be raised.
+  def node!(node)
     raise ArgumentError unless node.is_a?(Nokogiri::XML::Node)
 
-    node_whitelist = Set.new
+    if node.is_a?(Nokogiri::XML::Document)
+      unless @config[:elements].include?('html')
+        raise Error, 'When sanitizing a document, "<html>" must be allowlisted.'
+      end
+    end
 
-    unless @transformers[:breadth].empty?
-      traverse_breadth(node) {|n| transform_node!(n, node_whitelist, :breadth) }
+    node_allowlist = Set.new
+
+    traverse(node) do |n|
+      transform_node!(n, node_allowlist)
     end
 
-    traverse_depth(node) {|n| transform_node!(n, node_whitelist, :depth) }
     node
   end
 
+  # @deprecated Use {#node!} instead.
+  alias_method :clean_node!, :node!
+
   private
 
-  def transform_node!(node, node_whitelist, mode)
-    @transformers[mode].each do |transformer|
-      result = transformer.call({
-        :config         => @config,
-        :is_whitelisted => node_whitelist.include?(node),
-        :node           => node,
-        :node_name      => node.name.downcase,
-        :node_whitelist => node_whitelist,
-        :traversal_mode => mode
-      })
-
-      if result.is_a?(Hash) && result[:node_whitelist].respond_to?(:each)
-        node_whitelist.merge(result[:node_whitelist])
+  # Preprocesses HTML before parsing to remove undesirable Unicode chars.
+  def preprocess(html)
+    html = html.to_s.dup
+
+    unless html.encoding.name == 'UTF-8'
+      html.encode!('UTF-8',
+        :invalid => :replace,
+        :undef   => :replace)
+    end
+
+    html.gsub!(REGEX_UNSUITABLE_CHARS, '')
+    html
+  end
+
+  def to_html(node)
+    node.to_html(preserve_newline: true)
+  end
+
+  def transform_node!(node, node_allowlist)
+    @transformers.each do |transformer|
+      # Since transform_node! may be called in a tight loop to process thousands
+      # of items, we can optimize both memory and CPU performance by:
+      #
+      # 1. Reusing the same config hash for each transformer
+      # 2. Directly assigning values to hash instead of using merge!. Not only
+      # does merge! create a new hash, it is also 2.6x slower:
+      # https://github.com/JuanitoFatas/fast-ruby#hashmerge-vs-hashmerge-code
+      config = @transformer_config
+      config[:is_allowlisted] = config[:is_whitelisted] = node_allowlist.include?(node)
+      config[:node] = node
+      config[:node_name] = node.name.downcase
+      config[:node_allowlist] = config[:node_whitelist] = node_allowlist
+
+      result = transformer.call(**config)
+
+      if result.is_a?(Hash)
+        result_allowlist = result[:node_allowlist] || result[:node_whitelist]
+
+        if result_allowlist.respond_to?(:each)
+          node_allowlist.merge(result_allowlist)
+        end
       end
     end
 
     node
   end
 
-  # Performs breadth-first traversal, operating first on the root node, then
-  # traversing downwards.
-  def traverse_breadth(node, &block)
-    block.call(node)
-    node.children.each {|child| traverse_breadth(child, &block) }
-  end
+  # Performs top-down traversal of the given node, operating first on the node
+  # itself, then traversing each child (if any) in order.
+  def traverse(node, &block)
+    yield node
 
-  # Performs depth-first traversal, operating first on the deepest nodes in the
-  # document, then traversing upwards to the root.
-  def traverse_depth(node, &block)
-    node.children.each {|child| traverse_depth(child, &block) }
-    block.call(node)
+    child = node.child
+
+    while child do
+      prev = child.previous_sibling
+      traverse(child, &block)
+
+      if child.parent == node
+        child = child.next_sibling
+      else
+        # The child was unlinked or reparented, so traverse the previous node's
+        # next sibling, or the parent's first child if there is no previous
+        # node.
+        child = prev ? prev.next_sibling : node.child
+      end
+    end
   end
 
   class Error < StandardError; end

data/test/common.rb

@@ -0,0 +1,3 @@
+# encoding: utf-8
+require 'minitest/autorun'
+require_relative '../lib/sanitize'

data/test/test_clean_comment.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require_relative 'common'
+
+describe 'Sanitize::Transformers::CleanComment' do
+  make_my_diffs_pretty!
+  parallelize_me!
+
+  describe 'when :allow_comments is false' do
+    before do
+      @s = Sanitize.new(:allow_comments => false, :elements => ['div'])
+    end
+
+    it 'should remove comments' do
+      @s.fragment('foo <!-- comment --> bar').must_equal 'foo  bar'
+      @s.fragment('foo <!-- ').must_equal 'foo '
+      @s.fragment('foo <!-- - -> bar').must_equal 'foo '
+      @s.fragment("foo <!--\n\n\n\n-->bar").must_equal 'foo bar'
+      @s.fragment("foo <!-- <!-- <!-- --> --> -->bar").must_equal 'foo  --&gt; --&gt;bar'
+      @s.fragment("foo <div <!-- comment -->>bar</div>").must_equal 'foo <div>&gt;bar</div>'
+
+      # Special case: the comment markup is inside a <script>, which makes it
+      # text content and not an actual HTML comment.
+      @s.fragment("<script><!-- comment --></script>").must_equal ''
+
+      Sanitize.fragment("<script><!-- comment --></script>", :allow_comments => false, :elements => ['script'])
+        .must_equal '<script><!-- comment --></script>'
+    end
+  end
+
+  describe 'when :allow_comments is true' do
+    before do
+      @s = Sanitize.new(:allow_comments => true, :elements => ['div'])
+    end
+
+    it 'should allow comments' do
+      @s.fragment('foo <!-- comment --> bar').must_equal 'foo <!-- comment --> bar'
+      @s.fragment('foo <!-- ').must_equal 'foo <!-- -->'
+      @s.fragment('foo <!-- - -> bar').must_equal 'foo <!-- - -> bar-->'
+      @s.fragment("foo <!--\n\n\n\n-->bar").must_equal "foo <!--\n\n\n\n-->bar"
+      @s.fragment("foo <!-- <!-- <!-- --> --> -->bar").must_equal 'foo <!-- <!-- <!-- --> --&gt; --&gt;bar'
+      @s.fragment("foo <div <!-- comment -->>bar</div>").must_equal 'foo <div>&gt;bar</div>'
+
+      Sanitize.fragment("<script><!-- comment --></script>", :allow_comments => true, :elements => ['script'])
+        .must_equal '<script><!-- comment --></script>'
+    end
+  end
+end

data/test/test_clean_css.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+require_relative 'common'
+
+describe 'Sanitize::Transformers::CSS::CleanAttribute' do
+  make_my_diffs_pretty!
+  parallelize_me!
+
+  before do
+    @s = Sanitize.new(Sanitize::Config::RELAXED)
+  end
+
+  it 'should sanitize CSS properties in style attributes' do
+    @s.fragment(%[
+      <div style="color: #fff; width: expression(alert(1)); /* <-- evil! */"></div>
+    ].strip).must_equal %[
+      <div style="color: #fff;  /* <-- evil! */"></div>
+    ].strip
+  end
+
+  it 'should remove the style attribute if the sanitized CSS is empty' do
+    @s.fragment('<div style="width: expression(alert(1))"></div>').
+      must_equal '<div></div>'
+  end
+end
+
+describe 'Sanitize::Transformers::CSS::CleanElement' do
+  make_my_diffs_pretty!
+  parallelize_me!
+
+  before do
+    @s = Sanitize.new(Sanitize::Config::RELAXED)
+  end
+
+  it 'should sanitize CSS stylesheets in <style> elements' do
+    html = %[
+      <style>@import url(evil.css);
+      /* Yay CSS! */
+      .foo { color: #fff; }
+      #bar { background: url(yay.jpg); bogus: wtf; }
+      .evil { width: expression(xss()); }
+
+      @media screen (max-width:480px) {
+        .foo { width: 400px; }
+        #bar:not(.baz) { height: 100px; }
+      }
+      </style>
+    ].strip
+
+    @s.fragment(html).must_equal %[
+      <style>
+      /* Yay CSS! */
+      .foo { color: #fff; }
+      #bar { background: url(yay.jpg);  }
+      .evil {  }
+
+      @media screen (max-width:480px) {
+        .foo { width: 400px; }
+        #bar:not(.baz) { height: 100px; }
+      }
+      </style>
+    ].strip
+  end
+
+  it 'should remove the <style> element if the sanitized CSS is empty' do
+    @s.fragment('<style></style>').must_equal ''
+  end
+end

data/test/test_clean_doctype.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+require_relative 'common'
+
+describe 'Sanitize::Transformers::CleanDoctype' do
+  make_my_diffs_pretty!
+  parallelize_me!
+
+  describe 'when :allow_doctype is false' do
+    before do
+      @s = Sanitize.new(:allow_doctype => false, :elements => ['html'])
+    end
+
+    it 'should remove doctype declarations' do
+      @s.document('<!DOCTYPE html><html>foo</html>').must_equal "<html>foo</html>"
+      @s.fragment('<!DOCTYPE html>foo').must_equal 'foo'
+    end
+
+    it 'should not allow doctype definitions in fragments' do
+      @s.fragment('<!DOCTYPE html><html>foo</html>')
+        .must_equal "foo"
+
+      @s.fragment('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"><html>foo</html>')
+        .must_equal "foo"
+
+      @s.fragment("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n    \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\"><html>foo</html>")
+        .must_equal "foo"
+    end
+  end
+
+  describe 'when :allow_doctype is true' do
+    before do
+      @s = Sanitize.new(:allow_doctype => true, :elements => ['html'])
+    end
+
+    it 'should allow doctype declarations in documents' do
+      @s.document('<!DOCTYPE html><html>foo</html>')
+        .must_equal "<!DOCTYPE html><html>foo</html>"
+
+      @s.document('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"><html>foo</html>')
+        .must_equal "<!DOCTYPE html><html>foo</html>"
+
+      @s.document("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n    \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\"><html>foo</html>")
+        .must_equal "<!DOCTYPE html><html>foo</html>"
+    end
+
+    it 'should not allow obviously invalid doctype declarations in documents' do
+      @s.document('<!DOCTYPE blah blah blah><html>foo</html>')
+        .must_equal "<!DOCTYPE html><html>foo</html>"
+
+      @s.document('<!DOCTYPE blah><html>foo</html>')
+        .must_equal "<!DOCTYPE html><html>foo</html>"
+
+      @s.document('<!DOCTYPE html BLAH "-//W3C//DTD HTML 4.01//EN"><html>foo</html>')
+        .must_equal "<!DOCTYPE html><html>foo</html>"
+
+      @s.document('<!whatever><html>foo</html>')
+        .must_equal "<html>foo</html>"
+    end
+
+    it 'should not allow doctype definitions in fragments' do
+      @s.fragment('<!DOCTYPE html><html>foo</html>')
+        .must_equal "foo"
+
+      @s.fragment('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"><html>foo</html>')
+        .must_equal "foo"
+
+      @s.fragment("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n    \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\"><html>foo</html>")
+        .must_equal "foo"
+    end
+  end
+end