rails-deprecated_sanitizer 1.0.0

data/lib/rails/deprecated_sanitizer/html-scanner/html/tokenizer.rb

@@ -0,0 +1,107 @@
+require 'strscan'
+
+module HTML #:nodoc:
+
+  # A simple HTML tokenizer. It simply breaks a stream of text into tokens, where each
+  # token is a string. Each string represents either "text", or an HTML element.
+  #
+  # This currently assumes valid XHTML, which means no free < or > characters.
+  #
+  # Usage:
+  #
+  #   tokenizer = HTML::Tokenizer.new(text)
+  #   while token = tokenizer.next
+  #     p token
+  #   end
+  class Tokenizer #:nodoc:
+
+    # The current (byte) position in the text
+    attr_reader :position
+
+    # The current line number
+    attr_reader :line
+
+    # Create a new Tokenizer for the given text.
+    def initialize(text)
+      text.encode!
+      @scanner = StringScanner.new(text)
+      @position = 0
+      @line = 0
+      @current_line = 1
+    end
+
+    # Returns the next token in the sequence, or +nil+ if there are no more tokens in
+    # the stream.
+    def next
+      return nil if @scanner.eos?
+      @position = @scanner.pos
+      @line = @current_line
+      if @scanner.check(/<\S/)
+        update_current_line(scan_tag)
+      else
+        update_current_line(scan_text)
+      end
+    end
+
+    private
+
+      # Treat the text at the current position as a tag, and scan it. Supports
+      # comments, doctype tags, and regular tags, and ignores less-than and
+      # greater-than characters within quoted strings.
+      def scan_tag
+        tag = @scanner.getch
+        if @scanner.scan(/!--/) # comment
+          tag << @scanner.matched
+          tag << (@scanner.scan_until(/--\s*>/) || @scanner.scan_until(/\Z/))
+        elsif @scanner.scan(/!\[CDATA\[/)
+          tag << @scanner.matched
+          tag << (@scanner.scan_until(/\]\]>/) || @scanner.scan_until(/\Z/))
+        elsif @scanner.scan(/!/) # doctype
+          tag << @scanner.matched
+          tag << consume_quoted_regions
+        else
+          tag << consume_quoted_regions
+        end
+        tag
+      end
+
+      # Scan all text up to the next < character and return it.
+      def scan_text
+        "#{@scanner.getch}#{@scanner.scan(/[^<]*/)}"
+      end
+
+      # Counts the number of newlines in the text and updates the current line
+      # accordingly.
+      def update_current_line(text)
+        text.scan(/\r?\n/) { @current_line += 1 }
+      end
+
+      # Skips over quoted strings, so that less-than and greater-than characters
+      # within the strings are ignored.
+      def consume_quoted_regions
+        text = ""
+        loop do
+          match = @scanner.scan_until(/['"<>]/) or break
+
+          delim = @scanner.matched
+          if delim == "<"
+            match = match.chop
+            @scanner.pos -= 1
+          end
+
+          text << match
+          break if delim == "<" || delim == ">"
+
+          # consume the quoted region
+          while match = @scanner.scan_until(/[\\#{delim}]/)
+            text << match
+            break if @scanner.matched == delim
+            break if @scanner.eos?
+            text << @scanner.getch # skip the escaped character
+          end
+        end
+        text
+      end
+  end
+
+end

data/lib/rails/deprecated_sanitizer/html-scanner/html/version.rb

@@ -0,0 +1,11 @@
+module HTML #:nodoc:
+  module Version #:nodoc:
+
+    MAJOR = 0
+    MINOR = 5
+    TINY  = 3
+
+    STRING = [ MAJOR, MINOR, TINY ].join(".")
+
+  end
+end

data/lib/rails/deprecated_sanitizer/version.rb

@@ -0,0 +1,5 @@
+module Rails
+  module DeprecatedSanitizer
+    VERSION = "1.0.0"
+  end
+end

data/test/cdata_node_test.rb

@@ -0,0 +1,16 @@
+require 'test_helper'
+require 'rails/deprecated_sanitizer/html-scanner/html/node'
+
+class CDATANodeTest < ActiveSupport::TestCase
+  def setup
+    @node = HTML::CDATA.new(nil, 0, 0, "<p>howdy</p>")
+  end
+
+  def test_to_s
+    assert_equal "<![CDATA[<p>howdy</p>]]>", @node.to_s
+  end
+
+  def test_content
+    assert_equal "<p>howdy</p>", @node.content
+  end
+end

data/test/deprecated_sanitizer_test.rb

@@ -0,0 +1,30 @@
+require 'test_helper'
+require 'action_view'
+require 'action_view/helpers/sanitize_helper'
+
+class DeprecatedSanitizerTest < ActiveSupport::TestCase
+  def sanitize_helper
+    ActionView::Helpers::SanitizeHelper
+  end
+
+  test 'Action View sanitizer vendor is set to deprecated sanitizer' do
+    assert_equal Rails::DeprecatedSanitizer, sanitize_helper.sanitizer_vendor
+  end
+
+  test 'Action View sanitizer vendor returns constant from HTML module' do
+    assert_equal HTML::LinkSanitizer, sanitize_helper.sanitizer_vendor.link_sanitizer
+  end
+
+  test 'setting allowed tags modifies HTML::WhiteListSanitizers allowed tags' do
+    sanitize_helper.sanitized_allowed_tags = %w(horse)
+    assert_includes HTML::WhiteListSanitizer.allowed_tags, 'horse'
+  end
+
+  test 'setting allowed attributes modifies HTML::WhiteListSanitizers allowed attributes' do
+    attrs = %w(for your health)
+    sanitize_helper.sanitized_allowed_attributes = attrs
+    attrs.each do |attr|
+      assert_includes HTML::WhiteListSanitizer.allowed_attributes, attr
+    end
+  end
+end

data/test/document_test.rb

@@ -0,0 +1,149 @@
+require 'test_helper'
+require 'rails/deprecated_sanitizer/html-scanner'
+
+class DocumentTest < ActiveSupport::TestCase
+  def test_handle_doctype
+    doc = nil
+    assert_nothing_raised do
+      doc = HTML::Document.new <<-HTML.strip
+        <!DOCTYPE "blah" "blah" "blah">
+        <html>
+        </html>
+      HTML
+    end
+    assert_equal 3, doc.root.children.length
+    assert_equal %{<!DOCTYPE "blah" "blah" "blah">}, doc.root.children[0].content
+    assert_match %r{\s+}m, doc.root.children[1].content
+    assert_equal "html", doc.root.children[2].name
+  end
+
+  def test_find_img
+    doc = HTML::Document.new <<-HTML.strip
+      <html>
+        <body>
+          <p><img src="hello.gif"></p>
+        </body>
+      </html>
+    HTML
+    assert doc.find(:tag=>"img", :attributes=>{"src"=>"hello.gif"})
+  end
+
+  def test_find_all
+    doc = HTML::Document.new <<-HTML.strip
+      <html>
+        <body>
+          <p class="test"><img src="hello.gif"></p>
+          <div class="foo">
+            <p class="test">something</p>
+            <p>here is <em class="test">more</em></p>
+          </div>
+        </body>
+      </html>
+    HTML
+    all = doc.find_all :attributes => { :class => "test" }
+    assert_equal 3, all.length
+    assert_equal [ "p", "p", "em" ], all.map { |n| n.name }
+  end
+
+  def test_find_with_text
+    doc = HTML::Document.new <<-HTML.strip
+      <html>
+        <body>
+          <p>Some text</p>
+        </body>
+      </html>
+    HTML
+    assert doc.find(:content => "Some text")
+    assert doc.find(:tag => "p", :child => { :content => "Some text" })
+    assert doc.find(:tag => "p", :child => "Some text")
+    assert doc.find(:tag => "p", :content => "Some text")
+  end
+
+  def test_parse_xml
+    assert_nothing_raised { HTML::Document.new("<tags><tag/></tags>", true, true) }
+    assert_nothing_raised { HTML::Document.new("<outer><link>something</link></outer>", true, true) }
+  end
+
+  def test_parse_document
+    doc = HTML::Document.new(<<-HTML)
+      <div>
+        <h2>blah</h2>
+        <table>
+        </table>
+      </div>
+    HTML
+    assert_not_nil doc.find(:tag => "div", :children => { :count => 1, :only => { :tag => "table" } })
+  end
+
+  def test_tag_nesting_nothing_to_s
+    doc = HTML::Document.new("<tag></tag>")
+    assert_equal "<tag></tag>", doc.root.to_s
+  end
+
+  def test_tag_nesting_space_to_s
+    doc = HTML::Document.new("<tag> </tag>")
+    assert_equal "<tag> </tag>", doc.root.to_s
+  end
+
+  def test_tag_nesting_text_to_s
+    doc = HTML::Document.new("<tag>text</tag>")
+    assert_equal "<tag>text</tag>", doc.root.to_s
+  end
+
+  def test_tag_nesting_tag_to_s
+    doc = HTML::Document.new("<tag><nested /></tag>")
+    assert_equal "<tag><nested /></tag>", doc.root.to_s
+  end
+
+  def test_parse_cdata
+    doc = HTML::Document.new(<<-HTML)
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+  <head>
+    <title><![CDATA[<br>]]></title>
+   </head>
+  <body>
+    <p>this document has &lt;br&gt; for a title</p>
+  </body>
+</html>
+HTML
+
+    assert_nil doc.find(:tag => "title", :descendant => { :tag => "br" })
+    assert doc.find(:tag => "title", :child => "<br>")
+  end
+
+  def test_find_empty_tag
+    doc = HTML::Document.new("<div id='map'></div>")
+    assert_nil doc.find(:tag => "div", :attributes => { :id => "map" }, :content => /./)
+    assert doc.find(:tag => "div", :attributes => { :id => "map" }, :content => /\A\Z/)
+    assert doc.find(:tag => "div", :attributes => { :id => "map" }, :content => /^$/)
+    assert doc.find(:tag => "div", :attributes => { :id => "map" }, :content => "")
+    assert doc.find(:tag => "div", :attributes => { :id => "map" }, :content => nil)
+  end
+
+  def test_parse_invalid_document
+    assert_nothing_raised do
+      HTML::Document.new("<html>
+        <table>
+          <tr>
+            <td style=\"color: #FFFFFF; height: 17px; onclick=\"window.location.href='http://www.rmeinc.com/about_rme.aspx'\" style=\"cursor:pointer; height: 17px;\"; nowrap onclick=\"window.location.href='http://www.rmeinc.com/about_rme.aspx'\" onmouseout=\"this.bgColor='#0066cc'; this.style.color='#FFFFFF'\" onmouseover=\"this.bgColor='#ffffff'; this.style.color='#0033cc'\">About Us</td>
+          </tr>
+        </table>
+      </html>")
+    end
+  end
+
+  def test_invalid_document_raises_exception_when_strict
+    assert_raise RuntimeError do
+      HTML::Document.new("<html>
+        <table>
+          <tr>
+            <td style=\"color: #FFFFFF; height: 17px; onclick=\"window.location.href='http://www.rmeinc.com/about_rme.aspx'\" style=\"cursor:pointer; height: 17px;\"; nowrap onclick=\"window.location.href='http://www.rmeinc.com/about_rme.aspx'\" onmouseout=\"this.bgColor='#0066cc'; this.style.color='#FFFFFF'\" onmouseover=\"this.bgColor='#ffffff'; this.style.color='#0033cc'\">About Us</td>
+          </tr>
+        </table>
+      </html>", true)
+    end
+  end
+
+end

data/test/node_test.rb

@@ -0,0 +1,90 @@
+require 'test_helper'
+require 'rails/deprecated_sanitizer/html-scanner/html/node'
+
+class NodeTest < ActiveSupport::TestCase
+
+  class MockNode
+    def initialize(matched, value)
+      @matched = matched
+      @value = value
+    end
+
+    def find(conditions)
+      @matched && self
+    end
+
+    def to_s
+      @value.to_s
+    end
+  end
+
+  def setup
+    @node = HTML::Node.new("parent")
+    @node.children.concat [MockNode.new(false,1), MockNode.new(true,"two"), MockNode.new(false,:three)]
+  end
+
+  def test_match
+    assert !@node.match("foo")
+  end
+
+  def test_tag
+    assert !@node.tag?
+  end
+
+  def test_to_s
+    assert_equal "1twothree", @node.to_s
+  end
+
+  def test_find
+    assert_equal "two", @node.find('blah').to_s
+  end
+
+  def test_parse_strict
+    s = "<b foo='hello'' bar='baz'>"
+    assert_raise(RuntimeError) { HTML::Node.parse(nil,0,0,s) }
+  end
+
+  def test_parse_relaxed
+    s = "<b foo='hello'' bar='baz'>"
+    node = nil
+    assert_nothing_raised { node = HTML::Node.parse(nil,0,0,s,false) }
+    assert node.attributes.has_key?("foo")
+    assert !node.attributes.has_key?("bar")
+  end
+
+  def test_to_s_with_boolean_attrs
+    s = "<b foo bar>"
+    node = HTML::Node.parse(nil,0,0,s)
+    assert node.attributes.has_key?("foo")
+    assert node.attributes.has_key?("bar")
+    assert "<b foo bar>", node.to_s
+  end
+
+  def test_parse_with_unclosed_tag
+    s = "<span onmouseover='bang'"
+    node = nil
+    assert_nothing_raised { node = HTML::Node.parse(nil,0,0,s,false) }
+    assert node.attributes.has_key?("onmouseover")
+  end
+
+  def test_parse_with_valid_cdata_section
+    s = "<![CDATA[<span>contents</span>]]>"
+    node = nil
+    assert_nothing_raised { node = HTML::Node.parse(nil,0,0,s,false) }
+    assert_kind_of HTML::CDATA, node
+    assert_equal '<span>contents</span>', node.content
+  end
+
+  def test_parse_strict_with_unterminated_cdata_section
+    s = "<![CDATA[neverending..."
+    assert_raise(RuntimeError) { HTML::Node.parse(nil,0,0,s) }
+  end
+
+  def test_parse_relaxed_with_unterminated_cdata_section
+    s = "<![CDATA[neverending..."
+    node = nil
+    assert_nothing_raised { node = HTML::Node.parse(nil,0,0,s,false) }
+    assert_kind_of HTML::CDATA, node
+    assert_equal 'neverending...', node.content
+  end
+end

data/test/tag_node_test.rb

@@ -0,0 +1,244 @@
+require 'test_helper'
+require 'rails/deprecated_sanitizer/html-scanner/html/node'
+
+class TagNodeTest < ActiveSupport::TestCase
+  def test_open_without_attributes
+    node = tag("<tag>")
+    assert_equal "tag", node.name
+    assert_equal Hash.new, node.attributes
+    assert_nil node.closing
+  end
+
+  def test_open_with_attributes
+    node = tag("<TAG1 foo=hey_ho x:bar=\"blah blah\" BAZ='blah blah blah' >")
+    assert_equal "tag1", node.name
+    assert_equal "hey_ho", node["foo"]
+    assert_equal "blah blah", node["x:bar"]
+    assert_equal "blah blah blah", node["baz"]
+  end
+
+  def test_self_closing_without_attributes
+    node = tag("<tag/>")
+    assert_equal "tag", node.name
+    assert_equal Hash.new, node.attributes
+    assert_equal :self, node.closing
+  end
+
+  def test_self_closing_with_attributes
+    node = tag("<tag a=b/>")
+    assert_equal "tag", node.name
+    assert_equal( { "a" => "b" }, node.attributes )
+    assert_equal :self, node.closing
+  end
+
+  def test_closing_without_attributes
+    node = tag("</tag>")
+    assert_equal "tag", node.name
+    assert_nil node.attributes
+    assert_equal :close, node.closing
+  end
+
+  def test_bracket_op_when_no_attributes
+    node = tag("</tag>")
+    assert_nil node["foo"]
+  end
+
+  def test_bracket_op_when_attributes
+    node = tag("<tag a=b/>")
+    assert_equal "b", node["a"]
+  end
+
+  def test_attributes_with_escaped_quotes
+    node = tag("<tag a='b\\'c' b=\"bob \\\"float\\\"\">")
+    assert_equal "b\\'c", node["a"]
+    assert_equal "bob \\\"float\\\"", node["b"]
+  end
+
+  def test_to_s
+    node = tag("<a b=c d='f' g=\"h 'i'\" />")
+    node = node.to_s
+    assert node.include?('a')
+    assert node.include?('b="c"')
+    assert node.include?('d="f"')
+    assert node.include?('g="h')
+    assert node.include?('i')
+  end
+
+  def test_tag
+    assert tag("<tag>").tag?
+  end
+
+  def test_match_tag_as_string
+    assert tag("<tag>").match(:tag => "tag")
+    assert !tag("<tag>").match(:tag => "b")
+  end
+
+  def test_match_tag_as_regexp
+    assert tag("<tag>").match(:tag => /t.g/)
+    assert !tag("<tag>").match(:tag => /t[bqs]g/)
+  end
+
+  def test_match_attributes_as_string
+    t = tag("<tag a=something b=else />")
+    assert t.match(:attributes => {"a" => "something"})
+    assert t.match(:attributes => {"b" => "else"})
+  end
+
+  def test_match_attributes_as_regexp
+    t = tag("<tag a=something b=else />")
+    assert t.match(:attributes => {"a" => /^something$/})
+    assert t.match(:attributes => {"b" =>  /e.*e/})
+    assert t.match(:attributes => {"a" => /me..i/, "b" => /.ls.$/})
+  end
+
+  def test_match_attributes_as_number
+    t = tag("<tag a=15 b=3.1415 />")
+    assert t.match(:attributes => {"a" => 15})
+    assert t.match(:attributes => {"b" => 3.1415})
+    assert t.match(:attributes => {"a" => 15, "b" => 3.1415})
+  end
+
+  def test_match_attributes_exist
+    t = tag("<tag a=15 b=3.1415 />")
+    assert t.match(:attributes => {"a" => true})
+    assert t.match(:attributes => {"b" => true})
+    assert t.match(:attributes => {"a" => true, "b" => true})
+  end
+
+  def test_match_attributes_not_exist
+    t = tag("<tag a=15 b=3.1415 />")
+    assert t.match(:attributes => {"c" => false})
+    assert t.match(:attributes => {"c" => nil})
+    assert t.match(:attributes => {"a" => true, "c" => false})
+  end
+
+  def test_match_parent_success
+    t = tag("<tag a=15 b='hello'>", tag("<foo k='value'>"))
+    assert t.match(:parent => {:tag => "foo", :attributes => {"k" => /v.l/, "j" => false}})
+  end
+
+  def test_match_parent_fail
+    t = tag("<tag a=15 b='hello'>", tag("<foo k='value'>"))
+    assert !t.match(:parent => {:tag => /kafka/})
+  end
+
+  def test_match_child_success
+    t = tag("<tag x:k='something'>")
+    tag("<child v=john a=kelly>", t)
+    tag("<sib m=vaughn v=james>", t)
+    assert t.match(:child => { :tag => "sib", :attributes => {"v" => /j/}})
+    assert t.match(:child => { :attributes => {"a" => "kelly"}})
+  end
+
+  def test_match_child_fail
+    t = tag("<tag x:k='something'>")
+    tag("<child v=john a=kelly>", t)
+    tag("<sib m=vaughn v=james>", t)
+    assert !t.match(:child => { :tag => "sib", :attributes => {"v" => /r/}})
+    assert !t.match(:child => { :attributes => {"v" => false}})
+  end
+
+  def test_match_ancestor_success
+    t = tag("<tag x:k='something'>", tag("<parent v=john a=kelly>", tag("<grandparent m=vaughn v=james>")))
+    assert t.match(:ancestor => {:tag => "parent", :attributes => {"a" => /ll/}})
+    assert t.match(:ancestor => {:attributes => {"m" => "vaughn"}})
+  end
+
+  def test_match_ancestor_fail
+    t = tag("<tag x:k='something'>", tag("<parent v=john a=kelly>", tag("<grandparent m=vaughn v=james>")))
+    assert !t.match(:ancestor => {:tag => /^parent/, :attributes => {"v" => /m/}})
+    assert !t.match(:ancestor => {:attributes => {"v" => false}})
+  end
+
+  def test_match_descendant_success
+    tag("<grandchild m=vaughn v=james>", tag("<child v=john a=kelly>", t = tag("<tag x:k='something'>")))
+    assert t.match(:descendant => {:tag => "child", :attributes => {"a" => /ll/}})
+    assert t.match(:descendant => {:attributes => {"m" => "vaughn"}})
+  end
+
+  def test_match_descendant_fail
+    tag("<grandchild m=vaughn v=james>", tag("<child v=john a=kelly>", t = tag("<tag x:k='something'>")))
+    assert !t.match(:descendant => {:tag => /^child/, :attributes => {"v" => /m/}})
+    assert !t.match(:descendant => {:attributes => {"v" => false}})
+  end
+
+  def test_match_child_count
+    t = tag("<tag x:k='something'>")
+    tag("hello", t)
+    tag("<child v=john a=kelly>", t)
+    tag("<sib m=vaughn v=james>", t)
+    assert t.match(:children => { :count => 2 })
+    assert t.match(:children => { :count => 2..4 })
+    assert t.match(:children => { :less_than => 4 })
+    assert t.match(:children => { :greater_than => 1 })
+    assert !t.match(:children => { :count => 3 })
+  end
+
+  def test_conditions_as_strings
+    t = tag("<tag x:k='something'>")
+    assert t.match("tag" => "tag")
+    assert t.match("attributes" => { "x:k" => "something" })
+    assert !t.match("tag" => "gat")
+    assert !t.match("attributes" => { "x:j" => "something" })
+  end
+
+  def test_attributes_as_symbols
+    t = tag("<child v=john a=kelly>")
+    assert t.match(:attributes => { :v => /oh/ })
+    assert t.match(:attributes => { :a => /ll/ })
+  end
+
+  def test_match_sibling
+    t = tag("<tag x:k='something'>")
+    tag("hello", t)
+    tag("<span a=b>", t)
+    tag("world", t)
+    m = tag("<span k=r>", t)
+    tag("<span m=l>", t)
+
+    assert m.match(:sibling => {:tag => "span", :attributes => {:a => true}})
+    assert m.match(:sibling => {:tag => "span", :attributes => {:m => true}})
+    assert !m.match(:sibling => {:tag => "span", :attributes => {:k => true}})
+  end
+
+  def test_match_sibling_before
+    t = tag("<tag x:k='something'>")
+    tag("hello", t)
+    tag("<span a=b>", t)
+    tag("world", t)
+    m = tag("<span k=r>", t)
+    tag("<span m=l>", t)
+
+    assert m.match(:before => {:tag => "span", :attributes => {:m => true}})
+    assert !m.match(:before => {:tag => "span", :attributes => {:a => true}})
+    assert !m.match(:before => {:tag => "span", :attributes => {:k => true}})
+  end
+
+  def test_match_sibling_after
+    t = tag("<tag x:k='something'>")
+    tag("hello", t)
+    tag("<span a=b>", t)
+    tag("world", t)
+    m = tag("<span k=r>", t)
+    tag("<span m=l>", t)
+
+    assert m.match(:after => {:tag => "span", :attributes => {:a => true}})
+    assert !m.match(:after => {:tag => "span", :attributes => {:m => true}})
+    assert !m.match(:after => {:tag => "span", :attributes => {:k => true}})
+  end
+
+  def test_tag_to_s
+    t = tag("<b x='foo'>")
+    tag("hello", t)
+    tag("<hr />", t)
+    assert_equal %(<b x="foo">hello<hr /></b>), t.to_s
+  end
+
+  private
+
+    def tag(content, parent=nil)
+      node = HTML::Node.parse(parent,0,0,content)
+      parent.children << node if parent
+      node
+    end
+end