loofah 0.2.2 → 0.3.0

data/lib/loofah.rb

@@ -1,6 +1,5 @@
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))
 
-require 'rubygems'
 require 'nokogiri'
 
 require 'loofah/html5/whitelist'
@@ -11,8 +10,7 @@ require 'loofah/scrubber'
 require 'loofah/html/document'
 require 'loofah/html/document_fragment'
 
-require 'loofah/deprecated'
-
+require 'loofah/helpers'
 
 #
 # Loofah is an HTML sanitizer wrapped around Nokogiri[http://nokogiri.org], an excellent
@@ -161,7 +159,7 @@ require 'loofah/deprecated'
 #
 module Loofah
   # The version of Loofah you are using
-  VERSION = '0.2.2'
+  VERSION = '0.3.0'
 
   # The minimum required version of Nokogiri
   REQUIRED_NOKOGIRI_VERSION = '1.3.3'
@@ -196,8 +194,12 @@ if Nokogiri::VERSION < Loofah::REQUIRED_NOKOGIRI_VERSION
   raise RuntimeError, "Loofah requires Nokogiri #{Loofah::REQUIRED_NOKOGIRI_VERSION} or later (currently #{Nokogiri::VERSION})"
 end
 
-if defined? Rails.configuration
+if defined? Rails.configuration # rails 2.1 and later
   Rails.configuration.after_initialize do
     require 'loofah/active_record'
+    require 'loofah/xss_foliate'
   end
+elsif defined? ActiveRecord::Base # rails 2.0
+  require 'loofah/active_record'
+  require 'loofah/xss_foliate'
 end

data/lib/loofah/active_record.rb

@@ -20,8 +20,20 @@ module Loofah
   #
   module ActiveRecordExtension
     #
-    #  scrub an ActiveRecord attribute +attr+ as an HTML fragment
-    #  using the method specified in the required +:scrub+ option.
+    #  :call-seq:
+    #    html_fragment(attribute, :scrub => sanitization_method)
+    #
+    #  Scrub an ActiveRecord attribute +attribute+ as an HTML *fragment*
+    #  using the method specified by +sanitization_method+.
+    #
+    #  +sanitization_method+ must be one of:
+    #
+    #  * :string
+    #  * :prune
+    #  * :escape
+    #  * :whitewash
+    #
+    #  See Loofah for an explanation of each sanitization method.
     #
     def html_fragment(attr, options={})
       raise ArgumentError, "html_fragment requires :scrub option" unless method = options[:scrub]
@@ -31,8 +43,20 @@ module Loofah
     end
 
     #
-    #  scrub an ActiveRecord attribute +attr+ as an HTML document
-    #  using the method specified in the required +:scrub+ option.
+    #  :call-seq:
+    #    model.html_document(attribute, :scrub => sanitization_method)
+    #
+    #  Scrub an ActiveRecord attribute +attribute+ as an HTML *document*
+    #  using the method specified by +sanitization_method+.
+    #
+    #  +sanitization_method+ must be one of:
+    #
+    #  * :string
+    #  * :prune
+    #  * :escape
+    #  * :whitewash
+    #
+    #  See Loofah for an explanation of each sanitization method.
     #
     def html_document(attr, options={})
       raise ArgumentError, "html_document requires :scrub option" unless method = options[:scrub]

data/lib/loofah/helpers.rb

@@ -0,0 +1,23 @@
+module Loofah
+  module Helpers
+    class << self
+      #
+      #  A replacement for Rails's built-in +strip_tags+ helper.
+      #
+      #   Loofah::Helpers.strip_tags("<div>Hello <b>there</b></div>") # => "Hello there"
+      #
+      def strip_tags(string_or_io)
+        Loofah.fragment(string_or_io).text
+      end
+
+      #
+      #  A replacement for Rails's built-in +sanitize+ helper.
+      #
+      #   Loofah::Helpers.sanitize("<script src=http://ha.ckers.org/xss.js></script>") # => "&lt;script src=\"http://ha.ckers.org/xss.js\"&gt;&lt;/script&gt;"
+      #
+      def sanitize(string_or_io)
+        Loofah.scrub_fragment(string_or_io, :strip).to_s
+      end
+    end
+  end
+end

data/lib/loofah/xss_foliate.rb

@@ -0,0 +1,210 @@
+module Loofah
+  #
+  #  A replacement for
+  #  XssTerminate[http://github.com/look/xss_terminate/tree/master],
+  #  XssFoliate will strip all tags from your ActiveRecord models'
+  #  string and text attributes.
+  #
+  #  See Loofah::XssFoliate::ClassMethods for more information.
+  #
+  module XssFoliate
+    #
+    #  A replacement for
+    #  XssTerminate[http://github.com/look/xss_terminate/tree/master],
+    #  XssFoliate will strip all tags from your ActiveRecord models'
+    #  string and text attributes.
+    #
+    #  Please read the Loofah documentation for an explanation of the
+    #  different scrubbing methods.
+    #
+    #  If you'd like to scrub all fields in all your models (and perhaps *opt-out* in specific models):
+    #
+    #    # config/environment
+    #    LOOFAH_XSS_FOLIATE_ALL_MODELS = true
+    #    Rails::Initializer.run do |config|
+    #      config.gem "loofah"
+    #    end
+    #
+    #    # db/schema.rb
+    #    create_table "posts" do |t|
+    #      t.string  "title"
+    #      t.text    "body"
+    #      t.string  "author"
+    #    end
+    #
+    #    # app/model/post.rb
+    #    class Post < ActiveRecord::Base
+    #      #  by default, title, body and author will all be scrubbed down to their inner text
+    #    end
+    #
+    #  OR
+    #
+    #    # app/model/post.rb
+    #    class Post < ActiveRecord::Base
+    #      xss_foliate :except => :author  # opt-out of sanitizing author
+    #    end
+    #
+    #  OR
+    #
+    #      xss_foliate :strip => [:title, body]  # strip unsafe tags from both title and body
+    #
+    #  OR
+    #
+    #      xss_foliate :except => :title         # scrub body and author but not title
+    #
+    #  OR
+    #
+    #      # remove all tags from title, remove unsafe tags from body
+    #      xss_foliate :sanitize => :title, :scrub => :body
+    #
+    #  OR
+    #
+    #      # old xss_terminate code will work if you s/_terminate/_foliate/
+    #      # was: xss_terminate :except => [:title], :sanitize => [:body]
+    #      xss_foliate :except => [:title], :sanitize => [:body]
+    #
+    #  Alternatively, if you would like to *opt-in* to the models and attributes that are sanitized:
+    #
+    #    # config/environment.rb
+    #    LOOFAH_XSS_FOLIATE_ALL_MODELS = false # default, this line could be omitted
+    #    Rails::Initializer.run do |config|
+    #      config.gem "loofah"
+    #    end
+    #
+    #    # db/schema.rb
+    #    create_table "posts" do |t|
+    #      t.string  "title"
+    #      t.text    "body"
+    #      t.string  "author"
+    #    end
+    #
+    #    # app/model/post.rb
+    #    class Post < ActiveRecord::Base
+    #      xss_foliate  # scrub title, body and author down to their inner text
+    #    end
+    #
+    module ClassMethods
+      # :stopdoc:
+      VALID_OPTIONS = [:except, :strip, :escape, :prune, :text, :html5lib_sanitize, :sanitize]
+      ALIASED_OPTIONS = {:html5lib_sanitize => :escape, :sanitize => :strip}
+      REAL_OPTIONS = VALID_OPTIONS - ALIASED_OPTIONS.keys
+      # :startdoc:
+
+      #
+      #  Annotate your model with this method to specify which fields
+      #  you want scrubbed, and how you want them scrubbed. XssFoliate
+      #  assumes all character fields are HTML fragments (as opposed to
+      #  full documents, see the Loofah[http://loofah.rubyforge.org/]
+      #  documentation for a full explanation of the difference).
+      #
+      #  Example call:
+      #
+      #   xss_foliate :except => :author, :strip => :body, :prune => [:title, :description]
+      #
+      #  *Note* that the values in the options hash can be either an
+      #  array of attributes or a single attribute.
+      #
+      #  Options:
+      #
+      #   :except => [fields] # don't scrub these fields
+      #   :strip  => [fields] # strip unsafe tags from these fields
+      #   :escape => [fields] # escape unsafe tags from these fields
+      #   :prune  => [fields] # prune unsafe tags and subtrees from these fields
+      #   :text   => [fields] # remove everything except the inner text from these fields
+      #
+      #  XssTerminate compatibility options (note that the default
+      #  behavior in XssTerminate corresponds to :text)
+      #
+      #   :html5lib_sanitize => [fields] # same as :escape
+      #   :sanitize          => [fields] # same as :strip
+      #
+      #  The default is :text for all fields unless otherwise specified.
+      #
+      def xss_foliate(options = {})
+        callback_already_declared = \
+        if respond_to?(:before_validation_callback_chain)
+          # Rails 2.1 and later
+          before_validation_callback_chain.any? {|cb| cb.method == :xss_foliate_fields}
+        else
+          # Rails 2.0
+          cbs = read_inheritable_attribute(:before_validation)
+          (! cbs.nil?) && cbs.any? {|cb| cb == :xss_foliate_fields}
+        end
+
+        unless callback_already_declared
+          before_validation        :xss_foliate_fields
+          class_inheritable_reader :xss_foliate_options
+          include XssFoliate::InstanceMethods
+        end
+
+        options.keys.each do |option|
+          raise ArgumentError, "unknown xss_foliate option #{option}" unless VALID_OPTIONS.include?(option)
+        end
+
+        REAL_OPTIONS.each do |option|
+          options[option] = Array(options[option]).collect { |val| val.to_sym }
+        end
+
+        ALIASED_OPTIONS.each do |option, real|
+          options[real] += Array(options.delete(option)).collect { |val| val.to_sym } if options[option]
+        end
+
+        write_inheritable_attribute(:xss_foliate_options, options)
+      end
+
+      #
+      #  Class method to determine whether or not this model is applying
+      #  xss_foliation to its attributes. Could be useful in test suites.
+      #
+      def xss_foliated?
+        options = read_inheritable_attribute(:xss_foliate_options)
+        ! (options.nil? || options.empty?)
+      end
+    end
+
+    module InstanceMethods
+
+      def xss_foliate_fields # :nodoc:
+        # fix a bug with Rails internal AR::Base models that get loaded before
+        # the plugin, like CGI::Sessions::ActiveRecordStore::Session
+        return if xss_foliate_options.nil?
+
+        self.class.columns.each do |column|
+          next unless (column.type == :string || column.type == :text)
+
+          field = column.name.to_sym
+          value = self[field]
+
+          next if value.nil? || !value.is_a?(String)
+
+          if xss_foliate_options[:except].include?(field)
+            next
+
+          elsif xss_foliate_options[:strip].include?(field)
+            fragment = Loofah.scrub_fragment(value, :strip)
+            self[field] = fragment.nil? ? "" : fragment.to_s
+
+          elsif xss_foliate_options[:prune].include?(field)
+            fragment = Loofah.scrub_fragment(value, :prune)
+            self[field] = fragment.nil? ? "" : fragment.to_s
+
+          elsif xss_foliate_options[:escape].include?(field)
+            fragment = Loofah.scrub_fragment(value, :escape)
+            self[field] = fragment.nil? ? "" : fragment.to_s
+
+          else # :text
+            fragment = Loofah.scrub_fragment(value, :strip)
+            self[field] = fragment.nil? ? "" : fragment.text
+          end
+        end
+
+      end
+    end
+  end
+end
+
+ActiveRecord::Base.extend(Loofah::XssFoliate::ClassMethods)
+
+if defined?(LOOFAH_XSS_FOLIATE_ALL_MODELS) && LOOFAH_XSS_FOLIATE_ALL_MODELS
+  ActiveRecord::Base.xss_foliate
+end

data/test/test_active_record.rb

@@ -16,40 +16,82 @@ class TestActiveRecord < Test::Unit::TestCase
       end
     end
 
-    context "scrubbing field as a fragment" do
-      setup do
-        Post.html_fragment :html_string, :scrub => :prune
-        @post = Post.new :html_string => HTML_STRING, :plain_text => PLAIN_TEXT
+    context "scrubbing a single field as a fragment" do
+      context "using a symbol to indicate the attribute" do
+        setup do
+          Post.html_fragment :html_string, :scrub => :prune
+          assert ! Post.xss_foliated?
+          @post = Post.new :html_string => HTML_STRING, :plain_text => PLAIN_TEXT
+        end
+
+        should "scrub the specified field" do
+          Loofah.expects(:scrub_fragment).with(HTML_STRING, :prune).once
+          Loofah.expects(:scrub_fragment).with(PLAIN_TEXT, :prune).never
+          @post.valid?
+        end
+
+        should "only call scrub_fragment once" do
+          Loofah.expects(:scrub_fragment).once
+          @post.valid?
+        end
+
+        should "generate strings" do
+          @post.valid?
+          assert_equal String, @post.html_string.class
+          assert_equal HTML_STRING, @post.html_string
+        end
       end
 
-      should "scrub the specified field" do
-        Loofah.expects(:scrub_fragment).with(HTML_STRING, :prune).once
-        Loofah.expects(:scrub_fragment).with(PLAIN_TEXT, :prune).never
-        @post.valid?
-      end
-
-      should "generate strings" do
-        @post.valid?
-        assert_equal String, @post.html_string.class
-        assert_equal HTML_STRING, @post.html_string
+      context "using a string to indicate the attribute" do
+        setup do
+          Post.html_fragment 'html_string', :scrub => :prune
+          assert ! Post.xss_foliated?
+          @post = Post.new :html_string => HTML_STRING, :plain_text => PLAIN_TEXT
+        end
+
+        should "scrub the specified field" do
+          Loofah.expects(:scrub_fragment).with(HTML_STRING, :prune).once
+          Loofah.expects(:scrub_fragment).with(PLAIN_TEXT, :prune).never
+          @post.valid?
+        end
       end
     end
 
-    context "scrubbing field as a document" do
-      setup do
-        Post.html_document :html_string, :scrub => :strip
-        @post = Post.new :html_string => HTML_STRING, :plain_text => PLAIN_TEXT
-      end
-
-      should "scrub the specified field, but not other fields" do
-        Loofah.expects(:scrub_document).with(HTML_STRING, :strip).once
-        Loofah.expects(:scrub_document).with(PLAIN_TEXT, :strip).never
-        @post.valid?
+    context "scrubbing a single field as a document" do
+      context "using a symbol to indicate the attribute" do
+        setup do
+          Post.html_document :html_string, :scrub => :strip
+          @post = Post.new :html_string => HTML_STRING, :plain_text => PLAIN_TEXT
+        end
+
+        should "scrub the specified field, but not other fields" do
+          Loofah.expects(:scrub_document).with(HTML_STRING, :strip).once
+          Loofah.expects(:scrub_document).with(PLAIN_TEXT, :strip).never
+          @post.valid?
+        end
+
+        should "only call scrub_document once" do
+          Loofah.expects(:scrub_document).once
+          @post.valid?
+        end
+
+        should "generate strings" do
+          @post.valid?
+          assert_equal String, @post.html_string.class
+        end
       end
 
-      should "generate strings" do
-        @post.valid?
-        assert_equal String, @post.html_string.class
+      context "using a string to indicate the attribute" do
+        setup do
+          Post.html_document 'html_string', :scrub => :strip
+          @post = Post.new :html_string => HTML_STRING, :plain_text => PLAIN_TEXT
+        end
+
+        should "scrub the specified field, but not other fields" do
+          Loofah.expects(:scrub_document).with(HTML_STRING, :strip).once
+          Loofah.expects(:scrub_document).with(PLAIN_TEXT, :strip).never
+          @post.valid?
+        end
       end
     end
 

data/test/test_ad_hoc.rb

@@ -0,0 +1,185 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
+
+class TestAdHoc < Test::Unit::TestCase
+
+  def test_empty_string_with_escape
+    assert_equal "", Loofah.scrub_fragment("", :escape).to_xml
+  end
+
+  def test_empty_string_with_prune
+    assert_equal Loofah.scrub_document("", :prune).text, ""
+  end
+
+  def test_removal_of_illegal_tag
+    html = <<-HTML
+      following this there should be no jim tag
+      <jim>jim</jim>
+      was there?
+    HTML
+    sane = Nokogiri::HTML(Loofah.scrub_fragment(html, :escape).to_xml)
+    assert sane.xpath("//jim").empty?
+  end
+
+  def test_removal_of_illegal_attribute
+    html = "<p class=bar foo=bar abbr=bar />"
+    sane = Nokogiri::HTML(Loofah.scrub_fragment(html, :escape).to_xml)
+    node = sane.xpath("//p").first
+    assert node.attributes['class']
+    assert node.attributes['abbr']
+    assert_nil node.attributes['foo']
+  end
+
+  def test_removal_of_illegal_url_in_href
+    html = <<-HTML
+      <a href='jimbo://jim.jim/'>this link should have its href removed because of illegal url</a>
+      <a href='http://jim.jim/'>this link should be fine</a>
+    HTML
+    sane = Nokogiri::HTML(Loofah.scrub_fragment(html, :escape).to_xml)
+    nodes = sane.xpath("//a")
+    assert_nil nodes.first.attributes['href']
+    assert nodes.last.attributes['href']
+  end
+
+  def test_css_sanitization
+    html = "<p style='background-color: url(\"http://foo.com/\") ; background-color: #000 ;' />"
+    sane = Nokogiri::HTML(Loofah.scrub_fragment(html, :escape).to_xml)
+    assert_match(/#000/, sane.inner_html)
+    assert_no_match(/foo\.com/, sane.inner_html)
+  end
+
+  def test_fragment_with_no_tags
+    assert_equal "This fragment has no tags.", Loofah.scrub_fragment("This fragment has no tags.", :escape).to_xml
+  end
+
+  def test_fragment_in_p_tag
+    assert_equal "<p>This fragment is in a p.</p>", Loofah.scrub_fragment("<p>This fragment is in a p.</p>", :escape).to_xml
+  end
+
+  def test_fragment_in_p_tag_plus_stuff
+    assert_equal "<p>This fragment is in a p.</p>foo<strong>bar</strong>", Loofah.scrub_fragment("<p>This fragment is in a p.</p>foo<strong>bar</strong>", :escape).to_xml
+  end
+
+  def test_fragment_with_text_nodes_leading_and_trailing
+    assert_equal "text<p>fragment</p>text", Loofah.scrub_fragment("text<p>fragment</p>text", :escape).to_xml
+  end
+
+  def test_whitewash_on_fragment
+    html = "safe<frameset rows=\"*\"><frame src=\"http://example.com\"></frameset> <b>description</b>"
+    whitewashed = Loofah.scrub_document(html, :whitewash).to_s
+    assert_equal "<p>safe</p><b>description</b>", whitewashed.gsub("\n","")
+  end
+
+  MSWORD_HTML = <<-EOHTML
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8"><meta name="ProgId" content="Word.Document"><meta name="Generator" content="Microsoft Word 11"><meta name="Originator" content="Microsoft Word 11"><link rel="File-List" href="file:///C:%5CDOCUME%7E1%5CNICOLE%7E1%5CLOCALS%7E1%5CTemp%5Cmsohtml1%5C01%5Cclip_filelist.xml"><!--[if gte mso 9]><xml>
+<w:WordDocument>
+ <w:View>Normal</w:View>
+ <w:Zoom>0</w:Zoom>
+ <w:PunctuationKerning/>
+ <w:ValidateAgainstSchemas/>
+ <w:SaveIfXMLInvalid>false</w:SaveIfXMLInvalid>
+ <w:IgnoreMixedContent>false</w:IgnoreMixedContent>
+ <w:AlwaysShowPlaceholderText>false</w:AlwaysShowPlaceholderText>
+ <w:Compatibility>
+  <w:BreakWrappedTables/>
+  <w:SnapToGridInCell/>
+  <w:WrapTextWithPunct/>
+  <w:UseAsianBreakRules/>
+  <w:DontGrowAutofit/>
+ </w:Compatibility>
+ <w:BrowserLevel>MicrosoftInternetExplorer4</w:BrowserLevel>
+</w:WordDocument>
+</xml><![endif]--><!--[if gte mso 9]><xml>
+<w:LatentStyles DefLockedState="false" LatentStyleCount="156">
+</w:LatentStyles>
+</xml><![endif]--><style>
+<!--
+/* Style Definitions */
+p.MsoNormal, li.MsoNormal, div.MsoNormal
+{mso-style-parent:"";
+margin:0in;
+margin-bottom:.0001pt;
+mso-pagination:widow-orphan;
+font-size:12.0pt;
+font-family:"Times New Roman";
+mso-fareast-font-family:"Times New Roman";}
+@page Section1
+{size:8.5in 11.0in;
+margin:1.0in 1.25in 1.0in 1.25in;
+mso-header-margin:.5in;
+mso-footer-margin:.5in;
+mso-paper-source:0;}
+div.Section1
+{page:Section1;}
+-->
+</style><!--[if gte mso 10]>
+<style>
+/* Style Definitions */
+table.MsoNormalTable
+{mso-style-name:"Table Normal";
+mso-tstyle-rowband-size:0;
+mso-tstyle-colband-size:0;
+mso-style-noshow:yes;
+mso-style-parent:"";
+mso-padding-alt:0in 5.4pt 0in 5.4pt;
+mso-para-margin:0in;
+mso-para-margin-bottom:.0001pt;
+mso-pagination:widow-orphan;
+font-size:10.0pt;
+font-family:"Times New Roman";
+mso-ansi-language:#0400;
+mso-fareast-language:#0400;
+mso-bidi-language:#0400;}
+</style>
+<![endif]-->
+
+<p class="MsoNormal">Foo <b style="">BOLD<o:p></o:p></b></p>
+  EOHTML
+
+  def test_deprecated_whitewash_fragment_on_microsofty_markup
+    whitewashed = Loofah.scrub_fragment(MSWORD_HTML.chomp, :whitewash).to_s
+    assert_equal "<p>Foo <b>BOLD</b></p>", whitewashed
+  end
+
+  def test_deprecated_whitewash_on_microsofty_markup
+    whitewashed = Loofah.scrub_document(MSWORD_HTML, :whitewash).to_s
+    assert_equal "<p>Foo <b>BOLD</b></p>", whitewashed
+  end
+
+  def test_fragment_whitewash_on_microsofty_markup
+    whitewashed = Loofah.fragment(MSWORD_HTML.chomp).scrub!(:whitewash)
+    assert_equal "<p>Foo <b>BOLD</b></p>", whitewashed.to_s
+  end
+
+  def test_document_whitewash_on_microsofty_markup
+    whitewashed = Loofah.document(MSWORD_HTML.chomp).scrub!(:whitewash)
+    assert_equal "<p>Foo <b>BOLD</b></p>", whitewashed.to_s
+  end
+
+  def test_return_empty_string_when_nothing_left
+    assert_equal "", Loofah.scrub_document('<script>test</script>', :prune).text
+  end
+
+  def test_removal_of_all_tags
+    html = <<-HTML
+      What's up <strong>doc</strong>?
+    HTML
+    stripped = Loofah.scrub_document(html, :prune).text
+    assert_equal "What's up doc?".strip, stripped.strip
+  end
+
+  def test_dont_remove_whitespace
+    html = "Foo\nBar"
+    assert_equal html, Loofah.scrub_document(html, :prune).text
+  end
+
+  def test_dont_remove_whitespace_between_tags
+    html = "<p>Foo</p>\n<p>Bar</p>"
+    assert_equal "Foo\nBar", Loofah.scrub_document(html, :prune).text
+  end
+
+  def test_removal_of_entities
+    html = "<p>this is &lt; that &quot;&amp;&quot; the other &gt; boo&apos;ya</p>"
+    assert_equal 'this is < that "&" the other > boo\'ya', Loofah.scrub_document(html, :prune).text
+  end
+
+end