feed-normalizer 1.1.0 → 1.2.0

data/History.txt

@@ -0,0 +1,14 @@
+
+1.2.0
+
+ * Added HtmlCleaner - sanitizes HTML and removes 'bad' URIs to a level suitable
+   for 'safe' display inside a web browser. Can be used as a standalone library,
+   or as part of the Feed object. See Feed.clean! for details about cleaning a
+   Feed instance. Also see HtmlCleaner and its unit tests. Uses Hpricot.
+ * Added Feed-diffing. Differences between two feeds can be displayed using
+   Feed.diff. Works nicely with YAML for a readable diff.
+ * FeedNormalizer.parse now takes a hash for its arguments.
+ * Removed FN::Content.
+ * Now uses Hoe!
+
+

data/Manifest.txt

@@ -0,0 +1,19 @@
+History.txt
+License.txt
+Manifest.txt
+Rakefile
+Readme.txt
+lib/feed-normalizer.rb
+lib/html-cleaner.rb
+lib/parsers/rss.rb
+lib/parsers/simple-rss.rb
+lib/structures.rb
+test/data/atom03.xml
+test/data/atom10.xml
+test/data/rdf10.xml
+test/data/rss20.xml
+test/data/rss20diff.xml
+test/data/rss20diff_short.xml
+test/test_all.rb
+test/test_feednormalizer.rb
+test/test_htmlcleaner.rb

data/Rakefile

@@ -1,49 +1,25 @@
-require 'rubygems'
-require 'rake'
-require 'rake/testtask'
-require 'rake/rdoctask'
-require 'rake/clean'
-require 'rake/gempackagetask'
+require 'hoe'
 
-PKG_FILES = FileList[
-    "lib/**/*", "test/**/*", "[A-Z]*", "Rakefile", "html/**/*"
-]
-
-Gem::manage_gems
-
-task :default => [:test]
-task :package => [:test, :doc]
-
-spec = Gem::Specification.new do |s|
-  s.name = "feed-normalizer"
-  s.version = "1.1.0"
+Hoe.new("feed-normalizer", "1.2.0") do |s|
   s.author = "Andrew A. Smith"
   s.email = "andy@tinnedfruit.org"
-  s.homepage = "http://code.google.com/p/feed-normalizer/"
-  s.platform = Gem::Platform::RUBY
+  s.url = "http://feed-normalizer.rubyforge.org/"
   s.summary = "Extensible Ruby wrapper for Atom and RSS parsers"
-  s.files = PKG_FILES
-  s.require_path = "lib"
-  s.autorequire = "feed-normalizer"
-  s.has_rdoc = true
-  s.add_dependency  "simple-rss", ">= 1.1"
+  s.description = s.paragraphs_of('Readme.txt', 1..2).join("\n\n")
+  s.changes = s.paragraphs_of('History.txt', 0..1).join("\n\n")
+  s.extra_deps << ["simple-rss", ">= 1.1"]
+  s.extra_deps << ["hpricot", ">= 0.4"]
+  s.need_zip = true
+  s.need_tar = false
 end
 
-Rake::GemPackageTask.new(spec) do |pkg|
-  pkg.need_zip = true
-end
-
-Rake::TestTask.new do |t|
-  t.libs << "test"
-  t.test_files = FileList['test/*_test.rb']
-  t.verbose = true
-end
 
-desc "Create documentation"
-Rake::RDocTask.new("doc") do |rdoc|
-  rdoc.title = "Feed Normalizer"
-  rdoc.rdoc_dir = 'doc'
-  rdoc.rdoc_files.include('README')
-  rdoc.rdoc_files.include('lib/**/*.rb')
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new("rcov") do |t|
+    t.test_files = Dir['test/test_all.rb']
+  end
+rescue LoadError
+  nil
 end
 

data/{README → Readme.txt}

@@ -38,6 +38,17 @@ The feed representation stays the same, even though a different parser was used.
     feed.class # => FeedNormalizer::Feed
     feed.parser # => SimpleRSS
 
+== Cleaning / Sanitizing
+
+    feed.title # => "My Feed > Your Feed"
+    feed.entries.first.content # => "<p x='y'>Hello</p><object></object></html>"
+    feed.clean!
+
+All elements should now be either clean HTML, or HTML escaped strings.
+
+    feed.title # => "My Feed &gt; Your Feed"
+    feed.entries.first.content # => "<p>Hello</p>"
+
 == Extending
 
 Implement a parser wrapper by extending the FeedNormalizer::Parser class and overriding
@@ -49,4 +60,4 @@ See FeedNormalizer::RubyRssParser and FeedNormalizer::SimpleRssParser for exampl
 == Authors
 * Andrew A. Smith (andy@tinnedfruit.org)
 
-This library is released under the terms of the BSD License (see the LICENSE file for details).
+This library is released under the terms of the BSD License (see the License.txt file for details).

data/lib/feed-normalizer.rb

@@ -1,4 +1,5 @@
 require 'structures'
+require 'html-cleaner'
 
 module FeedNormalizer
 
@@ -88,21 +89,16 @@ module FeedNormalizer
     # used first, and if try_others is false, it is the only parser used,
     # otherwise all parsers in the ParserRegistry are attempted next, in
     # order of priority.
-    def self.parse(xml, forced_parser=nil, try_others=false)
+    def self.parse(xml, opts = {})
 
       # Get a string ASAP, as multiple read()'s will start returning nil..
       xml = xml.respond_to?(:read) ? xml.read : xml.to_s
 
-      if forced_parser
-        result = forced_parser.parse(xml)
+      if opts[:force_parser]
+        result = opts[:force_parser].parse(xml)
 
-        if result
-          return result
-        elsif !try_others
-          return nil
-        else
-          # fall through and continue with other parsers
-        end
+        return result if result
+        return nil if opts[:try_others] == false
       end
 
       ParserRegistry.parsers.each do |parser|

data/lib/html-cleaner.rb

@@ -0,0 +1,186 @@
+require 'hpricot'
+require 'cgi'
+
+module FeedNormalizer
+
+  # Various methods for cleaning up HTML and preparing it for safe public
+  # consumption.
+  #
+  # Documents used for refrence:
+  # - http://www.w3.org/TR/html4/index/attributes.html
+  # - http://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references
+  # - http://feedparser.org/docs/html-sanitization.html
+  # - http://code.whytheluckystiff.net/hpricot/wiki
+  class HtmlCleaner
+
+    # allowed html elements.
+    HTML_ELEMENTS = %w(
+      a abbr acronym address area b bdo big blockquote br button caption center
+      cite code col colgroup dd del dfn dir div dl dt em fieldset font h1 h2 h3
+      h4 h5 h6 hr i img ins kbd label legend li map menu ol optgroup p pre q s
+      samp small span strike strong sub sup table tbody td tfoot th thead tr tt
+      u ul var
+    )
+
+    # allowed attributes.
+    HTML_ATTRS = %w(
+      abbr accept accept-charset accesskey align alt axis border cellpadding
+      cellspacing char charoff charset checked cite class clear cols colspan
+      color compact coords datetime dir disabled for frame headers height href
+      hreflang hspace id ismap label lang longdesc maxlength media method
+      multiple name nohref noshade nowrap readonly rel rev rows rowspan rules
+      scope selected shape size span src start summary tabindex target title
+      type usemap valign value vspace width
+    )
+
+    # allowed attributes, but they can contain URIs, extra caution required.
+    # NOTE: That means this doesnt list *all* URI attrs, just the ones that are allowed.
+    HTML_URI_ATTRS = %w(
+      href src cite usemap longdesc
+    )
+
+    DODGY_URI_SCHEMES = %w(
+      javascript vbscript mocha livescript data
+    )
+
+    class << self
+
+      # Does this:
+      # - Unescape HTML
+      # - Parse HTML into tree
+      # - Find 'body' if present, and extract tree inside that tag, otherwise parse whole tree
+      # - Each tag:
+      #   - remove tag if not whitelisted
+      #   - escape HTML tag contents
+      #   - remove all attributes not on whitelist
+      #   - extra-scrub URI attrs; see dodgy_uri?
+      #
+      # Extra (i.e. unmatched) ending tags and comments are removed.
+      def clean(str)
+        str = unescapeHTML(str)
+
+        doc = Hpricot(str, :xhtml_strict => true)
+        doc = subtree(doc, :body)
+
+        # get all the tags in the document
+        tags = (doc/"*").collect {|e| e.name}
+
+        # Remove tags that aren't whitelisted.
+        remove_tags!(doc, tags - HTML_ELEMENTS)
+        remaining_tags = tags & HTML_ELEMENTS
+
+        # Remove attributes that aren't on the whitelist, or are suspicious URLs.
+        (doc/remaining_tags.join(",")).each do |element|
+          element.attributes.reject! do |attr,val|
+            !HTML_ATTRS.include?(attr) || (HTML_URI_ATTRS.include?(attr) && dodgy_uri?(val))
+          end
+
+          element.attributes = element.attributes.build_hash {|a,v| [a, add_entities(v)]}
+        end unless remaining_tags.empty?
+
+        doc.traverse_text {|t| t.set(add_entities(t.to_s))}
+
+        # Return the tree, without comments. Ugly way of removing comments,
+        # but can't see a way to do this in Hpricot yet.
+        doc.to_s.gsub(/<\!--.*-->/mi, '')
+      end
+
+      # For all other feed elements:
+      # - Unescape HTML.
+      # - Parse HTML into tree (taking 'body' as root, if present)
+      # - Takes text out of each tag, and escapes HTML.
+      # - Returns all text concatenated.
+      def flatten(str)
+        str.gsub!("\n", " ")
+        str = unescapeHTML(str)
+
+        doc = Hpricot(str, :xhtml_strict => true)
+        doc = subtree(doc, :body)
+
+        out = ""
+        doc.traverse_text {|t| out << add_entities(t.to_s)}
+
+        return out
+      end
+
+      # Returns true if the given string contains a suspicious URL,
+      # i.e. a javascript link.
+      #
+      # This method rejects javascript, vbscript, livescript, mocha and data URLs.
+      # It *could* be refined to only deny dangerous data URLs, however.
+      def dodgy_uri?(uri)
+
+        # special case for poorly-formed entities (missing ';')
+        # if these occur *anywhere* within the string, then throw it out.
+        return true if (uri =~ /&\#(\d+|x[0-9a-f]+)[^;\d]/mi)
+
+        # Try escaping as both HTML or URI encodings, and then trying
+        # each scheme regexp on each
+        [unescapeHTML(uri), CGI.unescape(uri)].each do |unesc_uri|
+          DODGY_URI_SCHEMES.each do |scheme|
+
+            regexp = "#{scheme}:".gsub(/./) do |char|
+              "([\000-\037\177\s]*)#{char}"
+            end
+
+            # regexp looks something like
+            # /\A([\000-\037\177\s]*)j([\000-\037\177\s]*)a([\000-\037\177\s]*)v([\000-\037\177\s]*)a([\000-\037\177\s]*)s([\000-\037\177\s]*)c([\000-\037\177\s]*)r([\000-\037\177\s]*)i([\000-\037\177\s]*)p([\000-\037\177\s]*)t([\000-\037\177\s]*):/mi
+            return true if (unesc_uri =~ %r{\A#{regexp}}mi)
+          end
+        end
+
+        nil
+      end
+
+      # unescapes HTML. If xml is true, also converts XML-only named entities to HTML.
+      def unescapeHTML(str, xml = true)
+        CGI.unescapeHTML(xml ? str.gsub("&apos;", "&#39;") : str)
+      end
+
+      # Adds entities where possible.
+      # Works like CGI.escapeHTML, but will not escape existing entities;
+      # i.e. &#123; will NOT become &amp;#123;
+      #
+      # This method could be improved by adding a whitelist of html entities.
+      def add_entities(str)
+        str.gsub(/\"/n, '&quot;').gsub(/>/n, '&gt;').gsub(/</n, '&lt;').gsub(/&(?!(\#\d+|\#x([0-9a-f]+)|\w{2,8});)/nmi, '&amp;')
+      end
+
+      private
+
+      # Everything below elment, or the just return the doc if element not present.
+      def subtree(doc, element)
+        doc.at("//#{element}/*") || doc
+      end
+
+      def remove_tags!(doc, tags)
+        (doc/tags.join(",")).remove unless tags.empty?
+      end
+
+    end
+  end
+end
+
+
+module Enumerable
+  def build_hash
+    result = {}
+    self.each do |elt|
+      key, value = yield elt
+      result[key] = value
+    end
+    result
+  end
+end
+
+# http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/207625
+#  Subject: A simple Hpricot text setter
+#  From: Chris Gehlker <canyonrat mac.com>
+#  Date: Fri, 11 Aug 2006 03:19:13 +0900
+class Hpricot::Text
+  def set(string)
+    @content = string
+    self.raw_string = string
+  end
+end
+

data/lib/parsers/rss.rb

@@ -52,6 +52,7 @@ module FeedNormalizer
         :date_published => :pubDate,
         :urls => :link,
         :description => :description,
+        :content => :description,
         :title => :title,
         :authors => :author
       }
@@ -62,7 +63,6 @@ module FeedNormalizer
 
         # custom item elements
         feed_entry.id = rss_item.guid.content if rss_item.respond_to?(:guid) && rss_item.guid
-        feed_entry.content.body = rss_item.description
         feed_entry.copyright = rss.copyright if rss_item.respond_to? :copyright
 
         feed.entries << feed_entry

data/lib/parsers/simple-rss.rb

@@ -53,6 +53,7 @@ module FeedNormalizer
         :date_published => [:pubDate, :published],
         :urls => :link,
         :description => [:description, :summary],
+        :content => [:content, :description],
         :title => :title,
         :authors => [:author, :contributor]
       }
@@ -64,7 +65,6 @@ module FeedNormalizer
         # custom entry elements
         feed_entry.id = atomrss_entry.guid || atomrss_entry[:id] # entries are a Hash..
         feed_entry.copyright = atomrss_entry.copyright || (atomrss.respond_to?(:copyright) ? atomrss.copyright : nil)
-        feed_entry.content.body = atomrss_entry.content || atomrss_entry.description
 
         feed.entries << feed_entry
       end
@@ -74,7 +74,7 @@ module FeedNormalizer
 
     def self.image(parser)
       if parser.respond_to?(:image) && parser.image
-        if parser.image.match /<url>/ # RSS image contains an <url> spec
+        if parser.image =~ /<url>/ # RSS image contains an <url> spec
           parser.image.scan(/<url>(.*)<\/url>/).to_s
         else
           parser.image # Atom contains just the url
@@ -90,9 +90,7 @@ module FeedNormalizer
 
     # gets the value returned from the method if it overriden, otherwise nil.
     def self.overridden_value(object, method)
-      # XXX: hack to find out if the id method is overriden
-      # Highly dependent upon Method's to_s :(
-      object.id if object.method(:id).to_s.match /SimpleRSS\#/
+      object.class.public_instance_methods(false).include? method
     end
 
   end

data/lib/structures.rb

@@ -10,14 +10,12 @@ module FeedNormalizer
     # Example:
     # Object contains an array called 'alphas', which looks like [:a, :b, :c].
     # Call object.alpha and :a is returned.
-    def method_missing(name)
-      if name.to_s =~ /[^s]$/ # doesnt end with 's'
-        plural = :"#{name}s"
-        if self.respond_to?(plural)
-          return self.send(plural).first
-        end
-      end
-      nil
+    def method_missing(name, *args)
+      return self.send(:"#{name}s").first rescue super(name, *args)
+    end
+
+    def respond_to?(x, y=false)
+      self.class::ELEMENTS.include?(x) || self.class::ELEMENTS.include?(:"#{x}s") || super(x, y)
     end
 
   end
@@ -34,55 +32,126 @@ module FeedNormalizer
           self.class::ELEMENTS.collect{|el| self.instance_variable_get("@#{el}")==other.instance_variable_get("@#{el}")}.all?)
     end
 
-  end
-
-  # Wraps content used in an Entry. type defaults to :text.
-  class Content
-    TYPE = [:text, :html, :xhtml]
-    attr_accessor :type, :body
+    # Returns the difference between two Feed instances as a hash.
+    # Any top-level differences in the Feed object as presented as:
+    #
+    #  { :title => [content, other_content] }
+    #
+    # For differences at the items level, an array of hashes shows the diffs
+    # on a per-entry basis. Only entries that differ will contain a hash:
+    #
+    #  { :items => [
+    #     {:title => ["An article tile", "A new article title"]},
+    #     {:title => ["one title", "a different title"]} ]}
+    #
+    # If the number of items in each feed are different, then the count of each
+    # is provided instead:
+    #
+    #  { :items => [4,5] }
+    #
+    # This method can also be useful for human-readable feed comparison if
+    # its output is dumped to YAML.
+    def diff(other, elements = self.class::ELEMENTS)
+      diffs = {}
+
+      elements.each do |element|
+        if other.respond_to?(element)
+          self_value = self.send(element)
+          other_value = other.send(element)
+
+          next if self_value == other_value
+
+          diffs[element] = if other_value.respond_to?(:diff)
+            self_value.diff(other_value)
+
+          elsif other_value.is_a?(Enumerable) && other_value.all?{|v| v.respond_to?(:diff)}
+
+            if self_value.size != other_value.size
+              [self_value.size, other_value.size]
+            else
+              enum_diffs = []
+              self_value.each_with_index do |val, index|
+                enum_diffs << val.diff(other_value[index], val.class::ELEMENTS)
+              end
+              enum_diffs.reject{|h| h.empty?}
+            end
+
+          else
+            [other_value, self_value] unless other_value == self_value
+          end
+        end
+      end
 
-    def initialize
-      @type = :text
+      diffs
     end
 
-    def to_s
-      body
-    end
+  end
 
-    def eql?(other)
-      self == (other)
-    end
+  module ElementCleaner
+    # Recursively cleans all elements in place.
+    #
+    # Only allow tags in whitelist. Always parse the html with a parser and delete
+    # all tags that arent on the list.
+    #
+    # For feed elements that can contain HTML:
+    # - feed.(title|description)
+    # - feed.entries[n].(title|description|content)
+    #
+    def clean!
+      self.class::SIMPLE_ELEMENTS.each do |element|
+        val = self.send(element)
 
-    # Equal if the body is the same. Ignores type.
-    def ==(other)
-      other.equal?(self) ||
-        (other.instance_of?(self.class) &&
-          self.body == other.body)
+        send("#{element}=", (val.is_a?(Array) ?
+          val.collect{|v| HtmlCleaner.flatten(v.to_s)} : HtmlCleaner.flatten(val.to_s)))
+      end
+
+      self.class::HTML_ELEMENTS.each do |element|
+        send("#{element}=", HtmlCleaner.clean(self.send(element).to_s))
+      end
+
+      self.class::BLENDED_ELEMENTS.each do |element|
+        self.send(element).collect{|v| v.clean!}
+      end
     end
   end
 
+
   # Represents a feed item entry.
   class Entry
-    include Singular, ElementEquality
+    include Singular, ElementEquality, ElementCleaner
+
+    HTML_ELEMENTS = [:content, :description, :title]
+    SIMPLE_ELEMENTS = [:date_published, :urls, :id, :authors, :copyright]
+    BLENDED_ELEMENTS = []
 
-    ELEMENTS = [:content, :date_published, :urls, :description, :title, :id, :authors, :copyright]
-    attr_accessor *ELEMENTS
+    ELEMENTS = HTML_ELEMENTS + SIMPLE_ELEMENTS + BLENDED_ELEMENTS
+
+    attr_accessor(*ELEMENTS)
 
     def initialize
       @urls = []
       @authors = []
-      @content = Content.new
     end
 
   end
 
   # Represents the root element of a feed.
   class Feed
-    include Singular, ElementEquality
+    include Singular, ElementEquality, ElementCleaner
+
+    # Elements that can contain HTML fragments.
+    HTML_ELEMENTS = [:title, :description]
 
-    ELEMENTS = [:title, :description, :id, :last_updated, :copyright, :authors, :urls, :image, :generator, :items]
-    attr_accessor *ELEMENTS
-    attr_accessor :parser
+    # Elements that contain 'plain' Strings, with HTML escaped.
+    SIMPLE_ELEMENTS = [:id, :last_updated, :copyright, :authors, :urls, :image, :generator]
+
+    # Elements that contain both HTML and escaped HTML.
+    BLENDED_ELEMENTS = [:items]
+
+    ELEMENTS = HTML_ELEMENTS + SIMPLE_ELEMENTS + BLENDED_ELEMENTS
+
+    attr_accessor(*ELEMENTS)
+    attr_accessor(:parser)
 
     alias :entries :items
 
@@ -95,6 +164,7 @@ module FeedNormalizer
     end
 
     def channel() self end
+
   end
 
 end

data/test/data/rss20diff.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<?xml-stylesheet title="XSL_formatting" type="text/xsl" href="/shared/bsp/xsl/rss/nolsol.xsl"?>
+<rss version="2.0">
+  <channel>
+    <title>diff</title>
+    <link>http://news.bbc.co.uk/go/rss/-/1/hi/technology/default.stm</link>
+    <description>Visit BBC News for up-to-the-minute news, breaking news, video, audio and feature stories. BBC News provides trusted World and UK news as well as local and regional perspectives. Also entertainment, business, science, technology and health news.</description>
+    <language>en-gb</language>
+    <lastBuildDate>Sat, 09 Sep 2006 14:57:06 GMT</lastBuildDate>
+    <copyright>Copyright: (C) British Broadcasting Corporation, see http://news.bbc.co.uk/1/hi/help/rss/4498287.stm for terms and conditions of reuse</copyright>
+    <docs>http://www.bbc.co.uk/syndication/</docs>
+    <ttl>15</ttl>
+
+    <image>
+      <title>BBC News</title>
+      <url>http://news.bbc.co.uk/nol/shared/img/bbc_news_120x60.gif</url>
+      <link>http://news.bbc.co.uk/go/rss/-/1/hi/technology/default.stm</link>
+    </image>
+
+    <item>
+      <title>diff</title>
+      <description>BBC Click investigates free security software and finds out who will protect PCs when Microsoft launches Vista.</description>
+      <link>http://news.bbc.co.uk/go/rss/-/1/hi/programmes/click_online/5326654.stm</link>
+      <guid isPermaLink="false">http://news.bbc.co.uk/1/hi/programmes/click_online/5326654.stm</guid>
+      <pubDate>Sat, 09 Sep 2006 12:45:35 GMT</pubDate>
+      <category>Click</category>
+    </item>
+
+    <item>
+      <title>diff</title>
+      <description>diff</description>
+      <link>diff</link>
+      <guid isPermaLink="false">http://news.bbc.co.uk/1/hi/technology/5328446.stm</guid>
+      <pubDate>Fri, 08 Sep 2006 16:18:08 GMT</pubDate>
+      <category>diff</category>
+    </item>
+
+    <item>
+      <title>MP3 player court order overturned</title>
+      <description>SanDisk puts its MP3 players back on display at a German electronics show after overturning a court injunction.</description>
+      <link>http://news.bbc.co.uk/go/rss/-/1/hi/technology/5326660.stm</link>
+      <guid isPermaLink="false">http://news.bbc.co.uk/1/hi/technology/5326660.stm</guid>
+      <pubDate>Fri, 08 Sep 2006 10:14:41 GMT</pubDate>
+      <category>Technology</category>
+    </item>
+
+  </channel>
+</rss>
+

data/test/data/rss20diff_short.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<?xml-stylesheet title="XSL_formatting" type="text/xsl" href="/shared/bsp/xsl/rss/nolsol.xsl"?>
+<rss version="2.0">
+  <channel>
+    <title>diff</title>
+    <link>http://news.bbc.co.uk/go/rss/-/1/hi/technology/default.stm</link>
+    <description>Visit BBC News for up-to-the-minute news, breaking news, video, audio and feature stories. BBC News provides trusted World and UK news as well as local and regional perspectives. Also entertainment, business, science, technology and health news.</description>
+    <language>en-gb</language>
+    <lastBuildDate>Sat, 09 Sep 2006 14:57:06 GMT</lastBuildDate>
+    <copyright>Copyright: (C) British Broadcasting Corporation, see http://news.bbc.co.uk/1/hi/help/rss/4498287.stm for terms and conditions of reuse</copyright>
+    <docs>http://www.bbc.co.uk/syndication/</docs>
+    <ttl>15</ttl>
+
+    <image>
+      <title>BBC News</title>
+      <url>http://news.bbc.co.uk/nol/shared/img/bbc_news_120x60.gif</url>
+      <link>http://news.bbc.co.uk/go/rss/-/1/hi/technology/default.stm</link>
+    </image>
+
+    <item>
+      <title>diff</title>
+      <description>BBC Click investigates free security software and finds out who will protect PCs when Microsoft launches Vista.</description>
+      <link>http://news.bbc.co.uk/go/rss/-/1/hi/programmes/click_online/5326654.stm</link>
+      <guid isPermaLink="false">http://news.bbc.co.uk/1/hi/programmes/click_online/5326654.stm</guid>
+      <pubDate>Sat, 09 Sep 2006 12:45:35 GMT</pubDate>
+      <category>Click</category>
+    </item>
+
+    <item>
+      <title>diff</title>
+      <description>A Japanese scientist who invented a sustainable form of light is awarded the Millennium Technology Prize.</description>
+      <link>diff</link>
+      <guid isPermaLink="false">http://news.bbc.co.uk/1/hi/technology/5328446.stm</guid>
+      <pubDate>Fri, 08 Sep 2006 16:18:08 GMT</pubDate>
+      <category>diff</category>
+    </item>
+
+  </channel>
+</rss>
+

data/test/test_all.rb

@@ -0,0 +1,6 @@
+$:.unshift(File.dirname(__FILE__))
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require 'test_htmlcleaner'
+require 'test_feednormalizer'
+

data/test/test_feednormalizer.rb

@@ -0,0 +1,137 @@
+require 'test/unit'
+require 'feed-normalizer'
+
+include FeedNormalizer
+
+class FeedNormalizerTest < Test::Unit::TestCase
+
+  XML_FILES = {}
+
+  data_dir = File.dirname(__FILE__) + '/data'
+
+  # Load up the xml files
+  Dir.open(data_dir).each do |fn|
+    next unless fn =~ /[.]xml$/
+    XML_FILES[fn.scan(/(.*)[.]/).to_s.to_sym] = File.read(data_dir + "/#{fn}")
+  end
+
+  def test_basic_parse
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20])
+  end
+
+  def test_force_parser
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20],
+      :force_parser => RubyRssParser, :try_others => true)
+  end
+
+  def test_force_parser_exclusive
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20],
+      :force_parser => RubyRssParser, :try_others => false)
+  end
+
+  def test_ruby_rss_parser
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20],
+      :force_parser => RubyRssParser, :try_others => false)
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rdf10],
+      :force_parser => RubyRssParser, :try_others => false)
+  end
+
+  def test_simple_rss_parser
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20],
+      :force_parser => SimpleRssParser, :try_others => false)
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10],
+      :force_parser => SimpleRssParser, :try_others => false)
+  end
+
+  def test_parser_failover_order
+    assert_equal SimpleRSS, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10], :force_parser => RubyRssParser).parser
+  end
+
+  def test_force_parser_fail
+    assert_nil FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10], :force_parser => RubyRssParser, :try_others => false)
+  end
+
+  def test_all_parsers_fail
+    assert_nil FeedNormalizer::FeedNormalizer.parse("This isn't RSS or Atom!")
+  end
+
+  def test_correct_parser_used
+    assert_equal RSS::Parser, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20]).parser
+    assert_equal SimpleRSS, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10]).parser
+  end
+
+  def test_rss
+    feed = FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20])
+
+    assert_equal "BBC News | Technology | UK Edition", feed.title
+    assert_equal ["http://news.bbc.co.uk/go/rss/-/1/hi/technology/default.stm"], feed.urls
+    assert_equal "MP3 player court order overturned", feed.entries.last.title
+    assert_equal "SanDisk puts its MP3 players back on display at a German electronics show after overturning a court injunction.", feed.entries.last.description
+    assert_equal "SanDisk puts its MP3 players back on display at a German electronics show after overturning a court injunction.", feed.entries.last.content
+  end
+
+  def test_simplerss
+    feed = FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
+
+    assert_equal "~:caboose", feed.title
+    assert_equal "http://habtm.com/xml/atom10/feed.xml", feed.url
+    assert_equal "Starfish - Easy Distribution of Site Maintenance", feed.entries.last.title
+    assert_equal "urn:uuid:6c028f36-f87a-4f53-b7e3-1f943d2341f0", feed.entries.last.id
+
+    assert !feed.entries.last.description.include?("google fame")
+    assert feed.entries.last.content.include?("google fame")
+  end
+
+  def test_sanity_check
+    XML_FILES.keys.each do |xml_file|
+      feed = FeedNormalizer::FeedNormalizer.parse(XML_FILES[xml_file])
+
+      assert [feed.title, feed.url, feed.entries.first.url].collect{|e| e.is_a?(String)}.all?, "Not everything was a String in #{xml_file}"
+      assert [feed.parser, feed.class].collect{|e| e.is_a?(Class)}.all?
+    end
+  end
+
+  def test_feed_equality
+    assert_equal FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20]), FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20])
+    assert_equal FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10]), FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
+    assert_not_equal FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom03]), FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
+    assert_not_equal FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20]), FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
+    assert_not_equal FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20]), FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20diff])
+  end
+
+  def test_feed_diff
+    feed = FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20])
+
+    diff = feed.diff(FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20diff]))
+    diff_short = feed.diff(FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20diff_short]))
+    no_diff = feed.diff(feed)
+
+    assert diff.keys.all? {|key| [:title, :items].include?(key)}
+    assert_equal 2, diff[:items].size
+
+    assert diff_short.keys.all? {|key| [:title, :items].include?(key)}
+    assert_equal [3,2], diff_short[:items]
+
+    assert no_diff.empty?
+  end
+
+  def test_marshal
+    feed = FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20])
+
+    assert_nothing_raised { Marshal.load(Marshal.dump(feed)) }
+  end
+
+  def test_method_missing
+    assert_raise(NoMethodError) { Feed.new(nil).nonexistant }
+  end
+
+  def test_clean
+    feed = FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
+
+    assert feed.entries.first.content !~ /\<p\>/
+    feed.clean!
+    assert feed.entries.first.content =~ /\<p\>/
+  end
+
+end
+

data/test/test_htmlcleaner.rb

@@ -0,0 +1,151 @@
+require 'test/unit'
+require 'html-cleaner'
+
+include FeedNormalizer
+
+class HtmlCleanerTest < Test::Unit::TestCase
+
+  def test_unescape
+    assert_equal "' ' &deg;", FeedNormalizer::HtmlCleaner.unescapeHTML("&apos; &#39; &deg;")
+    assert_equal "\" &deg;", FeedNormalizer::HtmlCleaner.unescapeHTML("&quot; &deg;")
+    assert_equal "\"\"\"\"", FeedNormalizer::HtmlCleaner.unescapeHTML("&#34;&#000000000000000000034;&#x22;&#x0000022;")
+    assert_equal "heavily subnet&#8217;d network,", FeedNormalizer::HtmlCleaner.unescapeHTML("heavily subnet&#8217;d network,")
+  end
+
+  def test_add_entities
+    assert_equal "x &gt; y", HtmlCleaner.add_entities("x > y")
+    assert_equal "1 &amp; 2", HtmlCleaner.add_entities("1 & 2")
+    assert_equal "&amp; &#123; &acute; &#x123;", HtmlCleaner.add_entities("& &#123; &acute; &#x123;")
+    assert_equal "&amp; &#123; &ACUTE; &#X123A; &#x80f;", HtmlCleaner.add_entities("& &#123; &ACUTE; &#X123A; &#x80f;")
+    assert_equal "heavily subnet&#8217;d network,", HtmlCleaner.add_entities("heavily subnet&#8217;d network,")
+  end
+
+  def test_html_clean
+    assert_equal "", HtmlCleaner.clean("")
+
+    assert_equal "<p>foo &gt; *</p>", HtmlCleaner.clean("<p>foo > *</p>")
+    assert_equal "<p>foo &gt; *</p>", HtmlCleaner.clean("<p>foo &gt; *</p>")
+
+    assert_equal "<p>para</p>", HtmlCleaner.clean("<p foo=bar>para</p>")
+    assert_equal "<p>para</p> outsider", HtmlCleaner.clean("<p foo=bar>para</p> outsider")
+
+    assert_equal "<p>para</p>", HtmlCleaner.clean("<p>para</p></notvalid>")
+    assert_equal "<p>para</p>", HtmlCleaner.clean("<p>para</p></body>")
+
+    assert_equal "<p>para</p>", HtmlCleaner.clean("<p>para</p><plaintext>")
+    assert_equal "<p>para</p>", HtmlCleaner.clean("<p>para</p><object><param></param></object>")
+    assert_equal "<p>para</p>", HtmlCleaner.clean("<p>para</p><iframe src='http://evil.example.org'></iframe>")
+    assert_equal "<p>para</p>", HtmlCleaner.clean("<p>para</p><iframe src='http://evil.example.org'>")
+
+    assert_equal "<p>para</p>", HtmlCleaner.clean("<p>para</p><invalid>invalid</invalid>")
+
+    assert_equal "<a href=\"http://example.org\">para</a>", HtmlCleaner.clean("<a href='http://example.org'>para</a>")
+    assert_equal "<a href=\"http://example.org/proc?a&amp;b\">para</a>", HtmlCleaner.clean("<a href='http://example.org/proc?a&b'>para</a>")
+
+    assert_equal "<p>two</p>", HtmlCleaner.clean("<p>para</p><body><p>two</p></body>")
+    assert_equal "<p>two</p>", HtmlCleaner.clean("<p>para</p><body><p>two</p>")
+    assert_equal "<p>para</p>&lt;bo /dy&gt;<p>two</p>", HtmlCleaner.clean("<p>para</p><bo /dy><p>two</p></body>")
+    assert_equal "<p>para</p>&lt;bo\\/dy&gt;<p>two</p>", HtmlCleaner.clean("<p>para</p><bo\\/dy><p>two</p></body>")
+    assert_equal "<p>para</p><p>two</p>", HtmlCleaner.clean("<p>para</p><body/><p>two</p></body>")
+
+    assert_equal "<p>one &amp; two</p>", HtmlCleaner.clean(HtmlCleaner.clean("<p>one & two</p>"))
+
+    assert_equal "<p id=\"p\">para</p>", HtmlCleaner.clean("<p id=\"p\" ignore=\"this\">para</p>")
+    assert_equal "<p id=\"p\">para</p>", HtmlCleaner.clean("<p id=\"p\" onclick=\"this\">para</p>")
+
+    assert_equal "<img src=\"http://example.org/pic\" />", HtmlCleaner.clean("<img src=\"http://example.org/pic\" />")
+    assert_equal "<img />", HtmlCleaner.clean("<img src=\"jav a script:call()\" />")
+
+    assert_equal "what's new", HtmlCleaner.clean("what&#000039;s new")
+    assert_equal "&quot;what's new?&quot;", HtmlCleaner.clean("\"what&apos;s new?\"")
+    assert_equal "&quot;what's new?&quot;", HtmlCleaner.clean("&quot;what&apos;s new?&quot;")
+
+    # Real-world examples from selected feeds
+    assert_equal "I have a heavily subnet&#8217;d/vlan&#8217;d network,", HtmlCleaner.clean("I have a heavily subnet&#8217;d/vlan&#8217;d network,")
+
+    assert_equal "<pre><blockquote>&lt;%= start_form_tag :action =&gt; &quot;create&quot; %&gt;</blockquote></pre>",
+                 HtmlCleaner.clean("<pre><blockquote>&lt;%= start_form_tag :action => \"create\" %></blockquote></pre>")
+
+    assert_equal "<a href=\"http://www.mcall.com/news/local/all-smashedmachine1107-cn,0,1574203.story?coll=all-news-hed\">[link]</a><a href=\"http://reddit.com/info/pyhc/comments\">[more]</a>",
+                 HtmlCleaner.clean("&lt;a href=\"http://www.mcall.com/news/local/all-smashedmachine1107-cn,0,1574203.story?coll=all-news-hed\"&gt;[link]&lt;/a&gt;&lt;a href=\"http://reddit.com/info/pyhc/comments\"&gt;[more]&lt;/a&gt;")
+
+
+    # Various exploits from the past
+    assert_equal "", HtmlCleaner.clean("<_img foo=\"<IFRAME width='80%' height='400' src='http://alive.znep.com/~marcs/passport/grabit.html'></IFRAME>\" >")
+    assert_equal "<a href=\"https://bugzilla.mozilla.org/attachment.cgi?id=&amp;action=force_internal_error&lt;script&gt;alert(document.cookie)&lt;/script&gt;\">link</a>",
+                 HtmlCleaner.clean("<a href=\"https://bugzilla.mozilla.org/attachment.cgi?id=&action=force_internal_error<script>alert(document.cookie)</script>\">link</a>")
+    assert_equal "<img src=\"doesntexist.jpg\" />", HtmlCleaner.clean("<img src='doesntexist.jpg' onerror='alert(document.cookie)'/>")
+    assert_equal "<img src=\"'doesntexist.jpg\" />", HtmlCleaner.clean("<img src=\"'doesntexist.jpg\" onmouseover=\"alert('img-ob-11');''\"/>")
+    assert_equal "&lt;IMG &quot;&quot;&quot;&gt;&quot;&gt;", HtmlCleaner.clean("<IMG \"\"\"><SCRIPT>alert(\"XSS\")</SCRIPT>\">")
+
+    # This doesnt come out as I would like, but the result is still safe.
+    # (Apparently, this would work in Gecko.)
+    assert HtmlCleaner.clean("<p onclick!\#$%&()*~+-_.,:;?@[/|\\]^=alert(\"XSS\")>para</p>") !~ /\<\>/
+    assert_equal "&lt;SCRIPT/XSS SRC=&quot;http://ha.ckers.org/xss.js&quot;&gt;", HtmlCleaner.clean("<SCRIPT/XSS SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT>")
+
+    assert_equal "", HtmlCleaner.clean("<!--[if gte IE 4]><SCRIPT>alert('XSS');</SCRIPT><![endif]-->")
+    assert_equal "<p></p>", HtmlCleaner.clean("<p><!--[if gte IE 4]><SCRIPT>alert('XSS');</SCRIPT><![endif]--></p>")
+    assert_equal "<p>hi</p><p></p>", HtmlCleaner.clean("<p>hi</p><p><!--[if gte IE 4]><SCRIPT>alert('XSS');</SCRIPT><![endif]--></p>")
+  end
+
+  def test_html_flatten
+    assert_equal "", HtmlCleaner.flatten("")
+
+    assert_equal "hello", HtmlCleaner.flatten("hello")
+    assert_equal "hello world", HtmlCleaner.flatten("hello\nworld")
+
+    assert_equal "A &gt; B : C", HtmlCleaner.flatten("A > B : C")
+    assert_equal "what's new", HtmlCleaner.flatten("what&#39;s new")
+    assert_equal "&quot;what's new?&quot;", HtmlCleaner.flatten("\"what&apos;s new?\"")
+
+    assert_equal "we&#8217;ve got &lt;a hre", HtmlCleaner.flatten("we&#8217;ve got <a hre")
+
+    assert_equal "http://example.org", HtmlCleaner.flatten("http://example.org")
+    assert_equal "http://example.org/proc?a&amp;b", HtmlCleaner.flatten("http://example.org/proc?a&b")
+
+    assert_equal "&quot;what's new?&quot;", HtmlCleaner.flatten(HtmlCleaner.flatten("\"what&apos;s new?\""))
+  end
+
+  def test_dodgy_uri
+    # All of these javascript urls work in IE6.
+    assert HtmlCleaner.dodgy_uri?("javascript:alert('HI');")
+    assert HtmlCleaner.dodgy_uri?(" &#106;&#97;&#118;&#97;&#115;&#99;&#114;&#105;&#112;&#116; \n :alert('HI');")
+    assert HtmlCleaner.dodgy_uri?("JaVaScRiPt:alert('HI');")
+    assert HtmlCleaner.dodgy_uri?("JaV   \naSc\nRiPt:alert('HI');")
+
+    # entities lacking ending ';'
+    # This only works if they're all packed together without spacing.
+    assert HtmlCleaner.dodgy_uri?("&#106&#97&#118&#97&#115&#99&#114&#105&#112&#116&#58&#97&#108&#101&#114&#116&#40&#39&#105&#109&#103&#45&#111&#98&#45&#50&#39&#41")
+    assert HtmlCleaner.dodgy_uri?("&#106&#97&#118&#97&#115&#99&#114&#105&#112&#116&#58&#97&#108&#101&#114&#116&#40&#39&#105&#109&#103&#45&#111&#98&#45&#50&#39 &#41 ; ")
+    # catch extra spacing anyway.. support for this is possible, depending where the spaces are.
+    assert HtmlCleaner.dodgy_uri?("&#106 &#97 &#118 &#97 &#115 &#99 &#114 &#105 &#112 &#116 &#58 &#97 &#108 &#101 &#114 &#116 &#40 &#39 &#105 &#109 &#103 &#45 &#111 &#98 &#45 &#50 &#39 &#41 ; ")
+    assert HtmlCleaner.dodgy_uri?("&#x06a &#97 &#118 &#97 &#115 &#99 &#114 &#105 &#112 &#116 &#58 &#97 &#108 &#101 &#114 &#116 &#40 &#39 &#105 &#109 &#103 &#45 &#111 &#98 &#45 &#50 &#39 &#41 ; ")
+    assert HtmlCleaner.dodgy_uri?("&#106avascript")
+    assert HtmlCleaner.dodgy_uri?("&#x06a;avascript")
+
+    # url-encoded
+    assert HtmlCleaner.dodgy_uri?("%6A%61%76%61%73%63%72%69%70%74%3A%61%6C%65%72%74%28%27%69%6D%67%2D%6F%62%2D%33%27%29")
+
+    # Other evil schemes
+    assert HtmlCleaner.dodgy_uri?("vbscript:MsgBox(\"hi\")")
+    assert HtmlCleaner.dodgy_uri?("mocha:alert('hi')")
+    assert HtmlCleaner.dodgy_uri?("livescript:alert('hi')")
+    assert HtmlCleaner.dodgy_uri?("data:text/html;base64,PHNjcmlwdD5hbGVydCgnWFNTJyk8L3NjcmlwdD4K")
+
+    # Various non-printing chars
+    assert HtmlCleaner.dodgy_uri?("javas\0cript:foo()")
+    assert HtmlCleaner.dodgy_uri?(" &#14; javascript:foo()")
+    assert HtmlCleaner.dodgy_uri?("jav&#x0A;ascript:foo()")
+    assert HtmlCleaner.dodgy_uri?("jav&#x09;ascript:foo()")
+    assert HtmlCleaner.dodgy_uri?("jav\tascript:foo()")
+
+    # The Good
+    assert_nil HtmlCleaner.dodgy_uri?("http://example.org")
+    assert_nil HtmlCleaner.dodgy_uri?("http://example.org/foo.html")
+    assert_nil HtmlCleaner.dodgy_uri?("http://example.org/foo.cgi?x=y&a=b")
+    assert_nil HtmlCleaner.dodgy_uri?("http://example.org/foo.cgi?x=y&amp;a=b")
+    assert_nil HtmlCleaner.dodgy_uri?("http://example.org/foo.cgi?x=y&#38;a=b")
+    assert_nil HtmlCleaner.dodgy_uri?("http://example.org/foo.cgi?x=y&#x56;a=b")
+  end
+
+end

metadata

@@ -3,16 +3,16 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: feed-normalizer
 version: !ruby/object:Gem::Version 
-  version: 1.1.0
-date: 2006-10-05 00:00:00 -07:00
+  version: 1.2.0
+date: 2006-11-29 00:00:00 -08:00
 summary: Extensible Ruby wrapper for Atom and RSS parsers
 require_paths: 
 - lib
 email: andy@tinnedfruit.org
-homepage: http://code.google.com/p/feed-normalizer/
-rubyforge_project: 
-description: 
-autorequire: feed-normalizer
+homepage: http://feed-normalizer.rubyforge.org/
+rubyforge_project: feed-normalizer
+description: An extensible Ruby wrapper for Atom and RSS parsers.  Feed normalizer wraps various RSS and Atom parsers, and returns a single unified object graph, regardless of the underlying feed format.
+autorequire: 
 default_executable: 
 bindir: bin
 has_rdoc: true
@@ -28,23 +28,27 @@ cert_chain:
 authors: 
 - Andrew A. Smith
 files: 
+- History.txt
+- License.txt
+- Manifest.txt
+- Rakefile
+- Readme.txt
 - lib/feed-normalizer.rb
-- lib/parsers
-- lib/structures.rb
+- lib/html-cleaner.rb
 - lib/parsers/rss.rb
 - lib/parsers/simple-rss.rb
-- test/base_test.rb
-- test/data
+- lib/structures.rb
 - test/data/atom03.xml
 - test/data/atom10.xml
 - test/data/rdf10.xml
 - test/data/rss20.xml
-- LICENSE
-- Rakefile
-- README
-- RELEASE
-test_files: []
-
+- test/data/rss20diff.xml
+- test/data/rss20diff_short.xml
+- test/test_all.rb
+- test/test_feednormalizer.rb
+- test/test_htmlcleaner.rb
+test_files: 
+- test/test_all.rb
 rdoc_options: []
 
 extra_rdoc_files: []
@@ -56,6 +60,15 @@ extensions: []
 requirements: []
 
 dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.6
+    version: 
 - !ruby/object:Gem::Dependency 
   name: simple-rss
   version_requirement: 
@@ -65,3 +78,12 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "1.1"
     version: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0.4"
+    version: 

data/RELEASE

@@ -1,13 +0,0 @@
-Steps to create a release:
-
- o Update Rakefile version
- o Create package
-    rake clobber
-    rake package
- o Tag release
-    svn copy https://feed-normalizer.googlecode.com/svn/trunk \
-             https://feed-normalizer.googlecode.com/svn/tags/RELEASE_[MAJOR]_[MINOR]_[REVISION]
- o Upload gem and zip to rubyforge
- o Update RAA
- o Post rubyforge news
-

data/test/base_test.rb

@@ -1,82 +0,0 @@
-$:.unshift(File.dirname(__FILE__) + '/../lib')
-
-require 'test/unit'
-require 'feed-normalizer'
-
-include FeedNormalizer
-
-class BaseTest < Test::Unit::TestCase
-
-  XML_FILES = {}
-
-  def setup
-    data_dir = File.dirname(__FILE__) + '/data'
-
-    # Load up the xml files
-    Dir.open(data_dir).each do |fn|
-      next unless fn =~ /[.]xml$/
-      XML_FILES[fn.scan(/(.*)[.]/).to_s.to_sym] = File.read(data_dir + "/#{fn}")
-    end
-  end
-
-
-  def test_basic_parse
-    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20])
-  end
-
-  def test_force_parser
-    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20], RubyRssParser, true)
-  end
-
-  def test_force_parser_exclusive
-    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20], RubyRssParser, false)
-  end
-
-  def test_ruby_rss_parser
-    assert_kind_of Feed, feed=FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20], RubyRssParser, false)
-    assert_kind_of Feed, feed=FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rdf10], RubyRssParser, false)
-  end
-
-  def test_simple_rss_parser
-    assert_kind_of Feed, feed=FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20], SimpleRssParser, false)
-    assert_kind_of Feed, feed=FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10], SimpleRssParser, false)
-  end
-
-  # Attempts to parse a feed that Ruby's RSS can't handle.
-  # SimpleRSS should provide the parsed feed.
-  def test_parser_failover_order
-    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
-  end
-
-  def test_all_parsers_fail
-    assert_nil FeedNormalizer::FeedNormalizer.parse("This isn't RSS or Atom!")
-  end
-
-  def test_correct_parser_used
-    assert_equal RSS::Parser, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20]).parser
-    assert_equal SimpleRSS, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10]).parser
-  end
-
-  def test_sanity_check
-    XML_FILES.keys.each do |xml_file|
-      feed = FeedNormalizer::FeedNormalizer.parse(XML_FILES[xml_file])
-
-      assert [feed.title, feed.url, feed.entries.first.url].collect{|e| e.is_a?(String)}.all?, "Not everything was a String in #{xml_file}"
-      assert [feed.parser, feed.class].collect{|e| e.is_a?(Class)}.all?
-    end
-  end
-
-  def test_feed_equality
-    assert_equal FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20]), FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20])
-    assert_equal FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10]), FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
-    assert_not_equal FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom03]), FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
-    assert_not_equal FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20]), FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
-
-    XML_FILES.keys.each do |xml_file|
-      feed = FeedNormalizer::FeedNormalizer.parse(XML_FILES[xml_file])
-      assert_equal feed, Marshal.load(Marshal.dump(feed))
-    end
-
-  end
-
-end