feed-normalizer 1.0.0

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2006, Andrew A. Smith
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+    * Neither the name of the copyright owner nor the names of its contributors
+      may be used to endorse or promote products derived from this software
+      without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README

@@ -0,0 +1,52 @@
+== Feed Normalizer
+
+An extensible Ruby wrapper for Atom and RSS parsers.
+
+Feed normalizer wraps various RSS and Atom parsers, and returns a single unified
+object graph, regardless of the underlying feed format.
+
+== Download
+
+* gem install feed-normalizer
+* http://rubyforge.org/projects/feed-normalizer
+* svn co http://feed-normalizer.googlecode.com/svn/trunk
+
+== Usage
+
+    require 'feed-normalizer'
+    require 'open-uri'
+
+    feed = FeedNormalizer::FeedNormalizer.parse open('http://www.iht.com/rss/frontpage.xml')
+
+    feed.title # => "International Herald Tribune"
+    feed.url # => "http://www.iht.com/pages/index.php"
+    feed.entries.first.url # => "http://www.iht.com/articles/2006/10/03/frontpage/web.1003UN.php"
+
+    feed.class # => FeedNormalizer::Feed
+    feed.parser # => RSS::Parser
+
+Now read an Atom feed, and the same class is returned, and the same terminology applies:
+
+    feed = FeedNormalizer::FeedNormalizer.parse open('http://www.atomenabled.org/atom.xml')
+
+    feed.title # => "AtomEnabled.org"
+    feed.url # => "http://www.atomenabled.org/atom.xml"
+    feed.entries.first.url # => "http://www.atomenabled.org/2006/09/moving-toward-atom.php"
+
+The feed representation stays the same, even though a different parser was used.
+
+    feed.class # => FeedNormalizer::Feed
+    feed.parser # => SimpleRSS
+
+== Extending
+
+Implement a parser wrapper by extending the FeedNormalizer::Parser class and overriding
+the public methods. Also note the helper methods in the root Parser object to make
+mapping of output from the particular parser to the Feed object easier.
+
+See FeedNormalizer::RubyRssParser and FeedNormalizer::SimpleRssParser for examples.
+
+== Authors
+* Andrew A. Smith (andy@tinnedfruit.org)
+
+This library is released under the terms of the BSD License (see the LICENSE file for details).

data/Rakefile

@@ -0,0 +1,49 @@
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/clean'
+require 'rake/gempackagetask'
+
+PKG_FILES = FileList[
+    "lib/**/*", "test/**/*", "[A-Z]*", "Rakefile", "html/**/*"
+]
+
+Gem::manage_gems
+
+task :default => [:test]
+task :package => [:test, :doc]
+
+spec = Gem::Specification.new do |s|
+  s.name = "feed-normalizer"
+  s.version = "1.0.0"
+  s.author = "Andrew A. Smith"
+  s.email = "andy@tinnedfruit.org"
+  s.homepage = "http://code.google.com/p/feed-normalizer/"
+  s.platform = Gem::Platform::RUBY
+  s.summary = "Extensible Ruby wrapper for Atom and RSS parsers"
+  s.files =  PKG_FILES
+  s.require_path = "lib"
+  s.autorequire = "feed-normalizer"
+  s.has_rdoc = true
+  s.add_dependency  "simple-rss", ">= 1.1"
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.need_zip = true
+end
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+end
+
+desc "Create documentation"
+Rake::RDocTask.new("doc") do |rdoc|
+  rdoc.title = "Feed Normalizer"
+  rdoc.rdoc_dir = 'doc'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/lib/feed-normalizer.rb

@@ -0,0 +1,128 @@
+require 'structures'
+
+module FeedNormalizer
+
+  # The root parser object. Every parser must extend this object.
+  class Parser
+
+    # Parser being used.
+    def self.parser
+      nil
+    end
+
+    # Parses the given feed, and returns a normalized representation.
+    # Returns nil if the feed could not be parsed.
+    def self.parse(feed)
+      nil
+    end
+
+    # Returns a number to indicate parser priority.
+    # The lower the number, the more likely the parser will be used first,
+    # and vice-versa.
+    def self.priority
+      0
+    end
+
+    protected
+
+    # Some utility methods that can be used by subclasses.
+
+    # sets value, or appends to an existing value
+    def self.map_functions!(mapping, src, dest)
+
+      mapping.each do |dest_function, src_functions|
+        src_functions = [src_functions].flatten # pack into array
+
+        src_functions.each do |src_function|
+          value = if src.respond_to?(src_function)
+            src.send(src_function)
+          elsif src.respond_to?(:has_key?)
+            src[src_function]
+          end
+
+          append_or_set!(value, dest, dest_function) if value
+        end
+
+      end
+    end
+
+    def self.append_or_set!(value, object, object_function)
+      if object.send(object_function).respond_to? :push
+        object.send(object_function).push(value)
+      else
+        object.send(:"#{object_function}=", value)
+      end
+    end
+
+    private
+
+    # Callback that ensures that every parser gets registered.
+    def self.inherited(subclass)
+      ParserRegistry.register(subclass)
+    end
+
+  end
+
+
+  # The parser registry keeps a list of current parsers that are available.
+  class ParserRegistry
+
+    @@parsers = []
+
+    def self.register(parser)
+      @@parsers << parser
+    end
+
+    # Returns a list of currently registered parsers, in order of priority.
+    def self.parsers
+      @@parsers.sort_by { |parser| parser.priority }
+    end
+
+  end
+
+
+  class FeedNormalizer
+
+    # Parses the given xml and attempts to return a normalized Feed object.
+    # Setting forced parser to a suitable parser will mean that parser is
+    # used first, and if try_others is false, it is the only parser used,
+    # otherwise all parsers in the ParserRegistry are attempted next, in
+    # order of priority.
+    def self.parse(xml, forced_parser=nil, try_others=false)
+
+      # Get a string ASAP, as multiple read()'s will start returning nil..
+      xml = xml.respond_to?(:read) ? xml.read : xml.to_s
+
+      if forced_parser
+        result = forced_parser.parse(xml)
+
+        if result
+          return result
+        elsif !try_others
+          return nil
+        else
+          # fall through and continue with other parsers
+        end
+      end
+
+      ParserRegistry.parsers.each do |parser|
+        result = parser.parse(xml)
+        return result if result
+      end
+
+      # if we got here, no parsers worked.
+      return nil
+    end
+  end
+
+
+  parser_dir = File.dirname(__FILE__) + '/parsers'
+
+  # Load up the parsers
+  Dir.open(parser_dir).each do |fn|
+    next unless fn =~ /[.]rb$/
+    require "parsers/#{fn}"
+  end
+
+end
+

data/lib/parsers/rss.rb

@@ -0,0 +1,74 @@
+require 'rss'
+
+module FeedNormalizer
+  class RubyRssParser < Parser
+
+    def self.parser
+      RSS::Parser
+    end
+
+    def self.parse(xml)
+      begin
+        rss = parser.parse(xml)
+      rescue Exception => e
+        #puts "Parser #{parser} failed because #{e.message.gsub("\n",', ')}"
+        return nil
+      end
+
+      rss ? package(rss) : nil
+    end
+
+    # Fairly high priority; a fast and strict parser.
+    def self.priority
+      100
+    end
+
+    protected
+
+    def self.package(rss)
+      feed = Feed.new(self)
+
+      # channel elements
+      feed_mapping = {
+        :generator => :generator,
+        :title => :title,
+        :urls => :link,
+        :description => :description,
+        :copyright => :copyright,
+        :authors => :managingEditor,
+        :last_updated => [:lastBuildDate, :pubDate],
+        :id => :guid
+      }
+
+      map_functions!(feed_mapping, rss.channel, feed)
+
+      # custom channel elements
+      feed.image = (rss.channel.image ? rss.channel.image.url : nil)
+
+      # item elements
+      item_mapping = {
+        :date_published => :pubDate,
+        :urls => :link,
+        :description => :description,
+        :title => :title,
+        :authors => :author
+      }
+
+      rss.channel.items.each do |rss_item|
+        feed_entry = Entry.new
+        map_functions!(item_mapping, rss_item, feed_entry)
+
+        # custom item elements
+        feed_entry.id = rss_item.guid.content
+        feed_entry.content.body = rss_item.description
+        feed_entry.copyright = rss.channel.copyright
+
+        feed.entries << feed_entry
+      end
+
+      feed
+    end
+
+  end
+end
+

data/lib/parsers/simple-rss.rb

@@ -0,0 +1,100 @@
+require 'simple-rss'
+
+module FeedNormalizer
+
+  # The SimpleRSS parser can handle both RSS and Atom feeds.
+  class SimpleRssParser < Parser
+
+    def self.parser
+      SimpleRSS
+    end
+
+    def self.parse(xml)
+      begin
+        atomrss = parser.parse(xml)
+      rescue Exception => e
+        #puts "Parser #{parser} failed because #{e.message.gsub("\n",', ')}"
+        return nil
+      end
+
+      package(atomrss)
+    end
+
+    # Fairly low priority; a slower, liberal parser.
+    def self.priority
+      900
+    end
+
+    protected
+
+    def self.package(atomrss)
+      feed = Feed.new(self)
+
+      # root elements
+      feed_mapping = {
+        :generator => :generator,
+        :title => :title,
+        :last_updated => [:updated, :lastBuildDate, :pubDate],
+        :copyright => [:copyright, :rights],
+        :authors => [:author, :webMaster, :managingEditor, :contributor],
+        :urls => :link,
+        :description => [:description, :subtitle]
+      }
+
+      map_functions!(feed_mapping, atomrss, feed)
+
+      # custom channel elements
+      feed.id = feed_id(atomrss)
+      feed.image = image(atomrss)
+
+
+      # entry elements
+      entry_mapping = {
+        :date_published => [:pubDate, :published],
+        :urls => :link,
+        :description => [:description, :summary],
+        :title => :title,
+        :authors => [:author, :contributor]
+      }
+
+      atomrss.entries.each do |atomrss_entry|
+        feed_entry = Entry.new
+        map_functions!(entry_mapping, atomrss_entry, feed_entry)
+
+        # custom entry elements
+        feed_entry.id = atomrss_entry.guid || atomrss_entry[:id] # entries are a Hash..
+        feed_entry.copyright = atomrss_entry.copyright || (atomrss.respond_to?(:copyright) ? atomrss.copyright : nil)
+        feed_entry.content.body = atomrss_entry.content || atomrss_entry.description
+
+        feed.entries << feed_entry
+      end
+
+      feed
+    end
+
+    def self.image(parser)
+      if parser.respond_to?(:image) && parser.image
+        if parser.image.match /<url>/ # RSS image contains an <url> spec
+          parser.image.scan(/<url>(.*)<\/url>/).to_s
+        else
+          parser.image # Atom contains just the url
+        end
+      elsif parser.respond_to?(:logo) && parser.logo
+        parser.logo
+      end
+    end
+
+    def self.feed_id(parser)
+      overridden_value(parser, :id) || "#{parser.link}"
+    end
+
+    # gets the value returned from the method if it overriden, otherwise nil.
+    def self.overridden_value(object, method)
+      # XXX: hack to find out if the id method is overriden
+      # Highly dependent upon Method's to_s :(
+      object.id if object.method(:id).to_s.match /SimpleRSS\#/
+    end
+
+  end
+end
+

data/lib/structures.rb

@@ -0,0 +1,74 @@
+
+module FeedNormalizer
+
+  module Singular
+
+    # If the method being called is a singular (in this simple case, does not
+    # end with an 's'), then it calls the plural method, and calls the first
+    # element. We're assuming that plural methods provide an array.
+    #
+    # Example:
+    # Object contains an array called 'alphas', which looks like [:a, :b, :c].
+    # Call object.alpha and :a is returned.
+    def method_missing(name)
+      if name.to_s =~ /[^s]$/ # doesnt end with 's'
+        plural = :"#{name}s"
+        if self.respond_to?(plural)
+          return self.send(plural).first
+        end
+      end
+      nil
+    end
+  end
+
+  # Wraps content used in an Entry. type defaults to :text.
+  class Content
+    TYPE = [:text, :html, :xhtml]
+    attr_accessor :type, :body
+
+    def initialize
+      @type = :text
+    end
+
+    def to_s
+      body
+    end
+  end
+
+  # Represents a feed item entry.
+  class Entry
+    include Singular
+
+    ELEMENTS = [:content, :date_published, :urls, :description, :title, :id, :authors, :copyright]
+    attr_accessor *ELEMENTS
+
+    def initialize
+      @urls = []
+      @authors = []
+      @content = Content.new
+    end
+  end
+
+  # Represents the root element of a feed.
+  class Feed
+    include Singular
+
+    ELEMENTS = [:title, :description, :id, :last_updated, :copyright, :authors, :urls, :image, :generator, :items]
+    attr_accessor *ELEMENTS
+    attr_accessor :parser
+
+    alias :entries :items
+
+    def initialize(wrapper)
+      # set up associations (i.e. arrays where needed)
+      @urls = []
+      @authors = []
+      @items = []
+      @parser = wrapper.parser
+    end
+
+    def channel() self end
+  end
+
+end
+

data/test/base_test.rb

@@ -0,0 +1,68 @@
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require 'test/unit'
+require 'feed-normalizer'
+
+include FeedNormalizer
+
+class BaseTest < Test::Unit::TestCase
+
+  XML_FILES = {}
+
+  def setup
+    data_dir = File.dirname(__FILE__) + '/data'
+
+    # Load up the xml files
+    Dir.open(data_dir).each do |fn|
+      next unless fn =~ /[.]xml$/
+      XML_FILES[fn.scan(/(.*)[.]/).to_s.to_sym] = File.read(data_dir + "/#{fn}")
+    end
+  end
+
+
+  def test_basic_parse
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20])
+  end
+
+  def test_force_parser
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20], RubyRssParser, true)
+  end
+
+  def test_force_parser_exclusive
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20], RubyRssParser, false)
+  end
+
+  def test_ruby_rss_parser
+    assert_kind_of Feed, feed=FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20], RubyRssParser, false)
+  end
+
+  def test_simple_rss_parser
+    assert_kind_of Feed, feed=FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20], SimpleRssParser, false)
+    assert_kind_of Feed, feed=FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10], SimpleRssParser, false)
+  end
+
+  # Attempts to parse a feed that Ruby's RSS can't handle.
+  # SimpleRSS should provide the parsed feed.
+  def test_parser_failover_order
+    assert_kind_of Feed, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10])
+  end
+
+  def test_all_parsers_fail
+    assert_nil FeedNormalizer::FeedNormalizer.parse("This isn't RSS or Atom!")
+  end
+
+  def test_correct_parser_used
+    assert_equal RSS::Parser, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:rss20]).parser
+    assert_equal SimpleRSS, FeedNormalizer::FeedNormalizer.parse(XML_FILES[:atom10]).parser
+  end
+
+  def test_sanity_check
+    XML_FILES.keys.each do |xml_file|
+      feed = FeedNormalizer::FeedNormalizer.parse(XML_FILES[xml_file])
+
+      assert [feed.title, feed.url, feed.entries.first.url].collect{|e| e.is_a?(String)}.all?, "Not everything was a String"
+      assert [feed.parser, feed.class].collect{|e| e.is_a?(Class)}.all?
+    end
+  end
+
+end

data/test/data/atom03.xml

@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed version="0.3"
+    xmlns="http://purl.org/atom/ns#"
+    xmlns:dc="http://purl.org/dc/elements/1.1/"
+    xml:lang="en">
+  <title>Cheap Stingy Bargains</title>
+  <link rel="alternate" type="text/html" href="http://www.cheapstingybargains.com" />
+  <tagline>Putting the "Squeeze" on high prices!</tagline>
+  <modified>2006-08-29T05:07:24Z</modified>
+  <copyright>Copyright 2006</copyright>
+  <generator url="http://wordpress.org/" version="2.0.3">WordPress</generator>
+  <entry>
+    <author>
+      <name>richard</name>
+    </author>
+    <title type="text/html" mode="escaped"><![CDATA[Levi Strauss Signature Girl&#8217;s Low Rise Slim Fit Flare Jeans $10]]></title>
+    <link rel="alternate" type="text/html" href="http://www.cheapstingybargains.com/24557/levi-strauss-signature-girls-low-rise-slim-fit-flare-jeans-10/" />
+    <id>http://www.cheapstingybargains.com/24557/levi-strauss-signature-girls-low-rise-slim-fit-flare-jeans-10/</id>
+    <modified>2006-08-29T02:31:03Z</modified>
+    <issued>2006-08-29T02:31:03Z</issued>
+
+    <dc:subject>Clothing</dc:subject>
+    <summary type="text/plain" mode="escaped"><![CDATA[
+Kmart has the Levi Strauss Signature Girl&#8217;s Low Rise Flare Jean for $10 after $5 instant savings (ends 9/2)
+Slim fit through hip and thigh, with zip-fly with button-through closure. Machine washable 99% Cotton/1% Spandex
+]]></summary>
+    <content type="text/html" mode="escaped" xml:base="http://www.cheapstingybargains.com/24557/levi-strauss-signature-girls-low-rise-slim-fit-flare-jeans-10/"><![CDATA[<p><a href="http://clickserve.cc-dt.com/link/tplclick?lid=41000000011334249&#038;pubid=21000000000053626" target=_"blank"><img  src="http://images.kmart.com/assets/images/product/productDetail/9990000058546711.jpg" width="150" height="150" border="0" style="float: right; margin: 0px 0px 5px 5px;" /></a><br />
+<strong>Kmart has the <a href="http://clickserve.cc-dt.com/link/tplclick?lid=41000000011334249&#038;pubid=21000000000053626" target=_"blank">Levi Strauss Signature Girl&#8217;s Low Rise Flare Jean</a> for $10 after $5 instant savings (ends 9/2)</strong></p>
+<p>Slim fit through hip and thigh, with zip-fly with button-through closure. Machine washable 99% Cotton/1% Spandex</p>
+]]></content>
+  </entry>
+  <entry>
+    <author>
+      <name>richard</name>
+    </author>
+    <title type="text/html" mode="escaped"><![CDATA[FragranceNet.com Coupon]]></title>
+    <link rel="alternate" type="text/html" href="http://www.cheapstingybargains.com/24556/fragrancenetcom-coupon/" />
+    <id>http://www.cheapstingybargains.com/24556/fragrancenetcom-coupon/</id>
+    <modified>2006-08-29T02:21:08Z</modified>
+    <issued>2006-08-29T02:21:08Z</issued>
+
+    <dc:subject>Coupon</dc:subject>
+    <dc:subject>General</dc:subject>
+    <summary type="text/plain" mode="escaped"><![CDATA[New Coupon from FragranceNet.com - $10 off orders of $75+  use Coupon Code LSLBY at check-out (ends 9/5)
+FragranceNet.com always offers deep discounts on all genuine brand name fragrances of up to 70% off retail with free shipping on all orders over $60
+
+]]></summary>
+    <content type="text/html" mode="escaped" xml:base="http://www.cheapstingybargains.com/24556/fragrancenetcom-coupon/"><![CDATA[<p>New Coupon from FragranceNet.com - <a target="_blank"  href="http://click.linksynergy.com/fs-bin/click?id=UnlJPFdznf8&#038;offerid=47781.10000314&#038;type=3&#038;subid=0"  >$10 off orders of $75+ </a> use Coupon Code <strong>LSLBY</strong> at check-out (ends 9/5)</p>
+<p>FragranceNet.com always offers deep discounts on all genuine brand name fragrances of up to 70% off retail with free shipping on all orders over $60
+</p>
+]]></content>
+  </entry>
+  <entry>
+    <author>
+      <name>Ayusha</name>
+    </author>
+    <title type="text/html" mode="escaped"><![CDATA[Astar LTV-37LS 37&#8243; Widescreen LCD TV with ATSC HD Tuner $1000]]></title>
+    <link rel="alternate" type="text/html" href="http://www.cheapstingybargains.com/23943/astar-ltv-37ls-37-widescreen-lcd-tv-1150-after-rebate/" />
+    <id>http://www.cheapstingybargains.com/23943/astar-ltv-37ls-37-widescreen-lcd-tv-1150-after-rebate/</id>
+    <modified>2006-08-29T02:13:38Z</modified>
+    <issued>2006-08-29T02:13:38Z</issued>
+
+    <dc:subject>Buy.com</dc:subject>
+    <dc:subject>TV</dc:subject>
+    <summary type="text/plain" mode="escaped"><![CDATA[New Low Price!
+
+Buy.com has the Astar LTV-37LS 37&#8243; Widescreen LCD TV for $1000 after $150 rebate (expires 9/9) w/ free shipping
+The Astar LTV-37LS 37&#8243; Widescreen LCD TV features 1366 x 768 resolution, 16:9  aspect ratio, 600cd/m2 brightness, 800:1 contrast ratio, built-in ATSC HD Tuner and 12ms response time.
+New customers [...]]]></summary>
+    <content type="text/html" mode="escaped" xml:base="http://www.cheapstingybargains.com/23943/astar-ltv-37ls-37-widescreen-lcd-tv-1150-after-rebate/"><![CDATA[<p><strong>New Low Price!</strong><br />
+<a href = "http://www.cheapstingybargains.com/jump.php?m=buy&#038;id=202883003" target="_blank"><img  src="http://ak.buy.com/db_assets/large_images/003/202883003.jpg" width="150" height="150" border="0" style="float: right; margin: 0px 0px 5px 5px;" /></a></p>
+<p><strong>Buy.com has the <a href="http://www.cheapstingybargains.com/jump.php?m=buy&#038;id=202883003" target="_blank" >Astar LTV-37LS 37&#8243; Widescreen LCD TV</a> for $1000 after $150 rebate (expires 9/9) w/ free shipping</strong></p>
+<p>The <strong>Astar LTV-37LS 37&#8243; Widescreen LCD TV</strong> features 1366 x 768 resolution, 16:9  aspect ratio, 600cd/m2 brightness, 800:1 contrast ratio, built-in ATSC HD Tuner and 12ms response time.     </p>
+<p>New customers of Buy.com may take an additional $15 discount by clicking <a href="http://www.anrdoezrs.net/click-1547706-10391416?&#038;URL=http%3A%2F%2Fwww%2Ebuy%2Ecom%2Fretail%2Fcoupon%2Easp%3Fprid%3D88301879" target=_"blank">$15 Coupon</a> and searching for <strong>202883003</strong>
+</p>
+]]></content>
+  </entry>
+  <entry>
+    <author>
+      <name>richard</name>
+    </author>
+    <title type="text/html" mode="escaped"><![CDATA[Acer Aspire Notebook (AS5003WLMI) w/ 15.4&#8243; WXGA display $500]]></title>
+    <link rel="alternate" type="text/html" href="http://www.cheapstingybargains.com/24555/acer-aspire-notebook-as5003wlmi-w-154-wxga-display-500/" />
+    <id>http://www.cheapstingybargains.com/24555/acer-aspire-notebook-as5003wlmi-w-154-wxga-display-500/</id>
+    <modified>2006-08-29T02:02:42Z</modified>
+    <issued>2006-08-29T02:02:42Z</issued>
+
+    <dc:subject>CircuitCity.com</dc:subject>
+    <dc:subject>Laptops &#038; Notebooks</dc:subject>
+    <summary type="text/plain" mode="escaped"><![CDATA[
+
+Circuit City has a Acer Aspire Laptop (AS5003WLMI)  for $500 after $100 instant savings and $150 rebate (expires 9/2) w/ free shipping
+Acer Aspire Laptop:
+AMD Turion 64 ML-32, 15.4&#8243; WXGA display, 512MB RAM,  80GB hard drive, Double-layer DVD drive, 3 USB 2.0 ports, XP Home
+]]></summary>
+    <content type="text/html" mode="escaped" xml:base="http://www.cheapstingybargains.com/24555/acer-aspire-notebook-as5003wlmi-w-154-wxga-display-500/"><![CDATA[<p><a href="http://www.cheapstingybargains.com/jump.php?m=circuit&#038;id=156638" target="_blank" rel="nofollow"><br />
+<img src="http://www.circuitcity.com/IMAGE/product/enlarged/aca/PC.ACA.AS5003WLMI.LT.JPG" width="210" height="140" border="0" style="float: right; margin: 0px 0px 5px 5px;" /></a></p>
+<p><strong>Circuit City has a <a href="http://www.cheapstingybargains.com/jump.php?m=circuit&#038;id=156638" target="_blank" rel="nofollow">Acer Aspire Laptop (AS5003WLMI) </a> for $500 after $100 instant savings and $150 rebate (expires 9/2) w/ free shipping</strong></p>
+<p><strong>Acer Aspire Laptop</strong>:<br />
+AMD Turion 64 ML-32, 15.4&#8243; WXGA display, 512MB RAM,  80GB hard drive, Double-layer DVD drive, 3 USB 2.0 ports, XP Home </p>
+]]></content>
+  </entry>
+  <entry>
+    <author>
+      <name>richard</name>
+    </author>
+    <title type="text/html" mode="escaped"><![CDATA[Astar LTV-32BG 32� LCD HD Television with built in HD Digital Tuner $700]]></title>
+    <link rel="alternate" type="text/html" href="http://www.cheapstingybargains.com/24554/astar-ltv-32bg-32%e2%80%9d-lcd-hd-television-with-built-in-hd-digital-tuner-700/" />
+    <id>http://www.cheapstingybargains.com/24554/astar-ltv-32bg-32%e2%80%9d-lcd-hd-television-with-built-in-hd-digital-tuner-700/</id>
+    <modified>2006-08-29T01:41:01Z</modified>
+    <issued>2006-08-29T01:41:01Z</issued>
+
+    <dc:subject>TV's &#038; HDTV</dc:subject>
+    <dc:subject>PCConnection</dc:subject>
+    <summary type="text/plain" mode="escaped"><![CDATA[
+PC Connection has the Astar LTV-32BG 32� LCD HDTV with built in HD Digital Tuner for $700 after $100 rebate (expires 9/9) w/ free shipping
+The Astar LTV-32BG 32� LCD HDTV displays HDTV broadcast signals received through its Component Video inputs, DVI input, or built-in ATSC HD Tuner. It features contemporary styling, built-in stereo [...]]]></summary>
+    <content type="text/html" mode="escaped" xml:base="http://www.cheapstingybargains.com/24554/astar-ltv-32bg-32%e2%80%9d-lcd-hd-television-with-built-in-hd-digital-tuner-700/"><![CDATA[<p><a href="http://clickserve.cc-dt.com/link/click?lid=41000000011336478" target=_"blank"><img  src="http://service.pcconnection.com/images/inhouse/6724384.jpg" width="150" height="150" border="0" style="float: right; margin: 0px 0px 5px 5px;" /></a><br />
+<strong>PC Connection has the <a href="http://clickserve.cc-dt.com/link/click?lid=41000000011336478" target=_"blank">Astar LTV-32BG 32� LCD HDTV with built in HD Digital Tuner</a> for $700 after $100 rebate (expires 9/9) w/ free shipping </strong> </p>
+<p>The <strong>Astar LTV-32BG 32� LCD HDTV</strong> displays HDTV broadcast signals received through its Component Video inputs, DVI input, or built-in ATSC HD Tuner. It features contemporary styling, built-in stereo speakers, classic easy-touch front-mounted control buttons, and a wide-screen 16:9 picture aspect ratio, 800:1 contrast ratio and 1366 x 768 resolution.
+</p>
+]]></content>
+  </entry>
+</feed>
+
+<!-- Dynamic Page Served (once) in 1.705 seconds -->
+<!-- Cached page served by WP-Cache -->

data/test/data/atom10.xml

@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xml:lang="en-US" xmlns="http://www.w3.org/2005/Atom">
+  <title>~:caboose</title>
+  <id>tag:habtm.com,2005:Typo</id>
+  <generator version="4.0" uri="http://www.typosphere.org">Typo</generator>
+  <link href="http://habtm.com/xml/atom10/feed.xml" rel="self" type="application/atom+xml"/>
+  <link href="http://habtm.com/" rel="alternate" type="text/html"/>
+  <updated>2006-08-16T11:59:44-05:00</updated>
+  <entry>
+    <author>
+      <name>technoweenie</name>
+    </author>
+    <id>urn:uuid:3d9fa115-1bb1-420e-9fca-478a8cb520b5</id>
+    <published>2006-08-16T11:58:00-05:00</published>
+    <updated>2006-08-16T11:59:44-05:00</updated>
+    <title type="html">A forum on Rails</title>
+    <link href="http://habtm.com/articles/2006/08/16/a-forum-on-rails" rel="alternate" type="text/html"/>
+    <category term="rails" scheme="http://habtm.com/articles/category/rails" label="rails"/>
+    <category term="ruby" scheme="http://habtm.com/articles/category/ruby" label="ruby"/>
+    <summary type="html">&lt;p&gt;Josh Goebel and I took an evening to bang out a little project: &lt;a href="http://beast.caboo.se/"&gt;Beast&lt;/a&gt;.  It&amp;#8217;s our minimal no-fluff Rails forum.  It&amp;#8217;s no beast of an application either, clocking in at 285 LOC and a 1:1.5 test ratio.  &lt;a href="http://svn.techno-weenie.net/projects/beast/"&gt;Check it out&lt;/a&gt;!&lt;/p&gt;</summary>
+    <content type="html">&lt;p&gt;Josh Goebel and I took an evening to bang out a little project: &lt;a href="http://beast.caboo.se/"&gt;Beast&lt;/a&gt;.  It&amp;#8217;s our minimal no-fluff Rails forum.  It&amp;#8217;s no beast of an application either, clocking in at 285 LOC and a 1:1.5 test ratio.  &lt;a href="http://svn.techno-weenie.net/projects/beast/"&gt;Check it out&lt;/a&gt;!&lt;/p&gt;</content>
+  </entry>
+  <entry>
+    <author>
+      <name>courtenay</name>
+    </author>
+    <id>urn:uuid:32816298-7855-4d65-a832-5f6a92b1a4ad</id>
+    <published>2006-08-11T12:11:13-05:00</published>
+    <updated>2006-08-11T12:11:13-05:00</updated>
+    <title type="html">Update on the documentation project</title>
+    <link href="http://habtm.com/articles/2006/08/11/update-on-the-documentation-project" rel="alternate" type="text/html"/>
+    <category term="rails" scheme="http://habtm.com/articles/category/rails" label="rails"/>
+    <category term="ruby" scheme="http://habtm.com/articles/category/ruby" label="ruby"/>
+    <summary type="html">&lt;p&gt;Please use the wiki - &lt;a href="http://caboose.stikipad.com/documentation"&gt;http://caboose.stikipad.com/documentation&lt;/a&gt; - to give us your ideas about how the documentation project can proceed. Thanks.&lt;/p&gt;</summary>
+    <content type="html">&lt;p&gt;Please use the wiki - &lt;a href="http://caboose.stikipad.com/documentation"&gt;http://caboose.stikipad.com/documentation&lt;/a&gt; - to give us your ideas about how the documentation project can proceed. Thanks.&lt;/p&gt;</content>
+  </entry>
+  <entry>
+    <author>
+      <name>KirinDave</name>
+    </author>
+    <id>urn:uuid:6c028f36-f87a-4f53-b7e3-1f943d2341f0</id>
+    <published>2006-08-10T01:09:02-05:00</published>
+    <updated>2006-08-10T01:09:02-05:00</updated>
+    <title type="html">Starfish - Easy Distribution of Site Maintenance</title>
+    <link href="http://habtm.com/articles/2006/08/10/starfish-easy-distribution-of-site-maintenance" rel="alternate" type="text/html"/>
+    <summary type="html">&lt;p&gt;Lots of Rails apps are very simple &lt;span class="caps"&gt;CRUD&lt;/span&gt; mappings, which is something Rails excels at. But, as more and more large, complex sites go live with Rails, there is a demand for more complex &amp;#8216;backend&amp;#8217; components in the system. Already we&amp;#8217;ve got &lt;a href="http://brainspl.at/articles/2006/05/15/backgoundrb-initial-release"&gt;BackgrounDRB&lt;/a&gt;, which is great, but enter Starfish, which is a tool for complex distributed tasks made easy.&lt;/p&gt;</summary>
+    <content type="html">&lt;p&gt;Lots of Rails apps are very simple &lt;span class="caps"&gt;CRUD&lt;/span&gt; mappings, which is something Rails excels at. But, as more and more large, complex sites go live with Rails, there is a demand for more complex &amp;#8216;backend&amp;#8217; components in the system. Already we&amp;#8217;ve got &lt;a href="http://brainspl.at/articles/2006/05/15/backgoundrb-initial-release"&gt;BackgrounDRB&lt;/a&gt;, which is great, but enter Starfish, which is a tool for complex distributed tasks made easy.&lt;/p&gt;&lt;p&gt;Lucas Carlson (a.k.a. cardmagic) and I have been using Starfish for various distributed tasks at &lt;a href="http://mog.com"&gt;mog.com&lt;/a&gt; for awhile now. I&amp;#8217;d like to show everyone how it works, as an incentive for Lucas to gem up the project and share it with the world.&lt;/p&gt;
+
+
+    &lt;p&gt;&lt;strong&gt;The Problem&lt;/strong&gt;: Due to a bug, many albums in our music database were assigned the wrong artist. We can fix this pretty easily by checking against other data, but there are a lot of albums to run this fix against and not a lot of downtime available each day.&lt;/p&gt;
+
+
+&lt;strong&gt;The Solution&lt;/strong&gt;: A starfish file that looks like this:
+&lt;pre&gt;
+ENV['RAILS_ENV'] ||= 'production'
+require File.dirname(__FILE__) + '/../config/boot'
+require File.dirname(__FILE__) + '/../config/environment'
+require 'user'
+require 'artist'
+require 'album'
+require 'user_collection_cleaning'
+
+class ScrubAlbumsTask
+  def self.service
+    @@service ||= ScrubAlbumsTask.new
+  end
+
+  def initialize
+    @various_artists = Artist.find_by_name( "Various Artists" )
+    @albums = Album.find(:all, :conditions =&amp;gt; ["artist_id = ?", @various_artists.id])
+  end
+
+  def next
+    @albums.shift.id
+  end
+
+  def has_data?
+    @albums.size &amp;gt; 0
+  end
+end
+
+server do |object|
+  object = ScrubAlbumsTask.service
+end
+
+client do |object|
+  if object.has_data?
+    album = Album.find(object.next)
+    album.repair_artist_by_majority!
+  end
+end
+&lt;/pre&gt;
+
+    &lt;p&gt;&lt;strong&gt;The Explanation&lt;/strong&gt;: We create a simple singleton-like object to do our dirty work. All it does is get ahold of the albums we need to fix (in &lt;tt&gt;initialize&lt;/tt&gt;) and then provide a simple way to get ahold of the object id to repair.&lt;/p&gt;
+
+
+    &lt;p&gt;The server object merely produces a drb proxy to our clients. The clients get an id from the queue, find the object in the database, then run the repair process on them.&lt;/p&gt;
+
+
+    &lt;p&gt;You can notice a few things we &lt;i&gt;didn&amp;#8217;t&lt;/i&gt; do. We didn&amp;#8217;t write any networking or distribution or threading code. We didn&amp;#8217;t need to partition the table in any arbitrary fashion. We didn&amp;#8217;t need to poll the database over and over to fetch new work units, the server keeps the state. We didn&amp;#8217;t even need to worry about redoing work, the way we wrote our server makes sure that doesn&amp;#8217;t happen.&lt;/p&gt;
+
+
+    &lt;p&gt;To invoke the system, we simply run &lt;tt&gt;starfish my_albums_task &amp;#38;&lt;/tt&gt;. The first time we do this, it will output a message that says &amp;#8220;Starting Server.&amp;#8221; Starfish is smart enough to find that there are no servers for a given task, and if that is the case, it forks one off. It then invokes a client process. If you invoke the task again, a new client will be added. We can run them on multiple machines, so long as they share a local network. No fancy &lt;span class="caps"&gt;CORBA&lt;/span&gt;-ish code needed. Starfish handles the details, and you can just kill the clients when they are done. Starfish protects its client block from signals and closes after a client finishes a task.&lt;/p&gt;
+
+
+    &lt;p&gt;The key to the speed of this system is that it parallelizes database access. Most database setups have very high read speeds, so running the repair processes in parallel isn&amp;#8217;t a significant burden on the system. Starfish is most useful in situations where you need to go record by record through a table and perform some specific–possibly slow–task.&lt;/p&gt;
+
+
+    &lt;p&gt;This is a very simple starfish task that I wrote in 20 minutes to handle this cleanup process. We use more sophsticated distributed systems for other aspects of our site, and starfish is scaling to work with them all. The most recent &lt;a href="http://rufy.com/starfish"&gt;darcs repo&lt;/a&gt; even has work to extend starfish to support MapReduce, of google fame.&lt;/p&gt;</content>
+  </entry>
+</feed>
+

data/test/data/rss20.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<?xml-stylesheet title="XSL_formatting" type="text/xsl" href="/shared/bsp/xsl/rss/nolsol.xsl"?>
+<rss version="2.0">
+  <channel>
+    <title>BBC News | Technology | UK Edition</title>
+    <link>http://news.bbc.co.uk/go/rss/-/1/hi/technology/default.stm</link>
+    <description>Visit BBC News for up-to-the-minute news, breaking news, video, audio and feature stories. BBC News provides trusted World and UK news as well as local and regional perspectives. Also entertainment, business, science, technology and health news.</description>
+    <language>en-gb</language>
+    <lastBuildDate>Sat, 09 Sep 2006 14:57:06 GMT</lastBuildDate>
+    <copyright>Copyright: (C) British Broadcasting Corporation, see http://news.bbc.co.uk/1/hi/help/rss/4498287.stm for terms and conditions of reuse</copyright>
+    <docs>http://www.bbc.co.uk/syndication/</docs>
+    <ttl>15</ttl>
+
+    <image>
+      <title>BBC News</title>
+      <url>http://news.bbc.co.uk/nol/shared/img/bbc_news_120x60.gif</url>
+      <link>http://news.bbc.co.uk/go/rss/-/1/hi/technology/default.stm</link>
+    </image>
+
+    <item>
+      <title>Concerns over security software</title>
+      <description>BBC Click investigates free security software and finds out who will protect PCs when Microsoft launches Vista.</description>
+      <link>http://news.bbc.co.uk/go/rss/-/1/hi/programmes/click_online/5326654.stm</link>
+      <guid isPermaLink="false">http://news.bbc.co.uk/1/hi/programmes/click_online/5326654.stm</guid>
+      <pubDate>Sat, 09 Sep 2006 12:45:35 GMT</pubDate>
+      <category>Click</category>
+    </item>
+
+    <item>
+      <title>Top prize for 'light' inventor</title>
+      <description>A Japanese scientist who invented a sustainable form of light is awarded the Millennium Technology Prize.</description>
+      <link>http://news.bbc.co.uk/go/rss/-/1/hi/technology/5328446.stm</link>
+      <guid isPermaLink="false">http://news.bbc.co.uk/1/hi/technology/5328446.stm</guid>
+      <pubDate>Fri, 08 Sep 2006 16:18:08 GMT</pubDate>
+      <category>Technology</category>
+    </item>
+
+    <item>
+      <title>MP3 player court order overturned</title>
+      <description>SanDisk puts its MP3 players back on display at a German electronics show after overturning a court injunction.</description>
+      <link>http://news.bbc.co.uk/go/rss/-/1/hi/technology/5326660.stm</link>
+      <guid isPermaLink="false">http://news.bbc.co.uk/1/hi/technology/5326660.stm</guid>
+      <pubDate>Fri, 08 Sep 2006 10:14:41 GMT</pubDate>
+      <category>Technology</category>
+    </item>
+
+  </channel>
+</rss>
+

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: feed-normalizer
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+date: 2006-10-03 00:00:00 -07:00
+summary: Extensible Ruby wrapper for Atom and RSS parsers
+require_paths: 
+- lib
+email: andy@tinnedfruit.org
+homepage: http://code.google.com/p/feed-normalizer/
+rubyforge_project: 
+description: 
+autorequire: feed-normalizer
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+- Andrew A. Smith
+files: 
+- lib/feed-normalizer.rb
+- lib/parsers
+- lib/structures.rb
+- lib/parsers/rss.rb
+- lib/parsers/simple-rss.rb
+- test/base_test.rb
+- test/data
+- test/data/atom03.xml
+- test/data/atom10.xml
+- test/data/rss20.xml
+- LICENSE
+- Rakefile
+- README
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: simple-rss
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "1.1"
+    version: