jeffrafter-feedzirra 0.0.4

data/lib/feedzirra/feed_entry_utilities.rb

@@ -0,0 +1,54 @@
+require 'digest/md5'
+
+module Feedzirra
+  module FeedEntryUtilities
+    module Sanitize
+      def sanitize!
+        self.replace(sanitize) unless self.nil?
+      end
+      
+      def sanitize
+        Dryopteris.sanitize(self)
+      end
+    end
+
+    attr_reader :published
+    
+    def parse_datetime(string)
+      DateTime.parse(string).feed_utils_to_gm_time
+    end
+    
+    def published=(val)
+      @published = parse_datetime(val)
+    end
+    
+    def content
+      @content.extend(Sanitize)
+    end
+    
+    def title
+      @title.extend(Sanitize)
+    end
+    
+    def author
+      @author.extend(Sanitize)
+    end
+    
+    def summary
+      @summary.extend(Sanitize)
+    end
+    
+    def sanitize!
+      self.title.sanitize!
+      self.author.sanitize!
+      self.content.sanitize!
+      self.summary.sanitize!
+    end
+    
+    def checksum
+      Digest::MD5.hexdigest("#{title}--#{url}--#{summary}--#{content}")
+    end    
+
+    alias_method :last_modified, :published
+  end
+end

data/lib/feedzirra/feed_utilities.rb

@@ -0,0 +1,71 @@
+module Feedzirra
+  module FeedUtilities
+    UPDATABLE_ATTRIBUTES = %w(title feed_url url last_modified)
+    
+    attr_writer   :new_entries, :updated, :last_modified
+    attr_accessor :etag
+
+    def last_modified
+      @last_modified ||= begin
+        entry = entries.reject {|e| e.published.nil? }.sort_by { |entry| entry.published if entry.published }.last
+        entry ? entry.published : nil
+      end
+    end
+    
+    def updated?
+      @updated
+    end
+    
+    def new_entries
+      @new_entries ||= []
+    end
+    
+    def has_new_entries?
+      new_entries.size > 0
+    end
+    
+    def update_from_feed(feed)
+      self.new_entries += find_new_entries_for(feed)
+      self.entries.unshift(*self.new_entries)
+      
+      updated! if UPDATABLE_ATTRIBUTES.any? { |name| update_attribute(feed, name) }
+    end
+    
+    def update_attribute(feed, name)
+      old_value, new_value = send(name), feed.send(name)
+      
+      if old_value != new_value
+        send("#{name}=", new_value)
+      end
+    end
+    
+    def sanitize_entries!
+      entries.each {|entry| entry.sanitize!}
+    end
+    
+    private
+    
+    def updated!
+      @updated = true
+    end
+    
+    def find_new_entries_for(feed)
+      # this implementation is a hack, which is why it's so ugly.
+      # it's to get around the fact that not all feeds have a published date.
+      # however, they're always ordered with the newest one first.
+      # So we go through the entries just parsed and insert each one as a new entry
+      # until we get to one that has the same url as the the newest for the feed
+      latest_entry = self.entries.first
+      found_new_entries = []
+      feed.entries.each do |entry|
+        break if entry.url == latest_entry.url
+        found_new_entries << entry
+      end
+      found_new_entries
+    end
+    
+    def existing_entry?(test_entry)
+      entries.any? { |entry| entry.url == test_entry.url }
+    end
+  end
+end

data/lib/feedzirra/rdf.rb

@@ -0,0 +1,15 @@
+module Feedzirra
+  class RDF
+    include SAXMachine
+    include FeedUtilities
+    element :title
+    element :link, :as => :url
+    elements :item, :as => :entries, :class => RDFEntry
+
+    attr_accessor :feed_url
+    
+    def self.able_to_parse?(xml)
+      xml =~ /(rdf\:RDF)|(#{Regexp.escape("http://purl.org/rss/1.0")})|(rss version\=\"0\.9.?\")/ || false
+    end
+  end
+end

data/lib/feedzirra/rdf_entry.rb

@@ -0,0 +1,12 @@
+module Feedzirra
+  class RDFEntry
+    include SAXMachine
+    include FeedEntryUtilities
+    element :title
+    element :link, :as => :url
+    element :"dc:creator", :as => :author
+    element :"content:encoded", :as => :content
+    element :description, :as => :summary
+    element :"dc:date", :as => :published
+  end
+end

data/lib/feedzirra/rss.rb

@@ -0,0 +1,15 @@
+module Feedzirra
+  class RSS
+    include SAXMachine
+    include FeedUtilities
+    element :title
+    element :link, :as => :url
+    elements :item, :as => :entries, :class => RSSEntry
+    
+    attr_accessor :feed_url
+    
+    def self.able_to_parse?(xml)
+      xml =~ /\<rss|rdf/
+    end
+  end
+end

data/lib/feedzirra/rss_entry.rb

@@ -0,0 +1,16 @@
+module Feedzirra
+  class RSSEntry
+    include SAXMachine
+    include FeedEntryUtilities
+    element :title
+    element :link, :as => :url
+
+    element :"dc:creator", :as => :author
+    element :"content:encoded", :as => :content
+    element :description, :as => :summary
+
+    element :pubDate, :as => :published
+    element :"dc:date", :as => :published
+    elements :category, :as => :categories
+  end
+end

data/spec/feedzirra/atom_entry_spec.rb

@@ -0,0 +1,37 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Feedzirra::AtomEntry do
+  before(:each) do
+    # I don't really like doing it this way because these unit test should only rely on AtomEntry,
+    # but this is actually how it should work. You would never just pass entry xml straight to the AtomEnry
+    @entry = Feedzirra::Atom.parse(sample_atom_feed).entries.first
+  end
+  
+  it "should parse the title" do
+    @entry.title.should == "AWS Job: Architect & Designer Position in Turkey"
+  end
+  
+  it "should parse the url" do
+    @entry.url.should == "http://aws.typepad.com/aws/2009/01/aws-job-architect-designer-position-in-turkey.html"
+  end
+  
+  it "should parse the author" do
+    @entry.author.should == "AWS Editor"
+  end
+  
+  it "should parse the content" do
+    @entry.content.should == sample_atom_entry_content
+  end
+  
+  it "should provide a summary" do
+    @entry.summary.should == "Late last year an entrepreneur from Turkey visited me at Amazon HQ in Seattle. We talked about his plans to use AWS as part of his new social video portal startup. I won't spill any beans before he's ready to..."
+  end
+  
+  it "should parse the published date" do
+    @entry.published.to_s.should == "Fri Jan 16 18:21:00 UTC 2009"
+  end
+
+  it "should parse the categories" do
+    @entry.categories.should == ['Turkey', 'Seattle']
+  end
+end

data/spec/feedzirra/atom_feed_burner_entry_spec.rb

@@ -0,0 +1,42 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Feedzirra::AtomFeedBurnerEntry do
+  before(:each) do
+    # I don't really like doing it this way because these unit test should only rely on AtomEntry,
+    # but this is actually how it should work. You would never just pass entry xml straight to the AtomEnry
+    @entry = Feedzirra::AtomFeedBurner.parse(sample_feedburner_atom_feed).entries.first
+  end
+  
+  it "should parse the title" do
+    @entry.title.should == "Making a Ruby C library even faster"
+  end
+  
+  it "should be able to fetch a url via the 'alternate' rel if no origLink exists" do
+    entry = Feedzirra::AtomFeedBurner.parse(File.read("#{File.dirname(__FILE__)}/../sample_feeds/PaulDixExplainsNothingAlternate.xml")).entries.first
+    entry.url.should == 'http://feeds.feedburner.com/~r/PaulDixExplainsNothing/~3/519925023/making-a-ruby-c-library-even-faster.html'
+  end
+
+  it "should parse the url" do
+    @entry.url.should == "http://www.pauldix.net/2009/01/making-a-ruby-c-library-even-faster.html"
+  end
+  
+  it "should parse the author" do
+    @entry.author.should == "Paul Dix"
+  end
+  
+  it "should parse the content" do
+    @entry.content.should == sample_feedburner_atom_entry_content
+  end
+  
+  it "should provide a summary" do
+    @entry.summary.should == "Last week I released the first version of a SAX based XML parsing library called SAX-Machine. It uses Nokogiri, which uses libxml, so it's pretty fast. However, I felt that it could be even faster. The only question was how..."
+  end
+  
+  it "should parse the published date" do
+    @entry.published.to_s.should == "Thu Jan 22 15:50:22 UTC 2009"
+  end
+
+  it "should parse the categories" do
+    @entry.categories.should == ['Ruby', 'Another Category']
+  end
+end

data/spec/feedzirra/atom_feed_burner_spec.rb

@@ -0,0 +1,39 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Feedzirra::AtomFeedBurner do
+  describe "#will_parse?" do
+    it "should return true for a feedburner atom feed" do
+      Feedzirra::AtomFeedBurner.should be_able_to_parse(sample_feedburner_atom_feed)
+    end
+    
+    it "should return false for an rdf feed" do
+      Feedzirra::AtomFeedBurner.should_not be_able_to_parse(sample_rdf_feed)
+    end
+    
+    it "should return false for a regular atom feed" do
+      Feedzirra::AtomFeedBurner.should_not be_able_to_parse(sample_atom_feed)
+    end
+  end
+
+  describe "parsing" do
+    before(:each) do
+      @feed = Feedzirra::AtomFeedBurner.parse(sample_feedburner_atom_feed)
+    end
+    
+    it "should parse the title" do
+      @feed.title.should == "Paul Dix Explains Nothing"
+    end
+    
+    it "should parse the url" do
+      @feed.url.should == "http://www.pauldix.net/"
+    end
+    
+    it "should parse the feed_url" do
+      @feed.feed_url.should == "http://feeds.feedburner.com/PaulDixExplainsNothing"
+    end
+    
+    it "should parse entries" do
+      @feed.entries.size.should == 5
+    end
+  end
+end

data/spec/feedzirra/atom_spec.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Feedzirra::Atom do
+  describe "#will_parse?" do
+    it "should return true for an atom feed" do
+      Feedzirra::Atom.should be_able_to_parse(sample_atom_feed)
+    end
+    
+    it "should return false for an rdf feed" do
+      Feedzirra::Atom.should_not be_able_to_parse(sample_rdf_feed)
+    end
+  end
+  
+  describe "parsing" do
+    before(:each) do
+      @feed = Feedzirra::Atom.parse(sample_atom_feed)
+    end
+    
+    it "should parse the title" do
+      @feed.title.should == "Amazon Web Services Blog"
+    end
+    
+    it "should parse the url" do
+      @feed.url.should == "http://aws.typepad.com/aws/"
+    end
+    
+    it "should parse the feed_url" do
+      @feed.feed_url.should == "http://aws.typepad.com/aws/atom.xml"
+    end
+    
+    it "should parse entries" do
+      @feed.entries.size.should == 10
+    end
+  end
+end

data/spec/feedzirra/feed_entry_utilities_spec.rb

@@ -0,0 +1,52 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Feedzirra::FeedUtilities do
+  before(:each) do
+    @klass = Class.new do
+      include Feedzirra::FeedEntryUtilities
+    end
+  end
+  
+  describe "handling dates" do
+    it "should parse an ISO 8601 formatted datetime into Time" do
+      time = @klass.new.parse_datetime("2008-02-20T8:05:00-010:00")
+      time.class.should == Time
+      time.to_s.should == "Wed Feb 20 18:05:00 UTC 2008"
+    end
+  end
+  
+  describe "sanitizing" do
+    before(:each) do
+      @feed = Feedzirra::Feed.parse(sample_atom_feed)
+      @entry = @feed.entries.first
+    end
+    
+    it "should provide a sanitized title" do
+      new_title = "<script>" + @entry.title
+      @entry.title = new_title
+      @entry.title.sanitize.should == Dryopteris.sanitize(new_title)
+    end
+    
+    it "should sanitize content in place" do
+      new_content = "<script>" + @entry.content
+      @entry.content = new_content.dup
+      @entry.content.sanitize!.should == Dryopteris.sanitize(new_content)
+      @entry.content.should == Dryopteris.sanitize(new_content)
+    end
+    
+    it "should sanitize things in place" do
+      @entry.title   += "<script>"
+      @entry.author  += "<script>"
+      @entry.content += "<script>"
+
+      cleaned_title   = Dryopteris.sanitize(@entry.title)
+      cleaned_author  = Dryopteris.sanitize(@entry.author)
+      cleaned_content = Dryopteris.sanitize(@entry.content)
+      
+      @entry.sanitize!
+      @entry.title.should   == cleaned_title
+      @entry.author.should  == cleaned_author
+      @entry.content.should == cleaned_content
+    end
+  end
+end

data/spec/feedzirra/feed_spec.rb

@@ -0,0 +1,251 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Feedzirra::Feed do
+  describe "#parse" do # many of these tests are redundant with the specific feed type tests, but I put them here for completeness
+    context "when there's an available parser" do
+      it "should parse an rdf feed" do
+        feed = Feedzirra::Feed.parse(sample_rdf_feed)
+        feed.title.should == "HREF Considered Harmful"
+        feed.entries.first.published.to_s.should == "Tue Sep 02 19:50:07 UTC 2008"
+        feed.entries.size.should == 10
+      end
+
+      it "should parse an rss feed" do
+        feed = Feedzirra::Feed.parse(sample_rss_feed)
+        feed.title.should == "Tender Lovemaking"
+        feed.entries.first.published.to_s.should == "Thu Dec 04 17:17:49 UTC 2008"
+        feed.entries.size.should == 10
+      end
+
+      it "should parse an atom feed" do
+        feed = Feedzirra::Feed.parse(sample_atom_feed)
+        feed.title.should == "Amazon Web Services Blog"
+        feed.entries.first.published.to_s.should == "Fri Jan 16 18:21:00 UTC 2009"
+        feed.entries.size.should == 10
+      end
+
+      it "should parse an feedburner atom feed" do
+        feed = Feedzirra::Feed.parse(sample_feedburner_atom_feed)
+        feed.title.should == "Paul Dix Explains Nothing"
+        feed.entries.first.published.to_s.should == "Thu Jan 22 15:50:22 UTC 2009"
+        feed.entries.size.should == 5
+      end      
+    end
+    
+    context "when there's no available parser" do
+      it "raises Feedzirra::NoParserAvailable" do
+        proc {
+          Feedzirra::Feed.parse("I'm an invalid feed")
+        }.should raise_error(Feedzirra::NoParserAvailable)
+      end      
+    end
+    
+    it "should parse an feedburner rss feed" do
+      feed = Feedzirra::Feed.parse(sample_rss_feed_burner_feed)
+      feed.title.should == "Sam Harris: Author, Philosopher, Essayist, Atheist"
+      feed.entries.first.published.to_s.should == "Tue Jan 13 17:20:28 UTC 2009"
+      feed.entries.size.should == 10
+    end
+  end
+  
+  describe "#determine_feed_parser_for_xml" do
+    it "should return the Feedzirra::Atom class for an atom feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_atom_feed).should == Feedzirra::Atom
+    end
+    
+    it "should return the Feedzirra::AtomFeedBurner class for an atom feedburner feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_feedburner_atom_feed).should == Feedzirra::AtomFeedBurner
+    end
+    
+    it "should return the Feedzirra::RSS class for an rdf/rss 1.0 feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rdf_feed).should == Feedzirra::RSS
+    end
+    
+    it "should return the Feedzirra::RSS class for an rss feedburner feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rss_feed_burner_feed).should == Feedzirra::RSS
+    end
+    
+    it "should return the Feedzirra::RSS object for an rss 2.0 feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rss_feed).should == Feedzirra::RSS
+    end
+  end
+  
+  describe "adding feed types" do
+    it "should prioritize added feed types over the built in ones" do
+      feed_text = "Atom asdf"
+      Feedzirra::Atom.should be_able_to_parse(feed_text)
+      new_feed_type = Class.new do
+        def self.able_to_parse?(val)
+          true
+        end
+      end
+      new_feed_type.should be_able_to_parse(feed_text)
+      Feedzirra::Feed.add_feed_class(new_feed_type)
+      Feedzirra::Feed.determine_feed_parser_for_xml(feed_text).should == new_feed_type
+      
+      # this is a hack so that this doesn't break the rest of the tests
+      Feedzirra::Feed.feed_classes.reject! {|o| o == new_feed_type }
+    end
+  end
+  
+  describe "header parsing" do
+    before(:each) do
+      @header = "HTTP/1.0 200 OK\r\nDate: Thu, 29 Jan 2009 03:55:24 GMT\r\nServer: Apache\r\nX-FB-Host: chi-write6\r\nLast-Modified: Wed, 28 Jan 2009 04:10:32 GMT\r\nETag: ziEyTl4q9GH04BR4jgkImd0GvSE\r\nP3P: CP=\"ALL DSP COR NID CUR OUR NOR\"\r\nConnection: close\r\nContent-Type: text/xml;charset=utf-8\r\n\r\n"
+    end
+    
+    it "should parse out an etag" do
+      Feedzirra::Feed.etag_from_header(@header).should == "ziEyTl4q9GH04BR4jgkImd0GvSE"
+    end
+    
+    it "should return nil if there is no etag in header" do
+      Feedzirra::Feed.etag_from_header("foo").should be_nil
+    end
+    
+    it "should parse out a last-modified date" do
+      Feedzirra::Feed.last_modified_from_header(@header).should == Time.parse("Wed, 28 Jan 2009 04:10:32 GMT")
+    end
+    
+    it "should return nil if there is no last-modified in header" do
+      Feedzirra::Feed.last_modified_from_header("foo").should be_nil
+    end
+  end
+  
+  describe "fetching feeds" do
+    before(:each) do
+      @paul_feed_url = "http://feeds.feedburner.com/PaulDixExplainsNothing"
+      @trotter_feed_url = "http://feeds2.feedburner.com/trottercashion"
+    end
+        
+    describe "handling many feeds" do
+      it "should break a large number into more manageable blocks of 40"
+      it "should add to the queue as feeds finish (instead of waiting for each block of 40 to finsih)"
+    end
+
+    describe "#fetch_raw" do
+      it "should take :user_agent as an option"
+      it "should take :if_modified_since as an option"
+      it "should take :if_none_match as an option"
+      it "should take an optional on_success lambda"
+      it "should take an optional on_failure lambda"
+      
+      it "should return raw xml" do
+        Feedzirra::Feed.fetch_raw(@paul_feed_url).should =~ /^#{Regexp.escape('<?xml version="1.0" encoding="UTF-8"?>')}/
+      end
+      
+      it "should take multiple feed urls and return a hash of urls and response xml" do
+        results = Feedzirra::Feed.fetch_raw([@paul_feed_url, @trotter_feed_url])
+        results.keys.should include(@paul_feed_url)
+        results.keys.should include(@trotter_feed_url)
+        results[@paul_feed_url].should =~ /Paul Dix/
+        results[@trotter_feed_url].should =~ /Trotter Cashion/
+      end
+      
+      it "should always return a hash when passed an array" do
+        results = Feedzirra::Feed.fetch_raw([@paul_feed_url])
+        results.class.should == Hash
+      end
+    end
+    
+    describe "#fetch_and_parse" do
+      it "should return a feed object for a single url" do
+        feed = Feedzirra::Feed.fetch_and_parse(@paul_feed_url)
+        feed.title.should == "Paul Dix Explains Nothing"
+      end
+      
+      it "should set the feed_url to the new url if redirected" do
+        feed = Feedzirra::Feed.fetch_and_parse("http://tinyurl.com/tenderlovemaking")
+        feed.feed_url.should == "http://tenderlovemaking.com/feed/"
+      end
+      
+      it "should set the feed_url for an rdf feed" do
+        feed = Feedzirra::Feed.fetch_and_parse("http://www.avibryant.com/rss.xml")
+        feed.feed_url.should == "http://www.avibryant.com/rss.xml"
+      end
+      
+      it "should set the feed_url for an rss feed" do
+        feed = Feedzirra::Feed.fetch_and_parse("http://tenderlovemaking.com/feed/")
+        feed.feed_url.should == "http://tenderlovemaking.com/feed/"
+      end
+      
+      it "should return a hash of feed objects with the passed in feed_url for the key and parsed feed for the value for multiple feeds" do
+        feeds = Feedzirra::Feed.fetch_and_parse([@paul_feed_url, @trotter_feed_url])
+        feeds.size.should == 2
+        feeds[@paul_feed_url].feed_url.should == @paul_feed_url
+        feeds[@trotter_feed_url].feed_url.should == @trotter_feed_url
+      end
+      
+      it "should always return a hash when passed an array" do
+        feeds = Feedzirra::Feed.fetch_and_parse([@paul_feed_url])
+        feeds.class.should == Hash
+      end
+      
+      it "should yeild the url and feed object to a :on_success lambda" do
+        successful_call_mock = mock("successful_call_mock")
+        successful_call_mock.should_receive(:call)
+        Feedzirra::Feed.fetch_and_parse(@paul_feed_url, :on_success => lambda { |feed_url, feed|
+          feed_url.should == @paul_feed_url
+          feed.class.should == Feedzirra::AtomFeedBurner
+          successful_call_mock.call})
+      end
+      
+      it "should yield the url, response_code, response_header, and response_body to a :on_failure lambda" do
+        failure_call_mock = mock("failure_call_mock")
+        failure_call_mock.should_receive(:call)
+        fail_url = "http://localhost"
+        Feedzirra::Feed.fetch_and_parse(fail_url, :on_failure => lambda {|feed_url, response_code, response_header, response_body|
+          feed_url.should == fail_url
+          response_code.should == 0
+          response_header.should == ""
+          response_body.should == ""
+          failure_call_mock.call})
+      end
+      
+      it "should return a not modified status for a feed with a :if_modified_since is past its last update" do
+        Feedzirra::Feed.fetch_and_parse(@paul_feed_url, :if_modified_since => Time.now).should == 304
+      end
+      
+      it "should set the etag from the header" # do
+       #        Feedzirra::Feed.fetch_and_parse(@paul_feed_url).etag.should_not == ""
+       #      end
+      
+      it "should set the last_modified from the header" # do
+       #        Feedzirra::Feed.fetch_and_parse(@paul_feed_url).last_modified.should.class == Time
+       #      end
+    end
+
+    describe "#update" do
+      it "should update and return a single feed object" do
+        feed = Feedzirra::Feed.fetch_and_parse(@paul_feed_url)
+        feed.entries.delete_at(0)
+        feed.last_modified = nil
+        feed.etag = nil
+        updated_feed = Feedzirra::Feed.update(feed)
+        updated_feed.new_entries.size.should == 1
+        updated_feed.should have_new_entries
+      end
+
+      it "should update a collection of feed objects" do
+        feeds = Feedzirra::Feed.fetch_and_parse([@paul_feed_url, @trotter_feed_url])
+        paul_entries_size    = feeds[@paul_feed_url].entries.size
+        trotter_entries_size = feeds[@trotter_feed_url].entries.size
+        
+        feeds.values.each do |feed|
+          feed.last_modified = nil
+          feed.etag = nil
+          feed.entries.delete_at(0)
+        end
+        updated_feeds = Feedzirra::Feed.update(feeds.values)
+        updated_feeds.detect {|f| f.feed_url == @paul_feed_url}.entries.size.should == paul_entries_size
+        updated_feeds.detect {|f| f.feed_url == @trotter_feed_url}.entries.size.should == trotter_entries_size
+      end
+      
+      it "should return the feed objects even when not updated" do
+        feeds = Feedzirra::Feed.fetch_and_parse([@paul_feed_url, @trotter_feed_url])
+        updated_feeds = Feedzirra::Feed.update(feeds.values)
+        updated_feeds.size.should == 2
+        updated_feeds.first.should_not be_updated
+        updated_feeds.last.should_not be_updated
+      end
+    end
+  end
+end