astro-feedzirra 0.0.8.20090419

data/lib/feedzirra/feed.rb

@@ -0,0 +1,321 @@
+module Feedzirra
+  class NoParserAvailable < StandardError; end
+  
+  class Feed
+    USER_AGENT = "feedzirra http://github.com/pauldix/feedzirra/tree/master"
+    
+    # Takes a raw XML feed and attempts to parse it. If no parser is available a Feedzirra::NoParserAvailable exception is raised.
+    #
+    # === Parameters
+    # [xml<String>] The XML that you would like parsed.
+    # === Returns
+    # An instance of the determined feed type. By default a Feedzirra::Atom, Feedzirra::AtomFeedBurner, Feedzirra::RDF, or Feedzirra::RSS object.
+    # === Raises
+    # Feedzirra::NoParserAvailable : If no valid parser classes could be found for the feed.
+    def self.parse(xml)
+      if parser = determine_feed_parser_for_xml(xml)
+        parser.parse(xml)
+      else
+        raise NoParserAvailable.new("No valid parser for XML.")
+      end
+    end
+
+    # Determines the correct parser class to use for parsing the feed.
+    # 
+    # === Parameters
+    # [xml<String>] The XML that you would like determine the parser for.
+    # === Returns
+    # The class name of the parser that can handle the XML.
+    def self.determine_feed_parser_for_xml(xml)
+      start_of_doc = xml.slice(0, 1000)
+      feed_classes.detect {|klass| klass.able_to_parse?(start_of_doc)}
+    end
+
+    # Adds a new feed parsing class that will be used for parsing.
+    #
+    # === Parameters
+    # [klass<Constant>] The class/constant that you want to register.
+    # === Returns
+    # A updated array of feed parser class names.
+    def self.add_feed_class(klass) 
+      feed_classes.unshift klass
+    end
+
+    # Provides a list of registered feed parsing classes.
+    #
+    # === Returns
+    # A array of class names.
+    def self.feed_classes
+      @feed_classes ||= [ITunesRSS, RSS, AtomFeedBurner, Atom]
+    end
+    
+    # Makes all entry types look for the passed in element to parse. This is actually just a call to 
+    # element (a SAXMachine call) in the class
+    #
+    # === Parameters
+    # [element_tag<String>]
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_entry_element(element_tag, options = {})
+      # need to think of a better way to do this. will break for people who want this behavior
+      # across their added classes
+      [RSSEntry, AtomFeedBurnerEntry, AtomEntry].each do |klass|
+        klass.send(:element, element_tag, options)
+      end
+    end
+    
+    # Makes all entry types look for the passed in elements to parse. This is actually just a call to 
+    # elements (a SAXMachine call) in the class
+    #
+    # === Parameters
+    # [element_tag<String>]
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_entry_elements(element_tag, options = {})
+      # need to think of a better way to do this. will break for people who want this behavior
+      # across their added classes
+      [RSSEntry, AtomFeedBurnerEntry, AtomEntry].each do |klass|
+        klass.send(:elements, element_tag, options)
+      end
+    end
+
+    # Fetches and returns the raw XML for each URL provided.
+    #
+    # === Parameters
+    # [urls<String> or <Array>] A single feed URL, or an array of feed URLs.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 :user_agent - String that overrides the default user agent.
+    #                 :if_modified_since - Time object representing when the feed was last updated.
+    #                 :if_none_match - String that's normally an etag for the request that was stored previously.
+    #                 :on_success - Block that gets executed after a successful request.
+    #                 :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # A String of XML if a single URL is passed.
+    # 
+    # A Hash if multiple URL's are passed. The key will be the URL, and the value the XML.
+    def self.fetch_raw(urls, options = {})
+      url_queue = [*urls]
+      multi = Curl::Multi.new
+      responses = {}
+      url_queue.each do |url|
+        easy = Curl::Easy.new(url) do |curl|
+          curl.headers["User-Agent"]        = (options[:user_agent] || USER_AGENT)
+          curl.headers["If-Modified-Since"] = options[:if_modified_since].httpdate if options.has_key?(:if_modified_since)
+          curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
+          curl.headers["Accept-encoding"]   = 'gzip, deflate'
+          curl.follow_location = true
+          curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
+
+          curl.on_success do |c|
+            responses[url] = decode_content(c)
+          end
+          curl.on_failure do |c|
+            responses[url] = c.response_code
+          end
+        end
+        multi.add(easy)
+      end
+
+      multi.perform
+      return urls.is_a?(String) ? responses.values.first : responses
+    end
+
+    # Fetches and returns the parsed XML for each URL provided.
+    #
+    # === Parameters
+    # [urls<String> or <Array>] A single feed URL, or an array of feed URLs.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :user_agent - String that overrides the default user agent.
+    #                 * :if_modified_since - Time object representing when the feed was last updated.
+    #                 * :if_none_match - String, an etag for the request that was stored previously.
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request. 
+    # === Returns
+    # A Feed object if a single URL is passed.
+    #
+    # A Hash if multiple URL's are passed. The key will be the URL, and the value the Feed object.
+    def self.fetch_and_parse(urls, options = {})
+      url_queue = [*urls]
+      multi = Curl::Multi.new
+      responses = {}
+      
+      # I broke these down so I would only try to do 30 simultaneously because 
+      # I was getting weird errors when doing a lot. As one finishes it pops another off the queue.
+      url_queue.slice!(0, 30).each do |url|
+        add_url_to_multi(multi, url, url_queue, responses, options)
+      end
+
+      multi.perform
+      return urls.is_a?(String) ? responses.values.first : responses
+    end
+
+    # Decodes the XML document if it was compressed.
+    #
+    # === Parameters
+    # [curl_request<Curl::Easy>] The Curl::Easy response object from the request.
+    # === Returns
+    # A decoded string of XML.
+    def self.decode_content(c)
+      if c.header_str.match(/Content-Encoding: gzip/)
+        begin
+          gz =  Zlib::GzipReader.new(StringIO.new(c.body_str))
+          xml = gz.read
+          gz.close
+        rescue Zlib::GzipFile::Error 
+          # Maybe this is not gzipped?
+          xml = c.body_str
+        end
+      elsif c.header_str.match(/Content-Encoding: deflate/)
+        xml = Zlib::Inflate.inflate(c.body_str)
+      else
+        xml = c.body_str
+      end
+
+      xml
+    end
+
+    # Updates each feed for each Feed object provided.
+    #
+    # === Parameters
+    # [feeds<Feed> or <Array>] A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :user_agent - String that overrides the default user agent.
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # A updated Feed object if a single URL is passed.
+    #
+    # A Hash if multiple Feeds are passed. The key will be the URL, and the value the updated Feed object.
+    def self.update(feeds, options = {})
+      feed_queue = [*feeds]
+      multi = Curl::Multi.new
+      responses = {}
+      
+      feed_queue.slice!(0, 30).each do |feed|
+        add_feed_to_multi(multi, feed, feed_queue, responses, options)
+      end
+    
+      multi.perform
+      return responses.size == 1 ? responses.values.first : responses.values
+    end
+    
+    # An abstraction for adding a feed by URL to the passed Curb::multi stack.
+    #
+    # === Parameters
+    # [multi<Curl::Multi>] The Curl::Multi object that the request should be added too.
+    # [url<String>] The URL of the feed that you would like to be fetched.
+    # [url_queue<Array>] An array of URLs that are queued for request.
+    # [responses<Hash>] Existing responses that you want the response from the request added to.
+    # [feeds<String> or <Array>] A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :user_agent - String that overrides the default user agent.
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # The updated Curl::Multi object with the request details added to it's stack.
+    def self.add_url_to_multi(multi, url, url_queue, responses, options)
+      easy = Curl::Easy.new(url) do |curl|
+        curl.headers["User-Agent"]        = (options[:user_agent] || USER_AGENT)
+        curl.headers["If-Modified-Since"] = options[:if_modified_since].httpdate if options.has_key?(:if_modified_since)
+        curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
+        curl.headers["Accept-encoding"]   = 'gzip, deflate'
+        curl.follow_location = true
+        curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
+        
+        curl.on_success do |c|
+          add_url_to_multi(multi, url_queue.shift, url_queue, responses, options) unless url_queue.empty?
+          xml = decode_content(c)
+          klass = determine_feed_parser_for_xml(xml)
+          
+          if klass
+            feed = klass.parse(xml)
+            feed.feed_url = c.last_effective_url
+            feed.etag = etag_from_header(c.header_str)
+            feed.last_modified = last_modified_from_header(c.header_str)
+            responses[url] = feed
+            options[:on_success].call(url, feed) if options.has_key?(:on_success)
+          else
+            # puts "Error determining parser for #{url} - #{c.last_effective_url}"
+            # raise NoParserAvailable.new("no valid parser for content.") (this would unfirtunately fail the whole 'multi', so it's not really useable)
+            options[:on_failure].call(url, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+          end
+        end
+        
+        curl.on_failure do |c|
+          add_url_to_multi(multi, url_queue.shift, url_queue, responses, options) unless url_queue.empty?
+          responses[url] = c.response_code
+          options[:on_failure].call(url, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+        end
+      end
+      multi.add(easy)
+    end
+    
+    # An abstraction for adding a feed by a Feed object to the passed Curb::multi stack.
+    #
+    # === Parameters
+    # [multi<Curl::Multi>] The Curl::Multi object that the request should be added too.
+    # [feed<Feed>] A feed object that you would like to be fetched.
+    # [url_queue<Array>] An array of feed objects that are queued for request.
+    # [responses<Hash>] Existing responses that you want the response from the request added to.
+    # [feeds<String>] or <Array> A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :user_agent - String that overrides the default user agent.
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # The updated Curl::Multi object with the request details added to it's stack.
+    def self.add_feed_to_multi(multi, feed, feed_queue, responses, options) 
+      easy = Curl::Easy.new(feed.feed_url) do |curl|
+        curl.headers["User-Agent"]        = (options[:user_agent] || USER_AGENT)
+        curl.headers["If-Modified-Since"] = feed.last_modified.httpdate if feed.last_modified
+        curl.headers["If-None-Match"]     = feed.etag if feed.etag
+        curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
+        curl.follow_location = true
+
+        curl.on_success do |c|
+          add_feed_to_multi(multi, feed_queue.shift, feed_queue, responses, options) unless feed_queue.empty?
+          updated_feed = Feed.parse(c.body_str)
+          updated_feed.feed_url = c.last_effective_url
+          updated_feed.etag = etag_from_header(c.header_str)
+          updated_feed.last_modified = last_modified_from_header(c.header_str)
+          feed.update_from_feed(updated_feed)
+          responses[feed.feed_url] = feed
+          options[:on_success].call(feed) if options.has_key?(:on_success)
+        end
+
+        curl.on_failure do |c|
+          add_feed_to_multi(multi, feed_queue.shift, feed_queue, responses, options) unless feed_queue.empty?
+          response_code = c.response_code
+          if response_code == 304 # it's not modified. this isn't an error condition
+            responses[feed.feed_url] = feed
+            options[:on_success].call(feed) if options.has_key?(:on_success)
+          else
+            responses[feed.url] = c.response_code
+            options[:on_failure].call(feed, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+          end
+        end
+      end
+      multi.add(easy)
+    end
+
+    # Determines the etag from the request headers.
+    # 
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A string of the etag or nil if it cannot be found in the headers.
+    def self.etag_from_header(header)
+      header =~ /.*ETag:\s(.*)\r/
+      $1
+    end
+
+    # Determines the last modified date from the request headers.
+    #
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A Time object of the last modified date or nil if it cannot be found in the headers.
+    def self.last_modified_from_header(header)
+      header =~ /.*Last-Modified:\s(.*)\r/
+      Time.parse($1) if $1
+    end
+  end
+end

data/lib/feedzirra/feed_entry_utilities.rb

@@ -0,0 +1,45 @@
+module Feedzirra
+  module FeedEntryUtilities    
+    def published
+      @published || @updated
+    end
+    
+    def parse_datetime(string)
+      begin
+        DateTime.parse(string).feed_utils_to_gm_time
+      rescue
+        puts "DATE CAN'T BE PARSED: #{string}"
+        nil
+      end
+    end
+    
+    ##
+    # Returns the id of the entry or its url if not id is present, as some formats don't support it
+    def id
+      @id || @url
+    end
+    
+    ##
+    # Writter for published. By default, we keep the "oldest" publish time found.
+    def published=(val)
+      parsed = parse_datetime(val)
+      @published = parsed if !@published || parsed < @published
+    end
+    
+    ##
+    # Writter for udapted. By default, we keep the most recenet update time found.
+    def updated=(val)
+      parsed = parse_datetime(val)
+      @updated = parsed if !@updated || parsed > @updated
+    end
+
+    def sanitize!
+      self.title.sanitize!   if self.title
+      self.author.sanitize!  if self.author
+      self.summary.sanitize! if self.summary
+      self.content.sanitize! if self.content
+    end
+    
+    alias_method :last_modified, :published
+  end
+end

data/lib/feedzirra/feed_utilities.rb

@@ -0,0 +1,71 @@
+module Feedzirra
+  module FeedUtilities
+    UPDATABLE_ATTRIBUTES = %w(title feed_url url last_modified)
+    
+    attr_writer   :new_entries, :updated, :last_modified
+    attr_accessor :etag
+
+    def last_modified
+      @last_modified ||= begin
+        entry = entries.reject {|e| e.published.nil? }.sort_by { |entry| entry.published if entry.published }.last
+        entry ? entry.published : nil
+      end
+    end
+    
+    def updated?
+      @updated
+    end
+    
+    def new_entries
+      @new_entries ||= []
+    end
+    
+    def has_new_entries?
+      new_entries.size > 0
+    end
+    
+    def update_from_feed(feed)
+      self.new_entries += find_new_entries_for(feed)
+      self.entries.unshift(*self.new_entries)
+      
+      updated! if UPDATABLE_ATTRIBUTES.any? { |name| update_attribute(feed, name) }
+    end
+    
+    def update_attribute(feed, name)
+      old_value, new_value = send(name), feed.send(name)
+      
+      if old_value != new_value
+        send("#{name}=", new_value)
+      end
+    end
+    
+    def sanitize_entries!
+      entries.each {|entry| entry.sanitize!}
+    end
+    
+    private
+    
+    def updated!
+      @updated = true
+    end
+    
+    def find_new_entries_for(feed)
+      # this implementation is a hack, which is why it's so ugly.
+      # it's to get around the fact that not all feeds have a published date.
+      # however, they're always ordered with the newest one first.
+      # So we go through the entries just parsed and insert each one as a new entry
+      # until we get to one that has the same url as the the newest for the feed
+      latest_entry = self.entries.first
+      found_new_entries = []
+      feed.entries.each do |entry|
+        break if entry.url == latest_entry.url
+        found_new_entries << entry
+      end
+      found_new_entries
+    end
+    
+    def existing_entry?(test_entry)
+      entries.any? { |entry| entry.url == test_entry.url }
+    end
+  end
+end

data/lib/feedzirra/itunes_rss.rb

@@ -0,0 +1,46 @@
+module Feedzirra
+  # iTunes is RSS 2.0 + some apple extensions
+  # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
+  class ITunesRSS
+    include SAXMachine
+    include FeedUtilities
+
+    attr_accessor :feed_url
+
+    # RSS 2.0 elements that need including
+    element :copyright
+    element :description
+    element :language
+    element :managingEditor
+    element :title
+    element :link, :as => :url
+
+    # If author is not present use managingEditor on the channel
+    element :"itunes:author", :as => :itunes_author
+    element :"itunes:block", :as => :itunes_block
+    element :"itunes:image", :value => :href, :as => :itunes_image
+    element :"itunes:explicit", :as => :itunes_explicit
+    element :"itunes:keywords", :as => :itunes_keywords
+    # New URL for the podcast feed
+    element :"itunes:new-feed-url", :as => :itunes_new_feed_url
+    element :"itunes:subtitle", :as => :itunes_subtitle
+    # If summary is not present, use the description tag
+    element :"itunes:summary", :as => :itunes_summary
+
+    # iTunes RSS feeds can have multiple main categories...
+    # ...and multiple sub-categories per category
+    # TODO subcategories not supported correctly - they are at the same level
+    #   as the main categories
+    elements :"itunes:category", :as => :itunes_categories, :value => :text
+
+    elements :"itunes:owner", :as => :itunes_owners, :class => ITunesRSSOwner
+
+    elements :item, :as => :entries, :class => ITunesRSSItem
+
+    def self.able_to_parse?(xml)
+      xml =~ /xmlns:itunes=\"http:\/\/www.itunes.com\/dtds\/podcast-1.0.dtd\"/
+    end
+
+  end
+
+end

data/lib/feedzirra/itunes_rss_item.rb

@@ -0,0 +1,28 @@
+module Feedzirra
+  # iTunes extensions to the standard RSS2.0 item
+  # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
+  class ITunesRSSItem
+    include SAXMachine
+    include FeedUtilities
+    element :author
+    element :guid
+    element :title
+    element :link, :as => :url
+    element :description, :as => :summary
+    element :pubDate, :as => :published
+
+    # If author is not present use author tag on the item
+    element :"itunes:author", :as => :itunes_author
+    element :"itunes:block", :as => :itunes_block
+    element :"itunes:duration", :as => :itunes_duration
+    element :"itunes:explicit", :as => :itunes_explicit
+    element :"itunes:keywords", :as => :itunes_keywords
+    element :"itunes:subtitle", :as => :itunes_subtitle
+    # If summary is not present, use the description tag
+    element :"itunes:summary", :as => :itunes_summary
+    element :enclosure, :value => :length, :as => :enclosure_length
+    element :enclosure, :value => :type, :as => :enclosure_type
+    element :enclosure, :value => :url, :as => :enclosure_url
+  end
+
+end

data/lib/feedzirra/itunes_rss_owner.rb

@@ -0,0 +1,8 @@
+module Feedzirra
+  class ITunesRSSOwner
+    include SAXMachine
+    include FeedUtilities
+    element :"itunes:name", :as => :name
+    element :"itunes:email", :as => :email
+  end
+end

data/lib/feedzirra/rss.rb

@@ -0,0 +1,23 @@
+module Feedzirra
+  # == Summary
+  # Parser for dealing with RSS feeds.
+  #
+  # == Attributes
+  # * title
+  # * feed_url
+  # * url
+  # * entries
+  class RSS
+    include SAXMachine
+    include FeedUtilities
+    element :title
+    element :link, :as => :url
+    elements :item, :as => :entries, :class => RSSEntry
+    
+    attr_accessor :feed_url
+    
+    def self.able_to_parse?(xml) #:nodoc:
+      xml =~ /\<rss|rdf/
+    end
+  end
+end

data/lib/feedzirra/rss_entry.rb

@@ -0,0 +1,35 @@
+module Feedzirra
+  # == Summary
+  # Parser for dealing with RDF feed entries.
+  #
+  # == Attributes
+  # * title
+  # * url
+  # * author
+  # * content
+  # * summary
+  # * published
+  # * categories
+  class RSSEntry
+    include SAXMachine
+    include FeedEntryUtilities
+    element :title
+    element :link, :as => :url
+
+    element :"dc:creator", :as => :author
+    element :"content:encoded", :as => :content
+    element :description, :as => :summary
+
+    element :pubDate, :as => :published
+    element :"dc:date", :as => :published
+    element :"dc:Date", :as => :published
+    element :"dcterms:created", :as => :published
+    
+    
+    element :"dcterms:modified", :as => :updated
+    element :issued, :as => :published
+    elements :category, :as => :categories
+    
+    element :guid, :as => :id
+  end
+end

data/lib/feedzirra.rb

@@ -0,0 +1,36 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
+
+gem 'activesupport'
+
+require 'zlib'
+require 'curb'
+require 'sax-machine'
+require 'dryopteris'
+require 'uri'
+require 'active_support/basic_object'
+require 'active_support/core_ext/object'
+require 'active_support/core_ext/time'
+
+require 'core_ext/date'
+require 'core_ext/string'
+
+require 'feedzirra/feed_utilities'
+require 'feedzirra/feed_entry_utilities'
+require 'feedzirra/feed'
+
+require 'feedzirra/push_parser'
+
+require 'feedzirra/rss_entry'
+require 'feedzirra/itunes_rss_owner'
+require 'feedzirra/itunes_rss_item'
+require 'feedzirra/atom_entry'
+require 'feedzirra/atom_feed_burner_entry'
+
+require 'feedzirra/rss'
+require 'feedzirra/itunes_rss'
+require 'feedzirra/atom'
+require 'feedzirra/atom_feed_burner'
+
+module Feedzirra
+  VERSION = "0.0.8"
+end

data/spec/feedzirra/atom_entry_spec.rb

@@ -0,0 +1,45 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Feedzirra::AtomEntry do
+  before(:each) do
+    # I don't really like doing it this way because these unit test should only rely on AtomEntry,
+    # but this is actually how it should work. You would never just pass entry xml straight to the AtomEnry
+    @entry = Feedzirra::Atom.parse(sample_atom_feed).entries.first
+  end
+  
+  it "should parse the title" do
+    @entry.title.should == "AWS Job: Architect & Designer Position in Turkey"
+  end
+  
+  it "should parse the url" do
+    @entry.url.should == "http://aws.typepad.com/aws/2009/01/aws-job-architect-designer-position-in-turkey.html"
+  end
+  
+  it "should parse the author" do
+    @entry.author.should == "AWS Editor"
+  end
+  
+  it "should parse the content" do
+    @entry.content.should == sample_atom_entry_content
+  end
+  
+  it "should provide a summary" do
+    @entry.summary.should == "Late last year an entrepreneur from Turkey visited me at Amazon HQ in Seattle. We talked about his plans to use AWS as part of his new social video portal startup. I won't spill any beans before he's ready to..."
+  end
+  
+  it "should parse the published date" do
+    @entry.published.to_s.should == "Fri Jan 16 18:21:00 UTC 2009"
+  end
+
+  it "should parse the categories" do
+    @entry.categories.should == ['Turkey', 'Seattle']
+  end
+  
+  it "should parse the updated date" do
+    @entry.updated.to_s.should == "Fri Jan 16 18:21:00 UTC 2009"
+  end
+  
+  it "should parse the id" do
+    @entry.id.should == "tag:typepad.com,2003:post-61484736"
+  end
+end