pauldix-feedzirra 0.0.3 → 0.0.5

data/README.textile

@@ -99,7 +99,7 @@ feeds = Feedzirra::Feed.fetch_and_parse(feeds_urls)
 # there will be a Fixnum of the http response code instead of a feed object
 
 # updating multiple feeds. it expects a collection of feed objects
-updated_feeds = Feedzirra::Feed.udpate(feeds.values)
+updated_feeds = Feedzirra::Feed.update(feeds.values)
 
 # defining custom behavior on failure or success. note that a return status of 304 (not updated) will call the on_success handler
 feed = Feedzirra::Feed.fetch_and_parse("http://feeds.feedburner.com/PaulDixExplainsNothing",

data/Rakefile

@@ -1,12 +1,51 @@
 require "spec"
 require "spec/rake/spectask"
+require 'rake/rdoctask'
 require 'lib/feedzirra.rb'
 
+# Grab recently touched specs
+def recent_specs(touched_since)
+  recent_specs = FileList['app/**/*'].map do |path|
+
+    if File.mtime(path) > touched_since
+      spec = File.join('spec', File.dirname(path).split("/")[1..-1].join('/'),
+        "#{File.basename(path, ".*")}_spec.rb")
+      spec if File.exists?(spec)
+    end
+  end.compact
+
+  recent_specs += FileList['spec/**/*_spec.rb'].select do |path|
+    File.mtime(path) > touched_since
+  end
+  recent_specs.uniq
+end
+
+# Tasks
 Spec::Rake::SpecTask.new do |t|
   t.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
   t.spec_files = FileList['spec/**/*_spec.rb']
 end
 
+desc 'Run recent specs'
+Spec::Rake::SpecTask.new("spec:recent") do |t|
+  t.spec_opts = ["--format","specdoc","--color"]
+  t.spec_files = recent_specs(Time.now - 600) # 10 min.
+end
+
+Spec::Rake::SpecTask.new('spec:rcov') do |t|
+  t.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.rcov = true
+  t.rcov_opts = ['--exclude', 'spec,/usr/lib/ruby,/usr/local,/var/lib,/Library', '--text-report']
+end
+
+Rake::RDocTask.new do |rd|
+  rd.title    = 'Feedzirra'
+  rd.rdoc_dir = 'rdoc'
+  rd.rdoc_files.include('README.rdoc', 'lib/feedzirra.rb', 'lib/feedzirra/**/*.rb')
+  rd.options = ["--quiet", "--opname", "index.html", "--line-numbers", "--inline-source", '--main', 'README.rdoc']
+end
+
 task :install do
   rm_rf "*.gem"
   puts `gem build feedzirra.gemspec`

data/lib/core_ext/string.rb

@@ -0,0 +1,9 @@
+class String
+  def sanitize!
+    self.replace(sanitize)
+  end
+  
+  def sanitize
+    Dryopteris.sanitize(self)
+  end
+end

data/lib/feedzirra/atom.rb

@@ -1,4 +1,12 @@
 module Feedzirra
+  # == Summary
+  # Parser for dealing with Atom feeds.
+  #
+  # == Attributes
+  # * title
+  # * feed_url
+  # * url
+  # * entries
   class Atom
     include SAXMachine
     include FeedUtilities
@@ -7,7 +15,7 @@ module Feedzirra
     element :link, :as => :feed_url, :value => :href, :with => {:type => "application/atom+xml"}
     elements :entry, :as => :entries, :class => AtomEntry
     
-    def self.able_to_parse?(xml)
+    def self.able_to_parse?(xml) #:nodoc:
       xml =~ /(Atom)|(#{Regexp.escape("http://purl.org/atom")})/
     end
   end

data/lib/feedzirra/atom_entry.rb

@@ -1,4 +1,15 @@
 module Feedzirra
+  # == Summary
+  # Parser for dealing with Atom feed entries.
+  #
+  # == Attributes
+  # * title
+  # * url
+  # * author
+  # * content
+  # * summary
+  # * published
+  # * categories
   class AtomEntry
     include SAXMachine
     include FeedEntryUtilities

data/lib/feedzirra/atom_feed_burner.rb

@@ -1,4 +1,12 @@
 module Feedzirra
+  # == Summary
+  # Parser for dealing with Feedburner Atom feeds.
+  #
+  # == Attributes
+  # * title
+  # * feed_url
+  # * url
+  # * entries
   class AtomFeedBurner
     include SAXMachine
     include FeedUtilities
@@ -7,7 +15,7 @@ module Feedzirra
     element :link, :as => :feed_url, :value => :href, :with => {:type => "application/atom+xml"}
     elements :entry, :as => :entries, :class => AtomFeedBurnerEntry
 
-    def self.able_to_parse?(xml)
+    def self.able_to_parse?(xml) #:nodoc:
       (xml =~ /Atom/ && xml =~ /feedburner/) || false
     end
   end

data/lib/feedzirra/atom_feed_burner_entry.rb

@@ -1,4 +1,15 @@
 module Feedzirra
+  # == Summary
+  # Parser for dealing with Feedburner Atom feed entries.
+  #
+  # == Attributes
+  # * title
+  # * url
+  # * author
+  # * content
+  # * summary
+  # * published
+  # * categories
   class AtomFeedBurnerEntry
     include SAXMachine
     include FeedEntryUtilities

data/lib/feedzirra/feed.rb

@@ -4,30 +4,93 @@ module Feedzirra
   class Feed
     USER_AGENT = "feedzirra http://github.com/pauldix/feedzirra/tree/master"
     
+    # Takes a raw XML feed and attempts to parse it. If no parser is available a Feedzirra::NoParserAvailable exception is raised.
+    #
+    # === Parameters
+    # [xml<String>] The XML that you would like parsed.
+    # === Returns
+    # An instance of the determined feed type. By default a Feedzirra::Atom, Feedzirra::AtomFeedBurner, Feedzirra::RDF, or Feedzirra::RSS object.
+    # === Raises
+    # Feedzirra::NoParserAvailable : If no valid parser classes could be found for the feed.
     def self.parse(xml)
       if parser = determine_feed_parser_for_xml(xml)
         parser.parse(xml)
       else
-        raise NoParserAvailable.new("no valid parser for content.")
+        raise NoParserAvailable.new("No valid parser for XML.")
       end
     end
 
+    # Determines the correct parser class to use for parsing the feed.
+    # 
+    # === Parameters
+    # [xml<String>] The XML that you would like determine the parser for.
+    # === Returns
+    # The class name of the parser that can handle the XML.
     def self.determine_feed_parser_for_xml(xml)
       start_of_doc = xml.slice(0, 1000)
       feed_classes.detect {|klass| klass.able_to_parse?(start_of_doc)}
     end
 
-    def self.add_feed_class(klass)
+    # Adds a new feed parsing class that will be used for parsing.
+    #
+    # === Parameters
+    # [klass<Constant>] The class/constant that you want to register.
+    # === Returns
+    # A updated array of feed parser class names.
+    def self.add_feed_class(klass) 
       feed_classes.unshift klass
     end
-    
+
+    # Provides a list of registered feed parsing classes.
+    #
+    # === Returns
+    # A array of class names.
     def self.feed_classes
-      @feed_classes ||= [RSS, AtomFeedBurner, Atom]
+      @feed_classes ||= [ITunesRSS, RSS, AtomFeedBurner, Atom]
+    end
+    
+    # Makes all entry types look for the passed in element to parse. This is actually just a call to 
+    # element (a SAXMachine call) in the class
+    #
+    # === Parameters
+    # [element_tag<String>]
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_entry_element(element_tag, options = {})
+      # need to think of a better way to do this. will break for people who want this behavior
+      # across their added classes
+      [RSSEntry, AtomFeedBurnerEntry, AtomEntry].each do |klass|
+        klass.send(:element, element_tag, options)
+      end
+    end
+    
+    # Makes all entry types look for the passed in elements to parse. This is actually just a call to 
+    # elements (a SAXMachine call) in the class
+    #
+    # === Parameters
+    # [element_tag<String>]
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_entry_elements(element_tag, options = {})
+      # need to think of a better way to do this. will break for people who want this behavior
+      # across their added classes
+      [RSSEntry, AtomFeedBurnerEntry, AtomEntry].each do |klass|
+        klass.send(:elements, element_tag, options)
+      end
     end
 
-    # can take a single url or an array of urls
-    # when passed a single url it returns the body of the response
-    # when passed an array of urls it returns a hash with the urls as keys and body of responses as values
+    # Fetches and returns the raw XML for each URL provided.
+    #
+    # === Parameters
+    # [urls<String> or <Array>] A single feed URL, or an array of feed URLs.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 :user_agent - String that overrides the default user agent.
+    #                 :if_modified_since - Time object representing when the feed was last updated.
+    #                 :if_none_match - String that's normally an etag for the request that was stored previously.
+    #                 :on_success - Block that gets executed after a successful request.
+    #                 :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # A String of XML if a single URL is passed.
+    # 
+    # A Hash if multiple URL's are passed. The key will be the URL, and the value the XML.
     def self.fetch_raw(urls, options = {})
       url_queue = [*urls]
       multi = Curl::Multi.new
@@ -39,6 +102,8 @@ module Feedzirra
           curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
           curl.headers["Accept-encoding"]   = 'gzip, deflate'
           curl.follow_location = true
+          curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
+
           curl.on_success do |c|
             responses[url] = decode_content(c)
           end
@@ -52,14 +117,28 @@ module Feedzirra
       multi.perform
       return urls.is_a?(String) ? responses.values.first : responses
     end
-    
+
+    # Fetches and returns the parsed XML for each URL provided.
+    #
+    # === Parameters
+    # [urls<String> or <Array>] A single feed URL, or an array of feed URLs.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :user_agent - String that overrides the default user agent.
+    #                 * :if_modified_since - Time object representing when the feed was last updated.
+    #                 * :if_none_match - String, an etag for the request that was stored previously.
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request. 
+    # === Returns
+    # A Feed object if a single URL is passed.
+    #
+    # A Hash if multiple URL's are passed. The key will be the URL, and the value the Feed object.
     def self.fetch_and_parse(urls, options = {})
       url_queue = [*urls]
       multi = Curl::Multi.new
-
+      responses = {}
+      
       # I broke these down so I would only try to do 30 simultaneously because 
       # I was getting weird errors when doing a lot. As one finishes it pops another off the queue.
-      responses = {}
       url_queue.slice!(0, 30).each do |url|
         add_url_to_multi(multi, url, url_queue, responses, options)
       end
@@ -67,25 +146,44 @@ module Feedzirra
       multi.perform
       return urls.is_a?(String) ? responses.values.first : responses
     end
-    
-    def self.decode_content(c)
-      if c.header_str.match(/Content-Encoding: gzip/)
-        gz =  Zlib::GzipReader.new(StringIO.new(c.body_str))
+
+    # Decodes the XML document if it was compressed.
+    #
+    # === Parameters
+    # [curl_request<Curl::Easy>] The Curl::Easy response object from the request.
+    # === Returns
+    # A decoded string of XML.
+    def self.decode_content(curl_request)
+      if curl_request.header_str.match(/Content-Encoding: gzip/)
+        gz =  Zlib::GzipReader.new(StringIO.new(curl_request.body_str))
         xml = gz.read
         gz.close
-      elsif c.header_str.match(/Content-Encoding: deflate/)
-        xml = Zlib::Deflate.inflate(c.body_str)
+      elsif curl_request.header_str.match(/Content-Encoding: deflate/)
+        xml = Zlib::Deflate.inflate(curl_request.body_str)
       else
-        xml = c.body_str
+        xml = curl_request.body_str
       end
-      
+
       xml
     end
-    
+
+    # Updates each feed for each Feed object provided.
+    #
+    # === Parameters
+    # [feeds<Feed> or <Array>] A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :user_agent - String that overrides the default user agent.
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # A updated Feed object if a single URL is passed.
+    #
+    # A Hash if multiple Feeds are passed. The key will be the URL, and the value the updated Feed object.
     def self.update(feeds, options = {})
       feed_queue = [*feeds]
       multi = Curl::Multi.new
       responses = {}
+      
       feed_queue.slice!(0, 30).each do |feed|
         add_feed_to_multi(multi, feed, feed_queue, responses, options)
       end
@@ -94,6 +192,20 @@ module Feedzirra
       return responses.size == 1 ? responses.values.first : responses.values
     end
     
+    # An abstraction for adding a feed by URL to the passed Curb::multi stack.
+    #
+    # === Parameters
+    # [multi<Curl::Multi>] The Curl::Multi object that the request should be added too.
+    # [url<String>] The URL of the feed that you would like to be fetched.
+    # [url_queue<Array>] An array of URLs that are queued for request.
+    # [responses<Hash>] Existing responses that you want the response from the request added to.
+    # [feeds<String> or <Array>] A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :user_agent - String that overrides the default user agent.
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # The updated Curl::Multi object with the request details added to it's stack.
     def self.add_url_to_multi(multi, url, url_queue, responses, options)
       easy = Curl::Easy.new(url) do |curl|
         curl.headers["User-Agent"]        = (options[:user_agent] || USER_AGENT)
@@ -101,10 +213,13 @@ module Feedzirra
         curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
         curl.headers["Accept-encoding"]   = 'gzip, deflate'
         curl.follow_location = true
+        curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
+        
         curl.on_success do |c|
           add_url_to_multi(multi, url_queue.shift, url_queue, responses, options) unless url_queue.empty?
           xml = decode_content(c)
           klass = determine_feed_parser_for_xml(xml)
+          
           if klass
             feed = klass.parse(xml)
             feed.feed_url = c.last_effective_url
@@ -113,9 +228,10 @@ module Feedzirra
             responses[url] = feed
             options[:on_success].call(url, feed) if options.has_key?(:on_success)
           else
-            puts "Error determining parser for #{url} - #{c.last_effective_url}"
+            raise NoParserAvailable.new("Error determining parser for #{url} - #{c.last_effective_url}.")
           end
         end
+        
         curl.on_failure do |c|
           add_url_to_multi(multi, url_queue.shift, url_queue, responses, options) unless url_queue.empty?
           responses[url] = c.response_code
@@ -125,16 +241,28 @@ module Feedzirra
       multi.add(easy)
     end
     
-    def self.add_feed_to_multi(multi, feed, feed_queue, responses, options)
-      # on_success = options[:on_success]
-      # on_failure = options[:on_failure]
-      # options[:on_success] = lambda do ||
-      
+    # An abstraction for adding a feed by a Feed object to the passed Curb::multi stack.
+    #
+    # === Parameters
+    # [multi<Curl::Multi>] The Curl::Multi object that the request should be added too.
+    # [feed<Feed>] A feed object that you would like to be fetched.
+    # [url_queue<Array>] An array of feed objects that are queued for request.
+    # [responses<Hash>] Existing responses that you want the response from the request added to.
+    # [feeds<String>] or <Array> A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :user_agent - String that overrides the default user agent.
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # The updated Curl::Multi object with the request details added to it's stack.
+    def self.add_feed_to_multi(multi, feed, feed_queue, responses, options) 
       easy = Curl::Easy.new(feed.feed_url) do |curl|
         curl.headers["User-Agent"]        = (options[:user_agent] || USER_AGENT)
         curl.headers["If-Modified-Since"] = feed.last_modified.httpdate if feed.last_modified
         curl.headers["If-None-Match"]     = feed.etag if feed.etag
+        curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
         curl.follow_location = true
+
         curl.on_success do |c|
           add_feed_to_multi(multi, feed_queue.shift, feed_queue, responses, options) unless feed_queue.empty?
           updated_feed = Feed.parse(c.body_str)
@@ -145,6 +273,7 @@ module Feedzirra
           responses[feed.feed_url] = feed
           options[:on_success].call(feed) if options.has_key?(:on_success)
         end
+
         curl.on_failure do |c|
           add_feed_to_multi(multi, feed_queue.shift, feed_queue, responses, options) unless feed_queue.empty?
           response_code = c.response_code
@@ -159,12 +288,24 @@ module Feedzirra
       end
       multi.add(easy)
     end
-    
+
+    # Determines the etag from the request headers.
+    # 
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A string of the etag or nil if it cannot be found in the headers.
     def self.etag_from_header(header)
       header =~ /.*ETag:\s(.*)\r/
       $1
     end
-    
+
+    # Determines the last modified date from the request headers.
+    #
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A Time object of the last modified date or nil if it cannot be found in the headers.
     def self.last_modified_from_header(header)
       header =~ /.*Last-Modified:\s(.*)\r/
       Time.parse($1) if $1

data/lib/feedzirra/feed_entry_utilities.rb

@@ -1,15 +1,5 @@
 module Feedzirra
   module FeedEntryUtilities
-    module Sanitize
-      def sanitize!
-        self.replace(sanitize)
-      end
-      
-      def sanitize
-        Dryopteris.sanitize(self)
-      end
-    end
-
     attr_reader :published
     
     def parse_datetime(string)
@@ -19,19 +9,7 @@ module Feedzirra
     def published=(val)
       @published = parse_datetime(val)
     end
-    
-    def content
-      @content.extend(Sanitize)
-    end
-    
-    def title
-      @title.extend(Sanitize)
-    end
-    
-    def author
-      @author.extend(Sanitize)
-    end
-    
+
     def sanitize!
       self.title.sanitize!
       self.author.sanitize!

data/lib/feedzirra/itunes_rss.rb

@@ -0,0 +1,46 @@
+module Feedzirra
+  # iTunes is RSS 2.0 + some apple extensions
+  # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
+  class ITunesRSS
+    include SAXMachine
+    include FeedUtilities
+
+    attr_accessor :feed_url
+
+    # RSS 2.0 elements that need including
+    element :copyright
+    element :description
+    element :language
+    element :managingEditor
+    element :title
+    element :link, :as => :url
+
+    # If author is not present use managingEditor on the channel
+    element :"itunes:author", :as => :itunes_author
+    element :"itunes:block", :as => :itunes_block
+    element :"itunes:image", :value => :href, :as => :itunes_image
+    element :"itunes:explicit", :as => :itunes_explicit
+    element :"itunes:keywords", :as => :itunes_keywords
+    # New URL for the podcast feed
+    element :"itunes:new-feed-url", :as => :itunes_new_feed_url
+    element :"itunes:subtitle", :as => :itunes_subtitle
+    # If summary is not present, use the description tag
+    element :"itunes:summary", :as => :itunes_summary
+
+    # iTunes RSS feeds can have multiple main categories...
+    # ...and multiple sub-categories per category
+    # TODO subcategories not supported correctly - they are at the same level
+    #   as the main categories
+    elements :"itunes:category", :as => :itunes_categories, :value => :text
+
+    elements :"itunes:owner", :as => :itunes_owners, :class => ITunesRSSOwner
+
+    elements :item, :as => :entries, :class => ITunesRSSItem
+
+    def self.able_to_parse?(xml)
+      xml =~ /xmlns:itunes=\"http:\/\/www.itunes.com\/dtds\/podcast-1.0.dtd\"/
+    end
+
+  end
+
+end

data/lib/feedzirra/itunes_rss_item.rb

@@ -0,0 +1,28 @@
+module Feedzirra
+  # iTunes extensions to the standard RSS2.0 item
+  # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
+  class ITunesRSSItem
+    include SAXMachine
+    include FeedUtilities
+    element :author
+    element :guid
+    element :title
+    element :link, :as => :url
+    element :description, :as => :summary
+    element :pubDate, :as => :published
+
+    # If author is not present use author tag on the item
+    element :"itunes:author", :as => :itunes_author
+    element :"itunes:block", :as => :itunes_block
+    element :"itunes:duration", :as => :itunes_duration
+    element :"itunes:explicit", :as => :itunes_explicit
+    element :"itunes:keywords", :as => :itunes_keywords
+    element :"itunes:subtitle", :as => :itunes_subtitle
+    # If summary is not present, use the description tag
+    element :"itunes:summary", :as => :itunes_summary
+    element :enclosure, :value => :length, :as => :enclosure_length
+    element :enclosure, :value => :type, :as => :enclosure_type
+    element :enclosure, :value => :url, :as => :enclosure_url
+  end
+
+end

data/lib/feedzirra/itunes_rss_owner.rb

@@ -0,0 +1,8 @@
+module Feedzirra
+  class ITunesRSSOwner
+    include SAXMachine
+    include FeedUtilities
+    element :"itunes:name", :as => :name
+    element :"itunes:email", :as => :email
+  end
+end

data/lib/feedzirra/rdf.rb

@@ -1,4 +1,12 @@
 module Feedzirra
+  # == Summary
+  # Parser for dealing with RDF feeds.
+  #
+  # == Attributes
+  # * title
+  # * feed_url
+  # * url
+  # * entries
   class RDF
     include SAXMachine
     include FeedUtilities
@@ -8,7 +16,7 @@ module Feedzirra
 
     attr_accessor :feed_url
     
-    def self.able_to_parse?(xml)
+    def self.able_to_parse?(xml) #:nodoc:
       xml =~ /(rdf\:RDF)|(#{Regexp.escape("http://purl.org/rss/1.0")})|(rss version\=\"0\.9.?\")/ || false
     end
   end

data/lib/feedzirra/rdf_entry.rb

@@ -1,4 +1,14 @@
 module Feedzirra
+  # == Summary
+  # Parser for dealing with RDF feed entries.
+  #
+  # == Attributes
+  # * title
+  # * url
+  # * author
+  # * content
+  # * summary
+  # * published
   class RDFEntry
     include SAXMachine
     include FeedEntryUtilities

data/lib/feedzirra/rss.rb

@@ -1,4 +1,12 @@
 module Feedzirra
+  # == Summary
+  # Parser for dealing with RSS feeds.
+  #
+  # == Attributes
+  # * title
+  # * feed_url
+  # * url
+  # * entries
   class RSS
     include SAXMachine
     include FeedUtilities
@@ -8,7 +16,7 @@ module Feedzirra
     
     attr_accessor :feed_url
     
-    def self.able_to_parse?(xml)
+    def self.able_to_parse?(xml) #:nodoc:
       xml =~ /\<rss|rdf/
     end
   end

data/lib/feedzirra/rss_entry.rb

@@ -1,4 +1,15 @@
 module Feedzirra
+  # == Summary
+  # Parser for dealing with RDF feed entries.
+  #
+  # == Attributes
+  # * title
+  # * url
+  # * author
+  # * content
+  # * summary
+  # * published
+  # * categories
   class RSSEntry
     include SAXMachine
     include FeedEntryUtilities