jsl-feedzirra 0.0.12.1

data/lib/feedzirra/http_multi.rb

@@ -0,0 +1,185 @@
+module Feedzirra
+  
+  # Handles HTTP requests for Feedzirra, including registration of on success and on failure 
+  # callbacks.
+  class HttpMulti
+        
+    attr_reader :options, :retrievables, :multi, :responses
+
+    DEFAULTS = {
+      :backend => {
+        :class => Feedzirra::Backend::Memory 
+      }  
+    }
+    
+    def initialize(*args)
+      @options          = DEFAULTS.merge(args.extract_options!)
+      @retrievables     = args.flatten
+      @multi            = Curl::Multi.new
+      @responses        = { }
+      @backend          = @options[:backend][:class].new
+    end
+
+    # Prepares the curl object and calls #perform
+    def run
+      prepare
+      @multi.perform
+    end
+
+    # Breaks the urls into chunks of 30 because of weird errors encountered on 
+    # entering more items.  As one finishes it pops another off the queue.
+    def prepare
+      retrievable_queue = @retrievables.dup
+      
+      retrievable_queue.slice!(0, 30).each do |retrievable|
+        add_to_multi(retrievable, retrievable_queue)
+      end
+    end
+        
+    # Generic method for building Curl::Multi object.  Retrievable may be a Feed or a 
+    # String URL.
+    def add_to_multi(retrievable, retrievable_queue)
+      if retrievable.respond_to?(:feed_url)
+        url = retrievable.feed_url
+      else
+        url = retrievable
+        retrievable = @backend.get(url)  # Try to fetch the last retrieval from backend
+      end
+
+      easy = build_curl_easy(url, retrievable, retrievable_queue)
+      @multi.add(easy)
+    end
+
+    # builds a Curl::Easy object that can be added to Curl::Multi.
+    def build_curl_easy(url, retrievable, retrievable_queue)
+
+      easy = Curl::Easy.new(url) do |curl|
+        curl = set_curl_configuration(curl, retrievable)
+
+        curl.on_success do |c|
+          on_success_handler(c, url, retrievable, retrievable_queue)
+        end
+
+        curl.on_failure do |c|
+          if c.response_code == 304
+            on_success_handler(c, url, retrievable, retrievable_queue)
+          else
+            on_failure_handler(c, url, retrievable, retrievable_queue)                          
+          end
+        end
+      end 
+           
+      easy
+    end
+
+    def set_updated_feed_entries!(retrievable, updated_feed)
+      if retrievable.respond_to?(:update_from_feed) 
+        retrievable.update_from_feed(updated_feed) 
+      else
+        # all elements are "new", since we weren't dealing with a Feed element.
+        updated_feed.new_entries = updated_feed.entries
+      end    
+    end
+
+    # Handles successful Curl responses.
+    def on_success_handler(curl, url, retrievable, retrievable_queue)
+      add_to_multi(retrievable_queue.shift, retrievable_queue) unless retrievable_queue.empty?
+      
+      begin      
+        if curl.response_code == 304
+          updated_feed = retrievable
+          updated_feed.new_entries = [ ]
+        else
+          updated_feed = parser_for_xml(curl.body_str).run
+          updated_feed.feed_url = curl.last_effective_url
+          updated_feed.etag = etag_from_header(curl.header_str)
+          updated_feed.last_modified = last_modified_from_header(curl.header_str)          
+          set_updated_feed_entries!(retrievable, updated_feed)
+        end
+
+        @backend.set(url, updated_feed)          
+
+        responses[url] = updated_feed
+        @options[:on_success].call(retrievable) if @options.has_key?(:on_success)
+
+      rescue Exception => e
+        puts "Caught exception, but we're throwing it away: #{e}"
+        @options[:on_failure].call(retrievable, curl.response_code, curl.header_str, curl.body_str) if @options.has_key?(:on_failure)
+      end      
+    end
+
+    # Handles failed Curl responses.
+    def on_failure_handler(curl, url, retrievable, retrievable_queue)      
+      add_to_multi(multi, retrievable_queue.shift, retrievable_queue, responses, options) unless retrievable_queue.empty?
+      responses[url] = curl.response_code
+      @options[:on_failure].call(retrievable, curl.response_code, curl.header_str, curl.body_str) if options.has_key?(:on_failure)
+    end
+
+    # Determines the etag from the request headers.
+    # 
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A string of the etag or nil if it cannot be found in the headers.
+    def etag_from_header(header)
+      header =~ /.*ETag:\s(.*)\r/
+      $1
+    end
+
+    # Determines the last modified date from the request headers.
+    #
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A Time object of the last modified date or nil if it cannot be found in the headers.
+    def last_modified_from_header(header)
+      header =~ /.*Last-Modified:\s(.*)\r/
+      Time.parse($1) if $1
+    end
+
+    def parser_for_xml(xml)
+      Feedzirra::FeedParser.new(xml)
+    end  
+    
+    
+    # Accepts a Curl::Easy object with an optional set of options and returns
+    # a Curl::Easy object with options merged into the defaults.
+    def set_curl_configuration(curl, retrievable = nil)
+      curl.headers["User-Agent"]        = @options[:user_agent] || Feedzirra::USER_AGENT
+      curl.headers["If-Modified-Since"] = @options[:if_modified_since].httpdate if @options.has_key?(:if_modified_since)
+      curl.headers["If-None-Match"]     = retrievable.etag if (retrievable.respond_to?(:etag) && retrievable.etag)
+      curl.headers["Accept-Encoding"]   = 'gzip, deflate' if @options.has_key?(:compress)
+
+      curl.follow_location = true
+      curl.userpwd = @options[:http_authentication].join(':') if @options.has_key?(:http_authentication)
+
+      curl
+    end
+
+    # Decodes the XML document if it was compressed.
+    #
+    # === Parameters
+    # [curl_request<Curl::Easy>] The Curl::Easy response object from the request.
+    # === Returns
+    # A decoded string of XML.
+    def decode_content(curl)
+      if curl.header_str.match(/Content-Encoding: gzip/)
+        begin
+          gz =  Zlib::GzipReader.new(StringIO.new(curl.body_str))
+          xml = gz.read
+          gz.close
+        rescue Zlib::GzipFile::Error 
+          # Maybe this is not gzipped?
+          xml = c.body_str
+        end
+      elsif curl.header_str.match(/Content-Encoding: deflate/)
+        xml = Zlib::Inflate.inflate(curl.body_str)
+      else
+        xml = curl.body_str
+      end
+
+      xml
+    end
+        
+  end    
+end

data/lib/feedzirra/parser/atom.rb

@@ -0,0 +1,26 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Atom feeds.
+    #
+    # == Attributes
+    # * title
+    # * feed_url
+    # * url
+    # * entries
+    class Atom
+      include SAXMachine
+      include FeedUtilities
+      element :title
+      element :link, :as => :url, :value => :href, :with => {:type => "text/html"}
+      element :link, :as => :feed_url, :value => :href, :with => {:type => "application/atom+xml"}
+      elements :entry, :as => :entries, :class => AtomEntry
+
+      def self.able_to_parse?(xml) #:nodoc:
+        xml =~ /(Atom)|(#{Regexp.escape("http://purl.org/atom")})/
+      end
+    end
+  end
+  
+end

data/lib/feedzirra/parser/atom_entry.rb

@@ -0,0 +1,34 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Atom feed entries.
+    #
+    # == Attributes
+    # * title
+    # * url
+    # * author
+    # * content
+    # * summary
+    # * published
+    # * categories
+    class AtomEntry
+      include SAXMachine
+      include FeedEntryUtilities
+      element :title
+      element :link, :as => :url, :value => :href, :with => {:type => "text/html", :rel => "alternate"}
+      element :name, :as => :author
+      element :content
+      element :summary
+      element :published
+      element :id
+      element :created, :as => :published
+      element :issued, :as => :published
+      element :updated
+      element :modified, :as => :updated
+      elements :category, :as => :categories, :value => :term
+    end
+
+  end
+  
+end

data/lib/feedzirra/parser/atom_feed_burner.rb

@@ -0,0 +1,27 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Feedburner Atom feeds.
+    #
+    # == Attributes
+    # * title
+    # * feed_url
+    # * url
+    # * entries
+    class AtomFeedBurner
+      include SAXMachine
+      include FeedUtilities
+      element :title
+      element :link, :as => :url, :value => :href, :with => {:type => "text/html"}
+      element :link, :as => :feed_url, :value => :href, :with => {:type => "application/atom+xml"}
+      elements :entry, :as => :entries, :class => AtomFeedBurnerEntry
+
+      def self.able_to_parse?(xml) #:nodoc:
+        (xml =~ /Atom/ && xml =~ /feedburner/) || false
+      end
+    end
+
+  end
+  
+end

data/lib/feedzirra/parser/atom_feed_burner_entry.rb

@@ -0,0 +1,35 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Feedburner Atom feed entries.
+    #
+    # == Attributes
+    # * title
+    # * url
+    # * author
+    # * content
+    # * summary
+    # * published
+    # * categories
+    class AtomFeedBurnerEntry
+      include SAXMachine
+      include FeedEntryUtilities
+      element :title
+      element :name, :as => :author
+      element :link, :as => :url, :value => :href, :with => {:type => "text/html", :rel => "alternate"}
+      element :"feedburner:origLink", :as => :url
+      element :summary
+      element :content
+      element :published
+      element :id
+      element :issued, :as => :published
+      element :created, :as => :published
+      element :updated
+      element :modified, :as => :updated
+      elements :category, :as => :categories, :value => :term
+    end
+
+  end
+    
+end

data/lib/feedzirra/parser/feed_entry_utilities.rb

@@ -0,0 +1,45 @@
+module Feedzirra
+  module FeedEntryUtilities    
+    def published
+      @published || @updated
+    end
+    
+    def parse_datetime(string)
+      begin
+        DateTime.parse(string).feed_utils_to_gm_time
+      rescue
+        puts "DATE CAN'T BE PARSED: #{string}"
+        nil
+      end
+    end
+    
+    ##
+    # Returns the id of the entry or its url if not id is present, as some formats don't support it
+    def id
+      @id || @url
+    end
+    
+    ##
+    # Writter for published. By default, we keep the "oldest" publish time found.
+    def published=(val)
+      parsed = parse_datetime(val)
+      @published = parsed if !@published || parsed < @published
+    end
+    
+    ##
+    # Writter for udapted. By default, we keep the most recenet update time found.
+    def updated=(val)
+      parsed = parse_datetime(val)
+      @updated = parsed if !@updated || parsed > @updated
+    end
+
+    def sanitize!
+      self.title.sanitize!   if self.title
+      self.author.sanitize!  if self.author
+      self.summary.sanitize! if self.summary
+      self.content.sanitize! if self.content
+    end
+    
+    alias_method :last_modified, :published
+  end
+end

data/lib/feedzirra/parser/feed_utilities.rb

@@ -0,0 +1,71 @@
+module Feedzirra
+  module FeedUtilities
+    UPDATABLE_ATTRIBUTES = %w(title feed_url url last_modified)
+    
+    attr_writer   :new_entries, :updated, :last_modified
+    attr_accessor :etag
+
+    def last_modified
+      @last_modified ||= begin
+        entry = entries.reject {|e| e.published.nil? }.sort_by { |entry| entry.published if entry.published }.last
+        entry ? entry.published : nil
+      end
+    end
+    
+    def updated?
+      @updated
+    end
+    
+    def new_entries
+      @new_entries ||= []
+    end
+    
+    def has_new_entries?
+      new_entries.size > 0
+    end
+    
+    def update_from_feed(feed)
+      self.new_entries += find_new_entries_for(feed)
+      self.entries.unshift(*self.new_entries)
+      
+      updated! if UPDATABLE_ATTRIBUTES.any? { |name| update_attribute(feed, name) }
+    end
+    
+    def update_attribute(feed, name)
+      old_value, new_value = send(name), feed.send(name)
+      
+      if old_value != new_value
+        send("#{name}=", new_value)
+      end
+    end
+    
+    def sanitize_entries!
+      entries.each {|entry| entry.sanitize!}
+    end
+    
+    private
+    
+    def updated!
+      @updated = true
+    end
+    
+    def find_new_entries_for(feed)
+      # this implementation is a hack, which is why it's so ugly.
+      # it's to get around the fact that not all feeds have a published date.
+      # however, they're always ordered with the newest one first.
+      # So we go through the entries just parsed and insert each one as a new entry
+      # until we get to one that has the same url as the the newest for the feed
+      latest_entry = self.entries.first
+      found_new_entries = []
+      feed.entries.each do |entry|
+        break if entry.url == latest_entry.url
+        found_new_entries << entry
+      end
+      found_new_entries
+    end
+    
+    def existing_entry?(test_entry)
+      entries.any? { |entry| entry.url == test_entry.url }
+    end
+  end
+end

data/lib/feedzirra/parser/itunes_rss.rb

@@ -0,0 +1,50 @@
+module Feedzirra
+  
+  module Parser
+    # iTunes is RSS 2.0 + some apple extensions
+    # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
+    class ITunesRSS
+      include SAXMachine
+      include FeedUtilities
+
+      attr_accessor :feed_url
+
+      # RSS 2.0 elements that need including
+      element :copyright
+      element :description
+      element :language
+      element :managingEditor
+      element :title
+      element :link, :as => :url
+
+      # If author is not present use managingEditor on the channel
+      element :"itunes:author", :as => :itunes_author
+      element :"itunes:block", :as => :itunes_block
+      element :"itunes:image", :value => :href, :as => :itunes_image
+      element :"itunes:explicit", :as => :itunes_explicit
+      element :"itunes:keywords", :as => :itunes_keywords
+      # New URL for the podcast feed
+      element :"itunes:new-feed-url", :as => :itunes_new_feed_url
+      element :"itunes:subtitle", :as => :itunes_subtitle
+      # If summary is not present, use the description tag
+      element :"itunes:summary", :as => :itunes_summary
+
+      # iTunes RSS feeds can have multiple main categories...
+      # ...and multiple sub-categories per category
+      # TODO subcategories not supported correctly - they are at the same level
+      #   as the main categories
+      elements :"itunes:category", :as => :itunes_categories, :value => :text
+
+      elements :"itunes:owner", :as => :itunes_owners, :class => ITunesRSSOwner
+
+      elements :item, :as => :entries, :class => ITunesRSSItem
+
+      def self.able_to_parse?(xml)
+        xml =~ /xmlns:itunes=\"http:\/\/www.itunes.com\/dtds\/podcast-1.0.dtd\"/
+      end
+
+    end
+    
+  end
+  
+end

data/lib/feedzirra/parser/itunes_rss_item.rb

@@ -0,0 +1,31 @@
+module Feedzirra
+  
+  module Parser
+    # iTunes extensions to the standard RSS2.0 item
+    # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
+    class ITunesRSSItem
+      include SAXMachine
+      include FeedUtilities
+      element :author
+      element :guid
+      element :title
+      element :link, :as => :url
+      element :description, :as => :summary
+      element :pubDate, :as => :published
+
+      # If author is not present use author tag on the item
+      element :"itunes:author", :as => :itunes_author
+      element :"itunes:block", :as => :itunes_block
+      element :"itunes:duration", :as => :itunes_duration
+      element :"itunes:explicit", :as => :itunes_explicit
+      element :"itunes:keywords", :as => :itunes_keywords
+      element :"itunes:subtitle", :as => :itunes_subtitle
+      # If summary is not present, use the description tag
+      element :"itunes:summary", :as => :itunes_summary
+      element :enclosure, :value => :length, :as => :enclosure_length
+      element :enclosure, :value => :type, :as => :enclosure_type
+      element :enclosure, :value => :url, :as => :enclosure_url
+    end
+  end
+  
+end

data/lib/feedzirra/parser/itunes_rss_owner.rb

@@ -0,0 +1,12 @@
+module Feedzirra
+  
+  module Parser
+    class ITunesRSSOwner
+      include SAXMachine
+      include FeedUtilities
+      element :"itunes:name", :as => :name
+      element :"itunes:email", :as => :email
+    end
+  end
+  
+end

data/lib/feedzirra/parser/rss.rb

@@ -0,0 +1,28 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with RSS feeds.
+    #
+    # == Attributes
+    # * title
+    # * feed_url
+    # * url
+    # * entries
+    class RSS
+      include SAXMachine
+      include FeedUtilities
+      element :title
+      element :link, :as => :url
+      elements :item, :as => :entries, :class => RSSEntry
+
+      attr_accessor :feed_url
+
+      def self.able_to_parse?(xml) #:nodoc:
+        xml =~ /\<rss|rdf/
+      end
+    end
+
+  end
+  
+end

data/lib/feedzirra/parser/rss_entry.rb

@@ -0,0 +1,40 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with RDF feed entries.
+    #
+    # == Attributes
+    # * title
+    # * url
+    # * author
+    # * content
+    # * summary
+    # * published
+    # * categories
+    class RSSEntry
+      include SAXMachine
+      include FeedEntryUtilities
+      element :title
+      element :link, :as => :url
+
+      element :"dc:creator", :as => :author
+      element :"content:encoded", :as => :content
+      element :description, :as => :summary
+
+      element :pubDate, :as => :published
+      element :"dc:date", :as => :published
+      element :"dc:Date", :as => :published
+      element :"dcterms:created", :as => :published
+
+
+      element :"dcterms:modified", :as => :updated
+      element :issued, :as => :published
+      elements :category, :as => :categories
+
+      element :guid, :as => :id
+    end
+
+  end
+  
+end

data/lib/feedzirra/reader.rb

@@ -0,0 +1,28 @@
+module Feedzirra
+  
+  # Class +Reader+ is an experimental interface for fetching and parsing +Feed+ objects,
+  # using pluggable and (optionally) persistent backends to maintain state for fetched feeds.
+  class Reader
+            
+    attr_reader :urls, :options
+        
+    def initialize(*args)
+      @options  = args.extract_options!
+      @urls     = args.flatten
+    end
+    
+    # Map all urls to Feed objects, pulling existing feeds from Backend where available,
+    # then slices and fetches feeds in chunks of 30.
+    def fetch
+      multi = multi_from(@urls, @options)
+      multi.run
+      multi.responses.size == 1 ? multi.responses.values.first : multi.responses.values 
+    end
+    
+    private
+    
+    def multi_from(urls, options)
+      Feedzirra::HttpMulti.new(@urls, @options)
+    end
+  end
+end

data/lib/feedzirra.rb

@@ -0,0 +1,44 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
+
+gem 'activesupport'
+
+require 'zlib'
+require 'curb'
+require 'sax-machine'
+require 'dryopteris'
+require 'uri'
+require 'active_support/basic_object'
+require 'active_support/core_ext/object'
+require 'active_support/core_ext/time'
+
+require 'core_ext/date'
+require 'core_ext/string'
+require 'core_ext/array'
+
+require 'feedzirra/backend/filesystem'
+require 'feedzirra/backend/memcache'
+require 'feedzirra/backend/memory'
+
+require 'feedzirra/http_multi'
+
+require 'feedzirra/parser/feed_utilities'
+require 'feedzirra/parser/feed_entry_utilities'
+require 'feedzirra/feed'
+require 'feedzirra/reader'
+require 'feedzirra/feed_parser'
+
+require 'feedzirra/parser/rss_entry'
+require 'feedzirra/parser/itunes_rss_owner'
+require 'feedzirra/parser/itunes_rss_item'
+require 'feedzirra/parser/atom_entry'
+require 'feedzirra/parser/atom_feed_burner_entry'
+
+require 'feedzirra/parser/rss'
+require 'feedzirra/parser/itunes_rss'
+require 'feedzirra/parser/atom'
+require 'feedzirra/parser/atom_feed_burner'
+
+module Feedzirra
+  USER_AGENT = "feedzirra http://github.com/pauldix/feedzirra/tree/master"
+  VERSION = "0.0.12"
+end