feedtools 0.2.6 → 0.2.7

data/CHANGELOG

@@ -1,6 +1,13 @@
+== FeedTools 0.2.7
+ * added support for Atom 1.0 output
+ * improved support for Atom parsing and handling
+ * added no content string option to keep the feed validator happy
+ * greatly improved handling of timestamps and feed item sorting
+ * added aliases for the Atom 1.0 elements that were renamed
+ * feed merging is now threaded
 == FeedTools 0.2.6
- * Added support for merging feeds
- * Vastly improved handling of feed creation
+ * added support for merging feeds
+ * vastly improved handling of feed creation
 == FeedTools 0.2.5
  * fixed multiple rows being created in the cache after a 301 redirection
  * fixed broken table creation for postgresql and sqlite

data/lib/feed_tools.rb

@@ -25,7 +25,7 @@ FEED_TOOLS_ENV = ENV['FEED_TOOLS_ENV'] ||
                  ENV['RAILS_ENV'] ||
                  'production' # :nodoc:
 
-FEED_TOOLS_VERSION = "0.2.6"
+FEED_TOOLS_VERSION = "0.2.7"
 
 $:.unshift(File.dirname(__FILE__))
 $:.unshift(File.dirname(__FILE__) + "/../../activerecord/lib")
@@ -305,6 +305,7 @@ module FeedTools
   @feed_cache = DatabaseFeedCache
   @user_agent = "FeedTools/#{FEED_TOOLS_VERSION} " + 
     "+http://www.sporkmonger.com/projects/feedtools/"
+  @no_content_string = "[no description]"
   
   # Returns the current caching mechanism.
   def FeedTools.feed_cache
@@ -377,6 +378,17 @@ module FeedTools
   def FeedTools.user_agent=(new_user_agent)
     @user_agent = new_user_agent
   end
+
+  # Returns the currently used no content string.
+  def FeedTools.no_content_string
+    return @no_content_string
+  end
+  
+  # Sets the no content string to use when a feed is missing a content element.
+  # Used only for xml output.
+  def FeedTools.no_content_string=(new_no_content_string)
+    @no_content_string = new_no_content_string
+  end
   
   # Returns true if the html tidy module can be used.
   #
@@ -548,11 +560,31 @@ module FeedTools
     end
   end
   
-  # Returns true if the parameter appears to be a valid url
-  def FeedTools.is_url?(url)
+  # Converts a url into a tag uri
+  def FeedTools.build_tag_uri(url, date)
+    unless url.kind_of? String
+      raise ArgumentError, "Expected String, got #{url.class.name}"
+    end
+    unless date.kind_of? Time
+      raise ArgumentError, "Expected Time, got #{date.class.name}"
+    end
+    tag_uri = normalize_url(url)
+    host = URI.parse(tag_uri).host
+    tag_uri.gsub!(/^(http|ftp|file):\/*/, "")
+    tag_uri.gsub!(/#/, "/")
+    tag_uri = "tag:#{host},#{date.strftime('%Y-%m-%d')}:" +
+      "#{tag_uri[(tag_uri.index(host) + host.size)..-1]}"
+    return tag_uri
+  end
+  
+  # Returns true if the parameter appears to be a valid uri
+  def FeedTools.is_uri?(url)
     return false if url.nil?
     begin
       uri = URI.parse(url)
+      if uri.scheme.nil? || uri.scheme == ""
+        return false
+      end
     rescue URI::InvalidURIError
       return false
     end
@@ -699,8 +731,18 @@ module FeedTools
   def FeedTools.build_merged_feed(url_array)
     return nil if url_array.nil?
     merged_feed = Feed.new
+    retrieved_feeds = []
+    feed_threads = []
     url_array.each do |feed_url|
-      feed = Feed.open(feed_url)
+      feed_threads << Thread.new do
+        feed = Feed.open(feed_url)
+        retrieved_feeds << feed
+      end
+    end
+    feed_threads.each do |thread|
+      thread.join
+    end
+    retrieved_feeds.each do |feed|
       merged_feed.entries.concat(
         feed.entries.collect do |entry|
           entry.title = "#{feed.title}: #{entry.title}"
@@ -1508,7 +1550,7 @@ module FeedTools
           end
         end
         if @link == "" || @link.nil?
-          if FeedTools.is_url? self.guid
+          if FeedTools.is_uri? self.guid
             @link = self.guid
           end
         end
@@ -1562,7 +1604,7 @@ module FeedTools
           if @icon == ""
             @icon = FeedTools.unescape_entities(
               XPath.first(icon_node, "text()").to_s)
-            unless FeedTools.is_url? @icon
+            unless FeedTools.is_uri? @icon
               @icon = ""
             end
           end
@@ -1581,19 +1623,20 @@ module FeedTools
     def author
       if @author.nil?
         @author = FeedTools::Feed::Author.new
-
-        author_node = XPath.first(channel_node, "author")
-        if author_node.nil?
-          author_node = XPath.first(channel_node, "managingEditor")
-        end
-        if author_node.nil?
-          author_node = XPath.first(channel_node, "dc:author")
-        end
-        if author_node.nil?
-          author_node = XPath.first(channel_node, "dc:creator")
-        end
-        if author_node.nil?
-          author_node = XPath.first(channel_node, "atom:author")
+        unless channel_node.nil?
+          author_node = XPath.first(channel_node, "author")
+          if author_node.nil?
+            author_node = XPath.first(channel_node, "managingEditor")
+          end
+          if author_node.nil?
+            author_node = XPath.first(channel_node, "dc:author")
+          end
+          if author_node.nil?
+            author_node = XPath.first(channel_node, "dc:creator")
+          end
+          if author_node.nil?
+            author_node = XPath.first(channel_node, "atom:author")
+          end
         end
         unless author_node.nil?
           @author.raw = FeedTools.unescape_entities(
@@ -1771,6 +1814,149 @@ module FeedTools
       return @itunes_author
     end
 
+    # Returns the feed item time
+    def time
+      if @time.nil?
+        unless channel_node.nil?
+          time_string = XPath.first(channel_node, "pubDate/text()").to_s
+          if time_string == ""
+            time_string = XPath.first(channel_node, "dc:date/text()").to_s
+          end
+          if time_string == ""
+            time_string = XPath.first(channel_node, "issued/text()").to_s
+          end
+          if time_string == ""
+            time_string = XPath.first(channel_node, "updated/text()").to_s
+          end
+          if time_string == ""
+            time_string = XPath.first(channel_node, "time/text()").to_s
+          end
+        end
+        begin
+          if time_string != nil && time_string != ""
+            @time = Time.parse(time_string) rescue self.succ_time
+          elsif time_string == nil
+            @time = self.succ_time
+          end
+          if @time == nil
+            @time = Time.now
+          end
+        rescue
+          @time = Time.now
+        end
+      end
+      return @time
+    end
+    
+    # Sets the feed item time
+    def time=(new_time)
+      @time = new_time
+    end
+    
+    # Returns 1 second after the previous item's time.
+    def succ_time #:nodoc:
+      begin
+        if feed.nil?
+          return nil
+        end
+        feed.items
+        unsorted_items = feed.instance_variable_get("@items")
+        item_index = unsorted_items.index(self)
+        if item_index.nil?
+          return nil
+        end
+        if item_index <= 0
+          return Time.now
+        end
+        previous_item = unsorted_items[item_index - 1]
+        return previous_item.time.succ
+      rescue
+        return nil
+      end
+    end
+    private :succ_time
+
+    # Returns the feed item updated time
+    def updated
+      if @updated.nil?
+        unless channel_node.nil?
+          updated_string = XPath.first(channel_node, "updated/text()").to_s
+          if updated_string == ""
+            updated_string = XPath.first(channel_node, "modified/text()").to_s
+          end
+        end
+        if updated_string != nil && updated_string != ""
+          @updated = Time.parse(updated_string) rescue nil
+        else
+          @updated = nil
+        end
+      end
+      return @updated
+    end
+    
+    # Sets the feed item updated time
+    def updated=(new_updated)
+      @updated = new_updated
+    end
+
+    # Returns the feed item issued time
+    def issued
+      if @issued.nil?
+        unless channel_node.nil?
+          issued_string = XPath.first(channel_node, "issued/text()").to_s
+          if issued_string == ""
+            issued_string = XPath.first(channel_node, "pubDate/text()").to_s
+          end
+          if issued_string == ""
+            issued_string = XPath.first(channel_node, "dc:date/text()").to_s
+          end
+          if issued_string == ""
+            issued_string = XPath.first(channel_node, "published/text()").to_s
+          end
+        end
+        if issued_string != nil && issued_string != ""
+          @issued = Time.parse(issued_string) rescue nil
+        else
+          @issued = nil
+        end
+      end
+      return @issued
+    end
+    
+    # Sets the feed item issued time
+    def issued=(new_issued)
+      @issued = new_issued
+    end
+
+    # Returns the feed item published time
+    def published
+      if @published.nil?
+        unless channel_node.nil?
+          published_string = XPath.first(channel_node, "published/text()").to_s
+          if published_string == ""
+            published_string = XPath.first(channel_node, "pubDate/text()").to_s
+          end
+          if published_string == ""
+            published_string = XPath.first(channel_node, "dc:date/text()").to_s
+          end
+          if published_string == ""
+            published_string = XPath.first(channel_node, "issued/text()").to_s
+          end
+        end
+        if published_string != nil && published_string != ""
+          @published = Time.parse(published_string) rescue nil
+        else
+          @published = nil
+        end
+      end
+      return @published
+    end
+    
+    # Sets the feed item published time
+    def published=(new_published)
+      @published = new_published
+    end
+
     # Returns a list of the feed's categories
     def categories
       if @categories.nil?
@@ -1889,15 +2075,20 @@ module FeedTools
     # Returns the feed's copyright information
     def copyright
       if @copyright.nil?
-        @copyright = XPath.first(channel_node, "copyright/text()").to_s
-        if @copyright == ""
-          @copyright = XPath.first(channel_node, "dc:rights/text()").to_s
-        end
-        if @copyright == ""
-          @copyright = XPath.first(channel_node, "copyrights/text()").to_s
+        unless channel_node.nil?
+          @copyright = XPath.first(channel_node, "copyright/text()").to_s
+          if @copyright == ""
+            @copyright = XPath.first(channel_node, "rights/text()").to_s
+          end
+          if @copyright == ""
+            @copyright = XPath.first(channel_node, "dc:rights/text()").to_s
+          end
+          if @copyright == ""
+            @copyright = XPath.first(channel_node, "copyrights/text()").to_s
+          end
+          @copyright = FeedTools.sanitize_html(@copyright, :strip)
+          @copyright = nil if @copyright == ""
         end
-        @copyright = FeedTools.sanitize_html(@copyright, :strip)
-        @copyright = nil if @copyright == ""
       end
       return @copyright
     end
@@ -2335,7 +2526,7 @@ module FeedTools
             end
           end
         end
-      elsif feed_type == "atom"
+      elsif feed_type == "atom" && version == 0.3
         # normal atom format
         return xml_builder.feed("xmlns" => "http://purl.org/atom/ns#",
             "version" => version,
@@ -2345,6 +2536,19 @@ module FeedTools
                 "mode" => "escaped",
                 "type" => "text/html")
           end
+          xml_builder.author do
+            unless self.author.nil? || self.author.name.nil?
+              xml_builder.name(self.author.name)
+            else
+              xml_builder.name("n/a")
+            end
+            unless self.author.nil? || self.author.email.nil?
+              xml_builder.email(self.author.email)
+            end
+            unless self.author.nil? || self.author.url.nil?
+              xml_builder.url(self.author.url)
+            end
+          end
           unless link.nil? || link == ""
             xml_builder.link("href" => link,
                 "rel" => "alternate",
@@ -2365,6 +2569,74 @@ module FeedTools
             end
           end
         end
+      elsif feed_type == "atom" && version == 1.0
+        # normal atom format
+        return xml_builder.feed("xmlns" => "http://www.w3.org/2005/Atom",
+            "xml:lang" => language) do
+          unless title.nil? || title == ""
+            xml_builder.title(title,
+                "type" => "html")
+          end
+          xml_builder.author do
+            unless self.author.nil? || self.author.name.nil?
+              xml_builder.name(self.author.name)
+            else
+              xml_builder.name("n/a")
+            end
+            unless self.author.nil? || self.author.email.nil?
+              xml_builder.email(self.author.email)
+            end
+            unless self.author.nil? || self.author.url.nil?
+              xml_builder.url(self.author.url)
+            end
+          end
+          unless self.url.nil? || self.url == ""
+            xml_builder.link("href" => self.url,
+                "rel" => "self",
+                "type" => "application/atom+xml")
+          end
+          unless self.link.nil? || self.link == ""
+            xml_builder.link("href" => self.link,
+                "rel" => "alternate",
+                "type" => "text/html",
+                "title" => self.title)
+          end
+          unless description.nil? || description == ""
+            xml_builder.subtitle(description,
+                "type" => "html")
+          else
+            xml_builder.subtitle(FeedTools.no_content_string,
+                "type" => "html")
+          end
+          if self.updated != nil
+            xml_builder.updated(self.updated.iso8601)
+          elsif self.time != nil
+            # Not technically correct, but a heck of a lot better
+            # than the Time.now fall-back.
+            xml_builder.updated(self.time.iso8601)
+          else
+            xml_builder.updated(Time.now.iso8601)
+          end
+          unless self.published.nil?
+            xml_builder.published(self.published.iso8601)            
+          end
+          xml_builder.generator("FeedTools - " +
+            "http://www.sporkmonger.com/projects/feedtools")
+          unless self.id.nil?
+            unless FeedTools.is_uri? self.id
+              raise "The unique id must be a URI."
+            end
+            xml_builder.id(self.id)
+          else
+            raise "Cannot build feed, missing feed unique id."
+          end
+          build_xml_hook(feed_type, version, xml_builder)
+          unless items.nil?
+            for item in items
+              item.build_xml(feed_type, version, xml_builder)
+            end
+          end
+        end
       end
     end
 
@@ -2678,6 +2950,15 @@ module FeedTools
         unless root_node.nil?
           repair_entities = false
           description_node = XPath.first(root_node, "description")
+          if description_node.nil?
+            description_node = XPath.first(root_node, "content:encoded")
+          end
+          if description_node.nil?
+            description_node = XPath.first(root_node, "content")
+          end
+          if description_node.nil?
+            description_node = XPath.first(root_node, "fullitem")
+          end
           if description_node.nil?
             description_node = XPath.first(root_node, "xhtml:body")
           end
@@ -2699,15 +2980,6 @@ module FeedTools
           if description_node.nil?
             description_node = XPath.first(root_node, "ABSTRACT")
           end
-          if description_node.nil?
-            description_node = XPath.first(root_node, "content:encoded")
-          end
-          if description_node.nil?
-            description_node = XPath.first(root_node, "content")
-          end
-          if description_node.nil?
-            description_node = XPath.first(root_node, "fullitem")
-          end
           if description_node.nil?
             description_node = XPath.first(root_node, "info")
             @bozo = true unless description_node.nil?
@@ -2853,7 +3125,7 @@ module FeedTools
           end
         end
         if @link == "" || @link.nil?
-          if FeedTools.is_url? self.guid
+          if FeedTools.is_uri? self.guid
             @link = self.guid
           end
         end
@@ -3016,9 +3288,20 @@ module FeedTools
     # Returns the feed item's copyright information
     def copyright
       if @copyright.nil?
-        @copyright = XPath.first(root_node, "dc:rights/text()").to_s
-        @copyright = FeedTools.sanitize_html(@copyright, :strip)
-        @copyright = nil if @copyright == ""
+        unless root_node.nil?
+          @copyright = XPath.first(root_node, "dc:rights/text()").to_s
+          if @copyright == ""
+            @copyright = XPath.first(root_node, "rights/text()").to_s
+          end
+          if @copyright == ""
+            @copyright = XPath.first(root_node, "copyright/text()").to_s
+          end
+          if @copyright == ""
+            @copyright = XPath.first(root_node, "copyrights/text()").to_s
+          end
+          @copyright = FeedTools.sanitize_html(@copyright, :strip)
+          @copyright = nil if @copyright == ""
+        end
       end
       return @copyright
     end
@@ -3400,19 +3683,20 @@ module FeedTools
     def author
       if @author.nil?
         @author = FeedTools::Feed::Author.new
-
-        author_node = XPath.first(root_node, "author")
-        if author_node.nil?
-          author_node = XPath.first(root_node, "managingEditor")
-        end
-        if author_node.nil?
-          author_node = XPath.first(root_node, "dc:author")
-        end
-        if author_node.nil?
-          author_node = XPath.first(root_node, "dc:creator")
-        end
-        if author_node.nil?
-          author_node = XPath.first(root_node, "atom:author")
+        unless root_node.nil?
+          author_node = XPath.first(root_node, "author")
+          if author_node.nil?
+            author_node = XPath.first(root_node, "managingEditor")
+          end
+          if author_node.nil?
+            author_node = XPath.first(root_node, "dc:author")
+          end
+          if author_node.nil?
+            author_node = XPath.first(root_node, "dc:creator")
+          end
+          if author_node.nil?
+            author_node = XPath.first(root_node, "atom:author")
+          end
         end
         unless author_node.nil?
           @author.raw = FeedTools.unescape_entities(
@@ -3621,14 +3905,22 @@ module FeedTools
     # Returns the feed item time
     def time
       if @time.nil?
-        time_string = XPath.first(root_node, "pubDate/text()").to_s
-        if time_string == ""
-          time_string = XPath.first(root_node, "dc:date/text()").to_s
-        end
-        if time_string == ""
-          time_string = XPath.first(root_node, "issued/text()").to_s
+        unless root_node.nil?
+          time_string = XPath.first(root_node, "pubDate/text()").to_s
+          if time_string == ""
+            time_string = XPath.first(root_node, "dc:date/text()").to_s
+          end
+          if time_string == ""
+            time_string = XPath.first(root_node, "issued/text()").to_s
+          end
+          if time_string == ""
+            time_string = XPath.first(root_node, "updated/text()").to_s
+          end
+          if time_string == ""
+            time_string = XPath.first(root_node, "time/text()").to_s
+          end
         end
-        if time_string != ""
+        if time_string != nil && time_string != ""
           @time = Time.parse(time_string) rescue Time.now
         elsif time_string == nil
           @time = Time.now
@@ -3641,7 +3933,59 @@ module FeedTools
     def time=(new_time)
       @time = new_time
     end
+
+    # Returns the feed item updated time
+    def updated
+      if @updated.nil?
+        unless root_node.nil?
+          updated_string = XPath.first(root_node, "updated/text()").to_s
+          if updated_string == ""
+            updated_string = XPath.first(root_node, "modified/text()").to_s
+          end
+        end
+        if updated_string != nil && updated_string != ""
+          @updated = Time.parse(updated_string) rescue nil
+        else
+          @updated = nil
+        end
+      end
+      return @updated
+    end
+    
+    # Sets the feed item updated time
+    def updated=(new_updated)
+      @updated = new_updated
+    end
+
+    # Returns the feed item issued time
+    def issued
+      if @issued.nil?
+        unless root_node.nil?
+          issued_string = XPath.first(root_node, "issued/text()").to_s
+          if issued_string == ""
+            issued_string = XPath.first(root_node, "published/text()").to_s
+          end
+          if issued_string == ""
+            issued_string = XPath.first(root_node, "pubDate/text()").to_s
+          end
+          if issued_string == ""
+            issued_string = XPath.first(root_node, "dc:date/text()").to_s
+          end
+        end
+        if issued_string != nil && issued_string != ""
+          @issued = Time.parse(issued_string) rescue nil
+        else
+          @issued = nil
+        end
+      end
+      return @issued
+    end
     
+    # Sets the feed item issued time
+    def issued=(new_issued)
+      @issued = new_issued
+    end
+        
     # Returns the url for posting comments
     def comments
       if @comments.nil?
@@ -3837,7 +4181,7 @@ module FeedTools
           end
           build_xml_hook(feed_type, version, xml_builder)
         end
-      elsif feed_type == "atom"
+      elsif feed_type == "atom" && version == 0.3
         # normal atom format
         return xml_builder.entry("xmlns" => "http://purl.org/atom/ns#") do
           unless title.nil? || title == ""
@@ -3845,6 +4189,19 @@ module FeedTools
                 "mode" => "escaped",
                 "type" => "text/html")
           end
+          xml_builder.author do
+            unless self.author.nil? || self.author.name.nil?
+              xml_builder.name(self.author.name)
+            else
+              xml_builder.name("n/a")
+            end
+            unless self.author.nil? || self.author.email.nil?
+              xml_builder.email(self.author.email)
+            end
+            unless self.author.nil? || self.author.url.nil?
+              xml_builder.url(self.author.url)
+            end
+          end
           unless link.nil? || link == ""
             xml_builder.link("href" => link,
                 "rel" => "alternate",
@@ -3866,6 +4223,74 @@ module FeedTools
           end
           build_xml_hook(feed_type, version, xml_builder)
         end
+      elsif feed_type == "atom" && version == 1.0
+        # normal atom format
+        return xml_builder.entry("xmlns" => "http://www.w3.org/2005/Atom") do
+          unless title.nil? || title == ""
+            xml_builder.title(title,
+                "type" => "html")
+          end
+          xml_builder.author do
+            unless self.author.nil? || self.author.name.nil?
+              xml_builder.name(self.author.name)
+            else
+              xml_builder.name("n/a")
+            end
+            unless self.author.nil? || self.author.email.nil?
+              xml_builder.email(self.author.email)
+            end
+            unless self.author.nil? || self.author.url.nil?
+              xml_builder.url(self.author.url)
+            end
+          end
+          unless link.nil? || link == ""
+            xml_builder.link("href" => link,
+                "rel" => "alternate",
+                "type" => "text/html",
+                "title" => title)
+          end
+          unless description.nil? || description == ""
+            xml_builder.content(description,
+                "type" => "html")
+          else
+            xml_builder.content(FeedTools.no_content_string,
+                "type" => "html")
+          end
+          if self.updated != nil
+            xml_builder.updated(self.updated.iso8601)
+          elsif self.time != nil
+            # Not technically correct, but a heck of a lot better
+            # than the Time.now fall-back.
+            xml_builder.updated(self.time.iso8601)
+          else
+            xml_builder.updated(Time.now.iso8601)
+          end
+          unless self.published.nil?
+            xml_builder.published(self.published.iso8601)            
+          end
+          if self.id != nil
+            unless FeedTools.is_uri? self.id
+              if self.time != nil && self.link != nil
+                xml_builder.id(FeedTools.build_tag_uri(self.link, self.time))
+              else
+                raise "The unique id must be a URI. " +
+                  "(Attempted to generate id, but failed.)"
+              end
+            else
+              xml_builder.id(self.id)
+            end
+          elsif self.time != nil && self.link != nil
+            xml_builder.id(FeedTools.build_tag_uri(self.link, self.time))
+          else
+            raise "Cannot build feed, missing feed unique id."
+          end
+          unless self.tags.nil? || self.tags.size == 0
+            for tag in self.tags
+              xml_builder.category("term" => tag)
+            end
+          end
+          build_xml_hook(feed_type, version, xml_builder)
+        end
       end
     end
     
@@ -3881,6 +4306,8 @@ module FeedTools
     alias_method :content=, :description=
     alias_method :guid, :id
     alias_method :guid=, :id=
+    alias_method :published, :issued
+    alias_method :published=, :issued=
   end
 end
 

data/rakefile

@@ -7,7 +7,7 @@ require 'rake/gempackagetask'
 require 'rake/contrib/rubyforgepublisher'
 
 PKG_NAME      = 'feedtools'
-PKG_VERSION   = '0.2.6'
+PKG_VERSION   = '0.2.7'
 PKG_FILE_NAME = "#{PKG_NAME}-#{PKG_VERSION}"
 
 RELEASE_NAME  = "REL #{PKG_VERSION}"

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: feedtools
 version: !ruby/object:Gem::Version 
-  version: 0.2.6
-date: 2005-09-01 00:00:00 -04:00
+  version: 0.2.7
+date: 2005-09-02 00:00:00 -04:00
 summary: "Parsing, generation, and caching system for xml news feeds."
 require_paths: 
   - lib