feedtools 0.2.16 → 0.2.17

data/CHANGELOG

@@ -1,3 +1,11 @@
+== FeedTools 0.2.17
+ * more fixes for timestamping of feed items
+ * fixed nil bug in root_node, feed_type, feed_version, build_xml
+ * build_xml method is now properly unit tested
+ * fixed issue with setting the Host http header
+ * namespaces are no longer hard-coded for feed generation
+ * better url normalization for file: style urls
+ * generated timestamps are now in UTC
 == FeedTools 0.2.16
  * atom feed generation now defaults to atom 1.0
  * removed erroneous published element at atom feed-level

data/lib/feed_tools/feed.rb

@@ -102,7 +102,7 @@ module FeedTools
       @http_headers = nil
       @xml_doc = nil
       @feed_data = nil
-      @feed_data_type = nil
+      @feed_data_type = :xml
       @root_node = nil
       @channel_node = nil
       @url = nil
@@ -110,6 +110,7 @@ module FeedTools
       @title = nil
       @description = nil
       @link = nil
+      @last_retrieved = nil
       @time_to_live = nil
       @items = nil
       @live = false
@@ -134,7 +135,7 @@ module FeedTools
       validate_options([ :cache_only ],
                        options.keys)
       options = { :cache_only => false }.merge(options)
-          
+      
       if options[:cache_only] && FeedTools.feed_cache.nil?
         raise(ArgumentError, "There is currently no caching mechanism set. " +
           "Cannot retrieve cached feeds.")
@@ -144,7 +145,7 @@ module FeedTools
       url = FeedTools.normalize_url(url)
 
       # create and load the new feed
-      feed = Feed.new
+      feed = FeedTools::Feed.new
       feed.url = url
       feed.update! unless options[:cache_only]
       return feed
@@ -232,97 +233,92 @@ module FeedTools
             # Uh, maybe try to fix it?
             feed_uri = URI.parse(FeedTools.normalize_url(feed_url))
           end
+          
+          begin
+            Net::HTTP.start(feed_uri.host, (feed_uri.port or 80)) do |http|
+              final_uri = feed_uri.path 
+              final_uri += ('?' + feed_uri.query) if feed_uri.query
+              http_headers = {} if no_headers
+              response = http.request_get(final_uri, http_headers)
 
-          # Borrowed from open-uri:
-          # According to RFC2616 14.23, Host: request-header field should be
-          # set to an origin server.
-          # But net/http wrongly set a proxy server if an absolute URI is
-          # specified as a request URI.
-          # So override it here explicitly.
-          http_headers['Host'] = feed_uri.host
-          http_headers['Host'] += ":#{feed_uri.port}" if feed_uri.port
-        
-          Net::HTTP.start(feed_uri.host, (feed_uri.port or 80)) do |http|
-            final_uri = feed_uri.path 
-            final_uri += ('?' + feed_uri.query) if feed_uri.query
-            http_headers = {} if no_headers
-            response = http.request_get(final_uri, http_headers)
-
-            case response
-            when Net::HTTPSuccess
-              # We've reached the final destination, process all previous
-              # redirections, and see if we need to update the url.
-              for redirected_response in response_chain
-                if redirected_response.last.code.to_i == 301
-                  # Reset the cache object or we may get duplicate entries
-                  self.cache_object = nil
-                  self.url = redirected_response.last['location']
-                else
-                  # Jump out as soon as we hit anything that isn't a
-                  # permanently moved redirection.
-                  break
-                end
-              end
-              response
-            when Net::HTTPRedirection
-              if response.code.to_i == 304
-                response.error!
-              else
-                if response['location'].nil?
-                  raise FeedAccessError,
-                    "No location to redirect to supplied: " + response.code
-                end
-                response_chain << [feed_url, response]
-                new_location = response['location']
-                if response_chain.assoc(new_location) != nil
-                  raise FeedAccessError, "Redirection loop detected."
+              case response
+              when Net::HTTPSuccess
+                # We've reached the final destination, process all previous
+                # redirections, and see if we need to update the url.
+                for redirected_response in response_chain
+                  if redirected_response.last.code.to_i == 301
+                    # Reset the cache object or we may get duplicate entries
+                    self.cache_object = nil
+                    self.url = redirected_response.last['location']
+                  else
+                    # Jump out as soon as we hit anything that isn't a
+                    # permanently moved redirection.
+                    break
+                  end
                 end
+                response
+              when Net::HTTPRedirection
+                if response.code.to_i == 304
+                  response.error!
+                else
+                  if response['location'].nil?
+                    raise FeedAccessError,
+                      "No location to redirect to supplied: " + response.code
+                  end
+                  response_chain << [feed_url, response]
+                  new_location = response['location']
+                  if response_chain.assoc(new_location) != nil
+                    raise FeedAccessError, "Redirection loop detected: #{new_location}"
+                  end
               
-                # Find out if we've already seen the url we've been
-                # redirected to.
-                found_redirect = false
-                begin
-                  cached_feed = FeedTools::Feed.open(new_location,
-                    :cache_only => true)
-                  if cached_feed.cache_object != nil &&
-                      cached_feed.cache_object.new_record? != true
-                    unless cached_feed.expired?
-                      # Copy the cached state, starting with the url
-                      self.url = cached_feed.url
-                      self.title = cached_feed.title
-                      self.link = cached_feed.link
-                      self.feed_data = cached_feed.feed_data
-                      self.feed_data_type = cached_feed.feed_data_type
-                      self.last_retrieved = cached_feed.last_retrieved
-                      self.http_headers = cached_feed.http_headers
-                      self.cache_object = cached_feed.cache_object
-                      @live = false
-                      found_redirect = true
+                  # Find out if we've already seen the url we've been
+                  # redirected to.
+                  found_redirect = false
+                  begin
+                    cached_feed = FeedTools::Feed.open(new_location,
+                      :cache_only => true)
+                    if cached_feed.cache_object != nil &&
+                        cached_feed.cache_object.new_record? != true
+                      unless cached_feed.expired?
+                        # Copy the cached state, starting with the url
+                        self.url = cached_feed.url
+                        self.title = cached_feed.title
+                        self.link = cached_feed.link
+                        self.feed_data = cached_feed.feed_data
+                        self.feed_data_type = cached_feed.feed_data_type
+                        self.last_retrieved = cached_feed.last_retrieved
+                        self.http_headers = cached_feed.http_headers
+                        self.cache_object = cached_feed.cache_object
+                        @live = false
+                        found_redirect = true
+                      end
                     end
+                  rescue
+                    # If anything goes wrong, ignore it.
+                  end
+                  unless found_redirect
+                    # TODO: deal with stupid people using relative urls
+                    # in Location header
+                    # =================================================
+                    http_fetch.call(new_location, http_headers,
+                      redirect_limit - 1, response_chain, no_headers)
+                  else
+                    response
                   end
-                rescue
-                  # If anything goes wrong, ignore it.
-                end
-                unless found_redirect
-                  # TODO: deal with stupid people using relative urls
-                  # in Location header
-                  # =================================================
-                  http_fetch.call(new_location, http_headers,
-                    redirect_limit - 1, response_chain, no_headers)
-                else
-                  response
                 end
-              end
-            else
-              class << response
-                def response_chain
-                  return @response_chain
+              else
+                class << response
+                  def response_chain
+                    return @response_chain
+                  end
                 end
+                response.instance_variable_set("@response_chain",
+                  response_chain)
+                response.error!
               end
-              response.instance_variable_set("@response_chain",
-                response_chain)
-              response.error!
             end
+          rescue SocketError
+            raise FeedAccessError, 'Socket error prevented feed retrieval'
           end
         end
       
@@ -360,7 +356,7 @@ module FeedTools
             self.http_response.each_header do |key, value|
               self.http_headers[key.downcase] = value
             end
-            self.last_retrieved = Time.now
+            self.last_retrieved = Time.now.gmtime
             self.feed_data = self.http_response.body
           end
         rescue FeedAccessError
@@ -401,7 +397,7 @@ module FeedTools
               self.http_headers[key.downcase] = value
             end
             if self.http_response.code.to_i == 304
-              self.last_retrieved = Time.now
+              self.last_retrieved = Time.now.gmtime
             end
           end
           @live = false
@@ -432,7 +428,7 @@ module FeedTools
           open(file_name) do |file|
             @http_response = nil
             @http_headers = {}
-            self.last_retrieved = Time.now
+            self.last_retrieved = Time.now.gmtime
             self.feed_data = file.read
             self.feed_data_type = :xml
           end
@@ -507,14 +503,25 @@ module FeedTools
         @xml_doc = nil
       else
         if @xml_doc.nil?
+          # INQUIRY: Is there any way of saying "dude, rescue *everything*"?
           begin
-            # TODO: :ignore_whitespace_nodes => :all
-            # Add that?
-            # ======================================
-            @xml_doc = Document.new(feed_data)
+            begin
+              # TODO: :ignore_whitespace_nodes => :all
+              # Add that?
+              # ======================================
+              @xml_doc = Document.new(feed_data)
+            rescue Exception
+              # Something failed especially badly, attempt to repair the
+              # xml with htree.
+              @xml_doc = HTree.parse(feed_data).to_rexml
+            rescue
+              # Something failed, attempt to repair the xml with htree.
+              @xml_doc = HTree.parse(feed_data).to_rexml
+            end
+          rescue Exception
+            @xml_doc = nil
           rescue
-            # Something failed, attempt to repair the xml with htree.
-            @xml_doc = HTree.parse(feed_data).to_rexml
+            @xml_doc = nil
           end
         end
       end
@@ -607,6 +614,9 @@ module FeedTools
     # "rss", "atom", "cdf", "!okay/news"
     def feed_type
       if @feed_type.nil?
+        if self.root_node.nil?
+          return nil
+        end
         case self.root_node.name.downcase
         when "feed"
           @feed_type = "atom"
@@ -633,6 +643,9 @@ module FeedTools
     # versions of RSS 0.91.
     def feed_version
       if @feed_version.nil?
+        if self.root_node.nil?
+          return nil
+        end
         version = nil
         begin
           version = XPath.first(root_node, "@version").to_s.strip.to_f
@@ -749,8 +762,8 @@ module FeedTools
           @url = nil if @url == ""
         end
         if override_url.call
-          @url = XPath.first(channel_node, "atom:link[@rel='self']/@href",
-            FEED_TOOLS_NAMESPACES_WITH_OLD_ATOM).to_s
+          @url = XPath.first(channel_node, "atom03:link[@rel='self']/@href",
+            FEED_TOOLS_NAMESPACES).to_s
           @url = nil if @url == ""
         end
         if override_url.call
@@ -921,7 +934,7 @@ module FeedTools
         unless @description.nil?
           @description = FeedTools.sanitize_html(@description, :strip)
           @description = FeedTools.unescape_entities(@description) if repair_entities
-          @description = FeedTools.tidy_html(@description) unless repair_entities
+          @description = FeedTools.tidy_html(@description)
         end
 
         @description = @description.strip unless @description.nil?
@@ -1305,10 +1318,10 @@ module FeedTools
           if time_string != nil && time_string != ""
             @time = Time.parse(time_string)
           else
-            @time = Time.now
+            @time = Time.now.gmtime
           end
         rescue
-          @time = Time.now
+          @time = Time.now.gmtime
         end
       end
       return @time
@@ -1838,7 +1851,7 @@ module FeedTools
     # server.
     def expired?
       return self.last_retrieved == nil ||
-        (self.last_retrieved + self.time_to_live) < Time.now
+        (self.last_retrieved + self.time_to_live) < Time.now.gmtime
     end
   
     # Forces this feed to expire.
@@ -1866,13 +1879,13 @@ module FeedTools
           version == 1.1)
         # RDF-based rss format
         return xml_builder.tag!("rdf:RDF",
-            "xmlns" => "http://purl.org/rss/1.0/",
-            "xmlns:rdf" => "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
-            "xmlns:dc" => "http://purl.org/dc/elements/1.1/",
-            "xmlns:syn" => "http://purl.org/rss/1.0/modules/syndication/",
-            "xmlns:taxo" => "http://purl.org/rss/1.0/modules/taxonomy/",
-            "xmlns:itunes" => "http://www.itunes.com/DTDs/Podcast-1.0.dtd",
-            "xmlns:media" => "http://search.yahoo.com/mrss") do
+            "xmlns" => FEED_TOOLS_NAMESPACES['rss10'],
+            "xmlns:rdf" => FEED_TOOLS_NAMESPACES['rdf'],
+            "xmlns:dc" => FEED_TOOLS_NAMESPACES['dc'],
+            "xmlns:syn" => FEED_TOOLS_NAMESPACES['syn'],
+            "xmlns:taxo" => FEED_TOOLS_NAMESPACES['taxo'],
+            "xmlns:itunes" => FEED_TOOLS_NAMESPACES['itunes'],
+            "xmlns:media" => FEED_TOOLS_NAMESPACES['media']) do
           channel_attributes = {}
           unless self.link.nil?
             channel_attributes["rdf:about"] = CGI.escapeHTML(self.link)
@@ -1955,13 +1968,12 @@ module FeedTools
       elsif feed_type == "rss"
         # normal rss format
         return xml_builder.rss("version" => "2.0",
-            "xmlns:rdf" => "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
-            "xmlns:dc" => "http://purl.org/dc/elements/1.1/",
-            "xmlns:taxo" => "http://purl.org/rss/1.0/modules/taxonomy/",
-            "xmlns:trackback" =>
-              "http://madskills.com/public/xml/rss/module/trackback/",
-            "xmlns:itunes" => "http://www.itunes.com/DTDs/Podcast-1.0.dtd",
-            "xmlns:media" => "http://search.yahoo.com/mrss") do
+            "xmlns:rdf" => FEED_TOOLS_NAMESPACES['rdf'],
+            "xmlns:dc" => FEED_TOOLS_NAMESPACES['dc'],
+            "xmlns:taxo" => FEED_TOOLS_NAMESPACES['taxo'],
+            "xmlns:trackback" => FEED_TOOLS_NAMESPACES['trackback'],
+            "xmlns:itunes" => FEED_TOOLS_NAMESPACES['itunes'],
+            "xmlns:media" => FEED_TOOLS_NAMESPACES['media']) do
           xml_builder.channel do
             unless title.nil? || title == ""
               xml_builder.title(title)
@@ -1985,7 +1997,7 @@ module FeedTools
         end
       elsif feed_type == "atom" && version == 0.3
         # normal atom format
-        return xml_builder.feed("xmlns" => "http://purl.org/atom/ns#",
+        return xml_builder.feed("xmlns" => FEED_TOOLS_NAMESPACES['atom03'],
             "version" => version,
             "xml:lang" => language) do
           unless title.nil? || title == ""
@@ -2028,7 +2040,7 @@ module FeedTools
         end
       elsif feed_type == "atom" && version == 1.0
         # normal atom format
-        return xml_builder.feed("xmlns" => "http://www.w3.org/2005/Atom",
+        return xml_builder.feed("xmlns" => FEED_TOOLS_NAMESPACES['atom'],
             "xml:lang" => language) do
           unless title.nil? || title == ""
             xml_builder.title(title,
@@ -2072,7 +2084,7 @@ module FeedTools
             # than the Time.now fall-back.
             xml_builder.updated(self.time.iso8601)
           else
-            xml_builder.updated(Time.now.iso8601)
+            xml_builder.updated(Time.now.gmtime.iso8601)
           end
           xml_builder.generator("FeedTools - " +
             "http://www.sporkmonger.com/projects/feedtools")

data/lib/feed_tools/feed_item.rb

@@ -127,12 +127,12 @@ module FeedTools
       super
       @feed = nil
       @feed_data = nil
-      @feed_data_type = nil
+      @feed_data_type = :xml
       @xml_doc = nil
       @root_node = nil
       @title = nil
       @id = nil
-      @time = nil
+      @time = Time.now.gmtime
     end
 
     # Returns the parent feed of this feed item
@@ -152,7 +152,35 @@ module FeedTools
 
     # Sets the feed item's data.
     def feed_data=(new_feed_data)
+      @time = nil
       @feed_data = new_feed_data
+      
+      # We need an immediate parse of the time so we don't mess up sort orders
+      unless root_node.nil?
+        repair_entities = false
+        time_node = XPath.first(root_node, "pubDate")
+        if time_node.nil?
+          time_node = XPath.first(root_node, "dc:date")
+        end
+        if time_node.nil?
+          time_node = XPath.first(root_node, "dc:date", FEED_TOOLS_NAMESPACES)
+        end
+        if time_node.nil?
+          time_node = XPath.first(root_node, "issued")
+        end
+        if time_node.nil?
+          time_node = XPath.first(root_node, "updated")
+        end
+        if time_node.nil?
+          time_node = XPath.first(root_node, "time")
+        end
+      end
+      unless time_node.nil?
+        begin
+          @time = Time.parse(time_node.inner_xml)
+        rescue
+        end
+      end
     end
 
     # Returns the feed item's data type.
@@ -193,6 +221,9 @@ module FeedTools
     # Returns the root node of the feed item.
     def root_node
       if @root_node.nil?
+        if xml.nil?
+          return nil
+        end
         @root_node = xml.root
       end
       return @root_node
@@ -369,7 +400,7 @@ module FeedTools
         unless @description.nil?
           @description = FeedTools.sanitize_html(@description, :strip)
           @description = FeedTools.unescape_entities(@description) if repair_entities
-          @description = FeedTools.tidy_html(@description) unless repair_entities
+          @description = FeedTools.tidy_html(@description)
         end
 
         @description = @description.strip unless @description.nil?
@@ -1272,12 +1303,18 @@ module FeedTools
             @time = Time.parse(time_string)
           else
             @time = succ_time
+            if @time.nil?
+              @time = prev_time
+            end
           end
         rescue
           @time = succ_time
+          if @time.nil?
+            @time = prev_time
+          end
         end
         if @time.nil?
-          @time = Time.now
+          @time = Time.now.gmtime
         end
       end
       return @time
@@ -1303,15 +1340,40 @@ module FeedTools
           return nil
         end
         if item_index <= 0
-          return Time.now
+          return nil
         end
         previous_item = unsorted_items[item_index - 1]
-        return previous_item.time.succ
+        return (previous_item.time + 1)
       rescue
         return nil
       end
     end
-    private :succ_time
+    #private :succ_time
+
+    # Returns 1 second before the succeeding item's time.
+    def prev_time #:nodoc:
+      begin
+        if feed.nil?
+          return nil
+        end
+        if feed.instance_variable_get("@items").nil?
+          feed.items
+        end
+        unsorted_items = feed.instance_variable_get("@items")
+        item_index = unsorted_items.index(self)
+        if item_index.nil?
+          return nil
+        end
+        if item_index >= (unsorted_items.size - 1)
+          return nil
+        end
+        succeeding_item = unsorted_items[item_index + 1]
+        return (succeeding_item.time - 1)
+      rescue
+        return nil
+      end
+    end
+    #private :prev_time
 
     # Returns the feed item updated time
     def updated
@@ -1398,10 +1460,17 @@ module FeedTools
       # =======================================
       if @tags.nil?
         @tags = []
+        if root_node.nil?
+          return @tags
+        end
         if @tags.nil? || @tags.size == 0
           @tags = []
           tag_list = XPath.match(root_node, "dc:subject/rdf:Bag/rdf:li/text()")
-          if tag_list.size > 1
+          if tag_list.nil? || tag_list.size == 0
+            tag_list = XPath.match(root_node,
+              "dc:subject/rdf:Bag/rdf:li/text()", FEED_TOOLS_NAMESPACES)
+          end
+          if tag_list != nil && tag_list.size > 1
             for tag in tag_list
               @tags << tag.to_s.downcase.strip
             end
@@ -1642,7 +1711,7 @@ module FeedTools
             # than the Time.now fall-back.
             xml_builder.updated(self.time.iso8601)
           else
-            xml_builder.updated(Time.now.iso8601)
+            xml_builder.updated(Time.now.gmtime.iso8601)
           end
           unless self.published.nil?
             xml_builder.published(self.published.iso8601)