feedtools 0.2.24 → 0.2.25

data/CHANGELOG

@@ -1,3 +1,15 @@
+== FeedTools 0.2.25
+ * fixed self reference href bug
+ * fixed primary link assignment heuristic
+ * link rel's now default to 'alternate' for atom feeds
+ * better handling of language attributes
+ * better support for threaded applications
+ * generated rss 2.0 now includes author, managingEditor, webMaster elements
+ * improved handling of author elements
+ * improved handling of http timeouts
+ * fixed issue with relative paths in the location header for http redirection
+ * monkey patches now work with Ruby 1.8.4's version of REXML
+ * REXML monkey patches moved into their own file
 == FeedTools 0.2.24
  * fixed autodiscovery bugs
  * updated autodiscovery to handle relative uris

data/lib/feed_tools.rb

@@ -151,6 +151,8 @@ begin
     end
   end
   
+  require 'feed_tools/monkey_patch'
+  
   require 'feed_tools/feed'
   require 'feed_tools/feed_item'
   require 'feed_tools/feed_structures'
@@ -348,301 +350,6 @@ module FeedTools
   end
 end
 
-module REXML # :nodoc:
-  class LiberalXPathParser < XPathParser # :nodoc:
-  private
-    def internal_parse(path_stack, nodeset) # :nodoc:
-      return nodeset if nodeset.size == 0 or path_stack.size == 0
-      case path_stack.shift
-      when :document
-        return [ nodeset[0].root.parent ]
-
-      when :qname
-        prefix = path_stack.shift.downcase
-        name = path_stack.shift.downcase
-        n = nodeset.clone
-        ns = @namespaces[prefix]
-        ns = ns ? ns : ''
-        n.delete_if do |node|
-          if node.node_type == :element and ns == ''
-            ns = node.namespace( prefix )
-          end
-          !(node.node_type == :element and
-            node.name.downcase == name.downcase and node.namespace == ns )
-        end
-        return n
-
-      when :any
-        n = nodeset.clone
-        n.delete_if { |node| node.node_type != :element }
-        return n
-
-      when :self
-        # THIS SPACE LEFT INTENTIONALLY BLANK
-
-      when :processing_instruction
-        target = path_stack.shift
-        n = nodeset.clone
-        n.delete_if do |node|
-          (node.node_type != :processing_instruction) or 
-          ( !target.nil? and ( node.target != target ) )
-        end
-        return n
-
-      when :text
-        n = nodeset.clone
-        n.delete_if do |node|
-          node.node_type != :text
-        end
-        return n
-
-      when :comment
-        n = nodeset.clone
-        n.delete_if do |node|
-          node.node_type != :comment
-        end
-        return n
-
-      when :node
-        return nodeset
-      
-      when :child
-        new_nodeset = []
-        nt = nil
-        for node in nodeset
-          nt = node.node_type
-          new_nodeset += node.children if nt == :element or nt == :document
-        end
-        return new_nodeset
-
-      when :literal
-        literal = path_stack.shift
-        if literal =~ /^\d+(\.\d+)?$/
-          return ($1 ? literal.to_f : literal.to_i) 
-        end
-        return literal
-        
-      when :attribute
-        new_nodeset = []
-        case path_stack.shift
-        when :qname
-          prefix = path_stack.shift
-          name = path_stack.shift.downcase
-          for element in nodeset
-            if element.node_type == :element
-              for attribute_name in element.attributes.keys
-                if attribute_name.downcase == name.downcase
-                  attrib = element.attribute( attribute_name,
-                    @namespaces[prefix] )
-                  new_nodeset << attrib if attrib
-                end
-              end
-            end
-          end
-        when :any
-          for element in nodeset
-            if element.node_type == :element
-              new_nodeset += element.attributes.to_a
-            end
-          end
-        end
-        return new_nodeset
-
-      when :parent
-        return internal_parse( path_stack,
-          nodeset.collect{|n| n.parent}.compact )
-
-      when :ancestor
-        new_nodeset = []
-        for node in nodeset
-          while node.parent
-            node = node.parent
-            new_nodeset << node unless new_nodeset.include? node
-          end
-        end
-        return new_nodeset
-
-      when :ancestor_or_self
-        new_nodeset = []
-        for node in nodeset
-          if node.node_type == :element
-            new_nodeset << node
-            while ( node.parent )
-              node = node.parent
-              new_nodeset << node unless new_nodeset.include? node
-            end
-          end
-        end
-        return new_nodeset
-
-      when :predicate
-        predicate = path_stack.shift
-        new_nodeset = []
-        Functions::size = nodeset.size
-        nodeset.size.times do |index|
-          node = nodeset[index]
-          Functions::node = node
-          Functions::index = index+1
-          result = Predicate( predicate, node )
-          if result.kind_of? Numeric
-            new_nodeset << node if result == (index+1)
-          elsif result.instance_of? Array
-            new_nodeset << node if result.size > 0
-          else
-            new_nodeset << node if result
-          end
-        end
-        return new_nodeset
-
-      when :descendant_or_self
-        rv = descendant_or_self( path_stack, nodeset )
-        path_stack.clear
-        return rv
-
-      when :descendant
-        results = []
-        nt = nil
-        for node in nodeset
-          nt = node.node_type
-          if nt == :element or nt == :document
-            results += internal_parse(
-              path_stack.clone.unshift( :descendant_or_self ),
-              node.children )
-          end
-        end
-        return results
-
-      when :following_sibling
-        results = []
-        for node in nodeset
-          all_siblings = node.parent.children
-          current_index = all_siblings.index( node )
-          following_siblings = all_siblings[ current_index+1 .. -1 ]
-          results += internal_parse( path_stack.clone, following_siblings )
-        end
-        return results
-
-      when :preceding_sibling
-        results = []
-        for node in nodeset
-          all_siblings = node.parent.children
-          current_index = all_siblings.index( node )
-          preceding_siblings = all_siblings[ 0 .. current_index-1 ]
-          results += internal_parse( path_stack.clone, preceding_siblings )
-        end
-        return results
-
-      when :preceding
-        new_nodeset = []
-        for node in nodeset
-          new_nodeset += preceding( node )
-        end
-        return new_nodeset
-
-      when :following
-        new_nodeset = []
-        for node in nodeset
-          new_nodeset += following( node )
-        end
-        return new_nodeset
-
-      when :namespace
-        new_set = []
-        for node in nodeset
-          if node.node_type == :element or node.node_type == :attribute
-            new_nodeset << node.namespace
-          end
-        end
-        return new_nodeset
-
-      when :variable
-        var_name = path_stack.shift
-        return @variables[ var_name ]
-
-      end
-      nodeset
-    end
-  end
-  
-  class XPath # :nodoc:
-    def self.liberal_match(element, path=nil, namespaces={},
-        variables={}) # :nodoc:
-			parser = LiberalXPathParser.new
-			parser.namespaces = namespaces
-			parser.variables = variables
-			path = "*" unless path
-			element = [element] unless element.kind_of? Array
-			parser.parse(path, element)
-    end
-
-    def self.liberal_first(element, path=nil, namespaces={},
-        variables={}) # :nodoc:
-      parser = LiberalXPathParser.new
-      parser.namespaces = namespaces
-      parser.variables = variables
-      path = "*" unless path
-      element = [element] unless element.kind_of? Array
-      parser.parse(path, element)[0]
-    end
-
-    def self.liberal_each(element, path=nil, namespaces={},
-        variables={}, &block) # :nodoc:
-			parser = LiberalXPathParser.new
-			parser.namespaces = namespaces
-			parser.variables = variables
-			path = "*" unless path
-			element = [element] unless element.kind_of? Array
-			parser.parse(path, element).each( &block )
-    end
-  end
-  
-  class Element # :nodoc:
-    unless REXML::Element.public_instance_methods.include? :inner_xml
-      def inner_xml # :nodoc:
-        result = ""
-        self.each_child do |child|
-          if child.kind_of? REXML::Comment
-            result << "<!--" + child.to_s + "-->"
-          else
-            result << child.to_s
-          end
-        end
-        return result.strip
-      end
-    else
-      warn("inner_xml method already exists.")
-    end
-    
-    def base_uri # :nodoc:
-      begin
-        base_attribute = FeedTools::XmlHelper.try_xpaths(self, [
-          '@xml:base'
-        ])
-        if parent == nil || parent.kind_of?(REXML::Document)
-          return nil if base_attribute == nil
-          return base_attribute.value
-        end
-        if base_attribute != nil && parent == nil
-          return base_attribute.value
-        elsif parent != nil && base_attribute == nil
-          return parent.base_uri
-        elsif parent != nil && base_attribute != nil
-          parent_base_uri = parent.base_uri
-          if parent_base_uri != nil
-            uri = URI.parse(parent_base_uri)
-            return (uri + base_attribute.value).to_s
-          else
-            return base_attribute.value
-          end
-        end
-        return nil
-      rescue
-        return nil
-      end
-    end
-  end
-end
-
 begin
   unless FeedTools.feed_cache.nil?
     FeedTools.feed_cache.initialize_cache

data/lib/feed_tools/feed.rb

@@ -78,6 +78,8 @@ module FeedTools
       # load the new feed
       feed.href = url
       feed.update! unless feed.configurations[:disable_update_from_remote]
+      Thread.pass
+      
       return feed
     end
     
@@ -101,6 +103,10 @@ module FeedTools
         # Don't do anything if this option is set
         return
       end
+      if !FeedTools.feed_cache.nil? &&
+          !FeedTools.feed_cache.set_up_correctly?
+        FeedTools.feed_cache.initialize_cache()
+      end
       if !FeedTools.feed_cache.nil? &&
           !FeedTools.feed_cache.set_up_correctly?
         raise "Your feed cache system is incorrectly set up.  " +
@@ -225,6 +231,7 @@ module FeedTools
                     end
     
                     @title = nil; self.title
+                    self.href
                     @link = nil; self.link
                   
                     self.last_retrieved = cached_feed.last_retrieved
@@ -254,7 +261,7 @@ module FeedTools
               self.http_headers[key.downcase] = value
             end
             self.last_retrieved = Time.now.gmtime
-            @live = true
+            @live = false
           else
             @live = false
           end
@@ -724,20 +731,24 @@ module FeedTools
           end
         end
         if override_href.call(@href) && self.feed_data != nil
-          # rdf:about is ordered last because a lot of people put the url to
-          # the feed inside it instead of a link to their blog.
-          # Ordering it last gives them as many chances as humanly possible
-          # for them to redeem themselves.  If the link turns out to be the
-          # same as the blog link, it will be reset to the original value.
           for link_object in self.links
             if link_object.rel == 'self'
-              if link_object.href != self.link
+              if link_object.href != self.link ||
+                  (link_object.href =~ /xml/ ||
+                  link_object.href =~ /atom/ ||
+                  link_object.href =~ /feed/)
                 @href = link_object.href
                 @href_overridden = true
+                @link = nil
                 return @href
               end
             end
           end
+          # rdf:about is ordered last because a lot of people put the url to
+          # the feed inside it instead of a link to their blog.
+          # Ordering it last gives them as many chances as humanly possible
+          # for them to redeem themselves.  If the link turns out to be the
+          # same as the blog link, it will be reset to the original value.
           @href = FeedTools::XmlHelper.try_xpaths(self.channel_node, [
             "admin:feed/@rdf:resource",
             "admin:feed/@resource",
@@ -770,6 +781,9 @@ module FeedTools
             @href = original_href
             @href_overridden = false
           end
+          if @href_overridden == true
+            @link = nil
+          end
         end
       end
       return @href
@@ -791,7 +805,8 @@ module FeedTools
           "atom:title",
           "title",
           "dc:title",
-          "channelTitle"
+          "channelTitle",
+          "TITLE"
         ])
         @title = FeedTools::HtmlHelper.process_text_construct(title_node,
           self.feed_type, self.feed_version)
@@ -822,6 +837,7 @@ module FeedTools
           "description",
           "summary",
           "abstract",
+          "ABSTRACT",
           "content:encoded",
           "encoded",
           "content",
@@ -947,19 +963,36 @@ module FeedTools
         max_score = 0
         for link_object in self.links.reverse
           score = 0
-          if FeedTools::HtmlHelper.html_type?(link_object.type)
-            score = score + 2
-          elsif link_object.type != nil
-            score = score - 1
+          next if link_object.href.nil?
+          if @href != nil && link_object.href == @href
+            score = score - 2
           end
-          if FeedTools::HtmlHelper.xml_type?(link_object.type)
-            score = score + 1
+          if link_object.type != nil
+            if (link_object.type =~ /image/ || link_object.type =~ /video/)
+              score = score - 2
+            end
+            if FeedTools::HtmlHelper.xml_type?(link_object.type)
+              score = score + 1
+            end
+            if FeedTools::HtmlHelper.html_type?(link_object.type)
+              score = score + 2
+            elsif link_object.type != nil
+              score = score - 1
+            end
+          end
+          if link_object.rel == "enclosure"
+            score = score - 2
           end
           if link_object.rel == "alternate"
             score = score + 1
           end
           if link_object.rel == "self"
             score = score - 1
+            if (link_object.href =~ /xml/ ||
+                link_object.href =~ /atom/ ||
+                link_object.href =~ /feed/)
+              score = score - 1
+            end
           end
           if score >= max_score
             max_score = score
@@ -1073,6 +1106,9 @@ module FeedTools
           unless link_object.rel.nil?
             link_object.rel = link_object.rel.downcase
           end
+          if link_object.rel.nil? && self.feed_type == "atom"
+            link_object.rel = "alternate"
+          end
           link_object.type = FeedTools::XmlHelper.try_xpaths(link_node, [
             "@atom10:type",
             "@atom03:type",
@@ -1110,6 +1146,7 @@ module FeedTools
               link_object.length = nil
             end
           end
+          @links = [] if @links.nil?
           @links << link_object
         end
       end
@@ -1128,10 +1165,13 @@ module FeedTools
           "@base"
         ], :select_result_value => true)
         if @base_uri.blank?
-          @base_uri =
-            FeedTools::GenericHelper.recursion_trap(:feed_base_uri) do
-              self.href
-            end
+          begin
+            @base_uri =
+              FeedTools::GenericHelper.recursion_trap(:feed_base_uri) do
+                self.href
+              end
+          rescue Exception
+          end
         end
         if !@base_uri.blank?
           @base_uri = FeedTools::UriHelper.normalize_url(@base_uri)
@@ -1307,6 +1347,21 @@ module FeedTools
               ], :select_result_value => true)
             )
           end
+          if @author.name.blank? && !@author.raw.blank? &&
+              !@author.email.blank?
+            name_scan = @author.raw.scan(
+              /"?([^"]*)"? ?[\(<].*#{@author.email}.*[\)>].*/)
+            if name_scan.flatten.size == 1
+              @author.name = name_scan.flatten[0].strip
+            end
+            if @author.name.blank?
+              name_scan = @author.raw.scan(
+                /.*#{@author.email} ?[\(<]"?([^"]*)"?[\)>].*/)
+              if name_scan.flatten.size == 1
+                @author.name = name_scan.flatten[0].strip
+              end
+            end
+          end
           @author.name = nil if @author.name.blank?
           @author.raw = nil if @author.raw.blank?
           @author.email = nil if @author.email.blank?
@@ -1935,7 +1990,13 @@ module FeedTools
         if @language.blank?
           @language = "en-us"
         end
+        @language.gsub!(/_/, "-")
         @language = @language.downcase
+        if @language.split('-').size > 1
+          @language =
+            "#{@language.split('-').first}-" +
+            "#{@language.split('-').last.upcase}"
+        end
       end
       return @language
     end
@@ -2267,6 +2328,12 @@ module FeedTools
             else
               xml_builder.description
             end
+            unless self.author.email.blank?
+              xml_builder.managingEditor(self.author.email)
+            end
+            unless self.publisher.email.blank?
+              xml_builder.webMaster(self.publisher.email)
+            end
             unless self.published.blank?
               xml_builder.pubDate(self.published.rfc822)
             end
@@ -2397,6 +2464,7 @@ module FeedTools
           end
           self.cache_object.http_headers = self.http_headers.to_yaml
           self.cache_object.last_retrieved = self.last_retrieved
+          Thread.pass
           self.cache_object.save
         end
       end