feedtools 0.2.23 → 0.2.24

data/lib/feed_tools/feed_item.rb

@@ -69,6 +69,24 @@ module FeedTools
       return parent_feed
     end
     
+    # Returns the load options for this feed.
+    def configurations
+      if @configurations.blank?
+        parent_feed = self.feed
+        if parent_feed != nil
+          @configurations = parent_feed.configurations.dup
+        else
+          @configurations = FeedTools.configurations.dup
+        end
+      end
+      return @configurations
+    end
+    
+    # Sets the load options for this feed.
+    def configurations=(new_configurations)
+      @configurations = new_configurations
+    end
+    
     # Returns the feed item's encoding.
     def encoding
       if @encoding.nil?
@@ -202,10 +220,10 @@ module FeedTools
         @title = FeedTools::HtmlHelper.process_text_construct(title_node,
           self.feed_type, self.feed_version)
         if self.feed_type == "atom" ||
-            FeedTools.configurations[:always_strip_wrapper_elements]
+            self.configurations[:always_strip_wrapper_elements]
           @title = FeedTools::HtmlHelper.strip_wrapper_element(@title)
         end
-        if !@title.blank? && FeedTools.configurations[:strip_comment_count]
+        if !@title.blank? && self.configurations[:strip_comment_count]
           # Some blogging tools include the number of comments in a post
           # in the title... this is supremely ugly, and breaks any
           # applications which expect the title to be static, so we're
@@ -257,7 +275,7 @@ module FeedTools
         @content = FeedTools::HtmlHelper.process_text_construct(content_node,
           self.feed_type, self.feed_version)
         if self.feed_type == "atom" ||
-            FeedTools.configurations[:always_strip_wrapper_elements]
+            self.configurations[:always_strip_wrapper_elements]
           @content = FeedTools::HtmlHelper.strip_wrapper_element(@content)
         end
         if @content.blank?
@@ -310,7 +328,7 @@ module FeedTools
         @summary = FeedTools::HtmlHelper.process_text_construct(summary_node,
           self.feed_type, self.feed_version)
         if self.feed_type == "atom" ||
-            FeedTools.configurations[:always_strip_wrapper_elements]
+            self.configurations[:always_strip_wrapper_elements]
           @summary = FeedTools::HtmlHelper.strip_wrapper_element(@summary)
         end
         if @summary.blank?
@@ -456,7 +474,7 @@ module FeedTools
           end
         rescue
         end
-        if FeedTools.configurations[:url_normalization_enabled]
+        if self.configurations[:url_normalization_enabled]
           @link = FeedTools::UriHelper.normalize_url(@link)
         end
       end
@@ -507,7 +525,7 @@ module FeedTools
             end
           rescue
           end
-          if FeedTools.configurations[:url_normalization_enabled]
+          if self.configurations[:url_normalization_enabled]
             link_object.href =
               FeedTools::UriHelper.normalize_url(link_object.href)
           end
@@ -640,7 +658,7 @@ module FeedTools
               end
             rescue
             end
-            if FeedTools.configurations[:url_normalization_enabled]
+            if self.configurations[:url_normalization_enabled]
               image.href = FeedTools::UriHelper.normalize_url(image.href)
             end            
             image.href.strip! unless image.href.nil?
@@ -688,7 +706,7 @@ module FeedTools
           "itunes:image/@href",
           "itunes:link[@rel='image']/@href"
         ], :select_result_value => true)
-        if FeedTools.configurations[:url_normalization_enabled]
+        if self.configurations[:url_normalization_enabled]
           @itunes_image_link = FeedTools::UriHelper.normalize_url(@itunes_image_link)
         end
       end
@@ -706,7 +724,7 @@ module FeedTools
         @media_thumbnail_link = FeedTools::XmlHelper.try_xpaths(self.root_node, [
           "media:thumbnail/@url"
         ], :select_result_value => true)
-        if FeedTools.configurations[:url_normalization_enabled]
+        if self.configurations[:url_normalization_enabled]
           @media_thumbnail_link = FeedTools::UriHelper.normalize_url(@media_thumbnail_link)
         end
       end
@@ -734,7 +752,7 @@ module FeedTools
         @rights = FeedTools::HtmlHelper.process_text_construct(rights_node,
           self.feed_type, self.feed_version)
         if self.feed_type == "atom" ||
-            FeedTools.configurations[:always_strip_wrapper_elements]
+            self.configurations[:always_strip_wrapper_elements]
           @rights = FeedTools::HtmlHelper.strip_wrapper_element(@rights)
         end
       end
@@ -1456,14 +1474,14 @@ module FeedTools
         begin
           if !time_string.blank?
             @time = Time.parse(time_string).gmtime
-          elsif FeedTools.configurations[:timestamp_estimation_enabled] &&
+          elsif self.configurations[:timestamp_estimation_enabled] &&
               !self.title.nil? &&
               (Time.parse(self.title) - Time.now).abs > 100
             @time = Time.parse(self.title).gmtime
           end
         rescue
         end
-        if FeedTools.configurations[:timestamp_estimation_enabled]
+        if self.configurations[:timestamp_estimation_enabled]
           if options[:estimate_timestamp]
             if @time.nil?
               begin
@@ -1615,7 +1633,7 @@ module FeedTools
           end
         rescue
         end
-        if FeedTools.configurations[:url_normalization_enabled]
+        if self.configurations[:url_normalization_enabled]
           @comments = FeedTools::UriHelper.normalize_url(@comments)
         end
       end

data/lib/feed_tools/feed_structures.rb

@@ -170,6 +170,10 @@ module FeedTools
     alias_method :url=, :href=
     alias_method :link, :href
     alias_method :link=, :href=
+    
+    def initialize
+      @expression = 'full'
+    end
   
     # Returns true if this is the default enclosure
     def is_default?
@@ -202,8 +206,7 @@ module FeedTools
     # Allowed values are 'sample', 'full', 'nonstop'.
     def expression=(new_expression)
       unless ['sample', 'full', 'nonstop'].include? new_expression.downcase
-        raise ArgumentError,
-          "Permitted values are 'sample', 'full', 'nonstop'."
+        return @expression
       end
       @expression = new_expression.downcase
     end

data/lib/feed_tools/helpers/debug_helper.rb

@@ -24,8 +24,7 @@
 module FeedTools
   module DebugHelper
     # Forces a stack_trace without interfering with the program
-    def stack_trace
-      
+    def self.stack_trace
       fork do
         ObjectSpace.each_object(Thread) do |th|
           th.raise Exception, "Stack Dump" unless Thread.current == th

data/lib/feed_tools/helpers/html_helper.rb

@@ -232,6 +232,17 @@ module FeedTools
       return tidy_html
     end
 
+    # Indents a text selection by a specified number of spaces.
+    def self.indent(text, spaces)
+      lines = text.split("\n")
+      buffer = ""
+      for line in lines
+        line = " " * spaces + line
+        buffer << line << "\n"
+      end
+      return buffer
+    end
+
     # Unindents a text selection by a specified number of spaces.
     def self.unindent(text, spaces)
       lines = text.split("\n")
@@ -301,10 +312,11 @@ module FeedTools
                   html_node.delete_element(child)
                 end
               end
-              for attribute in child.attributes.keys
-                if !(attribute =~ /^xmlns/)
-                  unless acceptable_attributes.include? attribute.downcase
-                    child.delete_attribute(attribute)
+              child.attributes.each_attribute do |attribute|
+                if !(attribute.value =~ /^xmlns(:.+)?$/)
+                  unless acceptable_attributes.include?(
+                      attribute.value.downcase)
+                    child.delete_attribute(attribute.value)
                   end
                 end
               end
@@ -364,8 +376,6 @@ module FeedTools
       ].include?(type)
     end
     
-#     can_be_relative_uri = ['link', 'id', 'wfw_comment', 'wfw_commentrss', 'docs', 'url', 'href', 'comments', 'license', 'icon', 'logo']
-    
     # Resolves all relative uris in a block of html.
     def self.resolve_relative_uris(html, base_uri_sources=[])
       relative_uri_attributes = [
@@ -398,23 +408,23 @@ module FeedTools
       html_doc = HTree.parse_xml("<root>" + html + "</root>").to_rexml
       
       resolve_node = lambda do |html_node|
-        if html_node.respond_to? :children
-          for child in html_node.children
-            if child.kind_of? REXML::Element
-              for element_attribute_pair in relative_uri_attributes
-                if child.name.downcase == element_attribute_pair[0]
-                  attribute = child.attribute(element_attribute_pair[1])
-                  if attribute != nil
-                    href = attribute.value
-                    href = FeedTools::UriHelper.resolve_relative_uri(
-                      href, [child.base_uri] | base_uri_sources)
-                    child.attribute(
-                      element_attribute_pair[1]).instance_variable_set(
-                        "@value", href)
-                  end
-                end
+        if html_node.kind_of? REXML::Element
+          for element_attribute_pair in relative_uri_attributes
+            if html_node.name.downcase == element_attribute_pair[0]
+              attribute = html_node.attribute(element_attribute_pair[1])
+              if attribute != nil
+                href = attribute.value
+                href = FeedTools::UriHelper.resolve_relative_uri(
+                  href, [html_node.base_uri] | base_uri_sources)
+                html_node.attribute(
+                  element_attribute_pair[1]).instance_variable_set(
+                    "@value", href)
               end
             end
+          end
+        end
+        if html_node.respond_to? :children
+          for child in html_node.children
             resolve_node.call(child)
           end
         end
@@ -428,36 +438,55 @@ module FeedTools
     # Returns a string containing normalized xhtml from within a REXML node.
     def self.extract_xhtml(rexml_node)
       rexml_node_dup = rexml_node.deep_clone
+      namespace_hash = FEED_TOOLS_NAMESPACES.dup
       normalize_namespaced_xhtml = lambda do |node, node_dup|
         if node.kind_of? REXML::Element
           node_namespace = node.namespace
-          # Massive hack, relies on REXML not changing
-          for index in 0...node.attributes.values.size
-            attribute = node.attributes.values[index]
-            attribute_dup = node_dup.attributes.values[index]
-            if attribute.namespace == FEED_TOOLS_NAMESPACES['xhtml']
-              attribute_dup.instance_variable_set(
-                "@expanded_name", attribute.name)
-            end
-            if node_namespace == FEED_TOOLS_NAMESPACES['xhtml']
-              if attribute.name == 'xmlns'
-                node_dup.attributes.delete('xmlns')
+          if node_namespace != namespace_hash['atom10'] &&
+              node_namespace != namespace_hash['atom03']
+            # Massive hack, relies on REXML not changing
+            for index in 0...node.attributes.values.size
+              attribute = node.attributes.values[index]
+              attribute_dup = node_dup.attributes.values[index]
+              if attribute.namespace == namespace_hash['xhtml']
+                attribute_dup.instance_variable_set(
+                  "@expanded_name", attribute.name)
+              end
+              if node_namespace == namespace_hash['xhtml']
+                if attribute.name == 'xmlns'
+                  node_dup.attributes.delete('xmlns')
+                end
               end
             end
-          end
-          if node_namespace == FEED_TOOLS_NAMESPACES['xhtml']
-            node_dup.instance_variable_set("@expanded_name", node.name)
-          end
-          if !node_namespace.blank? && node.prefix.blank?
-            if node.namespace != FEED_TOOLS_NAMESPACES['xhtml']
-              node_dup.add_namespace(node_namespace)
+            if node_namespace == namespace_hash['xhtml']
+              node_dup.instance_variable_set("@expanded_name", node.name)
+            end
+            if !node_namespace.blank? && node.prefix.blank?
+              if node_namespace != namespace_hash['xhtml']
+                prefix = nil
+                for known_prefix in namespace_hash.keys
+                  if namespace_hash[known_prefix] == node_namespace
+                    prefix = known_prefix
+                  end
+                end
+                if prefix.nil?
+                  prefix = "unknown" +
+                    Digest::SHA1.new(node_namespace).to_s[0..4]
+                  namespace_hash[prefix] = node_namespace
+                end
+                node_dup.instance_variable_set("@expanded_name",
+                  "#{prefix}:#{node.name}")
+                node_dup.instance_variable_set("@prefix",
+                  prefix)
+                node_dup.add_namespace(prefix, node_namespace)
+              end
             end
           end
         end
         for index in 0...node.children.size
           child = node.children[index]
-          child_dup = node_dup.children[index]
           if child.kind_of? REXML::Element
+            child_dup = node_dup.children[index]
             normalize_namespaced_xhtml.call(child, child_dup)
           end
         end
@@ -513,7 +542,9 @@ module FeedTools
           type == "application/xhtml+xml" ||
           content_node.namespace == FEED_TOOLS_NAMESPACES['xhtml']
         content = FeedTools::HtmlHelper.extract_xhtml(content_node)
-      elsif type == "escaped" || mode == "escaped"
+      elsif type == "escaped" || mode == "escaped" ||
+          type == "html" || mode == "html" ||
+          type == "text/html" || mode == "text/html"
         content = FeedTools::HtmlHelper.unescape_entities(
           content_node.inner_xml.strip)
       elsif type == "text" || mode == "text" ||
@@ -556,7 +587,7 @@ module FeedTools
         doc = REXML::Document.new(xhtml.to_s.strip)
         if doc.children.size == 1
           child = doc.children[0]
-          if child.name.downcase == "div"
+          if child.kind_of?(REXML::Element) && child.name.downcase == "div"
             return child.inner_xml.strip
           end
         end
@@ -574,7 +605,8 @@ module FeedTools
       # This is technically very, very wrong.  But it saves oodles of
       # clock cycles, and probably works 99.999% of the time.
       html_document = HTree.parse_xml(
-        FeedTools::HtmlHelper.tidy_html(html.gsub(/<body>(.|\n)*<\/body>/, ""))).to_rexml
+        FeedTools::HtmlHelper.tidy_html(
+          html.gsub(/<body.*?>(.|\n)*<\/body>/, "<body>-</body>"))).to_rexml
       html_node = nil
       head_node = nil
       link_nodes = []

data/lib/feed_tools/helpers/retrieval_helper.rb

@@ -22,20 +22,218 @@
 #++
 
 require 'feed_tools'
+require 'feed_tools/helpers/uri_helper'
 require 'net/http'
 
 # TODO: Not used yet, don't load since it'll only be a performance hit
 #  require 'net/https'
 #  require 'net/ftp'
 
-# Stolen from the Universal Feed Parser
-FEED_TOOLS_ACCEPT_HEADER = "application/atom+xml,application/rdf+xml," +
-  "application/rss+xml,application/x-netcdf,application/xml;" +
-  "q=0.9,text/xml;q=0.2,*/*;q=0.1"
-
-# TODO: Refactor http_fetch and other methods.
 module FeedTools
   # Methods for pulling remote data
   module RetrievalHelper
+    # Stolen from the Universal Feed Parser
+    ACCEPT_HEADER = "application/atom+xml,application/rdf+xml," +
+      "application/rss+xml,application/x-netcdf,application/xml;" +
+      "q=0.9,text/xml;q=0.2,*/*;q=0.1"
+    
+    # Makes an HTTP request and returns the HTTP response.  Optionally
+    # takes a block that determines whether or not to follow a redirect.
+    # The block will be passed the HTTP redirect response as an argument.
+    def self.http_request(http_operation, url, options={}, &block)
+      response = nil
+      
+      options = {
+        :feed_object => nil,
+        :form_data => nil,
+        :request_headers => {},
+        :follow_redirects => true,
+        :redirect_limit => 10,
+        :response_chain => []
+      }.merge(options)
+      
+      if options[:redirect_limit] == 0
+        raise FeedAccessError, 'Redirect too deep'
+      end
+      
+      if options[:response_chain].blank? ||
+          !options[:response_chain].kind_of?(Array)
+        options[:response_chain] = []
+      end
+      
+      if !options[:request_headers].kind_of?(Hash)
+        options[:request_headers] = {}
+      end
+      if !options[:form_data].kind_of?(Hash)
+        options[:form_data] = nil
+      end
+
+      if options[:request_headers].blank? && options[:feed_object] != nil
+        options[:request_headers] = {}
+        unless options[:feed_object].http_headers.nil?
+          unless options[:feed_object].http_headers['etag'].nil?
+            options[:request_headers]["If-None-Match"] =
+              options[:feed_object].http_headers['etag']
+          end
+          unless options[:feed_object].http_headers['last-modified'].nil?
+            options[:request_headers]["If-Modified-Since"] =
+              options[:feed_object].http_headers['last-modified']
+          end
+        end
+        unless options[:feed_object].configurations[:user_agent].nil?
+          options[:request_headers]["User-Agent"] =
+            options[:feed_object].configurations[:user_agent]
+        end
+      end
+      if options[:request_headers]["Accept"].nil?
+        options[:request_headers]["Accept"] =
+          FeedTools::RetrievalHelper::ACCEPT_HEADER
+      end
+      if options[:request_headers]["User-Agent"].nil?
+        options[:request_headers]["User-Agent"] =
+          FeedTools.configurations[:user_agent]
+      end
+      
+      uri = nil
+      begin
+        uri = URI.parse(url)
+      rescue URI::InvalidURIError
+        # Uh, maybe try to fix it?
+        uri = URI.parse(FeedTools::UriHelper.normalize_url(url))
+      end
+      
+      begin
+        proxy_address = nil
+        proxy_port = nil
+        proxy_user = nil
+        proxy_password = nil
+        
+        if options[:feed_object] != nil
+          proxy_address =
+            options[:feed_object].configurations[:proxy_address] || nil
+          proxy_port =
+            options[:feed_object].configurations[:proxy_port].to_i || nil
+          proxy_user =
+            options[:feed_object].configurations[:proxy_user].to_i || nil
+          proxy_password =
+            options[:feed_object].configurations[:proxy_password].to_i || nil
+        end        
+        
+        # No need to check for nil
+        http = Net::HTTP::Proxy(
+          proxy_address, proxy_port, proxy_user, proxy_password).new(
+            uri.host, (uri.port or 80))
+        
+        path = uri.path 
+        path += ('?' + uri.query) if uri.query
+        
+        request_params = [path, options[:request_headers]]
+        if http_operation == :post
+          options[:form_data] = {} if options[:form_data].blank?
+          request_params << options[:form_data]
+        end
+        response = http.send(http_operation, *request_params)
+        
+        case response
+        when Net::HTTPSuccess
+          if options[:feed_object] != nil
+            # We've reached the final destination, process all previous
+            # redirections, and see if we need to update the url.
+            for redirected_response in options[:response_chain]
+              if redirected_response.last.code.to_i == 301
+                # Reset the cache object or we may get duplicate entries
+
+                # TODO: verify this line is necessary!
+#=============================================================================
+                options[:feed_object].cache_object = nil
+                
+                options[:feed_object].href =
+                  redirected_response.last['location']
+              else
+                # Jump out as soon as we hit anything that isn't a
+                # permanently moved redirection.
+                break
+              end
+            end
+          end
+        when Net::HTTPNotModified
+          # Do nothing, we just don't want it processed as a redirection
+        when Net::HTTPRedirection
+          if response['location'].nil?
+            raise FeedAccessError,
+              "No location to redirect to supplied for " + response.code
+          end
+          options[:response_chain] << [url, response]
+
+          redirected_location = response['location']
+          redirected_location = FeedTools::UriHelper.resolve_relative_uri(
+            redirected_location, [uri.host])
+          
+          if options[:response_chain].assoc(redirected_location) != nil
+            raise FeedAccessError,
+              "Redirection loop detected: #{redirected_location}"
+          end
+          
+          # Let the block handle redirects
+          follow_redirect = true
+          if block != nil
+            follow_redirect = block.call(redirected_location, response)
+          end
+          
+          if follow_redirect
+            response = FeedTools::RetrievalHelper.http_request(
+              http_operation,
+              redirected_location, 
+              options.merge(
+                {:redirect_limit => (options[:redirect_limit] - 1)}),
+              &block)
+          end
+        end
+      rescue SocketError
+        raise FeedAccessError, 'Socket error prevented feed retrieval'
+      rescue Timeout::Error
+        raise FeedAccessError, 'Timeout while attempting to retrieve feed'
+      rescue Errno::ENETUNREACH
+        raise FeedAccessError, 'Network was unreachable'
+      rescue Errno::ECONNRESET
+        raise FeedAccessError, 'Connection was reset by peer'
+      end
+      
+      if response != nil
+        class << response
+          def response_chain
+            return @response_chain
+          end
+        end
+        response.instance_variable_set("@response_chain",
+          options[:response_chain])
+      end
+      
+      return response
+    end
+    
+    # Makes an HTTP GET request and returns the HTTP response.  Optionally
+    # takes a block that determines whether or not to follow a redirect.
+    # The block will be passed the HTTP redirect response as an argument.
+    def self.http_get(url, options={}, &block)
+      return FeedTools::RetrievalHelper.http_request(
+        :get, url, options, &block)
+    end
+
+    # Makes an HTTP POST request and returns the HTTP response.  Optionally
+    # takes a block that determines whether or not to follow a redirect.
+    # The block will be passed the HTTP redirect response as an argument.
+    def self.http_post(url, options={}, &block)
+      return FeedTools::RetrievalHelper.http_request(
+        :post, url, options, &block)
+    end
+    
+    # Makes an HTTP HEAD request and returns the HTTP response.  Optionally
+    # takes a block that determines whether or not to follow a redirect.
+    # The block will be passed the HTTP redirect response as an argument.
+    def http_head(url, options={}, &block)
+      return FeedTools::RetrievalHelper.http_request(
+        :head, url, options, &block)
+    end
   end
 end