feedtools 0.2.22 → 0.2.23

data/lib/feed_tools/helpers/retrieval_helper.rb

@@ -28,6 +28,11 @@ require 'net/http'
 #  require 'net/https'
 #  require 'net/ftp'
 
+# Stolen from the Universal Feed Parser
+FEED_TOOLS_ACCEPT_HEADER = "application/atom+xml,application/rdf+xml," +
+  "application/rss+xml,application/x-netcdf,application/xml;" +
+  "q=0.9,text/xml;q=0.2,*/*;q=0.1"
+
 # TODO: Refactor http_fetch and other methods.
 module FeedTools
   # Methods for pulling remote data

data/lib/feed_tools/helpers/uri_helper.rb

@@ -0,0 +1,223 @@
+#--
+# Copyright (c) 2005 Robert Aman
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'feed_tools'
+require 'uri'
+
+module FeedTools
+  # Generic url processing methods needed in numerous places throughout
+  # FeedTools
+  module UriHelper
+    # Returns true if the idn module can be used.
+    def self.idn_enabled?
+      # This is an override variable to keep idn from being used even if it
+      # is available.
+      if FeedTools.configurations[:idn_enabled] == false
+        return false
+      end
+      if @idn_enabled.nil? || @idn_enabled == false
+        @idn_enabled = false
+        begin
+          require 'idn'
+          if IDN::Idna.toASCII('http://www.詹姆斯.com/') ==
+            "http://www.xn--8ws00zhy3a.com/"
+            @idn_enabled = true
+          else
+            @idn_enabled = false
+          end
+        rescue LoadError
+          # Tidy not installed, disable features that rely on tidy.
+          @idn_enabled = false
+        end
+      end
+      return @idn_enabled
+    end
+    
+    # Attempts to ensures that the passed url is valid and sane.  Accepts very,
+    # very ugly urls and makes every effort to figure out what it was supposed
+    # to be.  Also translates from the feed: and rss: pseudo-protocols to the
+    # http: protocol.
+    def self.normalize_url(url)
+      if url.kind_of?(URI)
+        url = url.to_s
+      end
+      if url.blank?
+        return nil
+      end
+      normalized_url = CGI.unescape(url.strip)
+
+      # if a url begins with the '/' character, it only makes sense that they
+      # meant to be using a file:// url.  Fix it for them.
+      if normalized_url.length > 0 && normalized_url[0..0] == "/"
+        normalized_url = "file://" + normalized_url
+      end
+
+      # if a url begins with a drive letter followed by a colon, we're looking at
+      # a file:// url.  Fix it for them.
+      if normalized_url.length > 0 &&
+          normalized_url.scan(/^[a-zA-Z]:[\\\/]/).size > 0
+        normalized_url = "file:///" + normalized_url
+      end
+
+      # if a url begins with javascript:, it's quite possibly an attempt at
+      # doing something malicious.  Let's keep that from getting anywhere,
+      # shall we?
+      if (normalized_url.downcase =~ /javascript:/) != nil
+        return "#"
+      end
+
+      # deal with all of the many ugly possibilities involved in the rss:
+      # and feed: pseudo-protocols (incidentally, whose crazy idea was this
+      # mess?)
+      normalized_url.gsub!(/^http:\/*(feed:\/*)?/i, "http://")
+      normalized_url.gsub!(/^http:\/*(rss:\/*)?/i, "http://")
+      normalized_url.gsub!(/^feed:\/*(http:\/*)?/i, "http://")
+      normalized_url.gsub!(/^rss:\/*(http:\/*)?/i, "http://")
+      normalized_url.gsub!(/^file:\/*/i, "file:///")
+      normalized_url.gsub!(/^https:\/*/i, "https://")
+      # fix (very) bad urls (usually of the user-entered sort)
+      normalized_url.gsub!(/^http:\/*(http:\/*)*/i, "http://")
+
+      if (normalized_url =~ /^file:/i) == 0
+        # Adjust windows-style urls
+        normalized_url.gsub!(/^file:\/\/\/([a-zA-Z])\|/i, 'file:///\1:')
+        normalized_url.gsub!(/\\/, '/')
+      else
+        if (normalized_url =~ /^https?:\/\//i) == nil
+          normalized_url = "http://" + normalized_url
+        end
+        if normalized_url == "http://"
+          return nil
+        end
+        begin
+          scheme, host_part, path =
+            normalized_url.scan(/^(https?):\/\/([^\/]+)\/(.*)/i).flatten
+          if scheme != nil && host_part != nil && path != nil
+            scheme = scheme.downcase
+            if FeedTools::UriHelper.idn_enabled?
+              host_part =
+                IDN::Idna.toASCII(host_part)
+            end
+            new_path = ""
+            for index in 0...path.size
+              if path[index] <= 32 || path[index] >= 126
+                new_path << ("%" + path[index].to_s(16).upcase)
+              else
+                new_path << path[index..index]
+              end
+            end
+            path = new_path
+            normalized_url = scheme + "://" + host_part + "/" + path
+          end
+        rescue Object
+        end
+        begin
+          feed_uri = URI.parse(normalized_url)
+          if feed_uri.scheme == nil
+            feed_uri.scheme = "http"
+          end
+          if feed_uri.path.blank?
+            feed_uri.path = "/"
+          end
+          if (feed_uri.path =~ /^[\/]+/) == 0
+            feed_uri.path.gsub!(/^[\/]+/, "/")
+          end
+          while (feed_uri.path =~ /^\/\.\./)
+            feed_uri.path.gsub!(/^\/\.\./, "")
+          end
+          if feed_uri.path.blank?
+            feed_uri.path = "/"
+          end
+          feed_uri.host.downcase!
+          normalized_url = feed_uri.to_s
+        rescue URI::InvalidURIError
+        end
+      end
+
+      # We can't do a proper set of escaping, so this will
+      # have to do.
+      normalized_url.gsub!(/%20/, " ")
+      normalized_url.gsub!(/ /, "%20")
+
+      return normalized_url
+    end
+
+    # Resolves a relative uri
+    def self.resolve_relative_uri(relative_uri, base_uri_sources=[])
+      return relative_uri if base_uri_sources.blank?
+      return nil if relative_uri.nil?
+      begin
+        base_uri = URI.parse(
+          FeedTools::XmlHelper.select_not_blank(base_uri_sources))
+        resolved_uri = base_uri + relative_uri.to_s
+        return FeedTools::UriHelper.normalize_url(resolved_uri.to_s)
+      rescue
+        return relative_uri
+      end
+    end
+
+    # Converts a url into a tag uri
+    def self.build_tag_uri(url, date)
+      unless url.kind_of? String
+        raise ArgumentError, "Expected String, got #{url.class.name}"
+      end
+      unless date.kind_of? Time
+        raise ArgumentError, "Expected Time, got #{date.class.name}"
+      end
+      tag_uri = normalize_url(url)
+      unless FeedTools::UriHelper.is_uri?(tag_uri)
+        raise ArgumentError, "Must supply a valid URL."
+      end
+      host = URI.parse(tag_uri).host
+      tag_uri.gsub!(/^(http|ftp|file):\/*/, "")
+      tag_uri.gsub!(/#/, "/")
+      tag_uri = "tag:#{host},#{date.strftime('%Y-%m-%d')}:" +
+        "#{tag_uri[(tag_uri.index(host) + host.size)..-1]}"
+      return tag_uri
+    end
+
+    # Converts a url into a urn:uuid: uri
+    def self.build_urn_uri(url)
+      unless url.kind_of? String
+        raise ArgumentError, "Expected String, got #{url.class.name}"
+      end
+      normalized_url = normalize_url(url)
+      require 'uuidtools'
+      return UUID.sha1_create(UUID_URL_NAMESPACE, normalized_url).to_uri_string
+    end
+
+    # Returns true if the parameter appears to be a valid uri
+    def self.is_uri?(url)
+      return false if url.nil?
+      begin
+        uri = URI.parse(url)
+        if uri.scheme.blank?
+          return false
+        end
+      rescue URI::InvalidURIError
+        return false
+      end
+      return true
+    end
+  end
+end

data/lib/feed_tools/helpers/xml_helper.rb

@@ -0,0 +1,239 @@
+#--
+# Copyright (c) 2005 Robert Aman
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'feed_tools'
+require 'feed_tools/helpers/generic_helper'
+require 'rexml/document'
+
+module FeedTools
+  # Generic xml methods needed in numerous places throughout FeedTools
+  module XmlHelper    
+    # Selects the first non-blank result.
+    def self.select_not_blank(results, &block)
+      if results.kind_of? Array
+        for result in results
+          blank_result = false
+          if !block.nil?
+            blank_result = block.call(result)
+          else
+            blank_result = result.to_s.blank?
+          end
+          unless result.nil? || blank_result
+            return result
+          end
+        end
+      else
+        blank_result = false
+        if !block.nil?
+          blank_result = block.call(results)
+        else
+          blank_result = results.to_s.blank?
+        end
+        unless results.nil? || blank_result
+          return results
+        end
+      end
+      return nil
+    end
+    
+    # Runs through a list of XPath queries on an element or document and
+    # returns the first non-blank result.  Subsequent XPath queries will
+    # not be evaluated.
+    def self.try_xpaths(element, xpath_list,
+        options={}, &block)
+      FeedTools::GenericHelper.validate_options([ :select_result_value ],
+                       options.keys)
+      options = { :select_result_value => false }.merge(options)
+
+      result = nil
+      if element.nil?
+        return nil
+      end
+      for xpath in xpath_list
+        # Namespace aware
+        result = REXML::XPath.liberal_first(element, xpath,
+          FEED_TOOLS_NAMESPACES)
+        if options[:select_result_value] && !result.nil?
+          if result.respond_to?(:value)
+            result = result.value
+          else
+            result = result.to_s
+          end
+        end
+        blank_result = false
+        if block_given?
+          blank_result = yield(result)
+        else
+          blank_result = result.to_s.blank?
+        end
+        if !blank_result
+          if result.respond_to? :strip
+            result.strip!
+          end
+          return result
+        end
+        
+        # Namespace unaware
+        result = REXML::XPath.liberal_first(element, xpath)
+        if options[:select_result_value] && !result.nil?
+          if result.respond_to?(:value)
+            result = result.value
+          else
+            result = result.to_s
+          end
+        end
+        blank_result = false
+        if block_given?
+          blank_result = yield(result)
+        else
+          blank_result = result.to_s.blank?
+        end
+        if !blank_result
+          if result.respond_to? :strip
+            result.strip!
+          end
+          return result
+        end
+      end
+      return nil
+    end
+    
+    # Runs through a list of XPath queries on an element or document and
+    # returns all non-empty results.  Subsequent XPath queries will
+    # not be evaluated.
+    def self.try_xpaths_all(element, xpath_list, options={})
+      FeedTools::GenericHelper.validate_options([ :select_result_value ],
+                       options.keys)
+      options = { :select_result_value => false }.merge(options)
+
+      results = []
+      if element.nil?
+        return []
+      end
+      for xpath in xpath_list
+        # Namespace aware
+        results = REXML::XPath.liberal_match(element, xpath,
+          FEED_TOOLS_NAMESPACES)
+        if options[:select_result_value] && !results.nil? && !results.empty?
+          results =
+            results.map { |x| x.respond_to?(:value) ? x.value : x.to_s }
+        end
+        if results.blank?
+          results = REXML::XPath.liberal_match(element, xpath)
+        else
+          return results
+        end
+        
+        # Namespace unaware
+        if options[:select_result_value] && !results.nil? && !results.empty?
+          results =
+            results.map { |x| x.respond_to?(:value) ? x.value : x.to_s }
+        end
+        if !results.blank?
+          return results
+        end
+      end
+      for xpath in xpath_list
+        if xpath =~ /^\w+$/
+          results = []
+          for child in element.children
+            if child.class == REXML::Element
+              if child.name.downcase == xpath.downcase
+                results << child
+              end
+            end
+          end
+          if options[:select_result_value] && !results.nil? && !results.empty?
+            results =
+              results.map { |x| x.inner_xml }
+          end
+          if !results.blank?
+            return results
+          end
+        end
+      end
+      return []
+    end
+    
+    # Runs through a list of XPath queries on an element or document and
+    # returns all non-empty results, appending the results from each query
+    # onto the end of the results from the previous queries.
+    def self.combine_xpaths_all(element, xpath_list, options={})
+      FeedTools::GenericHelper.validate_options([ :select_result_value ],
+                       options.keys)
+      options = { :select_result_value => false }.merge(options)
+
+      all_results = []
+      result = []
+      if element.nil?
+        return []
+      end
+      for xpath in xpath_list
+        # Namespace aware
+        results = REXML::XPath.liberal_match(element, xpath,
+          FEED_TOOLS_NAMESPACES)
+        if options[:select_result_value] && !results.nil? && !results.empty?
+          results =
+            results.map { |x| x.respond_to?(:value) ? x.value : x.to_s }
+        end
+        if results.blank?
+          results = REXML::XPath.liberal_match(element, xpath)
+        else
+          all_results.concat(results)
+          next
+        end
+
+        # Namespace unaware
+        if options[:select_result_value] && !results.nil? && !results.empty?
+          results =
+            results.map { |x| x.respond_to?(:value) ? x.value : x.to_s }
+        end
+        if !results.blank?
+          all_results.concat(results)
+          next
+        end
+      end
+      for xpath in xpath_list
+        if xpath =~ /^\w+$/
+          results = []
+          for child in element.children
+            if child.class == REXML::Element
+              if child.name.downcase == xpath.downcase
+                results << child
+              end
+            end
+          end
+          if options[:select_result_value] && !results.nil? && !results.empty?
+            results =
+              results.map { |x| x.inner_xml }
+          end
+          if !results.blank?
+            all_results.concat(results)
+            next
+          end
+        end
+      end
+      return all_results.uniq
+    end
+  end
+end