feedtools 0.2.26 → 0.2.27

data/lib/feed_tools/helpers/generic_helper.rb

@@ -55,4 +55,4 @@ module FeedTools
       return result
     end
   end
-end
+end

data/lib/feed_tools/helpers/html_helper.rb

@@ -28,6 +28,30 @@ require 'rexml/document'
 module FeedTools
   # Methods for pulling remote data
   module HtmlHelper
+    
+    TIDY_OPTIONS = [
+      :add_xml_decl, :add_xml_space, :alt_text, :assume_xml_procins, :bare,
+      :clean, :css_prefix, :decorate_inferred_ul, :doctype,
+      :drop_empty_paras, :drop_font_tags, :drop_proprietary_attributes,
+      :enclose_block_text, :enclose_text, :escape_cdata, :fix_backslash,
+      :fix_bad_comments, :fix_uri, :hide_comments, :hide_endtags,
+      :indent_cdata, :input_xml, :join_classes, :join_styles,
+      :literal_attributes, :logical_emphasis, :lower_literals, :merge_divs,
+      :ncr, :new_blocklevel_tags, :new_empty_tags, :new_inline_tags,
+      :new_pre_tags, :numeric_entities, :output_html, :output_xhtml,
+      :output_xml, :preserve_entities, :quote_ampersand, :quote_marks,
+      :quote_nbsp, :repeated_attributes, :replace_color, :show_body_only,
+      :uppercase_attributes, :uppercase_tags, :word_2000,
+      :accessibility_check, :show_errors, :show_warnings, :break_before_br,
+      :indent, :indent_attributes, :indent_spaces, :markup,
+      :punctuation_wrap, :split, :tab_size, :vertical_space, :wrap,
+      :wrap_asp, :wrap_attributes, :wrap_jste, :wrap_php,
+      :wrap_script_literals, :wrap_sections, :ascii_chars, :char_encoding,
+      :input_encoding, :language, :newline, :output_bom, :output_encoding,
+      :error_file, :force_output, :gnu_emacs, :gnu_emacs_file, :keep_time,
+      :output_file, :quiet, :slide_style, :tidy_mark, :write_back
+    ]
+
     # Escapes all html entities
     def self.escape_entities(html)
       return nil if html.nil?
@@ -43,9 +67,12 @@ module FeedTools
       unescaped_html = html
       unescaped_html.gsub!(/&/, "&")
       unescaped_html.gsub!(/&/, "&")
-      unescaped_html = unescaped_html.gsub(/&#x\d+;/) do |hex|
-        "&#" + hex[3..-2].to_i(16).to_s + ";"
-      end
+      substitute_numerical_entities = Proc.new do |s|
+        m = $1
+        m = "0#{m}" if m[0] == ?x
+        [Integer(m)].pack('U*')
+       end
+      unescaped_html.gsub!(/&#0*((?:\d+)|(?:x[a-f0-9]+));/, &substitute_numerical_entities)
       unescaped_html = CGI.unescapeHTML(unescaped_html)
       unescaped_html.gsub!(/'/, "'")
       unescaped_html.gsub!(/"/, "\"")
@@ -140,7 +167,7 @@ module FeedTools
             end
             for path in libtidy_locations
               if File.exists? path
-                if File.ftype(path) == "file"
+                if File.ftype(path) == "file" || File.ftype(path) == "link"
                   Tidy.path = path
                   @tidy_enabled = true
                   break
@@ -178,12 +205,18 @@ module FeedTools
     # Tidys up the html
     def self.tidy_html(html, options = {})
       return nil if html.nil?
+      FeedTools::GenericHelper.validate_options(TIDY_OPTIONS, options.keys)
 
-      FeedTools::GenericHelper.validate_options([ :input_encoding,
-                                                  :output_encoding ],
-                       options.keys)
-      options = { :input_encoding => "utf-8",
-                  :output_encoding => "utf-8" }.merge(options)
+      options = {
+        :add_xml_decl => false,
+        :char_encoding => "utf8",
+        :doctype => "omit",
+        :indent => false,
+        :logical_emphasis => true,
+        :markup => true,
+        :show_warnings => false,
+        :wrap => 0
+      }.merge(options)
 
       if FeedTools::HtmlHelper.tidy_enabled?
         is_fragment = true
@@ -196,39 +229,26 @@ module FeedTools
           is_fragment = false
         end
 
+        options[:show_body_only] = true if is_fragment
+
         # Tidy sucks?
         # TODO: find the correct set of tidy options to set so
         # that *ugly* hacks like this aren't necessary.
         html = html.gsub(/\302\240/, "\240")
 
-        tidy_html = Tidy.open(:show_warnings=>false) do |tidy|
-          tidy.options.output_xml = true
-          tidy.options.markup = true
-          tidy.options.indent = true
-          tidy.options.wrap = 0
-          tidy.options.logical_emphasis = true
-          tidy.options.input_encoding = options[:input_encoding]
-          tidy.options.output_encoding = options[:output_encoding]
-          tidy.options.doctype = "omit"
+        tidy_html = Tidy.open(options) do |tidy|       
           xml = tidy.clean(html)
           xml
         end
-        if is_fragment
-          # Tidy sticks <html>...<body>[our html]</body>...</html> in.
-          # We don't want this.
-          tidy_html.strip!
-          tidy_html.gsub!(/^<html>(.|\n)*<body>/, "")
-          tidy_html.gsub!(/<\/body>(.|\n)*<\/html>$/, "")
-          tidy_html.gsub!("\t", "  ")
-          tidy_html = FeedTools::HtmlHelper.unindent(tidy_html, 4)
-          tidy_html.strip!
-        end
+        tidy_html.strip!
       else
         tidy_html = html
       end
+      
       if tidy_html.blank? && !html.blank?
         tidy_html = html.strip
       end
+      
       return tidy_html
     end
 
@@ -260,77 +280,6 @@ module FeedTools
       return buffer
     end
 
-    # Removes all dangerous html tags from the html formatted text.
-    # If mode is set to :escape, dangerous and unknown elements will
-    # be escaped.  If mode is set to :strip, dangerous and unknown
-    # elements and all children will be removed entirely.
-    # Dangerous or unknown attributes are always removed.
-    def self.sanitize_html(html, mode=:strip)
-      return nil if html.nil?
-
-      # Lists borrowed from Mark Pilgrim's feedparser
-      acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'b',
-        'big', 'blockquote', 'br', 'button', 'caption', 'center', 'cite',
-        'code', 'col', 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl',
-        'dt', 'em', 'fieldset', 'font', 'form', 'h1', 'h2', 'h3', 'h4',
-        'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', 'kbd', 'label', 'legend',
-        'li', 'map', 'menu', 'ol', 'optgroup', 'option', 'p', 'pre', 'q', 's',
-        'samp', 'select', 'small', 'span', 'strike', 'strong', 'sub', 'sup',
-        'table', 'tbody', 'td', 'textarea', 'tfoot', 'th', 'thead', 'tr', 'tt',
-        'u', 'ul', 'var']
-
-      acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
-        'action', 'align', 'alt', 'axis', 'border', 'cellpadding',
-        'cellspacing', 'char', 'charoff', 'charset', 'checked', 'cite', 'class',
-        'clear', 'cols', 'colspan', 'color', 'compact', 'coords', 'datetime',
-        'dir', 'disabled', 'enctype', 'for', 'frame', 'headers', 'height',
-        'href', 'hreflang', 'hspace', 'id', 'ismap', 'label', 'lang',
-        'longdesc', 'maxlength', 'media', 'method', 'multiple', 'name',
-        'nohref', 'noshade', 'nowrap', 'prompt', 'readonly', 'rel', 'rev',
-        'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size',
-        'span', 'src', 'start', 'summary', 'tabindex', 'target', 'title',
-        'type', 'usemap', 'valign', 'value', 'vspace', 'width']
-
-      # Replace with appropriate named entities
-      html.gsub!(/&#x26;/, "&amp;")
-      html.gsub!(/&#38;/, "&amp;")
-      html.gsub!(/&lt;!'/, "&amp;lt;!'")
-
-      # Hackity hack.  But it works, and it seems plenty fast enough.
-      html_doc = HTree.parse_xml("<root>" + html + "</root>").to_rexml
-
-      sanitize_node = lambda do |html_node|
-        if html_node.respond_to? :children
-          for child in html_node.children
-            if child.kind_of? REXML::Element
-              unless acceptable_elements.include? child.name.downcase
-                if mode == :strip
-                  html_node.delete_element(child)
-                else
-                  new_child = REXML::Text.new(CGI.escapeHTML(child.to_s))
-                  html_node.insert_after(child, new_child)
-                  html_node.delete_element(child)
-                end
-              end
-              child.attributes.each_attribute do |attribute|
-                if !(attribute.value =~ /^xmlns(:.+)?$/)
-                  unless acceptable_attributes.include?(
-                      attribute.value.downcase)
-                    child.delete_attribute(attribute.value)
-                  end
-                end
-              end
-            end
-            sanitize_node.call(child)
-          end
-        end
-        html_node
-      end
-      sanitize_node.call(html_doc.root)
-      html = html_doc.root.inner_xml
-      return html
-    end
-
     # Returns true if the type string provided indicates that something is
     # xml or xhtml content.
     def self.xml_type?(type)
@@ -405,20 +354,35 @@ module FeedTools
         ["q", "cite"],
         ["script", "src"]
       ]
-      html_doc = HTree.parse_xml("<root>" + html + "</root>").to_rexml
       
+      # HACK: Prevent the parser from freaking out if it sees this:
+      html.gsub!(/<!'/, "&lt;!'")
+      
+      if FeedTools.configurations[:sanitization_enabled]
+        fragments = HTML5::HTMLParser.parse_fragment(
+          html, :tokenizer => HTML5::HTMLSanitizer)
+      else
+        fragments = HTML5::HTMLParser.parse_fragment(html)
+      end
       resolve_node = lambda do |html_node|
         if html_node.kind_of? REXML::Element
-          for element_attribute_pair in relative_uri_attributes
-            if html_node.name.downcase == element_attribute_pair[0]
-              attribute = html_node.attribute(element_attribute_pair[1])
+          for element_name, attribute_name in relative_uri_attributes
+            if html_node.name.downcase == element_name
+              attribute = html_node.attribute(attribute_name)
               if attribute != nil
                 href = attribute.value
                 href = FeedTools::UriHelper.resolve_relative_uri(
                   href, [html_node.base_uri] | base_uri_sources)
-                html_node.attribute(
-                  element_attribute_pair[1]).instance_variable_set(
-                    "@value", href)
+                href = FeedTools::UriHelper.normalize_url(href)
+                html_node.attribute(attribute_name).instance_variable_set(
+                  "@value", href)
+                html_node.attribute(attribute_name).instance_variable_set(
+                  "@unnormalized", href)
+                html_node.attribute(attribute_name).instance_variable_set(
+                  "@normalized", href)
+                if html_node.attribute(attribute_name).value != href
+                  warn("Failed to update href to resolved value.")
+                end
               end
             end
           end
@@ -430,8 +394,12 @@ module FeedTools
         end
         html_node
       end
-      resolve_node.call(html_doc.root)
-      html = html_doc.root.inner_xml
+      fragments.each do |fragment|
+        resolve_node.call(fragment)
+      end
+      html = (fragments.map do |stuff|
+        stuff.to_s
+      end).join("")
       return html
     end
     
@@ -552,22 +520,16 @@ module FeedTools
         content = FeedTools::HtmlHelper.unescape_entities(
           content_node.inner_xml.strip)
       else
-        content = content_node.inner_xml.strip
-        repair_entities = true
+        content = FeedTools::HtmlHelper.unescape_entities(
+          content_node.inner_xml.strip)
       end
       if type == "text" || mode == "text" ||
           type == "text/plain" || mode == "text/plain"
         content = FeedTools::HtmlHelper.escape_entities(content)
       end        
       unless content.nil?
-        if FeedTools.configurations[:sanitization_enabled]
-          content = FeedTools::HtmlHelper.sanitize_html(content, :strip)
-        end
         content = FeedTools::HtmlHelper.resolve_relative_uris(content,
           [content_node.base_uri] | base_uri_sources)
-        if repair_entities
-          content = FeedTools::HtmlHelper.unescape_entities(content)
-        end
         content = FeedTools::HtmlHelper.tidy_html(content)
       end
       if FeedTools.configurations[:tab_spaces] != nil

data/lib/feed_tools/helpers/retrieval_helper.rb

@@ -108,21 +108,51 @@ module FeedTools
         proxy_user = nil
         proxy_password = nil
         
+        auth_user = nil
+        auth_password = nil
+        auth_scheme = nil
+        
         if options[:feed_object] != nil
           proxy_address =
             options[:feed_object].configurations[:proxy_address] || nil
           proxy_port =
             options[:feed_object].configurations[:proxy_port].to_i || nil
           proxy_user =
-            options[:feed_object].configurations[:proxy_user].to_i || nil
+            options[:feed_object].configurations[:proxy_user] || nil
           proxy_password =
-            options[:feed_object].configurations[:proxy_password].to_i || nil
+            options[:feed_object].configurations[:proxy_password] || nil
+
+          auth_user =
+            options[:feed_object].configurations[:auth_user] || nil
+          auth_password =
+            options[:feed_object].configurations[:auth_password] || nil
+          auth_scheme =
+            options[:feed_object].configurations[:auth_scheme] || nil
         end        
         
+        if (auth_user &&
+            (auth_scheme == nil || auth_scheme.to_s.to_sym == :basic))
+          options[:request_headers]["Authorization"] =
+            "Basic " + [
+              "#{auth_user}:#{auth_password}"
+            ].pack('m').delete("\r\n")
+        end
+        
         # No need to check for nil
         http = Net::HTTP::Proxy(
           proxy_address, proxy_port, proxy_user, proxy_password).new(
             uri.host, (uri.port or 80))
+
+        if options[:feed_object] != nil &&
+            options[:feed_object].configurations[:http_timeout] != nil
+          http.open_timeout = 
+            options[:feed_object].configurations[:http_timeout].to_f
+        elsif FeedTools.configurations[:http_timeout] != nil
+          http.open_timeout = FeedTools.configurations[:http_timeout].to_f
+        end
+        if http.open_timeout != nil && http.open_timeout == 0
+          http.open_timeout = nil
+        end
         
         path = uri.path 
         path += ('?' + uri.query) if uri.query
@@ -238,4 +268,4 @@ module FeedTools
         :head, url, options, &block)
     end
   end
-end
+end

data/lib/feed_tools/helpers/uri_helper.rb

@@ -59,13 +59,33 @@ module FeedTools
     # to be.  Also translates from the feed: and rss: pseudo-protocols to the
     # http: protocol.
     def self.normalize_url(url)
-      if url.kind_of?(URI)
+      if url.nil?
+        return nil
+      end
+      if !url.kind_of?(String)
         url = url.to_s
       end
       if url.blank?
-        return nil
+        return ""
+      end
+      normalized_url = url.strip
+
+      begin
+        normalized_url =
+          FeedTools::URI.convert_path(normalized_url.strip).normalize.to_s
+      rescue Exception
+      end
+      
+      begin
+        begin
+          normalized_url =
+            FeedTools::URI.parse(normalized_url.strip).normalize.to_s
+        rescue Exception
+          normalized_url = CGI.unescape(url.strip)
+        end
+      rescue Exception
+        normalized_url = url.strip
       end
-      normalized_url = CGI.unescape(url.strip)
 
       # if a url begins with the '/' character, it only makes sense that they
       # meant to be using a file:// url.  Fix it for them.
@@ -90,76 +110,40 @@ module FeedTools
       # deal with all of the many ugly possibilities involved in the rss:
       # and feed: pseudo-protocols (incidentally, whose crazy idea was this
       # mess?)
+      normalized_url.gsub!(/^htp:\/*/i, "http://")
       normalized_url.gsub!(/^http:\/*(feed:\/*)?/i, "http://")
       normalized_url.gsub!(/^http:\/*(rss:\/*)?/i, "http://")
       normalized_url.gsub!(/^feed:\/*(http:\/*)?/i, "http://")
       normalized_url.gsub!(/^rss:\/*(http:\/*)?/i, "http://")
       normalized_url.gsub!(/^file:\/*/i, "file:///")
       normalized_url.gsub!(/^https:\/*/i, "https://")
+      normalized_url.gsub!(/^mms:\/*/i, "http://")
       # fix (very) bad urls (usually of the user-entered sort)
       normalized_url.gsub!(/^http:\/*(http:\/*)*/i, "http://")
+      normalized_url.gsub!(/^http:\/*$/i, "")
 
       if (normalized_url =~ /^file:/i) == 0
         # Adjust windows-style urls
         normalized_url.gsub!(/^file:\/\/\/([a-zA-Z])\|/i, 'file:///\1:')
         normalized_url.gsub!(/\\/, '/')
       else
-        if (normalized_url =~ /^https?:\/\//i) == nil
+        if FeedTools::URI.parse(normalized_url).scheme == nil &&
+            normalized_url =~ /\./ &&
           normalized_url = "http://" + normalized_url
         end
         if normalized_url == "http://"
           return nil
         end
-        begin
-          scheme, host_part, path =
-            normalized_url.scan(/^(https?):\/\/([^\/]+)\/(.*)/i).flatten
-          if scheme != nil && host_part != nil && path != nil
-            scheme = scheme.downcase
-            if FeedTools::UriHelper.idn_enabled?
-              host_part =
-                IDN::Idna.toASCII(host_part)
-            end
-            new_path = ""
-            for index in 0...path.size
-              if path[index] <= 32 || path[index] >= 126
-                new_path << ("%" + path[index].to_s(16).upcase)
-              else
-                new_path << path[index..index]
-              end
-            end
-            path = new_path
-            normalized_url = scheme + "://" + host_part + "/" + path
-          end
-        rescue Object
-        end
-        begin
-          feed_uri = URI.parse(normalized_url)
-          if feed_uri.scheme == nil
-            feed_uri.scheme = "http"
-          end
-          if feed_uri.path.blank?
-            feed_uri.path = "/"
-          end
-          if (feed_uri.path =~ /^[\/]+/) == 0
-            feed_uri.path.gsub!(/^[\/]+/, "/")
-          end
-          while (feed_uri.path =~ /^\/\.\./)
-            feed_uri.path.gsub!(/^\/\.\./, "")
-          end
-          if feed_uri.path.blank?
-            feed_uri.path = "/"
-          end
-          feed_uri.host.downcase!
-          normalized_url = feed_uri.to_s
-        rescue URI::InvalidURIError
-        end
+      end
+      if normalized_url =~ /^https?:\/\/#/i
+        normalized_url.gsub!(/^https?:\/\/#/i, "#")
+      end
+      if normalized_url =~ /^https?:\/\/\?/i
+        normalized_url.gsub!(/^https?:\/\/\?/i, "?")
       end
 
-      # We can't do a proper set of escaping, so this will
-      # have to do.
-      normalized_url.gsub!(/%20/, " ")
-      normalized_url.gsub!(/ /, "%20")
-
+      normalized_url =
+        FeedTools::URI.parse(normalized_url.strip).normalize.to_s
       return normalized_url
     end
 
@@ -168,7 +152,15 @@ module FeedTools
       return relative_uri if base_uri_sources.blank?
       return nil if relative_uri.nil?
       begin
-        base_uri = URI.parse(
+        # Massive HACK to get around file protocol URIs being used to
+        # resolve relative URIs on feeds in the local file system.
+        # Better to leave these URIs unresolved and hope some other
+        # tool resolves them correctly.
+        base_uri_sources.reject! do |base_uri|
+          base_uri == nil ||
+            FeedTools::URI.parse(base_uri).scheme == "file"
+        end
+        base_uri = FeedTools::URI.parse(
           FeedTools::XmlHelper.select_not_blank(base_uri_sources))
         resolved_uri = base_uri
         if relative_uri.to_s != ''
@@ -207,7 +199,7 @@ module FeedTools
       end
       normalized_url = normalize_url(url)
       require 'uuidtools'
-      return UUID.sha1_create(UUID_URL_NAMESPACE, normalized_url).to_uri_string
+      return UUID.sha1_create(UUID_URL_NAMESPACE, normalized_url).to_uri.to_s
     end
 
     # Returns true if the parameter appears to be a valid uri