feedtools 0.2.18 → 0.2.19

data/CHANGELOG

@@ -1,3 +1,29 @@
+== FeedTools 0.2.19
+ * lousy encoding support (as opposed to none at all)
+ * xml processing instruction now correctly prefixes generated feeds
+ * attributes are escaped properly when generating feeds
+ * uppercase html is no longer sanitized for not being in the whitelist
+ * added alias method for assigning to entries
+ * changed the xpath querying to be much, much more DRY
+ * find_node and find_all_nodes are actually useful now
+ * full case-insensitivity implemented for the xpath helper methods
+ * fixed bug in tests where some assertion failures could affect other tests
+ * fixed bug where the feed item author would sometimes be parsed incorrectly
+ * fixed bug where the convertLineBreaks element would break feed entries
+ * default (i.e. preferred) methods will be Atom-style instead of RSS-style
+ * default feed output format changed to Atom 1.0
+ * itunes namespace corrected
+ * fixed images property when dealing with atom
+ * fixed atom link property
+ * improved timestamp handling
+ * whitespace nodes now ignored by REXML
+ * added option to disable timestamp estimation
+ * added option to limit time-to-live to some upper maximum
+ * enclosures included in feed generation
+ * no longer uses the cache at all for file:/// urls
+ * changed itunes:keywords to use commas
+ * testing now excludes cache testing by default, use "test_all" to include it
+ * more tests
 == FeedTools 0.2.18
  * no longer ever polls more often than once every 30 minutes
  * fixed overlooked improperly refactored enclosure code
@@ -6,6 +32,8 @@
  * test cases now implemented using helpers
  * fixed issue with timeouts
  * fixed stack overflow while estimating timestamps
+ * fixed some namespace issue with atom
+ * added base64 decoding support
 == FeedTools 0.2.17
  * more fixes for timestamping of feed items
  * fixed nil bug in root_node, feed_type, feed_version, build_xml

data/lib/feed_tools.rb

@@ -30,9 +30,9 @@ end
 
 FEED_TOOLS_ENV = ENV['FEED_TOOLS_ENV'] ||
                  ENV['RAILS_ENV'] ||
-                 'production' # :nodoc:
+                 'development' # :nodoc:
 
-FEED_TOOLS_VERSION = "0.2.18"
+FEED_TOOLS_VERSION = "0.2.19"
 
 FEED_TOOLS_NAMESPACES = {
   "admin" => "http://webns.net/mvcb/",
@@ -40,8 +40,10 @@ FEED_TOOLS_NAMESPACES = {
   "annotate" => "http://purl.org/rss/1.0/modules/annotate/",
   "atom10" => "http://www.w3.org/2005/Atom",
   "atom03" => "http://purl.org/atom/ns#",
+#  "atom-blog" => "http://purl.org/atom-blog/ns#",
   "audio" => "http://media.tangent.org/rss/1.0/",
   "blogChannel" => "http://backend.userland.com/blogChannelModule",
+  "blogger" => "http://www.blogger.com/atom/ns#",
   "cc" => "http://web.resource.org/cc/",
   "creativeCommons" => "http://backend.userland.com/creativeCommonsRssModule",
   "co" => "http://purl.org/rss/1.0/modules/company",
@@ -56,7 +58,7 @@ FEED_TOOLS_NAMESPACES = {
   "feedburner" => "http://rssnamespace.org/feedburner/ext/1.0",
   "foaf" => "http://xmlns.com/foaf/0.1/",
   "fm" => "http://freshmeat.net/rss/fm/",
-  "itunes" => "http://www.itunes.com/DTDs/Podcast-1.0.dtd",
+  "itunes" => "http://www.itunes.com/dtds/podcast-1.0.dtd",
   "l" => "http://purl.org/rss/1.0/modules/link/",
   "media" => "http://search.yahoo.com/mrss",
   "pingback" => "http://madskills.com/public/xml/rss/module/pingback/",
@@ -97,12 +99,7 @@ begin
 
   require 'rubygems'
 
-  begin
-    require 'builder'
-  rescue LoadError
-    # RubyGems version is not available, use included Builder
-    require 'feed_tools/vendor/builder'
-  end
+  require_gem('builder', '>= 1.2.4')
 
   begin
     require 'tidy'
@@ -113,8 +110,10 @@ begin
   require 'feed_tools/vendor/htree'
 
   require 'net/http'
-  require 'net/https'
-  require 'net/ftp'
+
+# TODO: Not used yet, don't load since it'll only be a performance hit
+#  require 'net/https'
+#  require 'net/ftp'
 
   require 'rexml/document'
 
@@ -125,7 +124,8 @@ begin
   require 'yaml'
   require 'base64'
 
-  require_gem('activerecord', '>= 1.10.1')
+  require_gem('activesupport', '>= 1.1.1')
+  require_gem('activerecord', '>= 1.11.1')
   require_gem('uuidtools', '>= 0.1.2')
 
   require 'feed_tools/feed'
@@ -155,25 +155,55 @@ end
 #  slashdot_feed.items.first.find_node("slash:hitparade/text()").value
 #  => "43,37,28,23,11,3,1"
 module FeedTools
+  @configurations = {}
+  
+  def FeedTools.load_configurations
+    if @configurations.blank?
+      config_hash = {}
+      @configurations = {
+        :feed_cache => "FeedTools::DatabaseFeedCache",
+        :user_agent => "FeedTools/#{FEED_TOOLS_VERSION} " + 
+          "+http://www.sporkmonger.com/projects/feedtools/",
+        :generator_name => "FeedTools/#{FEED_TOOLS_VERSION}",
+        :generator_href => "http://www.sporkmonger.com/projects/feedtools/",
+        :tidy_enabled => false,
+        :tidy_options => {},
+        :sanitize_with_nofollow => true,
+        :timestamp_estimation_enabled => true,
+        :url_normalization_enabled => true,
+        :strip_comment_count => false,
+        :max_ttl => 3.days.to_s,
+        :output_encoding => "utf-8",
+        :no_content_value => "[no description]"
+      }.merge(config_hash)
+    end
+    return @configurations
+  end
   
-  @force_tidy_enabled = true
-  @tidy_enabled = false
-  @feed_cache = DatabaseFeedCache
-  @user_agent = "FeedTools/#{FEED_TOOLS_VERSION} " + 
-    "+http://www.sporkmonger.com/projects/feedtools/"
-  @no_content_string = "[no description]"
+  # Resets configuration to a clean load
+  def FeedTools.reset_configurations
+    @configurations = nil
+    FeedTools.load_configurations
+  end
+  
+  # Returns the configuration hash for FeedTools
+  def FeedTools.configurations
+    if @configurations.blank?
+      FeedTools.load_configurations()
+    end
+    return @configurations
+  end
+  
+  # Sets the configuration hash for FeedTools
+  def FeedTools.configurations=(new_configurations)
+    @configurations = new_configurations
+  end
   
   # Error raised when a feed cannot be retrieved    
   class FeedAccessError < StandardError
   end
   
   # Returns the current caching mechanism.
-  def FeedTools.feed_cache
-    return @feed_cache
-  end
-  
-  # Sets the current caching mechanism.  If set to nil, disables caching.
-  # Default is the DatabaseFeedCache class.
   #
   # Objects of this class must accept the following messages:
   #  id
@@ -199,12 +229,26 @@ module FeedTools
   #  find_by_url
   #  initialize_cache
   #  connected?
-  def FeedTools.feed_cache=(new_feed_cache)
-    # TODO: ensure that the feed cache class actually does those things.
-    # ==================================================================
-    @feed_cache = new_feed_cache
+  def FeedTools.feed_cache
+    return nil if FeedTools.configurations[:feed_cache].blank?
+    class_name = FeedTools.configurations[:feed_cache].to_s
+    if @feed_cache.nil? || @feed_cache.to_s != class_name
+      begin
+        cache_class = eval(class_name)
+        if cache_class.kind_of?(Class)
+          @feed_cache = cache_class
+          return cache_class
+        else
+          return nil
+        end
+      rescue
+        return nil
+      end
+    else
+      return @feed_cache
+    end
   end
-    
+  
   # Returns true if FeedTools.feed_cache is not nil and a connection with
   # the cache has been successfully established.  Also returns false if an
   # error is raised while trying to determine the status of the cache.
@@ -216,28 +260,7 @@ module FeedTools
       return false
     end
   end
-  
-  # Returns the currently used user agent string.
-  def FeedTools.user_agent
-    return @user_agent
-  end
-  
-  # Sets the user agent string to send in the http headers.
-  def FeedTools.user_agent=(new_user_agent)
-    @user_agent = new_user_agent
-  end
-
-  # Returns the currently used no content string.
-  def FeedTools.no_content_string
-    return @no_content_string
-  end
-  
-  # Sets the no content string to use when a feed is missing a content element.
-  # Used only for xml output.
-  def FeedTools.no_content_string=(new_no_content_string)
-    @no_content_string = new_no_content_string
-  end
-  
+    
   # Returns true if the html tidy module can be used.
   #
   # Obviously, you need the tidy gem installed in order to run with html
@@ -255,7 +278,7 @@ module FeedTools
   def FeedTools.tidy_enabled?
     # This is an override variable to keep tidy from being used even if it
     # is available.
-    if @force_tidy_enabled == false
+    if FeedTools.configurations[:tidy_enabled] == false
       return false
     end
     if @tidy_enabled.nil? || @tidy_enabled == false
@@ -337,13 +360,6 @@ module FeedTools
     end
     return @tidy_enabled
   end
-  
-  # Turns html tidy support on or off.  Be aware, that setting this to true
-  # does not mean tidy will be enabled.  It simply means that tidy will be
-  # enabled if it is available to be enabled.
-  def FeedTools.tidy_enabled=(new_tidy_enabled)
-    @force_tidy_enabled = new_tidy_enabled
-  end
 
   # Attempts to ensures that the passed url is valid and sane.  Accepts very, very ugly urls
   # and makes every effort to figure out what it was supposed to be.  Also translates from
@@ -470,8 +486,8 @@ module FeedTools
   def FeedTools.escape_entities(html)
     return nil if html.nil?
     escaped_html = CGI.escapeHTML(html)
-    unescaped_html.gsub!(/'/, "&apos;")
-    unescaped_html.gsub!(/"/, "&quot;")
+    escaped_html.gsub!(/'/, "&apos;")
+    escaped_html.gsub!(/"/, "&quot;")
     return escaped_html
   end
   
@@ -540,6 +556,9 @@ module FeedTools
     else
       tidy_html = html
     end
+    if tidy_html.blank? && !html.blank?
+      tidy_html = html.strip
+    end
     return tidy_html
   end
 
@@ -586,7 +605,7 @@ module FeedTools
       if html_node.respond_to? :children
         for child in html_node.children
           if child.kind_of? REXML::Element
-            unless acceptable_elements.include? child.name
+            unless acceptable_elements.include? child.name.downcase
               if mode == :strip
                 html_node.delete_element(child)
               else
@@ -596,7 +615,7 @@ module FeedTools
               end
             end
             for attribute in child.attributes.keys
-              unless acceptable_attributes.include? attribute
+              unless acceptable_attributes.include? attribute.downcase
                 child.delete_attribute(attribute)
               end
             end
@@ -655,6 +674,252 @@ module FeedTools
 end
 
 module REXML # :nodoc:
+  class LiberalXPathParser < XPathParser # :nodoc:
+  private
+    def internal_parse(path_stack, nodeset) # :nodoc:
+      return nodeset if nodeset.size == 0 or path_stack.size == 0
+      case path_stack.shift
+      when :document
+        return [ nodeset[0].root.parent ]
+
+      when :qname
+        prefix = path_stack.shift.downcase
+        name = path_stack.shift.downcase
+        n = nodeset.clone
+        ns = @namespaces[prefix]
+        ns = ns ? ns : ''
+        n.delete_if do |node|
+          if node.node_type == :element and ns == ''
+            ns = node.namespace( prefix )
+          end
+          !(node.node_type == :element and
+            node.name.downcase == name and node.namespace == ns )
+        end
+        return n
+
+      when :any
+        n = nodeset.clone
+        n.delete_if { |node| node.node_type != :element }
+        return n
+
+      when :self
+        # THIS SPACE LEFT INTENTIONALLY BLANK
+
+      when :processing_instruction
+        target = path_stack.shift
+        n = nodeset.clone
+        n.delete_if do |node|
+          (node.node_type != :processing_instruction) or 
+          ( !target.nil? and ( node.target != target ) )
+        end
+        return n
+
+      when :text
+        n = nodeset.clone
+        n.delete_if do |node|
+          node.node_type != :text
+        end
+        return n
+
+      when :comment
+        n = nodeset.clone
+        n.delete_if do |node|
+          node.node_type != :comment
+        end
+        return n
+
+      when :node
+        return nodeset
+      
+      when :child
+        new_nodeset = []
+        nt = nil
+        for node in nodeset
+          nt = node.node_type
+          new_nodeset += node.children if nt == :element or nt == :document
+        end
+        return new_nodeset
+
+      when :literal
+        literal = path_stack.shift
+        if literal =~ /^\d+(\.\d+)?$/
+          return ($1 ? literal.to_f : literal.to_i) 
+        end
+        return literal
+        
+      when :attribute
+        new_nodeset = []
+        case path_stack.shift
+        when :qname
+          prefix = path_stack.shift
+          name = path_stack.shift.downcase
+          for element in nodeset
+            if element.node_type == :element
+              for attribute_name in element.attributes.keys
+                if attribute_name.downcase == name
+                  attrib = element.attribute( attribute_name,
+                    @namespaces[prefix] )
+                  new_nodeset << attrib if attrib
+                end
+              end
+            end
+          end
+        when :any
+          for element in nodeset
+            if element.node_type == :element
+              new_nodeset += element.attributes.to_a
+            end
+          end
+        end
+        return new_nodeset
+
+      when :parent
+        return internal_parse( path_stack, nodeset.collect{|n| n.parent}.compact )
+
+      when :ancestor
+        new_nodeset = []
+        for node in nodeset
+          while node.parent
+            node = node.parent
+            new_nodeset << node unless new_nodeset.include? node
+          end
+        end
+        return new_nodeset
+
+      when :ancestor_or_self
+        new_nodeset = []
+        for node in nodeset
+          if node.node_type == :element
+            new_nodeset << node
+            while ( node.parent )
+              node = node.parent
+              new_nodeset << node unless new_nodeset.include? node
+            end
+          end
+        end
+        return new_nodeset
+
+      when :predicate
+        predicate = path_stack.shift
+        new_nodeset = []
+        Functions::size = nodeset.size
+        nodeset.size.times do |index|
+          node = nodeset[index]
+          Functions::node = node
+          Functions::index = index+1
+          result = Predicate( predicate, node )
+          if result.kind_of? Numeric
+            new_nodeset << node if result == (index+1)
+          elsif result.instance_of? Array
+            new_nodeset << node if result.size > 0
+          else
+            new_nodeset << node if result
+          end
+        end
+        return new_nodeset
+
+      when :descendant_or_self
+        rv = descendant_or_self( path_stack, nodeset )
+        path_stack.clear
+        return rv
+
+      when :descendant
+        results = []
+        nt = nil
+        for node in nodeset
+          nt = node.node_type
+          if nt == :element or nt == :document
+            results += internal_parse(
+              path_stack.clone.unshift( :descendant_or_self ),
+              node.children )
+          end
+        end
+        return results
+
+      when :following_sibling
+        results = []
+        for node in nodeset
+          all_siblings = node.parent.children
+          current_index = all_siblings.index( node )
+          following_siblings = all_siblings[ current_index+1 .. -1 ]
+          results += internal_parse( path_stack.clone, following_siblings )
+        end
+        return results
+
+      when :preceding_sibling
+        results = []
+        for node in nodeset
+          all_siblings = node.parent.children
+          current_index = all_siblings.index( node )
+          preceding_siblings = all_siblings[ 0 .. current_index-1 ]
+          results += internal_parse( path_stack.clone, preceding_siblings )
+        end
+        return results
+
+      when :preceding
+        new_nodeset = []
+        for node in nodeset
+          new_nodeset += preceding( node )
+        end
+        return new_nodeset
+
+      when :following
+        new_nodeset = []
+        for node in nodeset
+          new_nodeset += following( node )
+        end
+        return new_nodeset
+
+      when :namespace
+        new_set = []
+        for node in nodeset
+          if node.node_type == :element or node.node_type == :attribute
+            new_nodeset << node.namespace
+          end
+        end
+        return new_nodeset
+
+      when :variable
+        var_name = path_stack.shift
+        return @variables[ var_name ]
+
+      end
+      nodeset
+    end
+  end
+  
+  class XPath # :nodoc:
+    def self.liberal_match(element, path=nil, namespaces={},
+        variables={}) # :nodoc:
+			parser = LiberalXPathParser.new
+			parser.namespaces = namespaces
+			parser.variables = variables
+			path = "*" unless path
+			element = [element] unless element.kind_of? Array
+			parser.parse(path, element)
+    end
+
+    def self.liberal_first(element, path=nil, namespaces={},
+        variables={}) # :nodoc:
+      parser = LiberalXPathParser.new
+      parser.namespaces = namespaces
+      parser.variables = variables
+      path = "*" unless path
+      element = [element] unless element.kind_of? Array
+      parser.parse(path, element)[0]
+    end
+
+    def self.liberal_each(element, path=nil, namespaces={},
+        variables={}, &block) # :nodoc:
+			parser = LiberalXPathParser.new
+			parser.namespaces = namespaces
+			parser.variables = variables
+			path = "*" unless path
+			element = [element] unless element.kind_of? Array
+			parser.parse(path, element).each( &block )
+    end
+  end
+  
   class Element # :nodoc:
     unless REXML::Element.public_instance_methods.include? :inner_xml
       def inner_xml # :nodoc: