feedtools 0.2.26 → 0.2.27

data/db/schema.mysql.sql

@@ -8,5 +8,7 @@
     `feed_data_type`  varchar(20) default NULL,
     `http_headers`    text default NULL,
     `last_retrieved`  datetime default NULL,
+    `time_to_live`    int(10) unsigned NULL,
+    `serialized`       longtext default NULL,
     PRIMARY KEY  (`id`)
   )

data/db/schema.postgresql.sql

@@ -7,5 +7,7 @@
     feed_data         text default NULL,
     feed_data_type    varchar(20) default NULL,
     http_headers      text default NULL,
-    last_retrieved    timestamp default NULL
+    last_retrieved    timestamp default NULL,
+    time_to_live      integer(10) default NULL,
+    serialized        text default NULL
   );

data/db/schema.sqlite.sql

@@ -7,5 +7,7 @@
     feed_data         TEXT DEFAULT NULL,
     feed_data_type    VARCHAR(20) DEFAULT NULL,
     http_headers      TEXT DEFAULT NULL,
-    last_retrieved    DATETIME DEFAULT NULL
+    last_retrieved    DATETIME DEFAULT NULL,
+    time_to_live      INTEGER DEFAULT NULL,
+    serialized        TEXT DEFAULT NULL
   );

data/lib/feed_tools.rb

@@ -33,6 +33,7 @@ FEED_TOOLS_ENV = ENV['FEED_TOOLS_ENV'] ||
                  'development' # :nodoc:
 
 FEED_TOOLS_NAMESPACES = {
+  "access" => "http://www.bloglines.com/about/specs/fac-1.0",
   "admin" => "http://webns.net/mvcb/",
   "ag" => "http://purl.org/rss/1.0/modules/aggregation/",
   "annotate" => "http://purl.org/rss/1.0/modules/annotate/",
@@ -54,10 +55,13 @@ FEED_TOOLS_NAMESPACES = {
   "ev" => "http://purl.org/rss/1.0/modules/event/",
   "icbm" => "http://postneo.com/icbm/",
   "image" => "http://purl.org/rss/1.0/modules/image/",
+  "indexing" => "urn:atom-extension:indexing",
   "feedburner" => "http://rssnamespace.org/feedburner/ext/1.0",
   "foaf" => "http://xmlns.com/foaf/0.1/",
   "foo" => "http://hsivonen.iki.fi/FooML",
   "fm" => "http://freshmeat.net/rss/fm/",
+  "gd" => "http://schemas.google.com/g/2005",
+  "gr" => "http://www.google.com/schemas/reader/atom/",
   "itunes" => "http://www.itunes.com/dtds/podcast-1.0.dtd",
   "l" => "http://purl.org/rss/1.0/modules/link/",
   "media" => "http://search.yahoo.com/mrss",
@@ -90,7 +94,8 @@ FEED_TOOLS_NAMESPACES = {
 }
 
 $:.unshift(File.dirname(__FILE__))
-$:.unshift(File.dirname(__FILE__) + "/feed_tools/vendor")
+$: << (File.dirname(__FILE__) + "/feed_tools/vendor")
+$: << (File.dirname(__FILE__) + "/feed_tools/vendor/html5/lib")
 
 begin
   require 'feed_tools/version'
@@ -104,8 +109,16 @@ begin
   end
 
   require 'rubygems'
-
-  require_gem('builder', '>= 1.2.4')
+  
+  if !defined?(Builder)
+    gem('builder', '>= 1.2.4')
+    require 'builder'
+  end
+  
+  if !defined?(ActiveRecord)
+    gem('activerecord', '>= 1.11.1')
+    require 'active_record'
+  end
 
   # Preload optional libraries.
   begin
@@ -117,7 +130,15 @@ begin
   rescue Object
   end  
 
-  require 'feed_tools/vendor/htree'
+  require 'html5'
+  require 'html5/html5parser'
+  require 'html5/liberalxmlparser'
+  require 'html5/treewalkers'
+  require 'html5/treebuilders'
+  require 'html5/serializer'
+  require 'html5/sanitizer'
+  
+  require 'feed_tools/vendor/uri'
 
   require 'net/http'
 
@@ -134,15 +155,8 @@ begin
   require 'yaml'
   require 'base64'
 
-  if !defined?(ActiveSupport)
-    require_gem('activesupport', '>= 1.1.1')
-  end
-  if !defined?(ActiveRecord)
-    require_gem('activerecord', '>= 1.11.1')
-  end
-
   begin
-    require_gem('uuidtools', '>= 0.1.2')
+    gem('uuidtools', '>= 0.1.2')
   rescue Gem::LoadError
     begin
       require 'uuidtools'
@@ -198,6 +212,10 @@ module FeedTools
         :proxy_port => nil,
         :proxy_user => nil,
         :proxy_password => nil,
+        :auth_user => nil,
+        :auth_password => nil,
+        :auth_scheme => nil,
+        :http_timeout => nil,
         :user_agent =>
           "FeedTools/#{FeedTools::FEED_TOOLS_VERSION::STRING} " + 
           "+http://www.sporkmonger.com/projects/feedtools/",
@@ -207,6 +225,8 @@ module FeedTools
           "http://www.sporkmonger.com/projects/feedtools/",
         :tidy_enabled => false,
         :tidy_options => {},
+        :lazy_parsing_enabled => true,
+        :serialization_enabled => false,
         :idn_enabled => true,
         :sanitization_enabled => true,
         :sanitize_with_nofollow => true,
@@ -217,6 +237,7 @@ module FeedTools
         :strip_comment_count => false,
         :tab_spaces => 2,
         :max_ttl => 3.days.to_s,
+        :default_ttl => 1.hour.to_s,
         :output_encoding => "utf-8"
       }.merge(config_hash)
     end
@@ -318,6 +339,7 @@ module FeedTools
     FeedTools::GenericHelper.validate_options([ :multi_threaded ],
                      options.keys)
     options = { :multi_threaded => false }.merge(options)
+    warn("FeedTools.build_merged_feed is deprecated.")
     return nil if url_array.nil?
     merged_feed = FeedTools::Feed.new
     retrieved_feeds = []
@@ -339,12 +361,13 @@ module FeedTools
       end
     end
     retrieved_feeds.each do |feed|
-      merged_feed.entries.concat(
+      merged_feed.entries = merged_feed.entries.concat(
         feed.entries.collect do |entry|
           new_entry = entry.dup
           new_entry.title = "#{feed.title}: #{entry.title}"
           new_entry
-        end )
+        end
+      )
     end
     return merged_feed
   end

data/lib/feed_tools/database_feed_cache.rb

@@ -51,6 +51,8 @@ module FeedTools
         ActiveRecord::Base.default_timezone = :utc
         ActiveRecord::Base.connection
       rescue
+      end
+      if !ActiveRecord::Base.connected?
         begin
           possible_config_files = [
             "./config/database.yml",
@@ -66,6 +68,7 @@ module FeedTools
           for file in possible_config_files
             if File.exists?(File.expand_path(file))
               database_config_file = file
+              @config_path = database_config_file
               break
             end
           end
@@ -84,6 +87,14 @@ module FeedTools
       end
       return nil
     end
+    
+    # Returns the path to the database.yml config file that FeedTools loaded.
+    def DatabaseFeedCache.config_path
+      if !defined?(@config_path) || @config_path.blank?
+        @config_path = nil
+      end
+      return @config_path
+    end
 
     # Returns true if a connection to the database has been established and the
     # required table structure is in place.
@@ -115,9 +126,9 @@ module FeedTools
     # True if the appropriate database table already exists
     def DatabaseFeedCache.table_exists?
       begin
-        ActiveRecord::Base.connection.execute("select id, href, title, " +
+        ActiveRecord::Base.connection.select_one("select id, href, title, " +
           "link, feed_data, feed_data_type, http_headers, last_retrieved " +
-          "from #{self.table_name()} limit 1")
+          "from #{self.table_name()}")
       rescue ActiveRecord::StatementInvalid
         return false
       rescue

data/lib/feed_tools/feed.rb

@@ -53,34 +53,112 @@ module FeedTools
       @live = false
       @encoding = nil
       @options = nil
+      @version = FeedTools::FEED_TOOLS_VERSION::STRING
+    end
+    
+    # Breaks any references that the feed may be keeping around, thus making
+    # the job of the garbage collector much, much easier.  Call this
+    # method prior to feeds going out of scope to prevent memory leaks.
+    def dispose()
+      self.entries.each do |entry|
+        entry.instance_variable_set("@root_node", nil)
+        entry.instance_variable_set("@feed", nil)
+        entry.instance_variable_set("@parent_feed", nil)
+        entry.dispose if entry.respond_to?(:dispose)
+      end
+      self.entries = []
+      
+      @cache_object = nil
+      @http_headers = nil
+      @xml_document = nil
+      @feed_data = nil
+      @feed_data_type = nil
+      @root_node = nil
+      @channel_node = nil
+      @href = nil
+      @id = nil
+      @title = nil
+      @subtitle = nil
+      @link = nil
+      @last_retrieved = nil
+      @time_to_live = nil
+      @entries = nil
+      @live = false
+      @encoding = nil
+      @options = nil
+
+      GC.start()
+      self
     end
           
     # Loads the feed specified by the url, pulling the data from the
     # cache if it hasn't expired.  Options supplied will override the
     # default options.
-    def Feed.open(url, options={})
+    def Feed.open(href, options={})
       FeedTools::GenericHelper.validate_options(
         FeedTools.configurations.keys, options.keys)
 
-      # create the new feed
-      feed = FeedTools::Feed.new
+      # clean up the url
+      href = FeedTools::UriHelper.normalize_url(href)
 
-      feed.configurations = FeedTools.configurations.merge(options)
+      feed_configurations = FeedTools.configurations.merge(options)
+      cache_object = nil
+      deserialized_feed = nil
       
-      if feed.configurations[:feed_cache] != nil && FeedTools.feed_cache.nil?
+      if feed_configurations[:feed_cache] != nil && FeedTools.feed_cache.nil?
         raise(ArgumentError, "There is currently no caching mechanism set. " +
           "Cannot retrieve cached feeds.")
+      elsif feed_configurations[:serialization_enabled] == true
+        # We've got a caching mechanism available
+        cache_object = FeedTools.feed_cache.find_by_href(href)
+        begin
+          if cache_object != nil && cache_object.serialized != nil
+            # If we've got a cache hit, deserialize
+            expired = true
+            if cache_object.time_to_live == nil
+              cache_object.time_to_live =
+                feed_configurations[:default_ttl].to_i
+              cache_object.save
+            end
+            if (cache_object.last_retrieved == nil)
+              expired = true
+            elsif (cache_object.time_to_live < 30.minutes)
+              expired =
+                (cache_object.last_retrieved + 30.minutes) < Time.now.gmtime
+            else
+              expired =
+                (cache_object.last_retrieved + cache_object.time_to_live) <
+                  Time.now.gmtime
+            end
+            if !expired
+              require 'yaml'
+              deserialized_feed = YAML.load(cache_object.serialized)
+              deserialized_feed.cache_object = cache_object
+              Thread.pass
+            end
+          end
+        rescue Exception
+        end
       end
       
-      # clean up the url
-      url = FeedTools::UriHelper.normalize_url(url)
+      if deserialized_feed == nil
+        # create the new feed
+        feed = FeedTools::Feed.new
+
+        feed.configurations = feed_configurations
 
-      # load the new feed
-      feed.href = url
-      feed.update! unless feed.configurations[:disable_update_from_remote]
-      Thread.pass
+        # load the new feed
+        feed.href = href
+        if cache_object != nil
+          feed.cache_object = cache_object
+        end
+        feed.update! unless feed.configurations[:disable_update_from_remote]
+        Thread.pass
       
-      return feed
+        return feed
+      else
+        return deserialized_feed
+      end
     end
     
     # Returns the load options for this feed.
@@ -99,10 +177,9 @@ module FeedTools
     # Loads the feed from the remote url if the feed has expired from the
     # cache or cannot be retrieved from the cache for some reason.
     def update!
-      if self.configurations[:disable_update_from_remote]
-        # Don't do anything if this option is set
-        return
-      end
+      # Don't do anything if this option is set
+      return if self.configurations[:disable_update_from_remote]
+
       if !FeedTools.feed_cache.nil? &&
           !FeedTools.feed_cache.set_up_correctly?
         FeedTools.feed_cache.initialize_cache()
@@ -129,25 +206,23 @@ module FeedTools
             self.http_headers['content-type'] =~ /application\/xhtml\+xml/
 
           autodiscovered_url = nil
-          autodiscovered_url =
-            FeedTools::HtmlHelper.extract_link_by_mime_type(self.feed_data,
-              "application/atom+xml")
-          if autodiscovered_url.nil?
+          ['atom', 'rss', 'rdf'].each do |type|
             autodiscovered_url =
               FeedTools::HtmlHelper.extract_link_by_mime_type(self.feed_data,
-                "application/rss+xml")
+                "application/#{type}+xml")
+            break unless autodiscovered_url.nil?
           end
-          if autodiscovered_url.nil?
-            autodiscovered_url =
-              FeedTools::HtmlHelper.extract_link_by_mime_type(self.feed_data,
-                "application/rdf+xml")
-          end
-          unless autodiscovered_url.nil?
+          
+          if autodiscovered_url != nil
             begin
               autodiscovered_url = FeedTools::UriHelper.resolve_relative_uri(
                 autodiscovered_url, [self.href])
             rescue Exception
             end
+            if self.href == autodiscovered_url
+              raise FeedAccessError,
+                "Autodiscovery loop detected: #{autodiscovered_url}"
+            end
             self.feed_data = nil
             self.href = autodiscovered_url
             if FeedTools.feed_cache.nil?
@@ -157,12 +232,24 @@ module FeedTools
                 FeedTools.feed_cache.find_by_href(autodiscovered_url)
             end
             self.update!
+          else
+            html_body = FeedTools::XmlHelper.try_xpaths(self.xml_document, [
+              "html/body"
+            ])
+            if html_body != nil
+              raise FeedAccessError,
+                "#{self.href} does not appear to be a feed."
+            end
           end
         else
           ugly_redirect = FeedTools::XmlHelper.try_xpaths(self.xml_document, [
             "redirect/newLocation/text()"
           ], :select_result_value => true)
           if !ugly_redirect.blank?
+            if self.href == ugly_redirect
+              raise FeedAccessError,
+                "Ugly redirect loop detected: #{ugly_redirect}"
+            end
             self.feed_data = nil
             self.href = ugly_redirect
             if FeedTools.feed_cache.nil?
@@ -187,6 +274,10 @@ module FeedTools
         @link = nil
         @time_to_live = nil
         @entries = nil
+        
+        if self.configurations[:lazy_parsing_enabled] == false
+          self.full_parse()
+        end
       end
     end
   
@@ -338,7 +429,101 @@ module FeedTools
         end
       end
     end
+    
+    # Does a full parse of the feed.
+    def full_parse
+      self.href
+
+      self.cache_object
       
+      self.http_headers
+      self.encoding
+      self.feed_data_utf_8
+      self.xml_document
+      self.root_node
+      self.channel_node
+      
+      self.base_uri
+      self.feed_type
+      self.feed_version
+
+      self.entries
+
+      self.id
+      self.title
+      self.subtitle
+      self.links
+      self.link
+      self.icon
+      self.favicon
+      self.author
+      self.publisher
+      self.time
+      self.updated
+      self.published
+      self.categories
+      self.images
+      self.rights
+      self.time_to_live
+      self.generator
+      self.language
+
+      self.docs
+      self.text_input
+      self.cloud
+
+      self.itunes_summary
+      self.itunes_subtitle
+      self.itunes_author
+
+      self.media_text
+
+      self.explicit?
+      
+      self.entries.each do |entry|
+        entry.full_parse()
+      end
+
+      nil
+    end
+    
+    # Does a full parse, then serializes the feed object directly to the
+    # cache.
+    def serialize_to_cache
+      @cache_object = nil
+      require 'yaml'
+      serialized_feed = YAML.dump(self.serializable)
+      if self.cache_object != nil
+        begin
+          self.cache_object.serialized = serialized_feed
+          self.cache_object.save
+        rescue Exception
+        end
+      end
+      return nil
+    end
+    
+    # Returns a duplicate object suitable for serialization
+    def serializable
+      self.full_parse()
+      entries_to_dump = self.entries
+      # This prevents errors due to temporarily having feed items with
+      # multiple parent feeds.
+      self.entries = []
+      feed_to_dump = self.dup
+      feed_to_dump.instance_variable_set("@xml_document", nil)
+      feed_to_dump.instance_variable_set("@root_node", nil)
+      feed_to_dump.instance_variable_set("@channel_node", nil)
+      feed_to_dump.entries = entries_to_dump.collect do |entry|
+        entry.serializable
+      end
+      self.entries = entries_to_dump
+      feed_to_dump.entries.each do |entry|
+        entry.instance_variable_set("@root_node", nil)
+      end
+      return feed_to_dump
+    end
+        
     # Returns the relevant information from an http request.
     def http_response
       return @http_response
@@ -360,8 +545,12 @@ module FeedTools
     # Returns the encoding that the feed was parsed with
     def encoding
       if @encoding.blank?
-        unless self.http_headers.blank?
-          @encoding = "utf-8"
+        if !self.http_headers.blank?
+          if self.http_headers['content-type'] =~ /charset=([\w\d-]+)/
+            @encoding = $1.downcase
+          else
+            @encoding = self.encoding_from_feed_data
+          end          
         else
           @encoding = self.encoding_from_feed_data
         end
@@ -377,7 +566,7 @@ module FeedTools
         return nil if raw_data.nil?
         encoding_from_xml_instruct = 
           raw_data.scan(
-            /^<\?xml [^>]*encoding="([\w]*)"[^>]*\?>/
+            /^<\?xml [^>]*encoding="([^\"]*)"[^>]*\?>/
           ).flatten.first
         unless encoding_from_xml_instruct.blank?
           encoding_from_xml_instruct.downcase!
@@ -446,11 +635,26 @@ module FeedTools
         if FeedTools.feed_cache.nil?
           self.cache_object = nil
         else
-          self.cache_object =
-            FeedTools.feed_cache.find_by_href(ugly_redirect)
+          begin
+            self.cache_object =
+              FeedTools.feed_cache.find_by_href(ugly_redirect)
+          rescue RuntimeError => error
+            if error.message =~ /sorry, too many clients already/
+              warn("There are too many connections to the database open.")
+            end
+            raise error
+          end
         end
         self.update!
       end
+      
+      # Get these things parsed in the correct order to avoid the retardedly
+      # painful corecursion issues.
+      self.href
+      @links = nil
+      @link = nil
+      self.links
+      self.link
     end
     
     # Returns the feed's raw data as utf-8.
@@ -462,11 +666,11 @@ module FeedTools
         else
           use_encoding = force_encoding
         end
-        if use_encoding != "utf-8"
+        if use_encoding != "utf-8" && use_encoding != nil && raw_data != nil
           begin
             @feed_data_utf_8 =
               Iconv.new('utf-8', use_encoding).iconv(raw_data)
-          rescue
+          rescue Exception => error
             return raw_data
           end
         else
@@ -495,24 +699,23 @@ module FeedTools
       unless self.cache_object.nil?
         self.cache_object.feed_data_type = new_feed_data_type
       end
+      if self.feed_data_type != :xml
+        @xml_document = nil
+      end
     end
-  
+
     # Returns a REXML Document of the feed_data
     def xml_document
-      if self.feed_data_type != :xml
-        @xml_document = nil
-      else
-        if @xml_document.nil?
+      if @xml_document.nil?
+        return nil if self.feed_data.blank?
+        if self.feed_data_type != :xml
+          @xml_document = nil
+        else
           begin
-            begin
-              @xml_document = REXML::Document.new(self.feed_data_utf_8)
-            rescue Object
-              # Something failed, attempt to repair the xml with htree.
-              @xml_document = HTree.parse(self.feed_data_utf_8).to_rexml
-            end
-          rescue Object
-            @xml_document = nil
-            raise
+            @xml_document = REXML::Document.new(self.feed_data_utf_8)
+          rescue Exception
+            # Something failed, attempt to repair the xml with htree.
+            @xml_document = HTree.parse(self.feed_data_utf_8).to_rexml
           end
         end
       end
@@ -551,7 +754,7 @@ module FeedTools
           else
             @root_node = self.xml_document.root
           end
-        rescue
+        rescue Exception
           return nil
         end
       end
@@ -564,7 +767,8 @@ module FeedTools
         @channel_node = FeedTools::XmlHelper.try_xpaths(self.root_node, [
           "channel",
           "CHANNEL",
-          "feedinfo"
+          "feedinfo",
+          "news"
         ])
         if @channel_node == nil
           @channel_node = self.root_node
@@ -584,9 +788,17 @@ module FeedTools
             if @href != nil
               begin
                 @cache_object = FeedTools.feed_cache.find_by_href(@href)
-              rescue
+              rescue RuntimeError => error
+                if error.message =~ /sorry, too many clients already/
+                  warn("There are too many connections to the database open.")
+                  raise error
+                else
+                  raise error
+                end
+              rescue => error
                 warn("The feed cache seems to be having trouble with the " +
                   "find_by_href method.  This may cause unexpected results.")
+                raise error
               end
             end
             if @cache_object.nil?
@@ -778,19 +990,33 @@ module FeedTools
           end
         end
         if override_href.call(@href) && self.feed_data != nil
-          for link_object in self.links
-            if link_object.rel == 'self'
-              if link_object.href != self.link ||
-                  (link_object.href =~ /xml/ ||
-                  link_object.href =~ /atom/ ||
-                  link_object.href =~ /feed/)
-                @href = link_object.href
-                @href_overridden = true
-                @link = nil
-                return @href
+          begin
+            links = FeedTools::GenericHelper.recursion_trap(:feed_href) do
+              self.links
+            end
+            link = FeedTools::GenericHelper.recursion_trap(:feed_href) do
+              self.link
+            end
+            if links != nil
+              for link_object in links
+                if link_object.rel == 'self'
+                  if link_object.href != link ||
+                      (link_object.href =~ /xml/ ||
+                      link_object.href =~ /atom/ ||
+                      link_object.href =~ /feed/)
+                    @href = link_object.href
+                    @href_overridden = true
+                    @links = nil
+                    @link = nil
+                    return @href
+                  end
+                end
               end
             end
+          rescue Exception
           end
+          @links = nil
+          @link = nil
           
           # rdf:about is ordered last because a lot of people put the url to
           # the feed inside it instead of a link to their blog.
@@ -804,7 +1030,8 @@ module FeedTools
             "feed/@resource",
             "@rdf:about",
             "@about",
-            "newLocation/text()"
+            "newLocation/text()",
+            "atom10:link[@rel='self']/@href"
           ], :select_result_value => true) do |result|
             override_href.call(FeedTools::UriHelper.normalize_url(result))
           end
@@ -831,6 +1058,7 @@ module FeedTools
             @href_overridden = false
           end
           if @href_overridden == true
+            @links = nil
             @link = nil
           end
         end
@@ -858,7 +1086,7 @@ module FeedTools
           "TITLE"
         ])
         @title = FeedTools::HtmlHelper.process_text_construct(title_node,
-          self.feed_type, self.feed_version)
+          self.feed_type, self.feed_version, [self.base_uri])
         if self.feed_type == "atom" ||
             self.configurations[:always_strip_wrapper_elements]
           @title = FeedTools::HtmlHelper.strip_wrapper_element(@title)
@@ -901,7 +1129,7 @@ module FeedTools
           "info"
         ])
         @subtitle = FeedTools::HtmlHelper.process_text_construct(
-          subtitle_node, self.feed_type, self.feed_version)
+          subtitle_node, self.feed_type, self.feed_version, [self.base_uri])
         if self.feed_type == "atom" ||
             self.configurations[:always_strip_wrapper_elements]
           @subtitle = FeedTools::HtmlHelper.strip_wrapper_element(@subtitle)
@@ -1119,6 +1347,13 @@ module FeedTools
             "@href",
             "text()"
           ], :select_result_value => true)
+          if link_object.href == "atom10:" ||
+              link_object.href == "atom03:" ||
+              link_object.href == "atom:"
+            link_object.href = FeedTools::XmlHelper.try_xpaths(link_node, [
+              "@href"
+            ], :select_result_value => true)
+          end
           if link_object.href.nil? && link_node.base_uri != nil
             link_object.href = ""
           end
@@ -1143,6 +1378,13 @@ module FeedTools
             "@atom:hreflang",
             "@hreflang"
           ], :select_result_value => true)
+          if link_object.hreflang == "atom10:" ||
+              link_object.hreflang == "atom03:" ||
+              link_object.hreflang == "atom:"
+            link_object.hreflang = FeedTools::XmlHelper.try_xpaths(link_node, [
+              "@hreflang"
+            ], :select_result_value => true)
+          end
           unless link_object.hreflang.nil?
             link_object.hreflang = link_object.hreflang.downcase
           end
@@ -1152,6 +1394,13 @@ module FeedTools
             "@atom:rel",
             "@rel"
           ], :select_result_value => true)
+          if link_object.rel == "atom10:" ||
+              link_object.rel == "atom03:" ||
+              link_object.rel == "atom:"
+            link_object.rel = FeedTools::XmlHelper.try_xpaths(link_node, [
+              "@rel"
+            ], :select_result_value => true)
+          end
           unless link_object.rel.nil?
             link_object.rel = link_object.rel.downcase
           end
@@ -1164,6 +1413,13 @@ module FeedTools
             "@atom:type",
             "@type"
           ], :select_result_value => true)
+          if link_object.type == "atom10:" ||
+              link_object.type == "atom03:" ||
+              link_object.type == "atom:"
+            link_object.type = FeedTools::XmlHelper.try_xpaths(link_node, [
+              "@type"
+            ], :select_result_value => true)
+          end
           unless link_object.type.nil?
             link_object.type = link_object.type.downcase
           end
@@ -1174,6 +1430,13 @@ module FeedTools
             "@title",
             "text()"
           ], :select_result_value => true)
+          if link_object.title == "atom10:" ||
+              link_object.title == "atom03:" ||
+              link_object.title == "atom:"
+            link_object.title = FeedTools::XmlHelper.try_xpaths(link_node, [
+              "@title"
+            ], :select_result_value => true)
+          end
           # This catches the ambiguities between atom, rss, and cdf
           if link_object.title == link_object.href
             link_object.title = nil
@@ -1184,6 +1447,13 @@ module FeedTools
             "@atom:length",
             "@length"
           ], :select_result_value => true)
+          if link_object.length == "atom10:" ||
+              link_object.length == "atom03:" ||
+              link_object.length == "atom:"
+            link_object.length = FeedTools::XmlHelper.try_xpaths(link_node, [
+              "@length"
+            ], :select_result_value => true)
+          end
           if !link_object.length.nil?
             link_object.length = link_object.length.to_i
           else
@@ -1211,7 +1481,10 @@ module FeedTools
     def base_uri
       if @base_uri.nil?
         @base_uri = FeedTools::XmlHelper.try_xpaths(self.channel_node, [
-          "@base"
+          "@base",
+          "base/@href",
+          "base/text()",
+          "@xml:base"
         ], :select_result_value => true)
         if @base_uri.blank?
           begin
@@ -1222,9 +1495,26 @@ module FeedTools
           rescue Exception
           end
         end
+        if @base_uri.blank?
+          @base_uri = FeedTools::XmlHelper.try_xpaths(self.root_node, [
+            "@xml:base"
+          ], :select_result_value => true)
+        end
         if !@base_uri.blank?
           @base_uri = FeedTools::UriHelper.normalize_url(@base_uri)
         end
+        if !@base_uri.blank?
+          parsed_uri = FeedTools::URI.parse(@base_uri)
+          # Feedburner is almost never the base uri that was intended
+          # Use the actual site instead
+          if parsed_uri.host =~ /feedburner/
+            site_uri =
+              FeedTools::GenericHelper.recursion_trap(:feed_base_uri) do
+                FeedTools::UriHelper.normalize_url(self.link)
+              end
+            @base_uri = site_uri if !site_uri.blank?
+          end
+        end
       end
       return @base_uri
     end
@@ -1425,6 +1715,12 @@ module FeedTools
             rescue
             end
           end
+          if FeedTools::XmlHelper.try_xpaths(author_node,
+              ["@gr:unknown-author"], :select_result_value => true) == "true"
+            if @author.name == "(author unknown)"
+              @author.name = nil
+            end
+          end
         end
         # Fallback on the itunes module if we didn't find an author name
         begin
@@ -1764,35 +2060,7 @@ module FeedTools
       end
       return @images
     end
-  
-    # Returns the feed's text input field
-    def text_input
-      if @text_input.nil?
-        @text_input = FeedTools::TextInput.new
-        text_input_node =
-          FeedTools::XmlHelper.try_xpaths(self.channel_node, ["textInput"])
-        unless text_input_node.nil?
-          @text_input.title =
-            FeedTools::XmlHelper.try_xpaths(text_input_node,
-              ["title/text()"],
-              :select_result_value => true)
-          @text_input.description =
-            FeedTools::XmlHelper.try_xpaths(text_input_node,
-              ["description/text()"],
-              :select_result_value => true)
-          @text_input.link =
-            FeedTools::XmlHelper.try_xpaths(text_input_node,
-              ["link/text()"],
-              :select_result_value => true)
-          @text_input.name =
-            FeedTools::XmlHelper.try_xpaths(text_input_node,
-              ["name/text()"],
-              :select_result_value => true)
-        end
-      end
-      return @text_input
-    end
-      
+
     # Returns the feed's copyright information
     def rights
       if @rights.nil?
@@ -1807,7 +2075,7 @@ module FeedTools
           "rights"
         ])
         @rights = FeedTools::HtmlHelper.process_text_construct(rights_node,
-          self.feed_type, self.feed_version)
+          self.feed_type, self.feed_version, [self.base_uri])
         if self.feed_type == "atom" ||
             self.configurations[:always_strip_wrapper_elements]
           @rights = FeedTools::HtmlHelper.strip_wrapper_element(@rights)
@@ -1821,12 +2089,24 @@ module FeedTools
       @rights = new_rights
     end
 
-    def license #:nodoc:
-      raise "Not implemented yet."
+    # Returns the first license link for the feed item.
+    def license
+      return self.licenses.first
+    end
+
+    # Returns all licenses linked from this feed item.
+    def licenses
+      if @licenses.nil?
+        @licenses = self.links.select do |link|
+          link.rel == "license"
+        end
+      end
+      return @licenses
     end
     
-    def license=(new_license) #:nodoc:
-      raise "Not implemented yet."
+    # Sets the feed item's licenses.
+    def licenses=(new_licenses)
+      @licenses = new_licenses
     end
     
     # Returns the number of seconds before the feed should expire
@@ -1910,14 +2190,14 @@ module FeedTools
               @time_to_live = @time_to_live + update_frequency_seconds.to_i
             end
             if @time_to_live == 0
-              @time_to_live = 1.hour
+              @time_to_live = self.configurations[:default_ttl].to_i
             end
           end
         end
       end
       if @time_to_live.nil? || @time_to_live == 0
         # Default to one hour
-        @time_to_live = 1.hour
+        @time_to_live = self.configurations[:default_ttl].to_i
       elsif self.configurations[:max_ttl] != nil &&
           self.configurations[:max_ttl] != 0 &&
           @time_to_live >= self.configurations[:max_ttl].to_i
@@ -1930,7 +2210,7 @@ module FeedTools
     # Sets the feed time to live
     def time_to_live=(new_time_to_live)
       @time_to_live = new_time_to_live.round
-      @time_to_live = 1.hour if @time_to_live < 1.hour
+      @time_to_live = 30.minutes if @time_to_live < 30.minutes
     end
 
     # Returns the feed's cloud
@@ -1966,6 +2246,34 @@ module FeedTools
       @cloud = new_cloud
     end
   
+    # Returns the feed's text input field
+    def text_input
+      if @text_input.nil?
+        @text_input = FeedTools::TextInput.new
+        text_input_node =
+          FeedTools::XmlHelper.try_xpaths(self.channel_node, ["textInput"])
+        unless text_input_node.nil?
+          @text_input.title =
+            FeedTools::XmlHelper.try_xpaths(text_input_node,
+              ["title/text()"],
+              :select_result_value => true)
+          @text_input.description =
+            FeedTools::XmlHelper.try_xpaths(text_input_node,
+              ["description/text()"],
+              :select_result_value => true)
+          @text_input.link =
+            FeedTools::XmlHelper.try_xpaths(text_input_node,
+              ["link/text()"],
+              :select_result_value => true)
+          @text_input.name =
+            FeedTools::XmlHelper.try_xpaths(text_input_node,
+              ["name/text()"],
+              :select_result_value => true)
+        end
+      end
+      return @text_input
+    end
+    
     # Returns the feed generator
     def generator
       if @generator.nil?
@@ -2096,7 +2404,8 @@ module FeedTools
             "atom10:entry",
             "atom03:entry",
             "atom:entry",
-            "entry"
+            "entry",
+            "story"
           ]),
           FeedTools::XmlHelper.try_xpaths_all(self.channel_node, [
             "rss10:item",
@@ -2104,7 +2413,8 @@ module FeedTools
             "rss11:items/item",
             "items/rss11:item",
             "items/item",
-            "item"
+            "item",
+            "story"
           ])
         ])
 
@@ -2234,6 +2544,17 @@ module FeedTools
     def build_xml(feed_type=(self.feed_type or "atom"), feed_version=nil,
         xml_builder=Builder::XmlMarkup.new(
           :indent => 2, :escape_attrs => false))
+      
+      if self.find_node("access:restriction/@relationship").to_s == "deny"
+        raise StandardError,
+          "Operation not permitted.  This feed denies redistribution."
+      elsif self.find_node("@indexing:index").to_s == "no"
+        raise StandardError,
+          "Operation not permitted.  This feed denies redistribution."
+      end
+      
+      self.full_parse()
+      
       xml_builder.instruct! :xml, :version => "1.0",
         :encoding => (self.configurations[:output_encoding] or "utf-8")
       if feed_type.nil?
@@ -2392,6 +2713,9 @@ module FeedTools
             unless self.copyright.blank?
               xml_builder.copyright(self.copyright)
             end
+            unless self.language.blank?
+              xml_builder.language(self.language)
+            end
             xml_builder.ttl((time_to_live / 1.minute).to_s)
             xml_builder.generator(
               self.configurations[:generator_href])
@@ -2467,6 +2791,8 @@ module FeedTools
             end
           elsif self.link != nil
             xml_builder.id(FeedTools::UriHelper.build_urn_uri(self.link))
+          elsif self.url != nil
+            xml_builder.id(FeedTools::UriHelper.build_urn_uri(self.url))
           else
             raise "Cannot build feed, missing feed unique id."
           end