feedtools 0.2.9 → 0.2.10

data/CHANGELOG

@@ -1,3 +1,8 @@
+== FeedTools 0.2.10
+ * http error messages now sport a list of redirections to simplify debugging
+ * automatic table creation removed
+ * database schema information included in rdoc and package files
+ * deals with servers returning 404 if passed a User-Agent http header
 == FeedTools 0.2.9
  * changed ordering of elements checked for the item content
  * added dependancy on uuidtools, uuids used in feed generation

data/db/schema.mysql.sql

@@ -0,0 +1,10 @@
+CREATE TABLE `feeds` (
+  `id`              int(10) unsigned NOT NULL auto_increment,
+  `url`             varchar(255) default NULL,
+  `title`           varchar(255) default NULL,
+  `link`            varchar(255) default NULL,
+  `xml_data`        longtext default NULL,
+  `http_headers`    text default NULL,
+  `last_retrieved`  datetime default NULL,
+  PRIMARY KEY  (`id`)
+)

data/db/schema.postgresql.sql

@@ -0,0 +1,9 @@
+CREATE TABLE feeds (
+  id                SERIAL PRIMARY KEY NOT NULL,
+  url               varchar(255) default NULL,
+  title             varchar(255) default NULL,
+  link              varchar(255) default NULL,
+  xml_data          text default NULL,
+  http_headers      text default NULL,
+  last_retrieved    timestamp default NULL
+);

data/db/schema.sqlite.sql

@@ -0,0 +1,9 @@
+CREATE TABLE 'feeds' (
+  'id'              INTEGER PRIMARY KEY NOT NULL,
+  'url'             VARCHAR(255) DEFAULT NULL,
+  'title'           VARCHAR(255) DEFAULT NULL,
+  'link'            VARCHAR(255) DEFAULT NULL,
+  'xml_data'        TEXT DEFAULT NULL,
+  'http_headers'    TEXT DEFAULT NULL,
+  'last_retrieved'  DATETIME DEFAULT NULL
+);

data/lib/database_feed_cache.rb

@@ -51,11 +51,6 @@ module FeedTools
         rescue
         end
       end
-      # Verify that the necessary database tables are in place
-      # and if they're missing, create them
-      unless DatabaseFeedCache.table_exists?
-        DatabaseFeedCache.create_table
-      end
       return nil
     end
 
@@ -85,61 +80,5 @@ module FeedTools
       end
       return true
     end
-  
-    # Creates the appropriate database table
-    # Deprecated.
-    def DatabaseFeedCache.create_table
-      warn("The table creation methods will be removed in a future version " +
-        "and any reliance on this method should be removed.")
-      unless DatabaseFeedCache.table_exists?
-        feeds_mysql = <<-SQL_END
-          CREATE TABLE `#{self.table_name()}` (
-            `id`              int(10) unsigned NOT NULL auto_increment,
-            `url`             varchar(255) default NULL,
-            `title`           varchar(255) default NULL,
-            `link`            varchar(255) default NULL,
-            `xml_data`        longtext default NULL,
-            `http_headers`    text default NULL,
-            `last_retrieved`  datetime default NULL,
-            PRIMARY KEY  (`id`)
-          )
-        SQL_END
-        feeds_sqlite = <<-SQL_END
-          CREATE TABLE '#{self.table_name()}' (
-            'id'              INTEGER PRIMARY KEY NOT NULL,
-            'url'             VARCHAR(255) DEFAULT NULL,
-            'title'           VARCHAR(255) DEFAULT NULL,
-            'link'            VARCHAR(255) DEFAULT NULL,
-            'xml_data'        TEXT DEFAULT NULL,
-            'http_headers'    TEXT DEFAULT NULL,
-            'last_retrieved'  DATETIME DEFAULT NULL
-          );
-        SQL_END
-        feeds_psql = <<-SQL_END
-          CREATE TABLE #{self.table_name()} (
-            id                SERIAL PRIMARY KEY NOT NULL,
-            url               varchar(255) default NULL,
-            title             varchar(255) default NULL,
-            link              varchar(255) default NULL,
-            xml_data          text default NULL,
-            http_headers      text default NULL,
-            last_retrieved    timestamp default NULL
-          );
-        SQL_END
-        table_creation_sql = nil
-        if configurations["adapter"] == "mysql"
-          table_creation_sql = feeds_mysql
-        elsif configurations["adapter"] == "sqlite"
-          table_creation_sql = feeds_sqlite
-        elsif configurations["adapter"] == "postgresql"
-          table_creation_sql = feeds_psql
-        end
-        if table_creation_sql.nil?
-          raise "Could not build #{self.table_name()} table."
-        else
-          connection.execute table_creation_sql
-        end
-      end
-    end
   end
 end

data/lib/feed_tools.rb

@@ -32,7 +32,7 @@ FEED_TOOLS_ENV = ENV['FEED_TOOLS_ENV'] ||
                  ENV['RAILS_ENV'] ||
                  'production' # :nodoc:
 
-FEED_TOOLS_VERSION = "0.2.9"
+FEED_TOOLS_VERSION = "0.2.10"
 
 $:.unshift(File.dirname(__FILE__))
 $:.unshift(File.dirname(__FILE__) + "/feed_tools/vendor")
@@ -75,7 +75,7 @@ require 'pp'
 require 'yaml'
 
 require_gem('activerecord', '>= 1.10.1')
-require_gem('uuidtools', '>= 0.1.1')
+require_gem('uuidtools', '>= 0.1.2')
 
 require 'database_feed_cache'
 
@@ -832,7 +832,7 @@ module FeedTools
 
         # The http feed access method
         http_fetch = lambda do |feed_url, http_headers, redirect_limit,
-            response_chain|
+            response_chain, no_headers|
           raise FeedAccessError, 'Redirect too deep' if redirect_limit == 0
           feed_uri = nil
           begin
@@ -850,10 +850,11 @@ module FeedTools
           # So override it here explicitly.
           http_headers['Host'] = feed_uri.host
           http_headers['Host'] += ":#{feed_uri.port}" if feed_uri.port
-
+          
           Net::HTTP.start(feed_uri.host, (feed_uri.port or 80)) do |http|
             final_uri = feed_uri.path 
             final_uri += ('?' + feed_uri.query) if feed_uri.query
+            http_headers = {} if no_headers
             response = http.request_get(final_uri, http_headers)
 
             case response
@@ -889,16 +890,50 @@ module FeedTools
                 # in Location header
                 # =================================================
                 http_fetch.call(new_location, http_headers,
-                  redirect_limit - 1, response_chain)
+                  redirect_limit - 1, response_chain, no_headers)
               end
             else
+              class << response
+                def response_chain
+                  return @response_chain
+                end
+              end
+              response.instance_variable_set("@response_chain",
+                response_chain)
               response.error!
             end
           end
         end
         
         begin
-          @http_response = http_fetch.call(self.url, headers, 10, [])
+          begin
+            @http_response = http_fetch.call(self.url, headers, 10, [], false)
+          rescue => error
+            if error.respond_to?(:response)
+              # You might not believe this, but...
+              #
+              # Under certain circumstances, web servers will try to block
+              # based on the User-Agent header.  This is *retarded*.  But
+              # we won't let their stupid error stop us!
+              #
+              # This is, of course, a quick-n-dirty hack.  But at least
+              # we get to blame other people's bad software and/or bad
+              # configuration files.
+              if error.response.code.to_i == 404 &&
+                  FeedTools.user_agent != nil
+                @http_response = http_fetch.call(self.url, {}, 10, [], true)
+                if @http_response != nil && @http_response.code.to_i == 200
+                  warn("The server appears to be blocking based on the " +
+                    "User-Agent header.  This is stupid, and you should " +
+                    "inform the webmaster of this.")
+                end
+              else
+                raise error
+              end
+            else
+              raise error
+            end
+          end
           @http_headers = {}
           self.http_response.each_header do |header|
             self.http_headers[header.first.downcase] = header.last
@@ -948,7 +983,14 @@ module FeedTools
           end
           @live = false
           if self.xml_data.nil?
-            raise
+            if error.respond_to?(:response) &&
+                error.response.respond_to?(:response_chain)
+              redirects = error.response.response_chain.map do |pair|
+                pair.first
+              end
+              error.message << (" - Redirects: " + redirects.inspect)
+            end
+            raise error
           end
         end
       elsif retrieval_method == "https"

data/rakefile

@@ -7,7 +7,7 @@ require 'rake/gempackagetask'
 require 'rake/contrib/rubyforgepublisher'
 
 PKG_NAME      = 'feedtools'
-PKG_VERSION   = '0.2.9'
+PKG_VERSION   = '0.2.10'
 PKG_FILE_NAME = "#{PKG_NAME}-#{PKG_VERSION}"
 
 RELEASE_NAME  = "REL #{PKG_VERSION}"
@@ -16,7 +16,8 @@ RUBY_FORGE_PROJECT = "feedtools"
 RUBY_FORGE_USER    = "vacindak"
 
 PKG_FILES = FileList[
-    "lib/**/*", "test/**/*", "examples/**/*", "doc/**/*", "[A-Z]*", "install.rb", "rakefile"
+    "lib/**/*", "test/**/*", "examples/**/*", "doc/**/*", "db/**/*",
+    "[A-Z]*", "install.rb", "rakefile"
 ].exclude(/\bCVS\b|~$/).exclude(/database\.yml/)
 
 desc "Default Task"
@@ -39,11 +40,12 @@ Rake::RDocTask.new { |rdoc|
   rdoc.template = "#{ENV['template']}.rb" if ENV['template']
   rdoc.rdoc_files.include('README', 'CHANGELOG')
   rdoc.rdoc_files.include('lib/**/*.rb')
+  rdoc.rdoc_files.include('db/**/*.sql')
 }
 
 # Create compressed packages
 
-dist_dirs = [ "lib", "test" ]
+dist_dirs = [ "lib", "test", "db" ]
 
 spec = Gem::Specification.new do |s|
   s.name = PKG_NAME
@@ -59,7 +61,7 @@ spec = Gem::Specification.new do |s|
   end
   
   s.add_dependency('activerecord', '>= 1.10.1')
-  s.add_dependency('uuidtools', '>= 0.1.1')
+  s.add_dependency('uuidtools', '>= 0.1.2')
 
   s.require_path = 'lib'
   s.autorequire = 'feed_tools'

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: feedtools
 version: !ruby/object:Gem::Version 
-  version: 0.2.9
-date: 2005-09-25 00:00:00 -04:00
+  version: 0.2.10
+date: 2005-09-26 00:00:00 -04:00
 summary: "Parsing, generation, and caching system for xml news feeds."
 require_paths: 
   - lib
@@ -78,6 +78,9 @@ files:
   - test/helper_test.rb
   - test/nonstandard_test.rb
   - test/rss_test.rb
+  - db/schema.mysql.sql
+  - db/schema.postgresql.sql
+  - db/schema.sqlite.sql
 test_files: []
 rdoc_options: 
   - "--main"
@@ -106,5 +109,5 @@ dependencies:
         - 
           - ">="
           - !ruby/object:Gem::Version 
-            version: 0.1.1
+            version: 0.1.2
       version: