spidr 0.4.1 → 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 31e83cba8fd67a2527641b404f82773d60b5fb97
+  data.tar.gz: cbd735b652d209cd49a6990eedf3de6f7a22e385
+SHA512:
+  metadata.gz: d33742df9e9a4ec8090d4934de3562036e149195b3567ac1143c4637012876d86a18618e9f89251506ed8aa1d9c85cc18ed324774d4da29038e975827698f265
+  data.tar.gz: 24b08172be0184f7c68fbc63b31eaac55b0c55d70b35b8983fbbb1a3ce871e157b0bbf7d598625ef37ec3fe420c7372bc5fdaf7dd4b7131eac6e6e23e465e475

data/ChangeLog.md

@@ -1,21 +1,36 @@
+### 0.5.0 / 2016-01-03
+
+* Added support for respecting `robots.txt` files.
+
+      Spidr.site('http://reddit.com/', robots: true)
+
+* Added {Spidr.robots=} and {Spidr.robots?}.
+* Added {Spidr::Page#each_mailto} and {Spidr::Page#mailtos}.
+* Fixed a bug in {Spidr::Agent.host} that limited spidering to only `http://`
+  URIs.
+* Rescue `Zlib::Error` to catch `Zlib::DataError` and `Zlib::BufError`
+  exceptions caused by web servers that use incompatible gzip compression.
+* Fixed a bug in {URI.expand_path} where `/../foo` was being expanded to `foo`
+  instead of `/foo`.
+
 ### 0.4.1 / 2011-12-08
 
 * Catch `OpenSSL::SSL::SSLError` exceptions when initiated HTTPS Sessions.
 
 ### 0.4.0 / 2011-08-07
 
-* Added {Spidr::Headers#content_charset}.
-* Pass the Page `url` and `content_charset` to Nokogiri in {Spidr::Body#doc}.
+* Added `Spidr::Headers#content_charset`.
+* Pass the Page `url` and `content_charset` to Nokogiri in `Spidr::Body#doc`.
   This ensures that Nokogiri will preserve the body encoding.
-* Made {Spidr::Headers#is_content_type?} public.
-* Allow {Spidr::Headers#is_content_type?} to match the full Content-Type
+* Made `Spidr::Headers#is_content_type?` public.
+* Allow `Spidr::Headers#is_content_type?` to match the full Content-Type
   or the sub-type.
 
 ### 0.3.2 / 2011-06-20
 
-* Added separate intitialize methods for {Spidr::Actions}, {Spidr::Events},
-  {Spidr::Filters} and {Spidr::Sanitizers}.
-* Aliased {Spidr::Events#urls_like} to {Spidr::Events#every_url_like}.
+* Added separate intitialize methods for `Spidr::Actions`, `Spidr::Events`,
+  `Spidr::Filters` and `Spidr::Sanitizers`.
+* Aliased `Spidr::Events#urls_like` to `Spidr::Events#every_url_like`.
 * Reduce usage of `self.included` and `module_eval`.
 * Reduce usage of nested-blocks.
 * Reduce usage of `return`.
@@ -28,21 +43,21 @@
 
 * Switched from Jeweler to [Ore](http://github.com/ruby-ore/ore).
 * Split all header related methods out of {Spidr::Page} and into
-  {Spidr::Headers}.
+  `Spidr::Headers`.
 * Split all body related methods out of {Spidr::Page} and into
-  {Spidr::Body}.
+  `Spidr::Body`.
 * Split all link related methods out of {Spidr::Page} and into
-  {Spidr::Links}.
-* Added {Spidr::Headers#directory?}.
-* Added {Spidr::Headers#json?}.
-* Added {Spidr::Links#each_url}.
-* Added {Spidr::Links#each_link}.
-* Added {Spidr::Links#each_redirect}.
-* Added {Spidr::Links#each_meta_redirect}.
-* Aliased {Spidr::Headers#raw_cookie} to {Spidr::Headers#cookie}.
-* Aliased {Spidr::Body#to_s} to {Spidr::Body#body}.
-* Also check for `application/xml` in {Spidr::Headers#xml?}.
-* Catch all exceptions when merging URIs in {Spidr::Links#to_absolute}.
+  `Spidr::Links`.
+* Added `Spidr::Headers#directory?`.
+* Added `Spidr::Headers#json?`.
+* Added `Spidr::Links#each_url`.
+* Added `Spidr::Links#each_link`.
+* Added `Spidr::Links#each_redirect`.
+* Added `Spidr::Links#each_meta_redirect`.
+* Aliased `Spidr::Headers#raw_cookie` to `Spidr::Headers#cookie`.
+* Aliased `Spidr::Body#to_s` to `Spidr::Body#body`.
+* Also check for `application/xml` in `Spidr::Headers#xml?`.
+* Catch all exceptions when merging URIs in `Spidr::Links#to_absolute`.
 * Always prepend a `/` to all FTP URI paths. Fixes a Ruby 1.8 specific
   bug, where it expects an absolute path for all FTP URIs.
 * Refactored {URI.expand_path}.
@@ -73,10 +88,10 @@
 
 ### 0.2.4 / 2010-05-05
 
-* Added {Spidr::Filters#visit_urls}.
-* Added {Spidr::Filters#visit_urls_like}.
-* Added {Spidr::Filters#ignore_urls}.
-* Added {Spidr::Filters#ignore_urls_like}.
+* Added `Spidr::Filters#visit_urls`.
+* Added `Spidr::Filters#visit_urls_like`.
+* Added `Spidr::Filters#ignore_urls`.
+* Added `Spidr::Filters#ignore_urls_like`.
 * Added `Spidr::Page#is_content_type?`.
 * Default `Spidr::Page#body` to an empty String.
 * Default `Spidr::Page#content_type` to an empty String.
@@ -89,7 +104,7 @@
 
 * Migrated to Jeweler, for the packaging and releasing RubyGems.
 * Switched to MarkDown formatted YARD documentation.
-* Added {Spidr::Events#every_link}.
+* Added `Spidr::Events#every_link`.
 * Added {Spidr::SessionCache#active?}.
 * Added specs for {Spidr::SessionCache}.
 
@@ -102,7 +117,7 @@
 * Added `Spidr::Page#cookie`.
 * Added `Spidr::Page#cookies`.
 * Added `Spidr::Page#cookie_params`.
-* Added {Spidr::Sanitizers}.
+* Added `Spidr::Sanitizers`.
 * Added {Spidr::SessionCache}.
 * Added {Spidr::CookieJar} (thanks Nick Plante).
 * Added {Spidr::AuthStore} (thanks Nick Plante).
@@ -112,31 +127,31 @@
 
 ### 0.2.1 / 2009-11-25
 
-* Added {Spidr::Events#every_ok_page}.
-* Added {Spidr::Events#every_redirect_page}.
-* Added {Spidr::Events#every_timedout_page}.
-* Added {Spidr::Events#every_bad_request_page}.
-* Added {Spidr::Events#every_unauthorized_page}.
-* Added {Spidr::Events#every_forbidden_page}.
-* Added {Spidr::Events#every_missing_page}.
-* Added {Spidr::Events#every_internal_server_error_page}.
-* Added {Spidr::Events#every_txt_page}.
-* Added {Spidr::Events#every_html_page}.
-* Added {Spidr::Events#every_xml_page}.
-* Added {Spidr::Events#every_xsl_page}.
-* Added {Spidr::Events#every_doc}.
-* Added {Spidr::Events#every_html_doc}.
-* Added {Spidr::Events#every_xml_doc}.
-* Added {Spidr::Events#every_xsl_doc}.
-* Added {Spidr::Events#every_rss_doc}.
-* Added {Spidr::Events#every_atom_doc}.
-* Added {Spidr::Events#every_javascript_page}.
-* Added {Spidr::Events#every_css_page}.
-* Added {Spidr::Events#every_rss_page}.
-* Added {Spidr::Events#every_atom_page}.
-* Added {Spidr::Events#every_ms_word_page}.
-* Added {Spidr::Events#every_pdf_page}.
-* Added {Spidr::Events#every_zip_page}.
+* Added `Spidr::Events#every_ok_page`.
+* Added `Spidr::Events#every_redirect_page`.
+* Added `Spidr::Events#every_timedout_page`.
+* Added `Spidr::Events#every_bad_request_page`.
+* Added `Spidr::Events#every_unauthorized_page`.
+* Added `Spidr::Events#every_forbidden_page`.
+* Added `Spidr::Events#every_missing_page`.
+* Added `Spidr::Events#every_internal_server_error_page`.
+* Added `Spidr::Events#every_txt_page`.
+* Added `Spidr::Events#every_html_page`.
+* Added `Spidr::Events#every_xml_page`.
+* Added `Spidr::Events#every_xsl_page`.
+* Added `Spidr::Events#every_doc`.
+* Added `Spidr::Events#every_html_doc`.
+* Added `Spidr::Events#every_xml_doc`.
+* Added `Spidr::Events#every_xsl_doc`.
+* Added `Spidr::Events#every_rss_doc`.
+* Added `Spidr::Events#every_atom_doc`.
+* Added `Spidr::Events#every_javascript_page`.
+* Added `Spidr::Events#every_css_page`.
+* Added `Spidr::Events#every_rss_page`.
+* Added `Spidr::Events#every_atom_page`.
+* Added `Spidr::Events#every_ms_word_page`.
+* Added `Spidr::Events#every_pdf_page`.
+* Added `Spidr::Events#every_zip_page`.
 * Fixed a bug where {Spidr::Agent#delay} was not being used to delay
   requesting pages.
 * Spider `link` and `script` tags in HTML pages (thanks Nick Plante).
@@ -160,11 +175,11 @@
 * Aliased `Spidr::Page#forbidden?` to `Spidr::Page#is_forbidden?`.
 * Aliased `Spidr::Page#missing?` to `Spidr::Page#is_missing?`.
 * Split URL filtering code out of {Spidr::Agent} and into
-  {Spidr::Filters}.
+  `Spidr::Filters`.
 * Split URL / Page event code out of {Spidr::Agent} and into
-  {Spidr::Events}.
+  `Spidr::Events`.
 * Split pause! / continue! / skip_link! / skip_page! methods out of
-  {Spidr::Agent} and into {Spidr::Actions}.
+  {Spidr::Agent} and into `Spidr::Actions`.
 * Fixed a bug in `Spidr::Page#code`, where it was not returning an Integer.
 * Make sure `Spidr::Page#doc` returns `Nokogiri::XML::Document` objects for
   RSS/RDF/Atom pages as well.

data/Gemfile

@@ -6,12 +6,16 @@ end
 
 gemspec
 
+
+gem 'robots', group: :robots
+
 group :development do
-  gem 'rake',         '~> 0.8'
+  gem 'rake'
+  gem 'rubygems-tasks', '~> 0.2'
 
-  gem 'ore-tasks',    '~> 0.4'
-  gem 'rspec',        '~> 2.4'
-  gem 'wsoc',         '~> 0.1.3'
+  gem 'wsoc',     '~> 0.1.3'
+  gem 'rspec',    '~> 3.0'
 
-  gem 'kramdown',     '~> 0.12'
+  gem 'kramdown', '~> 0.12'
+  gem 'yard',     '~> 0.8'
 end

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2008-2011 Hal Brodigan
+Copyright (c) 2008-2016 Hal Brodigan
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -1,8 +1,8 @@
 # Spidr
 
-* [Homepage](http://spidr.rubyforge.org/)
-* [Source](http://github.com/postmodern/spidr)
-* [Issues](http://github.com/postmodern/spidr/issues)
+* [Homepage](https://github.com/postmodern/spidr#readme)
+* [Source](https://github.com/postmodern/spidr)
+* [Issues](https://github.com/postmodern/spidr/issues)
 * [Mailing List](http://groups.google.com/group/spidr)
 * [IRC](http://webchat.freenode.net/?channels=spidr&uio=d4)
 
@@ -15,9 +15,9 @@ and easy to use.
 ## Features
 
 * Follows:
-  * a tags.
-  * iframe tags.
-  * frame tags.
+  * `a` tags.
+  * `iframe` tags.
+  * `frame` tags.
   * Cookie protected links.
   * HTTP 300, 301, 302, 303 and 307 Redirects.
   * Meta-Refresh Redirects.
@@ -51,36 +51,40 @@ Start spidering from a URL:
 
 Spider a host:
 
-    Spidr.host('coderrr.wordpress.com')
+    Spidr.host('solnic.eu')
 
 Spider a site:
 
-    Spidr.site('http://rubyflow.com/')
+    Spidr.site('http://www.rubyflow.com/')
 
 Spider multiple hosts:
 
     Spidr.start_at(
       'http://company.com/',
-      :hosts => [
+      hosts: [
         'company.com',
-        /host\d\.company\.com/
+        /host[\d]+\.company\.com/
       ]
     )
 
 Do not spider certain links:
 
-    Spidr.site('http://matasano.com/', :ignore_links => [/log/])
+    Spidr.site('http://company.com/', ignore_links: [%{^/blog/}])
 
 Do not spider links on certain ports:
 
+    Spidr.site('http://company.com/', ignore_ports: [8000, 8010, 8080])
+
+Do not spider links blacklisted in robots.txt:
+
     Spidr.site(
-      'http://sketchy.content.com/',
-      :ignore_ports => [8000, 8010, 8080]
+      'http://company.com/',
+      robots: true
     )
 
 Print out visited URLs:
 
-    Spidr.site('http://rubyinside.org/') do |spider|
+    Spidr.site('http://www.rubyinside.com/') do |spider|
       spider.every_url { |url| puts url }
     end
 
@@ -96,7 +100,7 @@ Build a URL map of a site:
 
 Print out the URLs that could not be requested:
 
-    Spidr.site('http://sketchy.content.com/') do |spider|
+    Spidr.site('http://company.com/') do |spider|
       spider.every_failed_url { |url| puts url }
     end
 
@@ -118,22 +122,22 @@ Finds all pages which have broken links:
 
 Search HTML and XML pages:
 
-    Spidr.site('http://company.withablog.com/') do |spider|
+    Spidr.site('http://company.com/') do |spider|
       spider.every_page do |page|
-        puts "[-] #{page.url}"
+        puts ">>> #{page.url}"
 
         page.search('//meta').each do |meta|
           name = (meta.attributes['name'] || meta.attributes['http-equiv'])
           value = meta.attributes['content']
 
-          puts "    #{name} = #{value}"
+          puts "  #{name} = #{value}"
         end
       end
     end
 
 Print out the titles from every page:
 
-    Spidr.site('http://www.rubypulse.com/') do |spider|
+    Spidr.site('https://www.ruby-lang.org/') do |spider|
       spider.every_html_page do |page|
         puts page.title
       end
@@ -143,7 +147,7 @@ Find what kinds of web servers a host is using, by accessing the headers:
 
     servers = Set[]
 
-    Spidr.host('generic.company.com') do |spider|
+    Spidr.host('company.com') do |spider|
       spider.all_headers do |headers|
         servers << headers['server']
       end
@@ -151,7 +155,7 @@ Find what kinds of web servers a host is using, by accessing the headers:
 
 Pause the spider on a forbidden page:
 
-    spider = Spidr.host('overnight.startup.com') do |spider|
+    spider = Spidr.host('company.com') do |spider|
       spider.every_forbidden_page do |page|
         spider.pause!
       end
@@ -159,7 +163,7 @@ Pause the spider on a forbidden page:
 
 Skip the processing of a page:
 
-    Spidr.host('sketchy.content.com') do |spider|
+    Spidr.host('company.com') do |spider|
       spider.every_missing_page do |page|
         spider.skip_page!
       end
@@ -167,7 +171,7 @@ Skip the processing of a page:
 
 Skip the processing of links:
 
-    Spidr.host('sketchy.content.com') do |spider|
+    Spidr.host('company.com') do |spider|
       spider.every_url do |url|
         if url.path.split('/').find { |dir| dir.to_i > 1000 }
           spider.skip_link!
@@ -177,14 +181,18 @@ Skip the processing of links:
 
 ## Requirements
 
-* [nokogiri](http://nokogiri.rubyforge.org/) ~> 1.3
+* [ruby] >= 1.9.1
+* [nokogiri] ~> 1.3
 
 ## Install
 
-    $ sudo gem install spidr
+    $ gem install spidr
 
 ## License
 
-Copyright (c) 2008-2011 Hal Brodigan
+Copyright (c) 2008-2016 Hal Brodigan
 
 See {file:LICENSE.txt} for license information.
+
+[ruby]: https://www.ruby-lang.org/
+[nokogiri]: http://www.nokogiri.org/

data/Rakefile

@@ -1,25 +1,14 @@
 require 'rubygems'
 
 begin
-  require 'bundler'
+  require 'bundler/setup'
 rescue LoadError => e
-  STDERR.puts e.message
-  STDERR.puts "Run `gem install bundler` to install Bundler."
-  exit e.status_code
-end
-
-begin
-  Bundler.setup(:development)
-rescue Bundler::BundlerError => e
-  STDERR.puts e.message
-  STDERR.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
+  abort e.message
 end
 
 require 'rake'
-
-require 'ore/tasks'
-Ore::Tasks.new
+require 'rubygems/tasks'
+Gem::Tasks.new
 
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new

data/gemspec.yml

@@ -8,12 +8,13 @@ description:
 license: MIT
 authors: Postmodern
 email: postmodern.mod3@gmail.com
-homepage: http://github.com/postmodern/spidr
+homepage: https://github.com/postmodern/spidr#readme
 has_yard: true
 
+required_ruby_version: ">= 1.9.1"
+
 dependencies:
   nokogiri: ~> 1.3
 
 development_dependencies:
   bundler: ~> 1.0
-  yard: ~> 0.7

data/lib/spidr/agent.rb

@@ -1,7 +1,7 @@
-require 'spidr/sanitizers'
-require 'spidr/filters'
-require 'spidr/events'
-require 'spidr/actions'
+require 'spidr/agent/sanitizers'
+require 'spidr/agent/filters'
+require 'spidr/agent/events'
+require 'spidr/agent/actions'
 require 'spidr/page'
 require 'spidr/session_cache'
 require 'spidr/cookie_jar'
@@ -12,48 +12,72 @@ require 'openssl'
 require 'net/http'
 require 'set'
 
+begin
+  require 'robots'
+rescue LoadError
+end
+
 module Spidr
   class Agent
 
-    include Sanitizers
-    include Filters
-    include Events
-    include Actions
-
     # HTTP Host Header to use
+    #
+    # @return [String]
     attr_accessor :host_header
 
     # HTTP Host Headers to use for specific hosts
+    #
+    # @return [Hash{String,Regexp => String}]
     attr_reader :host_headers
 
     # User-Agent to use
+    #
+    # @return [String]
     attr_accessor :user_agent
 
     # HTTP Authentication credentials
+    #
+    # @return [AuthStore]
     attr_accessor :authorized
 
     # Referer to use
+    #
+    # @return [String]
     attr_accessor :referer
 
     # Delay in between fetching pages
+    #
+    # @return [Integer]
     attr_accessor :delay
 
     # History containing visited URLs
+    #
+    # @return [Set<URI::HTTP>]
     attr_reader :history
 
     # List of unreachable URLs
+    #
+    # @return [Set<URI::HTTP>]
     attr_reader :failures
 
     # Queue of URLs to visit
+    #
+    # @return [Array<URI::HTTP>]
     attr_reader :queue
 
     # Cached cookies
+    #
+    # @return [CookieJar]
     attr_reader :cookies
     
     # Maximum depth
+    #
+    # @return [Integer]
     attr_reader :max_depth
 
     # The visited URLs and their depth within a site
+    #
+    # @return [Hash{URI::HTTP => Integer}]
     attr_reader :levels
 
     #
@@ -101,6 +125,9 @@ module Spidr
     # @option options [Integer] :max_depth
     #   The maximum link depth to follow.
     #
+    # @option options [Boolean] :robots (Spidr.robots?)
+    #   Specifies whether `robots.txt` should be honored.
+    #
     # @yield [agent]
     #   If a block is given, it will be passed the newly created agent
     #   for further configuration.
@@ -108,8 +135,13 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
+    # @see #initialize_sanitizers
+    # @see #initialize_filters
+    # @see #initialize_actions
+    # @see #initialize_events
+    #
     def initialize(options={})
-      @host_header = options[:host_header]
+      @host_header  = options[:host_header]
       @host_headers = {}
 
       if options[:host_headers]
@@ -117,21 +149,29 @@ module Spidr
       end
 
       @user_agent = options.fetch(:user_agent,Spidr.user_agent)
-      @referer = options[:referer]
+      @referer    = options[:referer]
 
-      @sessions = SessionCache.new(options.fetch(:proxy,Spidr.proxy))
-      @cookies = CookieJar.new
+      @sessions   = SessionCache.new(options.fetch(:proxy,Spidr.proxy))
+      @cookies    = CookieJar.new
       @authorized = AuthStore.new
 
-      @running = false
-      @delay = options.fetch(:delay,0)
-      @history = Set[]
+      @running  = false
+      @delay    = options.fetch(:delay,0)
+      @history  = Set[]
       @failures = Set[]
-      @queue = []
+      @queue    = []
 
-      @levels = Hash.new(0)
+      @levels    = Hash.new(0)
       @max_depth = options[:max_depth]
 
+      if options.fetch(:robots,Spidr.robots?)
+        unless Object.const_defined?(:Robots)
+          raise(ArgumentError,":robots option given but unable to require 'robots' gem")
+        end
+
+        @robots = Robots.new(@user_agent)
+      end
+
       initialize_sanitizers(options)
       initialize_filters(options)
       initialize_actions(options)
@@ -156,6 +196,9 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
+    # @see #initialize
+    # @see #start_at
+    #
     def self.start_at(url,options={},&block)
       agent = new(options,&block)
       agent.start_at(url)
@@ -177,17 +220,19 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
+    # @see #initialize
+    #
     def self.site(url,options={},&block)
       url = URI(url.to_s) unless url.kind_of?(URI)
 
-      agent = new(options.merge(:host => url.host),&block)
+      agent = new(options.merge(host: url.host),&block)
       agent.start_at(url)
     end
 
     #
     # Creates a new agent and spiders the given host.
     #
-    # @param [String]
+    # @param [String] name
     #   The host-name to spider.
     #
     # @param [Hash] options
@@ -200,8 +245,11 @@ module Spidr
     # @yieldparam [Agent] agent
     #   The newly created agent.
     #
+    # @see #initialize
+    #
     def self.host(name,options={},&block)
-      site(URI::HTTP.build(:host => name, :path => '/'),options,&block)
+      agent = new(options.merge(host: name),&block)
+      agent.start_at(URI::HTTP.build(host: name, path: '/'))
     end
 
     #
@@ -315,11 +363,9 @@ module Spidr
       @history.clear
 
       new_history.each do |url|
-        @history << unless url.kind_of?(URI)
-                      URI(url.to_s)
-                    else
-                      url
-                    end
+        url = URI(url.to_s) unless url.kind_of?(URI)
+
+        @history << url
       end
 
       return @history
@@ -362,10 +408,23 @@ module Spidr
       return @history.include?(url)
     end
 
+    #
+    # Determines whether a URL is allowed by the robot policy.
+    #
+    # @param [URI::HTTP, String] url
+    #   The URL to check.
+    #
+    # @return [Boolean]
+    #   Specifies whether a URL is allowed by the robot policy.
+    #
+    def robot_allowed?(url)
+      @robots ? @robots.allowed?(url) : true
+    end
+
     #
     # Sets the list of failed URLs.
     #
-    # @param [#each]
+    # @param [#each] new_failures
     #   The new list of failed URLs.
     #
     # @return [Array<URI::HTTP>]
@@ -378,11 +437,9 @@ module Spidr
       @failures.clear
 
       new_failures.each do |url|
-        @failures << unless url.kind_of?(URI)
-                       URI(url.to_s)
-                     else
-                       url
-                     end
+        url = URI(url.to_s) unless url.kind_of?(URI)
+
+        @failures << url
       end
 
       return @failures
@@ -408,7 +465,7 @@ module Spidr
     #
     # Sets the queue of URLs to visit.
     #
-    # @param [#each]
+    # @param [#each] new_queue
     #   The new list of URLs to visit.
     #
     # @return [Array<URI::HTTP>]
@@ -421,11 +478,9 @@ module Spidr
       @queue.clear
 
       new_queue.each do |url|
-        @queue << unless url.kind_of?(URI)
-                    URI(url.to_s)
-                  else
-                    url
-                  end
+        url = URI(url.to_s) unless url.kind_of?(URI)
+
+        @queue << url
       end
 
       return @queue
@@ -542,7 +597,7 @@ module Spidr
     # @since 0.2.2
     #
     def post_page(url,post_data='')
-      url = URI(url.to_s)
+      url = URI(url.to_s) unless url.kind_of?(URI)
 
       prepare_request(url) do |session,path,headers|
         new_page = Page.new(url,session.post(path,post_data,headers))
@@ -616,7 +671,7 @@ module Spidr
     #   the `queue` of the agent.
     #
     def to_hash
-      {:history => @history, :queue => @queue}
+      {history: @history, queue: @queue}
     end
 
     protected
@@ -666,9 +721,9 @@ module Spidr
         end
       end
 
-      headers['Host'] ||= @host_header if @host_header
+      headers['Host']     ||= @host_header if @host_header
       headers['User-Agent'] = @user_agent if @user_agent
-      headers['Referer'] = @referer if @referer
+      headers['Referer']    = @referer if @referer
 
       if (authorization = @authorized.for_url(url))
         headers['Authorization'] = "Basic #{authorization}"
@@ -687,7 +742,8 @@ module Spidr
              SocketError,
              IOError,
              OpenSSL::SSL::SSLError,
-             Net::HTTPBadResponse
+             Net::HTTPBadResponse,
+             Zlib::Error
 
         @sessions.kill!(url)
 
@@ -722,7 +778,8 @@ module Spidr
        visit_port?(url.port) &&
        visit_link?(url.to_s) &&
        visit_url?(url) &&
-       visit_ext?(url.path)
+       visit_ext?(url.path) &&
+       robot_allowed?(url.to_s)
     end
 
     #