mechanize 2.0.pre.1 → 2.0.pre.2

data/test/test_robots.rb

@@ -0,0 +1,87 @@
+require "helper"
+
+class TestRobots < Test::Unit::TestCase
+  def setup
+    @agent = Mechanize.new
+    @robot = Mechanize.new { |a|
+      a.robots = true
+    }
+  end
+
+  def test_mechanize_webrobots_http_get
+    robotstxt = @agent.__send__(:webrobots_http_get, 'http://localhost/robots.txt')
+    assert_not_equal '', robotstxt
+
+    robotstxt = @agent.__send__(:webrobots_http_get, 'http://localhost/response_code?code=404')
+    assert_equal '', robotstxt
+  end
+
+  def test_robots
+    assert_equal "Welcome!", @robot.get("http://localhost/robots.html").title
+
+    assert_raise(Mechanize::RobotsDisallowedError) {
+      @robot.get("http://localhost/norobots.html")
+    }
+  end
+
+  def test_robots_allowed_eh
+    allowed    = URI.parse 'http://localhost/robots.html'
+    disallowed = URI.parse 'http://localhost/norobots.html'
+    assert  @agent.robots_allowed?(allowed)
+    assert !@agent.robots_allowed?(disallowed)
+
+    assert !@agent.robots_disallowed?(allowed)
+    assert  @agent.robots_disallowed?(disallowed)
+  end
+
+  def test_noindex
+    assert_nothing_raised {
+      @agent.get("http://localhost/noindex.html")
+    }
+
+    assert @robot.robots_allowed?(URI.parse("http://localhost/noindex.html"))
+    assert_raise(Mechanize::RobotsDisallowedError) {
+      @robot.get("http://localhost/noindex.html")
+    }
+  end
+
+  def test_nofollow
+    page = @agent.get("http://localhost/nofollow.html")
+
+    assert_nothing_raised {
+      page.links[0].click
+    }
+    assert_nothing_raised {
+      page.links[1].click
+    }
+
+    page = @robot.get("http://localhost/nofollow.html")
+
+    assert_raise(Mechanize::RobotsDisallowedError) {
+      page.links[0].click
+    }
+    assert_raise(Mechanize::RobotsDisallowedError) {
+      page.links[1].click
+    }
+  end
+
+  def test_rel_nofollow
+    page = @agent.get("http://localhost/rel_nofollow.html")
+
+    assert_nothing_raised {
+      page.links[0].click
+    }
+    assert_nothing_raised {
+      page.links[1].click
+    }
+
+    page = @robot.get("http://localhost/rel_nofollow.html")
+
+    assert_nothing_raised {
+      page.links[0].click
+    }
+    assert_raise(Mechanize::RobotsDisallowedError) {
+      page.links[1].click
+    }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification 
 name: mechanize
 version: !ruby/object:Gem::Version 
-  hash: 1923831931
+  hash: 1923831933
   prerelease: 
   segments: 
   - 2
   - 0
   - pre
-  - 1
-  version: 2.0.pre.1
+  - 2
+  version: 2.0.pre.2
 platform: ruby
 authors: 
 - Eric Hodel
@@ -39,7 +39,7 @@ cert_chain:
   x52qPcexcYZR7w==
   -----END CERTIFICATE-----
 
-date: 2011-04-10 00:00:00 Z
+date: 2011-04-18 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
@@ -95,9 +95,32 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: hoe
+  name: webrobots
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 0
+        - 0
+        version: "0.0"
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 19
+        segments: 
+        - 0
+        - 0
+        - 6
+        version: 0.0.6
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -109,7 +132,7 @@ dependencies:
         - 4
         version: 2.9.4
   type: :development
-  version_requirements: *id004
+  version_requirements: *id005
 description: |-
   The Mechanize library is used for automating interaction with websites.
   Mechanize automatically stores and sends cookies, follows redirects,
@@ -181,8 +204,8 @@ files:
 - lib/mechanize/redirect_limit_reached_error.rb
 - lib/mechanize/redirect_not_get_or_head_error.rb
 - lib/mechanize/response_code_error.rb
+- lib/mechanize/robots_disallowed_error.rb
 - lib/mechanize/unsupported_scheme_error.rb
-- lib/mechanize/uri_resolver.rb
 - lib/mechanize/util.rb
 - test/data/htpasswd
 - test/data/server.crt
@@ -193,6 +216,7 @@ files:
 - test/htdocs/alt_text.html
 - test/htdocs/bad_form_test.html
 - test/htdocs/button.jpg
+- test/htdocs/canonical_uri.html
 - test/htdocs/dir with spaces/foo.html
 - test/htdocs/empty_form.html
 - test/htdocs/file_upload.html
@@ -214,8 +238,14 @@ files:
 - test/htdocs/link with space.html
 - test/htdocs/meta_cookie.html
 - test/htdocs/no_title_test.html
+- test/htdocs/nofollow.html
+- test/htdocs/noindex.html
+- test/htdocs/norobots.html
 - test/htdocs/rails_3_encoding_hack_form_test.html
+- test/htdocs/rel_nofollow.html
 - test/htdocs/relative/tc_relative_links.html
+- test/htdocs/robots.html
+- test/htdocs/robots.txt
 - test/htdocs/tc_bad_charset.html
 - test/htdocs/tc_bad_links.html
 - test/htdocs/tc_base_images.html
@@ -241,25 +271,12 @@ files:
 - test/htdocs/unusual______.html
 - test/servlets.rb
 - test/ssl_server.rb
-- test/test_authenticate.rb
-- test/test_bad_links.rb
-- test/test_blank_form.rb
-- test/test_checkboxes.rb
-- test/test_content_type.rb
 - test/test_cookies.rb
-- test/test_encoded_links.rb
-- test/test_errors.rb
-- test/test_field_precedence.rb
-- test/test_follow_meta.rb
 - test/test_form_action.rb
 - test/test_form_as_hash.rb
 - test/test_form_button.rb
 - test/test_form_no_inputname.rb
-- test/test_forms.rb
 - test/test_frames.rb
-- test/test_get_headers.rb
-- test/test_gzipping.rb
-- test/test_hash_api.rb
 - test/test_headers.rb
 - test/test_history.rb
 - test/test_history_added.rb
@@ -267,17 +284,20 @@ files:
 - test/test_if_modified_since.rb
 - test/test_images.rb
 - test/test_links.rb
-- test/test_mech.rb
-- test/test_mech_proxy.rb
 - test/test_mechanize.rb
 - test/test_mechanize_cookie.rb
 - test/test_mechanize_cookie_jar.rb
 - test/test_mechanize_file.rb
 - test/test_mechanize_file_request.rb
 - test/test_mechanize_file_response.rb
+- test/test_mechanize_form.rb
+- test/test_mechanize_form_check_box.rb
+- test/test_mechanize_form_encoding.rb
+- test/test_mechanize_form_field.rb
 - test/test_mechanize_form_image_button.rb
 - test/test_mechanize_page.rb
-- test/test_mechanize_uri_resolver.rb
+- test/test_mechanize_redirect_not_get_or_head_error.rb
+- test/test_mechanize_subclass.rb
 - test/test_mechanize_util.rb
 - test/test_meta.rb
 - test/test_multi_select.rb
@@ -288,11 +308,11 @@ files:
 - test/test_pretty_print.rb
 - test/test_radiobutton.rb
 - test/test_redirect_limit_reached.rb
-- test/test_redirect_verb_handling.rb
 - test/test_referer.rb
 - test/test_relative_links.rb
 - test/test_request.rb
 - test/test_response_code.rb
+- test/test_robots.rb
 - test/test_save_file.rb
 - test/test_scheme.rb
 - test/test_select.rb
@@ -301,7 +321,6 @@ files:
 - test/test_select_noopts.rb
 - test/test_set_fields.rb
 - test/test_ssl_server.rb
-- test/test_subclass.rb
 - test/test_textarea.rb
 - test/test_upload.rb
 - test/test_verbs.rb
@@ -343,25 +362,12 @@ signing_key:
 specification_version: 3
 summary: The Mechanize library is used for automating interaction with websites
 test_files: 
-- test/test_authenticate.rb
-- test/test_bad_links.rb
-- test/test_blank_form.rb
-- test/test_checkboxes.rb
-- test/test_content_type.rb
 - test/test_cookies.rb
-- test/test_encoded_links.rb
-- test/test_errors.rb
-- test/test_field_precedence.rb
-- test/test_follow_meta.rb
 - test/test_form_action.rb
 - test/test_form_as_hash.rb
 - test/test_form_button.rb
 - test/test_form_no_inputname.rb
-- test/test_forms.rb
 - test/test_frames.rb
-- test/test_get_headers.rb
-- test/test_gzipping.rb
-- test/test_hash_api.rb
 - test/test_headers.rb
 - test/test_history.rb
 - test/test_history_added.rb
@@ -369,17 +375,20 @@ test_files:
 - test/test_if_modified_since.rb
 - test/test_images.rb
 - test/test_links.rb
-- test/test_mech.rb
-- test/test_mech_proxy.rb
 - test/test_mechanize.rb
 - test/test_mechanize_cookie.rb
 - test/test_mechanize_cookie_jar.rb
 - test/test_mechanize_file.rb
 - test/test_mechanize_file_request.rb
 - test/test_mechanize_file_response.rb
+- test/test_mechanize_form.rb
+- test/test_mechanize_form_check_box.rb
+- test/test_mechanize_form_encoding.rb
+- test/test_mechanize_form_field.rb
 - test/test_mechanize_form_image_button.rb
 - test/test_mechanize_page.rb
-- test/test_mechanize_uri_resolver.rb
+- test/test_mechanize_redirect_not_get_or_head_error.rb
+- test/test_mechanize_subclass.rb
 - test/test_mechanize_util.rb
 - test/test_meta.rb
 - test/test_multi_select.rb
@@ -390,11 +399,11 @@ test_files:
 - test/test_pretty_print.rb
 - test/test_radiobutton.rb
 - test/test_redirect_limit_reached.rb
-- test/test_redirect_verb_handling.rb
 - test/test_referer.rb
 - test/test_relative_links.rb
 - test/test_request.rb
 - test/test_response_code.rb
+- test/test_robots.rb
 - test/test_save_file.rb
 - test/test_scheme.rb
 - test/test_select.rb
@@ -403,7 +412,6 @@ test_files:
 - test/test_select_noopts.rb
 - test/test_set_fields.rb
 - test/test_ssl_server.rb
-- test/test_subclass.rb
 - test/test_textarea.rb
 - test/test_upload.rb
 - test/test_verbs.rb

data/lib/mechanize/uri_resolver.rb

@@ -1,82 +0,0 @@
-class Mechanize::URIResolver
-
-  attr_reader :scheme_handlers
-
-  def initialize
-    @scheme_handlers = Hash.new { |h, scheme|
-      h[scheme] = lambda { |link, page|
-        raise Mechanize::UnsupportedSchemeError, scheme
-      }
-    }
-
-    @scheme_handlers['http']      = lambda { |link, page| link }
-    @scheme_handlers['https']     = @scheme_handlers['http']
-    @scheme_handlers['relative']  = @scheme_handlers['http']
-    @scheme_handlers['file']      = @scheme_handlers['http']
-  end
-
-  def resolve uri, referer = nil
-    uri = uri.dup if uri.is_a?(URI)
-
-    unless uri.is_a?(URI)
-      uri = uri.to_s.strip.gsub(/[^#{0.chr}-#{126.chr}]/o) { |match|
-        if RUBY_VERSION >= "1.9.0"
-          Mechanize::Util.uri_escape(match)
-        else
-          sprintf('%%%X', match.unpack($KCODE == 'UTF8' ? 'U' : 'C')[0])
-        end
-      }
-
-      unescaped = uri.split(/(?:%[0-9A-Fa-f]{2})+|#/)
-      escaped   = uri.scan(/(?:%[0-9A-Fa-f]{2})+|#/)
-
-      escaped_uri = Mechanize::Util.html_unescape(
-        unescaped.zip(escaped).map { |x,y|
-          "#{WEBrick::HTTPUtils.escape(x)}#{y}"
-        }.join('')
-      )
-
-      begin
-        uri = URI.parse(escaped_uri)
-      rescue
-        uri = URI.parse(WEBrick::HTTPUtils.escape(escaped_uri))
-      end
-    end
-
-    scheme = uri.relative? ? 'relative' : uri.scheme.downcase
-    uri = @scheme_handlers[scheme].call(uri, referer)
-
-    if referer && referer.uri
-      if uri.path.length == 0 && uri.relative?
-        uri.path = referer.uri.path
-      end
-    end
-
-    uri.path = '/' if uri.path.length == 0
-
-    if uri.relative?
-      raise ArgumentError, "absolute URL needed (not #{uri})" unless
-        referer && referer.uri
-
-      base = nil
-      if referer.respond_to?(:bases) && referer.parser
-        base = referer.bases.last
-      end
-
-      uri = ((base && base.uri && base.uri.absolute?) ?
-             base.uri :
-             referer.uri) + uri
-      uri = referer.uri + uri
-      # Strip initial "/.." bits from the path
-      uri.path.sub!(/^(\/\.\.)+(?=\/)/, '')
-    end
-
-    unless ['http', 'https', 'file'].include?(uri.scheme.downcase)
-      raise ArgumentError, "unsupported scheme: #{uri.scheme}"
-    end
-
-    uri
-  end
-
-end
-

data/test/test_authenticate.rb

@@ -1,71 +0,0 @@
-require "helper"
-require 'pp'
-
-class BasicAuthTest < Test::Unit::TestCase
-  def setup
-    @agent = Mechanize.new
-  end
-
-  def test_auth_success
-    @agent.basic_auth('user', 'pass')
-    page = @agent.get("http://localhost/basic_auth")
-    assert_equal('You are authenticated', page.body)
-  end
-
-  def test_digest_auth_success
-    @agent.basic_auth('user', 'pass')
-    page = @agent.get("http://localhost/digest_auth")
-    assert_equal('You are authenticated', page.body)
-  end
-
-  def test_no_duplicate_headers
-    block_called = false
-    @agent.pre_connect_hooks << lambda { |_, request|
-      block_called = true
-      request.to_hash.each do |k,v|
-        assert_equal(1, v.length)
-      end
-    }
-    @agent.basic_auth('user', 'pass')
-    @agent.get("http://localhost/digest_auth")
-    assert block_called
-  end
-
-  def test_post_auth_success
-    class << @agent
-      alias :old_fetch_page :fetch_page
-      attr_accessor :requests
-      def fetch_page(uri, method, *args)
-        @requests ||= []
-        x = old_fetch_page(uri, method, *args)
-        @requests << method
-        x
-      end
-    end
-    @agent.basic_auth('user', 'pass')
-    page = @agent.post("http://localhost/basic_auth")
-    assert_equal('You are authenticated', page.body)
-    assert_equal(2, @agent.requests.length)
-    r1 = @agent.requests[0]
-    r2 = @agent.requests[1]
-    assert_equal(r1, r2)
-  end
-
-  def test_auth_bad_user_pass
-    @agent.basic_auth('aaron', 'aaron')
-    begin
-      @agent.get("http://localhost/basic_auth")
-    rescue Mechanize::ResponseCodeError => e
-      assert_equal("401", e.response_code)
-    end
-  end
-
-  def test_auth_failure
-    begin
-      @agent.get("http://localhost/basic_auth")
-    rescue Mechanize::ResponseCodeError => e
-      assert_equal("401", e.response_code)
-    end
-  end
-
-end