mechanize 2.0.pre.1 → 2.0.pre.2

data.tar.gz.sig

@@ -1,2 +1,2 @@
-W�y�@����r�?� 	\��{X9�=�KҪ���f�U#�s�H������9=%v�$�j�\�6�,V��I�1|[ΘF�U�r���H�� ����A���x�1cX��	�XH��0r	S�=�kv�1He�I֋6��hBk�+�;��EW`X�h��	~�I�۳&�������E�.')r�UY^\���X�<��h ��\ۯ�Z
-���)e�)�A��3wo�m�k:;���%	Ѿ�7ŗ�H�ݟ�+;N��IJ�̒���:
+����:�0*��r��jkb�V�w!��\y�	�������]��F}6<��R@x�8D	�f^�[�^d��
+�*����%94��6�ˉ���g8ϗ�o�~�ƛ̰����X?�B��8�6s�����:�S�M�ѡ\φC>�9xR���R��@v�&�^�&�*$���;)��on۱�m`��y��b�/���j�ѱ�n��Q��f+��g8W `�a�

data/CHANGELOG.rdoc

@@ -1,11 +1,11 @@
 = Mechanize CHANGELOG
 
-=== 2.0.pre.1 / 2011-04-09
+=== 2.0.pre.2 / 2011-04-17
 
 Mechanize is now under the MIT license
 
 * API changes
-  * WWW::Mechanize has been removed.
+  * WWW::Mechanize has been removed.  Use Mechanize.
   * Pre connect hooks are now called with the agent and the request.  See
     Mechanize#pre_connect_hooks.
   * Post connect hooks are now called with the agent and the response.  See
@@ -23,6 +23,16 @@ Mechanize is now under the MIT license
   * The User-Agent header has changed.  It no longer includes the WWW- prefix
     and now includes the ruby version.  The URL has been updated as well.
   * Mechanize now requires ruby 1.8.7 or newer.
+  * Hpricot support has been removed as webrobots requires nokogiri.
+  * Mechanize#get no longer accepts the referer as the second argument.
+  * Mechanize#get no longer allows the HTTP method to be changed (:verb
+    option).
+
+* Deprecations
+  * Mechanize#get with an options hash is deprecated and will be removed after
+    October, 2011.
+  * Mechanize::Util::to_native_charset is deprecated as it is no longer used
+    by Mechanize.
 
 * New Features
 
@@ -39,6 +49,17 @@ Mechanize is now under the MIT license
   * Mechanize now allows a certificate and key to be passed directly.  GH #71
   * Mechanize::Form::MultiSelectList now implements #option_with and
     #options_with.  GH #42
+  * Add Mechanize::Page::Link#rel and #rel?(kind) to read and test the rel
+    attribute.
+  * Add Mechanize::Page#canonical_uri to read a </tt><link
+    rel="canonical"></tt> tag.
+  * Add support for Robots Exclusion Protocol (i.e. robots.txt) and
+    nofollow/noindex in meta tags and the rel attribute.  Automatic
+    exclusion can be turned on by setting:
+      agent.robots = true
+  * Manual robots.txt test can be performed with
+    Mechanize#robots_allowed? and #robots_disallowed?.
+  * Mechanize::Form now supports the accept-charset attribute.  GH #96
 
 * Bug Fixes:
 
@@ -63,6 +84,7 @@ Mechanize is now under the MIT license
   * Content-Encoding: x-gzip is now treated like gzip per RFC 2616.
   * Mechanize now unescapes URIs for meta refresh.  GH #68
   * Mechanize now has more robust HTML charset detection.  GH #43
+  * Mechanize::Form::Textarea is now created from a textarea element.  GH #94
 
 === 1.0.0
 

data/Manifest.txt

@@ -46,8 +46,8 @@ lib/mechanize/pluggable_parsers.rb
 lib/mechanize/redirect_limit_reached_error.rb
 lib/mechanize/redirect_not_get_or_head_error.rb
 lib/mechanize/response_code_error.rb
+lib/mechanize/robots_disallowed_error.rb
 lib/mechanize/unsupported_scheme_error.rb
-lib/mechanize/uri_resolver.rb
 lib/mechanize/util.rb
 test/data/htpasswd
 test/data/server.crt
@@ -58,6 +58,7 @@ test/helper.rb
 test/htdocs/alt_text.html
 test/htdocs/bad_form_test.html
 test/htdocs/button.jpg
+test/htdocs/canonical_uri.html
 test/htdocs/dir with spaces/foo.html
 test/htdocs/empty_form.html
 test/htdocs/file_upload.html
@@ -79,8 +80,14 @@ test/htdocs/index.html
 test/htdocs/link with space.html
 test/htdocs/meta_cookie.html
 test/htdocs/no_title_test.html
+test/htdocs/nofollow.html
+test/htdocs/noindex.html
+test/htdocs/norobots.html
 test/htdocs/rails_3_encoding_hack_form_test.html
+test/htdocs/rel_nofollow.html
 test/htdocs/relative/tc_relative_links.html
+test/htdocs/robots.html
+test/htdocs/robots.txt
 test/htdocs/tc_bad_charset.html
 test/htdocs/tc_bad_links.html
 test/htdocs/tc_base_images.html
@@ -106,25 +113,12 @@ test/htdocs/test_click.html
 test/htdocs/unusual______.html
 test/servlets.rb
 test/ssl_server.rb
-test/test_authenticate.rb
-test/test_bad_links.rb
-test/test_blank_form.rb
-test/test_checkboxes.rb
-test/test_content_type.rb
 test/test_cookies.rb
-test/test_encoded_links.rb
-test/test_errors.rb
-test/test_field_precedence.rb
-test/test_follow_meta.rb
 test/test_form_action.rb
 test/test_form_as_hash.rb
 test/test_form_button.rb
 test/test_form_no_inputname.rb
-test/test_forms.rb
 test/test_frames.rb
-test/test_get_headers.rb
-test/test_gzipping.rb
-test/test_hash_api.rb
 test/test_headers.rb
 test/test_history.rb
 test/test_history_added.rb
@@ -132,17 +126,20 @@ test/test_html_unscape_forms.rb
 test/test_if_modified_since.rb
 test/test_images.rb
 test/test_links.rb
-test/test_mech.rb
-test/test_mech_proxy.rb
 test/test_mechanize.rb
 test/test_mechanize_cookie.rb
 test/test_mechanize_cookie_jar.rb
 test/test_mechanize_file.rb
 test/test_mechanize_file_request.rb
 test/test_mechanize_file_response.rb
+test/test_mechanize_form.rb
+test/test_mechanize_form_check_box.rb
+test/test_mechanize_form_encoding.rb
+test/test_mechanize_form_field.rb
 test/test_mechanize_form_image_button.rb
 test/test_mechanize_page.rb
-test/test_mechanize_uri_resolver.rb
+test/test_mechanize_redirect_not_get_or_head_error.rb
+test/test_mechanize_subclass.rb
 test/test_mechanize_util.rb
 test/test_meta.rb
 test/test_multi_select.rb
@@ -153,11 +150,11 @@ test/test_post_form.rb
 test/test_pretty_print.rb
 test/test_radiobutton.rb
 test/test_redirect_limit_reached.rb
-test/test_redirect_verb_handling.rb
 test/test_referer.rb
 test/test_relative_links.rb
 test/test_request.rb
 test/test_response_code.rb
+test/test_robots.rb
 test/test_save_file.rb
 test/test_scheme.rb
 test/test_select.rb
@@ -166,7 +163,6 @@ test/test_select_none.rb
 test/test_select_noopts.rb
 test/test_set_fields.rb
 test/test_ssl_server.rb
-test/test_subclass.rb
 test/test_textarea.rb
 test/test_upload.rb
 test/test_verbs.rb

data/Rakefile

@@ -12,9 +12,12 @@ Hoe.spec 'mechanize' do
   self.readme_file      = 'README.rdoc'
   self.history_file     = 'CHANGELOG.rdoc'
   self.extra_rdoc_files += Dir['*.rdoc']
-  self.extra_deps       << ['nokogiri', '~> 1.4']
-  self.extra_deps       << ['net-http-persistent', '~> 1.6']
-  self.extra_deps       << ['net-http-digest_auth', '~> 1.1', '>= 1.1.1']
+
+  self.extra_deps << ['nokogiri',             '~> 1.4']
+  self.extra_deps << ['net-http-persistent',  '~> 1.6']
+  self.extra_deps << ['net-http-digest_auth', '~> 1.1', '>= 1.1.1']
+  self.extra_deps << ['webrobots',            '~> 0.0', '>= 0.0.6']
+
   self.spec_extras[:required_ruby_version] = '>= 1.8.7'
 end
 

data/lib/mechanize.rb

@@ -12,7 +12,6 @@ require 'uri'
 require 'webrick/httputils'
 require 'zlib'
 
-
 # = Synopsis
 # The Mechanize library is used for automating interaction with a website.  It
 # can follow links, and submit forms.  Form fields can be populated and
@@ -71,7 +70,7 @@ class Mechanize
   attr_accessor  :read_timeout
 
   # The identification string for the client initiating a web request
-  attr_accessor :user_agent
+  attr_reader :user_agent
 
   # The value of watch_for_set is passed to pluggable parsers for retrieved
   # content
@@ -96,6 +95,15 @@ class Mechanize
   # redirects are followed.
   attr_accessor :redirect_ok
 
+  # Says this agent should consult the site's robots.txt for each access.
+  attr_reader :robots
+
+  def robots=(value)
+    require 'webrobots' if value
+    @webrobots = nil if value != @robots
+    @robots = value
+  end
+
   # Disables HTTP/1.1 gzip compression (enabled by default)
   attr_accessor :gzip_enabled
 
@@ -198,6 +206,9 @@ class Mechanize
     @follow_meta_refresh  = false
     @redirection_limit    = 20
 
+    @robots         = false
+    @webrobots      = nil
+
     # Connection Cache & Keep alive
     @keep_alive_time  = 300
     @keep_alive       = true
@@ -208,8 +219,16 @@ class Mechanize
     @proxy_user = nil
     @proxy_pass = nil
 
-    @resolver = Mechanize::URIResolver.new
-    @scheme_handlers = @resolver.scheme_handlers
+    @scheme_handlers = Hash.new { |h, scheme|
+      h[scheme] = lambda { |link, page|
+        raise Mechanize::UnsupportedSchemeError, scheme
+      }
+    }
+
+    @scheme_handlers['http']      = lambda { |link, page| link }
+    @scheme_handlers['https']     = @scheme_handlers['http']
+    @scheme_handlers['relative']  = @scheme_handlers['http']
+    @scheme_handlers['file']      = @scheme_handlers['http']
 
     @pre_connect_hooks = []
     @post_connect_hooks = []
@@ -243,6 +262,11 @@ class Mechanize
     nil
   end
 
+  def user_agent=(value)
+    @webrobots = nil if value != @user_agent
+    @user_agent = value
+  end
+
   # Set the user agent for the Mechanize object.
   # See AGENT_ALIASES
   def user_agent_alias=(al)
@@ -263,17 +287,15 @@ class Mechanize
   alias :basic_auth :auth
 
   # Fetches the URL passed in and returns a page.
-  def get(options, parameters = [], referer = nil)
+  def get(uri, parameters = [], referer = nil, headers = {})
     method = :get
 
-    unless options.is_a? Hash
-      url = options
-      unless parameters.respond_to?(:each) # FIXME: Remove this in 0.8.0
-        referer = parameters
-        parameters = []
-      end
-    else
-      raise ArgumentError, "url must be specified" unless url = options[:url]
+    if Hash === uri then
+      options = uri
+      location = Gem.location_of_caller.join ':'
+      warn "#{location}: Mechanize#get with options hash is deprecated and will be removed October 2011"
+
+      raise ArgumentError, "url must be specified" unless uri = options[:url]
       parameters = options[:params] || []
       referer    = options[:referer]
       headers    = options[:headers]
@@ -281,7 +303,7 @@ class Mechanize
     end
 
     unless referer
-      if url.to_s =~ %r{\Ahttps?://}
+      if uri.to_s =~ %r{\Ahttps?://}
         referer = Page.new(nil, {'content-type'=>'text/html'})
       else
         referer = current_page || Page.new(nil, {'content-type'=>'text/html'})
@@ -299,7 +321,7 @@ class Mechanize
 
     # fetch the page
     headers ||= {}
-    page = fetch_page url, method, headers, parameters, referer
+    page = fetch_page uri, method, headers, parameters, referer
     add_to_history(page)
     yield page if block_given?
     page
@@ -347,6 +369,14 @@ class Mechanize
   # Mechanize::Page::Link object passed in. Returns the page fetched.
   def click(link)
     case link
+    when Page::Link
+      referer = link.page || current_page()
+      if robots
+        if (referer.is_a?(Page) && referer.parser.nofollow?) || link.rel?('nofollow')
+          raise RobotsDisallowedError.new(link.href)
+        end
+      end
+      get link.href, [], referer
     when String, Regexp
       if real_link = page.link_with(:text => link)
         click real_link
@@ -359,10 +389,10 @@ class Mechanize
         submit form, button if form
       end
     else
-      referer = link.page rescue referer = nil
+      referer = current_page()
       href = link.respond_to?(:href) ? link.href :
         (link['href'] || link['src'])
-      get(:url => href, :referer => (referer || current_page()))
+      get href, [], referer
     end
   end
 
@@ -421,11 +451,10 @@ class Mechanize
     when 'POST'
       post_form(form.action, form, headers)
     when 'GET'
-      get(  :url      => form.action.gsub(/\?[^\?]*$/, ''),
-            :params   => form.build_query,
-            :headers  => headers,
-            :referer  => form.page
-            )
+      get(form.action.gsub(/\?[^\?]*$/, ''),
+          form.build_query,
+          form.page,
+          headers)
     else
       raise ArgumentError, "unsupported method: #{form.method.upcase}"
     end
@@ -473,6 +502,36 @@ class Mechanize
     end
   end
 
+  # Tests if this agent is allowed to access +url+, consulting the
+  # site's robots.txt.
+  def robots_allowed?(uri)
+    return true if uri.request_uri == '/robots.txt'
+
+    webrobots.allowed?(uri)
+  end
+
+  # Equivalent to !robots_allowed?(url).
+  def robots_disallowed?(url)
+    !webrobots.allowed?(url)
+  end
+
+  # Returns an error object if there is an error in fetching or
+  # parsing robots.txt of the site +url+.
+  def robots_error(url)
+    webrobots.error(url)
+  end
+
+  # Raises the error if there is an error in fetching or parsing
+  # robots.txt of the site +url+.
+  def robots_error!(url)
+    webrobots.error!(url)
+  end
+
+  # Removes robots.txt cache for the site +url+.
+  def robots_reset(url)
+    webrobots.reset(url)
+  end
+
   alias :page :current_page
 
   def connection_for uri
@@ -608,6 +667,69 @@ class Mechanize
     request['User-Agent'] = @user_agent if @user_agent
   end
 
+  def resolve(uri, referer = current_page())
+    uri = uri.dup if uri.is_a?(URI)
+
+    unless uri.is_a?(URI)
+      uri = uri.to_s.strip.gsub(/[^#{0.chr}-#{126.chr}]/o) { |match|
+        if RUBY_VERSION >= "1.9.0"
+          Mechanize::Util.uri_escape(match)
+        else
+          sprintf('%%%X', match.unpack($KCODE == 'UTF8' ? 'U' : 'C')[0])
+        end
+      }
+
+      unescaped = uri.split(/(?:%[0-9A-Fa-f]{2})+|#/)
+      escaped   = uri.scan(/(?:%[0-9A-Fa-f]{2})+|#/)
+
+      escaped_uri = Mechanize::Util.html_unescape(
+        unescaped.zip(escaped).map { |x,y|
+          "#{WEBrick::HTTPUtils.escape(x)}#{y}"
+        }.join('')
+      )
+
+      begin
+        uri = URI.parse(escaped_uri)
+      rescue
+        uri = URI.parse(WEBrick::HTTPUtils.escape(escaped_uri))
+      end
+    end
+
+    scheme = uri.relative? ? 'relative' : uri.scheme.downcase
+    uri = @scheme_handlers[scheme].call(uri, referer)
+
+    if referer && referer.uri
+      if uri.path.length == 0 && uri.relative?
+        uri.path = referer.uri.path
+      end
+    end
+
+    uri.path = '/' if uri.path.length == 0
+
+    if uri.relative?
+      raise ArgumentError, "absolute URL needed (not #{uri})" unless
+        referer && referer.uri
+
+      base = nil
+      if referer.respond_to?(:bases) && referer.parser
+        base = referer.bases.last
+      end
+
+      uri = ((base && base.uri && base.uri.absolute?) ?
+             base.uri :
+             referer.uri) + uri
+      uri = referer.uri + uri
+      # Strip initial "/.." bits from the path
+      uri.path.sub!(/^(\/\.\.)+(?=\/)/, '')
+    end
+
+    unless ['http', 'https', 'file'].include?(uri.scheme.downcase)
+      raise ArgumentError, "unsupported scheme: #{uri.scheme}"
+    end
+
+    uri
+  end
+
   def resolve_parameters uri, method, parameters
     case method
     when :head, :get, :delete, :trace then
@@ -707,7 +829,7 @@ class Mechanize
     response.read_body { |part|
       total += part.length
       body.write(part)
-      log.debug("Read #{total} bytes") if log
+      log.debug("Read #{part.length} bytes (#{total} total)") if log
     }
 
     body.rewind
@@ -814,8 +936,15 @@ class Mechanize
 
   private
 
-  def resolve(url, referer = current_page())
-    @resolver.resolve(url, referer).to_s
+  def webrobots_http_get(uri)
+    get_file(uri)
+  rescue Mechanize::ResponseCodeError => e
+    return '' if e.response_code == '404'
+    raise e
+  end
+
+  def webrobots
+    @webrobots ||= WebRobots.new(@user_agent, :http_get => method(:webrobots_http_get))
   end
 
   def set_http proxy = nil
@@ -868,7 +997,7 @@ class Mechanize
                  referer = current_page, redirects = 0
     referer_uri = referer ? referer.uri : nil
 
-    uri = @resolver.resolve uri, referer
+    uri = resolve uri, referer
 
     uri, params = resolve_parameters uri, method, params
 
@@ -889,6 +1018,11 @@ class Mechanize
 
     pre_connect request
 
+    # Consult robots.txt
+    if robots && uri.is_a?(URI::HTTP)
+      robots_allowed?(uri) or raise RobotsDisallowedError.new(uri)
+    end
+
     # Add If-Modified-Since if page is in history
     if (page = visited_page(uri)) and page.response['Last-Modified']
       request['If-Modified-Since'] = page.response['Last-Modified']
@@ -921,7 +1055,13 @@ class Mechanize
     return meta if meta
 
     case response
-    when Net::HTTPSuccess, Mechanize::FileResponse
+    when Net::HTTPSuccess
+      if robots && page.is_a?(Page)
+        page.parser.noindex? and raise RobotsDisallowedError.new(uri)
+      end
+
+      page
+    when Mechanize::FileResponse
       page
     when Net::HTTPNotModified
       log.debug("Got cached page") if log
@@ -958,7 +1098,7 @@ require 'mechanize/pluggable_parsers'
 require 'mechanize/redirect_limit_reached_error'
 require 'mechanize/redirect_not_get_or_head_error'
 require 'mechanize/response_code_error'
+require 'mechanize/robots_disallowed_error'
 require 'mechanize/unsupported_scheme_error'
-require 'mechanize/uri_resolver'
 require 'mechanize/util'