mechanize 2.7.3 → 2.7.4

data/lib/mechanize/directory_saver.rb

@@ -16,7 +16,8 @@ class Mechanize::DirectorySaver < Mechanize::Download
   # Creates a DirectorySaver subclass that will save responses to the given
   # +directory+. If +options+ includes a +decode_filename+ value set to +true+
   # then the downloaded filename will be ran through +CGI.unescape+ before
-  # being saved.
+  # being saved. If +options+ includes a +overwrite+ value set to +true+ then
+  # downloaded file will be overwritten if two files with the same names exist.
 
   def self.save_to directory, options = {}
     directory = File.expand_path directory
@@ -41,6 +42,13 @@ class Mechanize::DirectorySaver < Mechanize::Download
     @options[:decode_filename]
   end
 
+  ##
+  # Checks if +overwrite+ parameter is set to true
+
+  def self.overwrite?
+    @options[:overwrite]
+  end
+
   ##
   # Saves the +body_io+ into the directory specified for this DirectorySaver
   # by save_to.  The filename is chosen by Mechanize::Parser#extract_filename.
@@ -58,7 +66,11 @@ class Mechanize::DirectorySaver < Mechanize::Download
     @filename = CGI.unescape(@filename) if self.class.decode_filename?
     path = File.join directory, @filename
 
-    save path
+    if self.class.overwrite?
+      save! path
+    else
+      save path
+    end
   end
 
 end

data/lib/mechanize/element_matcher.rb

@@ -3,17 +3,28 @@ module Mechanize::ElementMatcher
   def elements_with singular, plural = "#{singular}s"
     class_eval <<-CODE
       def #{plural}_with criteria = {}
-        criteria = if String === criteria then
-                     {:name => criteria}
-                   else
-                     Hash[criteria.map do |k, v|
-                            k = :dom_id if k.to_sym == :id
-                            k = :dom_class if k.to_sym == :class
-                            [k, v]
-                          end]
-                   end
-
-        f = select_#{plural}(criteria.delete(:search)).find_all do |thing|
+        selector = method = nil
+        if String === criteria then
+          criteria = {:name => criteria}
+        else
+          criteria = criteria.each_with_object({}) { |(k, v), h|
+            case k = k.to_sym
+            when :id
+              h[:dom_id] = v
+            when :class
+              h[:dom_class] = v
+            when :search, :xpath, :css
+              if v
+                selector = v
+                method = k
+              end
+            else
+              h[k] = v
+            end
+          }
+        end
+
+        f = select_#{plural}(selector, method).find_all do |thing|
           criteria.all? do |k,v|
             v === thing.send(k)
           end
@@ -35,11 +46,11 @@ module Mechanize::ElementMatcher
         f
       end
 
-      def select_#{plural} selector
+      def select_#{plural} selector, method = :search
         if selector.nil? then
           #{plural}
         else
-          nodes = search(selector)
+          nodes = __send__(method, selector)
           #{plural}.find_all do |element|
             nodes.include?(element.node)
           end

data/lib/mechanize/file_response.rb

@@ -69,9 +69,7 @@ class Mechanize::FileResponse
     }
     body << %w[</body></html>]
 
-    body = body.join "\n"
-    body.force_encoding Encoding::BINARY if body.respond_to? :force_encoding
-    body
+    body.join("\n").force_encoding(Encoding::BINARY)
   end
 
   def directory?

data/lib/mechanize/form.rb

@@ -330,6 +330,13 @@ class Mechanize::Form
     @clicked_buttons << button
   end
 
+  # This method allows the same form to be submitted second time
+  # with the different submit button being clicked.
+  def reset
+    # In the future, should add more functionality here to reset the form values to their defaults.
+    @clicked_buttons = []
+  end
+
   # This method calculates the request data to be sent back to the server
   # for this form, depending on if this is a regular post, get, or a
   # multi-part post,
@@ -348,8 +355,7 @@ class Mechanize::Form
       params.concat @file_uploads.map { |f| file_to_multipart(f) }
 
       params.map do |part|
-        part.force_encoding('ASCII-8BIT') if part.respond_to? :force_encoding
-        "--#{boundary}\r\n#{part}"
+        "--#{boundary}\r\n#{part.force_encoding(Encoding::ASCII_8BIT)}"
       end.join('') +
         "--#{boundary}--\r\n"
     else

data/lib/mechanize/http/agent.rb

@@ -167,7 +167,7 @@ class Mechanize::HTTP::Agent
 
     @scheme_handlers = Hash.new { |h, scheme|
       h[scheme] = lambda { |link, page|
-        raise Mechanize::UnsupportedSchemeError, scheme
+        raise Mechanize::UnsupportedSchemeError.new(scheme, link)
       }
     }
 
@@ -213,6 +213,10 @@ class Mechanize::HTTP::Agent
   # +method+ is used to retrieve it, along with the HTTP +headers+, request
   # +params+ and HTTP +referer+.
   #
+  # The final URI to access is built with +uri+ and +params+, the
+  # latter of which is formatted into a string using
+  # Mechanize::Util.build_query_string, which see.
+  #
   # +redirects+ tracks the number of redirects experienced when retrieving the
   # page.  If it is over the redirection_limit an error will be raised.
 
@@ -246,9 +250,13 @@ class Mechanize::HTTP::Agent
       request['If-Modified-Since'] = last_modified
     end if @conditional_requests
 
-    # Specify timeouts if given
-    connection.open_timeout = @open_timeout if @open_timeout
-    connection.read_timeout = @read_timeout if @read_timeout
+    # Specify timeouts if supplied and our connection supports them
+    if @open_timeout && connection.respond_to?(:open_timeout=)
+      connection.open_timeout = @open_timeout
+    end
+    if @read_timeout && connection.respond_to?(:read_timeout=)
+      connection.read_timeout = @read_timeout
+    end
 
     request_log request
 
@@ -515,6 +523,8 @@ class Mechanize::HTTP::Agent
 
   def request_auth request, uri
     base_uri = uri + '/'
+    base_uri.user     = nil
+    base_uri.password = nil
     schemes = @authenticate_methods[base_uri]
 
     if realm = schemes[:digest].find { |r| r.uri == base_uri } then
@@ -530,9 +540,7 @@ class Mechanize::HTTP::Agent
   def request_auth_digest request, uri, realm, base_uri, iis
     challenge = @digest_challenges[realm]
 
-    user, password, = @auth_store.credentials_for uri, realm.realm
-    uri.user     = user
-    uri.password = password
+    uri.user, uri.password, = @auth_store.credentials_for uri, realm.realm
 
     auth = @digest_auth.auth_header uri, challenge.to_s, request.method, iis
     request['Authorization'] = auth
@@ -608,17 +616,7 @@ class Mechanize::HTTP::Agent
       end
 
       url.gsub!(/[^#{0.chr}-#{126.chr}]/o) { |match|
-        if RUBY_VERSION >= "1.9.0"
-          Mechanize::Util.uri_escape(match)
-        else
-          begin
-            sprintf('%%%X', match.unpack($KCODE == 'UTF8' ? 'U' : 'C').first)
-          rescue ArgumentError
-            # workaround for ruby 1.8 with -Ku but ISO-8859-1 characters in
-            # URIs.  See #227.  I can't wait to drop 1.8 support
-            sprintf('%%%X', match.unpack('C').first)
-          end
-        end
+        Mechanize::Util.uri_escape(match)
       }
 
       escaped_url = Mechanize::Util.html_unescape(
@@ -679,6 +677,18 @@ class Mechanize::HTTP::Agent
     uri
   end
 
+  def secure_resolve!(uri, referer = current_page)
+    new_uri = resolve(uri, referer)
+
+    if (referer_uri = referer && referer.uri) &&
+       referer_uri.scheme != 'file'.freeze &&
+       new_uri.scheme == 'file'.freeze
+      raise Mechanize::Error, "insecure redirect to a file URI"
+    end
+
+    new_uri
+  end
+
   def resolve_parameters uri, method, parameters
     case method
     when :head, :get, :delete, :trace then
@@ -740,7 +750,7 @@ class Mechanize::HTTP::Agent
 
       if existing_realms.include? realm
         message = 'Digest authentication failed'
-        raise Mechanize::UnauthorizedError.new(page, challeges, message)
+        raise Mechanize::UnauthorizedError.new(page, challenges, message)
       end
 
       existing_realms << realm
@@ -822,7 +832,7 @@ class Mechanize::HTTP::Agent
   ensure
     begin
       if Tempfile === body_io and
-         (StringIO === out_io or out_io.path != body_io.path) then
+         (StringIO === out_io or (out_io and out_io.path != body_io.path)) then
         body_io.close!
       end
     rescue IOError
@@ -861,7 +871,7 @@ class Mechanize::HTTP::Agent
   def response_follow_meta_refresh response, uri, page, redirects
     delay, new_url = get_meta_refresh(response, uri, page)
     return nil unless delay
-    new_url = new_url ? resolve(new_url, page) : uri
+    new_url = new_url ? secure_resolve!(new_url, page) : uri
 
     raise Mechanize::RedirectLimitReachedError.new(page, redirects) if
       redirects + 1 > @redirection_limit
@@ -893,10 +903,9 @@ class Mechanize::HTTP::Agent
     if use_tempfile? content_length then
       body_io = make_tempfile 'mechanize-raw'
     else
-      body_io = StringIO.new
+      body_io = StringIO.new.set_encoding(Encoding::BINARY)
     end
 
-    body_io.set_encoding Encoding::BINARY if body_io.respond_to? :set_encoding
     total = 0
 
     begin
@@ -970,8 +979,9 @@ class Mechanize::HTTP::Agent
       headers.delete key
     end
 
+    new_uri = secure_resolve! response['Location'].to_s, page
+
     @history.push(page, page.uri)
-    new_uri = resolve response['Location'].to_s, page
 
     fetch new_uri, redirect_method, headers, [], referer, redirects + 1
   end
@@ -1086,12 +1096,12 @@ class Mechanize::HTTP::Agent
   # SSL version to use
   def ssl_version
     @http.ssl_version
-  end if RUBY_VERSION > '1.9'
+  end
 
   # Sets the SSL version to use
   def ssl_version= ssl_version
     @http.ssl_version = ssl_version
-  end if RUBY_VERSION > '1.9'
+  end
 
   # A callback for additional certificate verification.  See
   # OpenSSL::SSL::SSLContext#verify_callback
@@ -1141,9 +1151,7 @@ class Mechanize::HTTP::Agent
   # processing.
 
   def auto_io name, read_size, input_io
-    out_io = StringIO.new
-
-    out_io.set_encoding Encoding::BINARY if out_io.respond_to? :set_encoding
+    out_io = StringIO.new.set_encoding(Encoding::BINARY)
 
     until input_io.eof? do
       if StringIO === out_io and use_tempfile? out_io.size then
@@ -1216,7 +1224,7 @@ class Mechanize::HTTP::Agent
   def make_tempfile name
     io = Tempfile.new name
     io.unlink
-    io.binmode if io.respond_to? :binmode
+    io.binmode
     io
   end
 

data/lib/mechanize/http/auth_store.rb

@@ -93,6 +93,8 @@ only to a particular server you specify.
     uri = URI uri unless URI === uri
 
     uri += '/'
+    uri.user = nil
+    uri.password = nil
 
     realms = @auth_accounts[uri]
 

data/lib/mechanize/http/www_authenticate_parser.rb

@@ -128,7 +128,7 @@ class Mechanize::HTTP::WWWAuthenticateParser
 
   def auth_param
     return nil unless name = token
-    return nil unless @scanner.scan(/=/)
+    return nil unless @scanner.scan(/ *= */)
 
     value = if @scanner.peek(1) == '"' then
               quoted_string

data/lib/mechanize/page.rb

@@ -42,7 +42,7 @@ class Mechanize::Page < Mechanize::File
     if body
       # Force the encoding to be 8BIT so we can perform regular expressions.
       # We'll set it to the detected encoding later
-      body.force_encoding 'ASCII-8BIT' if body.respond_to? :force_encoding
+      body.force_encoding(Encoding::ASCII_8BIT)
 
       @encodings.concat self.class.meta_charset body
 
@@ -92,7 +92,9 @@ class Mechanize::Page < Mechanize::File
   end
 
   def encoding
-    parser.respond_to?(:encoding) ? parser.encoding : nil
+    parser.encoding
+  rescue NoMethodError
+    nil
   end
 
   # Return whether parser result has errors related to encoding or not.
@@ -109,15 +111,17 @@ class Mechanize::Page < Mechanize::File
 
   def parser
     return @parser if @parser
-    return nil unless @body
+    return unless @body
+
+    url = @uri && @uri.to_s
 
-    if @encoding then
-      @parser = @mech.html_parser.parse html_body, nil, @encoding
-    elsif mech.force_default_encoding then
-      @parser = @mech.html_parser.parse html_body, nil, @mech.default_encoding
+    if @encoding
+      @parser = mech.html_parser.parse html_body, url, @encoding
+    elsif mech.force_default_encoding
+      @parser = mech.html_parser.parse html_body, url, @mech.default_encoding
     else
       @encodings.reverse_each do |encoding|
-        @parser = @mech.html_parser.parse html_body, nil, encoding
+        @parser = mech.html_parser.parse html_body, url, encoding
 
         break unless encoding_error? @parser
       end
@@ -192,39 +196,71 @@ class Mechanize::Page < Mechanize::File
   ##
   # :method: search
   #
-  # Search for +paths+ in the page using Nokogiri's #search.  The +paths+ can
-  # be XPath or CSS and an optional Hash of namespaces may be appended.
+  # Shorthand for +parser.search+.
   #
-  # See Nokogiri::XML::Node#search for further details.
+  # See Nokogiri::XML::Node#search for details.
 
-  def_delegator :parser, :search, :search
+  ##
+  # :method: css
+  #
+  # Shorthand for +parser.css+.
+  #
+  # See also Nokogiri::XML::Node#css for details.
 
-  alias / search
+  ##
+  # :method: xpath
+  #
+  # Shorthand for +parser.xpath+.
+  #
+  # See also Nokogiri::XML::Node#xpath for details.
 
   ##
   # :method: at
   #
-  # Search through the page for +path+ under +namespace+ using Nokogiri's #at.
-  # The +path+ may be either a CSS or XPath expression.
+  # Shorthand for +parser.at+.
+  #
+  # See also Nokogiri::XML::Node#at for details.
+
+  ##
+  # :method: at_css
+  #
+  # Shorthand for +parser.at_css+.
+  #
+  # See also Nokogiri::XML::Node#at_css for details.
+
+  ##
+  # :method: at_xpath
   #
-  # See also Nokogiri::XML::Node#at
+  # Shorthand for +parser.at_xpath+.
+  #
+  # See also Nokogiri::XML::Node#at_xpath for details.
 
-  def_delegator :parser, :at, :at
+  def_delegators :parser, :search, :css, :xpath, :at, :at_css, :at_xpath
+
+  alias / search
+  alias % at
 
   ##
   # :method: form_with
   #
   # :call-seq:
   #   form_with(criteria)
+  #   form_with(criteria) { |form| ... }
   #
-  # Find a single form matching +criteria+.
-  # Example:
-  #   page.form_with(:action => '/post/login.php') do |f|
+  # Find a single form matching +criteria+.  See +forms_with+ for
+  # details of +criteria+.
+  #
+  # Examples:
+  #   page.form_with(action: '/post/login.php') do |f|
   #     ...
   #   end
 
   ##
-  # :mehtod: form_with!(criteria)
+  # :method: form_with!(criteria)
+  #
+  # :call-seq:
+  #   form_with!(criteria)
+  #   form_with!(criteria) { |form| ... }
   #
   # Same as +form_with+ but raises an ElementNotFoundError if no button matches
   # +criteria+
@@ -232,11 +268,33 @@ class Mechanize::Page < Mechanize::File
   ##
   # :method: forms_with
   #
-  # :call-seq: forms_with(criteria)
+  # :call-seq:
+  #   forms_with(name)
+  #   forms_with(name: name_matcher, id: id_matcher, class: class_matcher,
+  #              search: search_expression, xpath: xpath_expression, css: css_expression,
+  #              action: action_matcher, ...)
+  #
+  # Find all forms form matching criteria.  If a string is given, it
+  # is taken as a name attribute value.  If a hash is given, forms
+  # are narrowed by the key-value pairs as follows.
+  #
+  # :id, :dom_id: selects forms with a #dom_id value that matches this
+  # value.
+  #
+  # :class, :dom_class: selects forms with a #dom_class value that
+  # matches this value.
+  #
+  # :search: only selects forms matching this selector expression.
+  #
+  # :xpath: only selects forms matching this XPath expression.
+  #
+  # :css: only selects forms matching this CSS selector expression.
+  #
+  # :action, :method, etc.: narrows forms by a given attribute value
+  # using the === operator.
   #
-  # Find all forms form matching +criteria+.
   # Example:
-  #   page.forms_with(:action => '/post/login.php').each do |f|
+  #   page.forms_with(css: '#content table.login_box form', method: /\APOST\z/i, ).each do |f|
   #     ...
   #   end
 
@@ -245,14 +303,22 @@ class Mechanize::Page < Mechanize::File
   ##
   # :method: link_with
   #
-  # :call-seq: link_with(criteria)
+  # :call-seq:
+  #   link_with(criteria)
+  #   link_with(criteria) { |link| ... }
+  #
+  # Find a single link matching +criteria+.  See +forms_with+ for
+  # details of +criteria+, where for "form(s)" read "link(s)".
   #
-  # Find a single link matching +criteria+.
   # Example:
-  #   page.link_with(:href => /foo/).click
+  #   page.link_with(href: /foo/).click
 
   ##
-  # :mehtod: link_with!(criteria)
+  # :method: link_with!
+  #
+  # :call-seq:
+  #   link_with!(criteria)
+  #   link_with!(criteria) { |link| ... }
   #
   # Same as +link_with+ but raises an ElementNotFoundError if no button matches
   # +criteria+
@@ -263,9 +329,11 @@ class Mechanize::Page < Mechanize::File
   # :call-seq:
   #   links_with(criteria)
   #
-  # Find all links matching +criteria+.
+  # Find all links matching +criteria+.  See +forms_with+ for details
+  # of +criteria+, where for "form(s)" read "link(s)".
+  #
   # Example:
-  #   page.links_with(:href => /foo/).each do |link|
+  #   page.links_with(href: /foo/).each do |link|
   #     puts link.href
   #   end
 
@@ -274,14 +342,22 @@ class Mechanize::Page < Mechanize::File
   ##
   # :method: base_with
   #
-  # :call-seq: base_with(criteria)
+  # :call-seq:
+  #   base_with(criteria)
+  #   base_with(criteria) { |base| ... }
+  #
+  # Find a single base tag matching +criteria+.  See +forms_with+ for
+  # details of +criteria+, where for "form(s)" read "base tag(s)".
   #
-  # Find a single base tag matching +criteria+.
   # Example:
-  #   page.base_with(:href => /foo/).click
+  #   page.base_with(href: /foo/).click
 
   ##
-  # :mehtod: base_with!(criteria)
+  # :method: base_with!(criteria)
+  #
+  # :call-seq:
+  #   base_with!(criteria)
+  #   base_with!(criteria) { |base| ... }
   #
   # Same as +base_with+ but raises an ElementNotFoundError if no button matches
   # +criteria+
@@ -291,9 +367,11 @@ class Mechanize::Page < Mechanize::File
   #
   # :call-seq: bases_with(criteria)
   #
-  # Find all base tags matching +criteria+.
+  # Find all base tags matching +criteria+.  See +forms_with+ for
+  # details of +criteria+, where for "form(s)" read "base tag(s)".
+  #
   # Example:
-  #   page.bases_with(:href => /foo/).each do |base|
+  #   page.bases_with(href: /foo/).each do |base|
   #     puts base.href
   #   end
 
@@ -302,14 +380,22 @@ class Mechanize::Page < Mechanize::File
   ##
   # :method: frame_with
   #
-  # :call-seq: frame_with(criteria)
+  # :call-seq:
+  #   frame_with(criteria)
+  #   frame_with(criteria) { |frame| ... }
+  #
+  # Find a single frame tag matching +criteria+.  See +forms_with+ for
+  # details of +criteria+, where for "form(s)" read "frame tag(s)".
   #
-  # Find a single frame tag matching +criteria+.
   # Example:
-  #   page.frame_with(:src => /foo/).click
+  #   page.frame_with(src: /foo/).click
 
   ##
-  # :mehtod: frame_with!(criteria)
+  # :method: frame_with!
+  #
+  # :call-seq:
+  #   frame_with!(criteria)
+  #   frame_with!(criteria) { |frame| ... }
   #
   # Same as +frame_with+ but raises an ElementNotFoundError if no button matches
   # +criteria+
@@ -319,9 +405,11 @@ class Mechanize::Page < Mechanize::File
   #
   # :call-seq: frames_with(criteria)
   #
-  # Find all frame tags matching +criteria+.
+  # Find all frame tags matching +criteria+.  See +forms_with+ for
+  # details of +criteria+, where for "form(s)" read "frame tag(s)".
+  #
   # Example:
-  #   page.frames_with(:src => /foo/).each do |frame|
+  #   page.frames_with(src: /foo/).each do |frame|
   #     p frame.src
   #   end
 
@@ -330,14 +418,22 @@ class Mechanize::Page < Mechanize::File
   ##
   # :method: iframe_with
   #
-  # :call-seq: iframe_with(criteria)
+  # :call-seq:
+  #   iframe_with(criteria)
+  #   iframe_with(criteria) { |iframe| ... }
+  #
+  # Find a single iframe tag matching +criteria+.  See +forms_with+ for
+  # details of +criteria+, where for "form(s)" read "iframe tag(s)".
   #
-  # Find a single iframe tag matching +criteria+.
   # Example:
-  #   page.iframe_with(:src => /foo/).click
+  #   page.iframe_with(src: /foo/).click
 
   ##
-  # :mehtod: iframe_with!(criteria)
+  # :method: iframe_with!
+  #
+  # :call-seq:
+  #   iframe_with!(criteria)
+  #   iframe_with!(criteria) { |iframe| ... }
   #
   # Same as +iframe_with+ but raises an ElementNotFoundError if no button
   # matches +criteria+
@@ -347,9 +443,11 @@ class Mechanize::Page < Mechanize::File
   #
   # :call-seq: iframes_with(criteria)
   #
-  # Find all iframe tags matching +criteria+.
+  # Find all iframe tags matching +criteria+.  See +forms_with+ for
+  # details of +criteria+, where for "form(s)" read "iframe tag(s)".
+  #
   # Example:
-  #   page.iframes_with(:src => /foo/).each do |iframe|
+  #   page.iframes_with(src: /foo/).each do |iframe|
   #     p iframe.src
   #   end
 
@@ -358,14 +456,22 @@ class Mechanize::Page < Mechanize::File
   ##
   # :method: image_with
   #
-  # :call-seq: image_with(criteria)
+  # :call-seq:
+  #   image_with(criteria)
+  #   image_with(criteria) { |image| ... }
+  #
+  # Find a single image matching +criteria+.  See +forms_with+ for
+  # details of +criteria+, where for "form(s)" read "image(s)".
   #
-  # Find a single image matching +criteria+.
   # Example:
-  #   page.image_with(:alt => /main/).fetch.save
+  #   page.image_with(alt: /main/).fetch.save
 
   ##
-  # :mehtod: image_with!(criteria)
+  # :method: image_with!
+  #
+  # :call-seq:
+  #   image_with!(criteria)
+  #   image_with!(criteria) { |image| ... }
   #
   # Same as +image_with+ but raises an ElementNotFoundError if no button matches
   # +criteria+
@@ -375,9 +481,11 @@ class Mechanize::Page < Mechanize::File
   #
   # :call-seq: images_with(criteria)
   #
-  # Find all images matching +criteria+.
+  # Find all images matching +criteria+.  See +forms_with+ for
+  # details of +criteria+, where for "form(s)" read "image(s)".
+  #
   # Example:
-  #   page.images_with(:src => /jpg\Z/).each do |img|
+  #   page.images_with(src: /jpg\Z/).each do |img|
   #     img.fetch.save
   #   end