mechanize 2.5.1 → 2.6.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 65bb3082549e10c6ed19080676ab618e0e663f24
+  data.tar.gz: 4e1bab1272afcb0003e819641091507cb1b0080c
+SHA512:
+  metadata.gz: dcbc3f181d1b07d4a659e234c0cd769d48f8c7a1cf275905c7a4121ed6efc4661defe764c6a32b43b1ae3aa83ed5fcbf0bf100bdfe0ab3128cf9df156022f986
+  data.tar.gz: de01e70e18111ab5626dbf33d1e508b0fbb65acce86280edfa3f99f3d0f057f18a2aba96c6280261e582aeaf44dabd91324788bc2ae266ea30f89d7955272225

data/.travis.yml

@@ -0,0 +1,16 @@
+---
+after_script:
+- rake travis:after -t
+before_script:
+- gem install hoe-travis --no-rdoc --no-ri
+- rake travis:before -t
+language: ruby
+notifications:
+  email:
+  - drbrain@segment7.net
+  - ljjarvis@gmail.com
+rvm:
+- 1.8.7
+- 1.9.2
+- 1.9.3
+script: rake travis

data/CHANGELOG.rdoc

@@ -1,5 +1,58 @@
 = Mechanize CHANGELOG
 
+=== 2.6.0
+
+* New Features
+  * Mechanize#start and Mechanize#shutdown (Thanks, Damian Janowski!)
+  * Added Mechanize::Agent#allowed_error_codes for setting an Array
+    of status codes which should not raise an error. #248 Laurence Rowe.
+  * Added `File.save!` for overwriting existing files #219.
+  * DirectorySaver::save_to now accepts an option to decode filename. #262
+  * element(s)_with methods now accept a :search option for doing xpath/css
+    selector searching. #287 Philippe Bourgau
+  * Added httponly option for Mechanize::Cookie #242 by Paolo Perego.
+  * Added Mechanize::XmlFile as a default pluggable parser for handling
+    XML responses. #289
+
+* Minor enhancements
+  * Added Mechanize::Download#save_as as an alias to #save. #246
+  * Fix documentation for `Mechanize::Page` element matchers. #269
+  * Added Mechanize::Form::Field#raw_value for fetching a fields value
+    before it's sent through Mechanize::Util.html_unescape. #283
+  * Added iPad and Android user agents.  #277 by sambit, #278 by seansay.
+
+* Bug fix
+  * Mechanize#cert and Mechanize#key now return the values set by
+    #cert= and #key=. #244, #245 (Thanks, Robert Gogolok!)
+  * Mechanize no longer submits disabled form fields.  #276 by Bogdan Gusiev,
+    #279 by Ricardo Valeriano.
+  * Mechanize::File#save now behaves like Mechanize::Download#save in
+    that it will create the parent directory before saving.
+    #272, #280 by Ryan Kowalick
+  * Ensure `application/xml` is registered as an XML parser in
+    `PluggableParser`, not just `text/xml`. #266 James Gregory
+  * Mechanize now writes cookiestxt with a prefixed dot for wildcard domain
+    handling. #295 by Mike Morearty.
+
+=== 2.5.2
+
+* New Features
+  * Mechanize::CookieJar#save_as takes a keyword option "session" to say
+    that session cookies should be saved.  Based on #230 by Jim Jones.
+
+* Minor enhancements
+  * Added Mechanize#follow_redirect= as an alias to redirect_ok=.
+
+* Bug fix
+  * Fixed casing of the Mac Firefox user-agent alias to match Linux Firefox.
+    In mechanize 3 the old "Mac FireFox" user-agent alias will be removed.
+    Pull request #231 by Gavin Miller.
+  * Mechanize now authenticates using the raw challenge, not a reconstructed
+    one, to avoid dealing with quoting rules of RFC 2617.  Fixes failures in
+    #231 due to net-http-digest_auth 1.2.1
+  * Fixed Content-Disposition parameter parser to be case insensitive. #233
+  * Fixed redirection counting in following meta refresh. #240
+
 === 2.5.1
 
 * Bug fix
@@ -860,4 +913,4 @@ Mechanize is now under the MIT license
 * Added a response code exception
 * Thanks to Brian Ellin (brianellin@gmail.com) for:
   Added support for CA files, and support for 301 response codes
-  
+

data/Manifest.txt

@@ -1,4 +1,5 @@
 .autotest
+.travis.yml
 CHANGELOG.rdoc
 EXAMPLES.rdoc
 GUIDE.rdoc
@@ -98,6 +99,7 @@ lib/mechanize/test_case/verb_servlet.rb
 lib/mechanize/unauthorized_error.rb
 lib/mechanize/unsupported_scheme_error.rb
 lib/mechanize/util.rb
+lib/mechanize/xml_file.rb
 test/data/htpasswd
 test/data/server.crt
 test/data/server.csr
@@ -140,6 +142,8 @@ test/htdocs/tc_checkboxes.html
 test/htdocs/tc_encoded_links.html
 test/htdocs/tc_field_precedence.html
 test/htdocs/tc_follow_meta.html
+test/htdocs/tc_follow_meta_loop_1.html
+test/htdocs/tc_follow_meta_loop_2.html
 test/htdocs/tc_form_action.html
 test/htdocs/tc_links.html
 test/htdocs/tc_meta_in_body.html
@@ -194,4 +198,5 @@ test/test_mechanize_redirect_not_get_or_head_error.rb
 test/test_mechanize_response_read_error.rb
 test/test_mechanize_subclass.rb
 test/test_mechanize_util.rb
+test/test_mechanize_xml_file.rb
 test/test_multi_select.rb

data/README.rdoc

@@ -1,9 +1,9 @@
 = Mechanize
 
 * http://mechanize.rubyforge.org
-* https://github.com/tenderlove/mechanize
+* https://github.com/sparklemotion/mechanize
 
-== DESCRIPTION
+== Description
 
 The Mechanize library is used for automating interaction with websites.
 Mechanize automatically stores and sends cookies, follows redirects,
@@ -16,15 +16,15 @@ a history.
 * ruby 1.8.7, 1.9.2, or 1.9.3
 * nokogiri[http://nokogiri.rubyforge.org]
 
-== SUPPORT:
+== Support:
 
 The mechanize mailing list is available here:
 
-  * http://rubyforge.org/mailman/listinfo/mechanize-users
+* http://rubyforge.org/mailman/listinfo/mechanize-users
 
 The bug tracker is available here:
 
-  * https://github.com/tenderlove/mechanize/issues
+* https://github.com/sparklemotion/mechanize/issues
 
 == Examples
 

data/Rakefile

@@ -3,25 +3,27 @@ require 'hoe'
 
 Hoe.plugin :git
 Hoe.plugin :minitest
+Hoe.plugin :travis
 
 hoe = Hoe.spec 'mechanize' do
   developer 'Eric Hodel',      'drbrain@segment7.net'
   developer 'Aaron Patterson', 'aaronp@rubyforge.org'
   developer 'Mike Dalessio',   'mike.dalessio@gmail.com'
   developer 'Akinori MUSHA',   'knu@idaemons.org'
+  developer 'Lee Jarvis',      'ljjarvis@gmail.com'
 
   self.readme_file      = 'README.rdoc'
   self.history_file     = 'CHANGELOG.rdoc'
   self.extra_rdoc_files += Dir['*.rdoc']
 
-  rdoc_locations << 'drbrain@rubyforge.org:/var/www/gforge-projects/mechanize/'
+  rdoc_locations << 'rubyforge.org:/var/www/gforge-projects/mechanize/'
 
   self.extra_deps << ['net-http-digest_auth', '~> 1.1', '>= 1.1.1']
   self.extra_deps << ['net-http-persistent',  '~> 2.5', '>= 2.5.2']
   self.extra_deps << ['mime-types',           '~> 1.17', '>= 1.17.2']
   self.extra_deps << ['nokogiri',             '~> 1.4']
   self.extra_deps << ['ntlm-http',            '~> 0.1', '>= 0.1.1']
-  self.extra_deps << ['webrobots',            '~> 0.0', '>= 0.0.9']
+  self.extra_deps << ['webrobots',            '<  0.2', '>= 0.0.9']
   self.extra_deps << ['domain_name',          '~> 0.5', '>= 0.5.1']
 
   self.spec_extras[:required_ruby_version] = '>= 1.8.7'

data/lib/mechanize.rb

@@ -37,7 +37,7 @@ require 'zlib'
 # == Issues with mechanize
 #
 # If you think you have a bug with mechanize, but aren't sure, please file a
-# ticket at https://github.com/tenderlove/mechanize/issues
+# ticket at https://github.com/sparklemotion/mechanize/issues
 #
 # Here are some common problems you may experience with mechanize
 #
@@ -73,7 +73,7 @@ class Mechanize
   ##
   # The version of Mechanize you are using.
 
-  VERSION = '2.5.1'
+  VERSION = '2.6.0'
 
   ##
   # Base mechanize error class
@@ -106,6 +106,8 @@ class Mechanize
   # * Windows IE 9
   # * Windows Mozilla
   # * iPhone (3.0)
+  # * iPad
+  # * Android (Motorola Xoom)
   #
   # Example:
   #
@@ -113,11 +115,11 @@ class Mechanize
   #   agent.user_agent_alias = 'Mac Safari'
 
   AGENT_ALIASES = {
-    'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/tenderlove/mechanize/)",
+    'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/sparklemotion/mechanize/)",
     'Linux Firefox' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.1) Gecko/20100122 firefox/3.6.1',
     'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)',
     'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
-    'Mac FireFox' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6',
+    'Mac Firefox' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6',
     'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401',
     'Mac Safari 4' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; de-at) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10',
     'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.51.22 (KHTML, like Gecko) Version/5.1.1 Safari/534.51.22',
@@ -127,14 +129,36 @@ class Mechanize
     'Windows IE 9' => 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
     'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6',
     'iPhone' => 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3',
+    'iPad' => 'Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10',
+    'Android' => 'Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13'
   }
 
+  AGENT_ALIASES['Mac FireFox'] = AGENT_ALIASES['Mac Firefox']
+  AGENT_ALIASES['Linux FireFox'] = AGENT_ALIASES['Linux Firefox']
+
   def self.inherited(child) # :nodoc:
-    child.html_parser ||= html_parser
-    child.log ||= log
+    child.html_parser = html_parser
+    child.log = log
     super
   end
 
+  ##
+  # Creates a new Mechanize instance and yields it to the given block.
+  #
+  # After the block executes, the instance is cleaned up. This includes
+  # closing all open connections.
+  #
+  #   Mechanize.start do |m|
+  #     m.get("http://example.com")
+  #   end
+
+  def self.start
+    instance = new
+    yield(instance)
+  ensure
+    instance.shutdown
+  end
+
   ##
   # Creates a new mechanize instance.  If a block is given, the created
   # instance is yielded to the block for setting up pre-connection state such
@@ -443,7 +467,7 @@ class Mechanize
   #   agent.post('http://example.com/', "<message>hello</message>",
   #              'Content-Type' => 'application/xml')
 
-  def post(uri, query={}, headers={})
+  def post(uri, query = {}, headers = {})
     return request_with_entity(:post, uri, query, headers) if String === query
 
     node = {}
@@ -507,7 +531,7 @@ class Mechanize
   #
   #   agent.submit(page.forms.first, page.forms.first.buttons.first)
 
-  def submit(form, button=nil, headers={})
+  def submit(form, button = nil, headers = {})
     form.add_button_to_query(button) if button
 
     case form.method.upcase
@@ -542,6 +566,7 @@ class Mechanize
   # keep-alives, compression, redirects and headers.
 
   @html_parser = Nokogiri::HTML
+  @log = nil
 
   class << self
 
@@ -884,6 +909,8 @@ Use of #auth and #basic_auth are deprecated due to a security vulnerability.
     @agent.redirect_ok = follow
   end
 
+  alias follow_redirect= redirect_ok=
+
   ##
   # Maximum number of redirections to follow
 
@@ -1017,7 +1044,7 @@ Use of #auth and #basic_auth are deprecated due to a security vulnerability.
   # An OpenSSL client certificate or the path to a certificate file.
 
   def cert
-    @agent.cert
+    @agent.certificate
   end
 
   ##
@@ -1076,7 +1103,7 @@ Use of #auth and #basic_auth are deprecated due to a security vulnerability.
   # An OpenSSL private key or the path to a private key
 
   def key
-    @agent.key
+    @agent.private_key
   end
 
   ##
@@ -1207,6 +1234,22 @@ Use of #auth and #basic_auth are deprecated due to a security vulnerability.
     @agent.set_proxy address, port, user, password
   end
 
+  ##
+  # Clears history and cookies.
+
+  def reset
+    @agent.reset
+  end
+
+  ##
+  # Shuts down this session by clearing browsing state and closing all
+  # persistent connections.
+
+  def shutdown
+    reset
+    @agent.shutdown
+  end
+
   private
 
   ##

data/lib/mechanize/cookie.rb

@@ -10,6 +10,7 @@ class Mechanize::Cookie
   attr_accessor :comment, :max_age
 
   attr_accessor :session
+  attr_accessor :httponly
 
   attr_accessor :created_at
   attr_accessor :accessed_at
@@ -33,7 +34,7 @@ class Mechanize::Cookie
     @version = 0     # Netscape Cookie
 
     @domain = @path = @secure = @comment = @max_age =
-      @expires = @comment_url = @discard = @port = nil
+      @expires = @comment_url = @discard = @port = @httponly = nil
 
     @created_at = @accessed_at = Time.now
     case args.size
@@ -78,7 +79,7 @@ class Mechanize::Cookie
     # array of Cookie objects.  Note that this array may contain
     # nil's when some of the cookie-pairs are malformed.
     def parse(uri, str, log = Mechanize.log)
-      return str.split(/,(?=[^;,]*=)|,$/).map { |c|
+      return str.to_s.split(/,(?=[^;,]*=)|,$/).map { |c|
         cookie_elem = c.split(/;+/)
         first_elem = cookie_elem.shift
         first_elem.strip!
@@ -97,6 +98,7 @@ class Mechanize::Cookie
           next unless key
           value = WEBrick::HTTPUtils.dequote(value.strip) if value
 
+
           case key.downcase
           when 'domain'
             next unless value && !value.empty?
@@ -134,6 +136,8 @@ class Mechanize::Cookie
               log.warn("Couldn't parse version '#{value}'") if log
               cookie.version = nil
             end
+          when 'httponly'
+            cookie.httponly = true
           when 'secure'
             cookie.secure = true
           end
@@ -142,6 +146,7 @@ class Mechanize::Cookie
         cookie.path    ||= (uri + './').path
         cookie.secure  ||= false
         cookie.domain  ||= uri.host
+        cookie.httponly ||= false
 
         # RFC 6265 4.1.2.2
         cookie.expires   = Time.now + cookie.max_age if cookie.max_age
@@ -193,6 +198,7 @@ class Mechanize::Cookie
   end
 
   alias secure? secure
+  alias httponly? httponly
 
   def acceptable_from_uri?(uri)
     host = DomainName.new(uri.host)

data/lib/mechanize/cookie_jar.rb

@@ -70,14 +70,37 @@ class Mechanize::CookieJar
     }
   end
 
-  # Save the cookie jar to a file in the format specified.
+  # call-seq:
+  #   jar.save_as(file, format = :yaml)
+  #   jar.save_as(file, options)
   #
-  # Available formats:
-  # :yaml  <- YAML structure
-  # :cookiestxt  <- Mozilla's cookies.txt format
-  def save_as(file, format = :yaml)
+  # Save the cookie jar to a file in the format specified and return
+  # self.
+  #
+  # Available option keywords are below:
+  #
+  # * +format+
+  #   [<tt>:yaml</tt>]
+  #     YAML structure (default)
+  #   [<tt>:cookiestxt</tt>]
+  #     Mozilla's cookies.txt format
+  # * +session+
+  #   [+true+]
+  #     Save session cookies as well.
+  #   [+false+]
+  #     Do not save session cookies. (default)
+  def save_as(file, options = nil)
+    if Symbol === options
+      format = options
+      session = false
+    else
+      options ||= {}
+      format = options[:format] || :yaml
+      session = !!options[:session]
+    end
+
     jar = dup
-    jar.cleanup true
+    jar.cleanup !session
 
     open(file, 'w') { |f|
       case format
@@ -164,9 +187,12 @@ class Mechanize::CookieJar
 
   # Write cookies to Mozilla cookies.txt-style IO stream
   def dump_cookiestxt(io)
+    io.puts "# HTTP Cookie File"
+    io.puts "# This file was generated by Ruby Mechanize " \
+      "#{Mechanize::VERSION} https://github.com/sparklemotion/mechanize.\n\n"
     to_a.each do |cookie|
       io.puts([
-        cookie.domain,
+        (cookie.for_domain? ? "." : "") + cookie.domain,
         cookie.for_domain? ? "TRUE" : "FALSE",
         cookie.path,
         cookie.secure ? "TRUE" : "FALSE",

data/lib/mechanize/directory_saver.rb

@@ -10,16 +10,20 @@
 class Mechanize::DirectorySaver < Mechanize::Download
 
   @directory = nil
+  @options = {}
 
   ##
   # Creates a DirectorySaver subclass that will save responses to the given
-  # +directory+.
+  # +directory+. If +options+ includes a +decode_filename+ value set to +true+
+  # then the downloaded filename will be ran through +CGI.unescape+ before
+  # being saved.
 
-  def self.save_to directory
+  def self.save_to directory, options = {}
     directory = File.expand_path directory
 
     Class.new self do |klass|
       klass.instance_variable_set :@directory, directory
+      klass.instance_variable_set :@options, options
     end
   end
 
@@ -30,6 +34,13 @@ class Mechanize::DirectorySaver < Mechanize::Download
     @directory
   end
 
+  ##
+  # True if downloaded files should have their names decoded before saving.
+
+  def self.decode_filename?
+    @options[:decode_filename]
+  end
+
   ##
   # Saves the +body_io+ into the directory specified for this DirectorySaver
   # by save_to.  The filename is chosen by Mechanize::Parser#extract_filename.
@@ -44,6 +55,7 @@ class Mechanize::DirectorySaver < Mechanize::Download
 
     super
 
+    @filename = CGI.unescape(@filename) if self.class.decode_filename?
     path = File.join directory, @filename
 
     save path