mechanize 2.6.0 → 2.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65bb3082549e10c6ed19080676ab618e0e663f24
-  data.tar.gz: 4e1bab1272afcb0003e819641091507cb1b0080c
+  metadata.gz: 697096d6b8d2bd29662a4a9e2241b5bc22ccb70d
+  data.tar.gz: 1a4348dbc2f838ee019917249e7f0c4348035a9a
 SHA512:
-  metadata.gz: dcbc3f181d1b07d4a659e234c0cd769d48f8c7a1cf275905c7a4121ed6efc4661defe764c6a32b43b1ae3aa83ed5fcbf0bf100bdfe0ab3128cf9df156022f986
-  data.tar.gz: de01e70e18111ab5626dbf33d1e508b0fbb65acce86280edfa3f99f3d0f057f18a2aba96c6280261e582aeaf44dabd91324788bc2ae266ea30f89d7955272225
+  metadata.gz: 3a9802905e414093174c09110703e042cde32de5abfde18c8f7bd1d02b6f414dda4dfb897792a7e7201fa3ed52923fdbeb777059ca6075d7c9bb0469cad96cbb
+  data.tar.gz: 5f93e7092082c7cd7c33701e566f621299f4604360e85a61e9f900cb9f81a8971ebf2a2e21cdeeb9e00f9bf582b7bc8db7252fe4dea180e6ace8dc04523f466f

data/.travis.yml

@@ -9,8 +9,24 @@ notifications:
   email:
   - drbrain@segment7.net
   - ljjarvis@gmail.com
+  - knu@idaemons.org
 rvm:
 - 1.8.7
 - 1.9.2
 - 1.9.3
+- 2.0.0
+- ruby-head
+- jruby-18mode
+- jruby-19mode
+- jruby-head
+- rbx-18mode
+- rbx-19mode
 script: rake travis
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: jruby-18mode
+    - rvm: jruby-19mode
+    - rvm: jruby-head
+    - rvm: rbx-18mode
+    - rvm: rbx-19mode

data/CHANGELOG.rdoc

@@ -1,5 +1,19 @@
 = Mechanize CHANGELOG
 
+=== 2.7.0
+
+* New Features
+  * Mechanize::Agent#response_read will now raise a
+    Mechanize::ResponseReadError instead of an EOFError and avoid losing
+    requested content. #296.
+  * Depend on http-cookie, add backwards compatible deprecations.
+    #257 Akinori MUSHA.
+  * Added `Download#save!` for overwriting existing files. #300 Sean Kim.
+
+* Bug fix
+  * Ensure page URLs with whitespace in them are escaped #313 @pacop.
+  * Added a workaround for a bug of URI#+ that led to failure in resolving a relative path containing double slash like "/../http://.../". #304
+
 === 2.6.0
 
 * New Features

data/Manifest.txt

@@ -62,6 +62,7 @@ lib/mechanize/page/link.rb
 lib/mechanize/page/meta_refresh.rb
 lib/mechanize/parser.rb
 lib/mechanize/pluggable_parsers.rb
+lib/mechanize/prependable.rb
 lib/mechanize/redirect_limit_reached_error.rb
 lib/mechanize/redirect_not_get_or_head_error.rb
 lib/mechanize/response_code_error.rb

data/README.rdoc

@@ -13,7 +13,7 @@ a history.
 
 == Dependencies
 
-* ruby 1.8.7, 1.9.2, or 1.9.3
+* ruby 1.8.7, 1.9.2, 1.9.3, or 2.0.0
 * nokogiri[http://nokogiri.rubyforge.org]
 
 == Support:
@@ -28,7 +28,7 @@ The bug tracker is available here:
 
 == Examples
 
-If you are just starting, check out the GUIDE.  Also, check out the EXAMPLES
+If you are just starting, check out the GUIDE[http://mechanize.rubyforge.org/GUIDE_rdoc.html].  Also, check out the EXAMPLES[http://mechanize.rubyforge.org/EXAMPLES_rdoc.html]
 file.
 
 == Developers
@@ -67,9 +67,10 @@ This library comes with a shameless plug for employing me
 
 == Acknowledgments
 
-This library was heavily influenced by its namesake in the perl world.  A big
-thanks goes to Andy Lester (andy@petdance.com), the author of the original
-perl Mechanize which is available here[http://search.cpan.org/~petdance/WWW-Mechanize/].  Ruby Mechanize would not be around without you!
+This library was heavily influenced by its namesake in the Perl world.  A big
+thanks goes to {Andy Lester}[http://petdance.com],
+the author of the original Perl module WWW::Mechanize which is available
+here[http://search.cpan.org/dist/WWW-Mechanize/].  Ruby Mechanize would not be around without you!
 
 Thank you to Michael Neumann for starting the Ruby version.  Thanks to everyone
 who's helped out in various ways.  Finally, thank you to the people using this
@@ -77,5 +78,5 @@ library!
 
 == License
 
-This library is distributed under the MIT license.  Please see the LICENSE file.
+This library is distributed under the MIT license.  Please see the LICENSE[http://mechanize.rubyforge.org/LICENSE_rdoc.html] file.
 

data/Rakefile

@@ -21,11 +21,14 @@ hoe = Hoe.spec 'mechanize' do
   self.extra_deps << ['net-http-digest_auth', '~> 1.1', '>= 1.1.1']
   self.extra_deps << ['net-http-persistent',  '~> 2.5', '>= 2.5.2']
   self.extra_deps << ['mime-types',           '~> 1.17', '>= 1.17.2']
+  self.extra_deps << ['http-cookie',          '~> 1.0.0']
   self.extra_deps << ['nokogiri',             '~> 1.4']
   self.extra_deps << ['ntlm-http',            '~> 0.1', '>= 0.1.1']
   self.extra_deps << ['webrobots',            '<  0.2', '>= 0.0.9']
   self.extra_deps << ['domain_name',          '~> 0.5', '>= 0.5.1']
 
+  self.extra_dev_deps << ['minitest', '~> 5.0']
+
   self.spec_extras[:required_ruby_version] = '>= 1.8.7'
 end
 

data/lib/mechanize.rb

@@ -73,7 +73,7 @@ class Mechanize
   ##
   # The version of Mechanize you are using.
 
-  VERSION = '2.6.0'
+  VERSION = '2.7.0'
 
   ##
   # Base mechanize error class

data/lib/mechanize/cookie.rb

@@ -1,244 +1,73 @@
-require 'time'
-require 'webrick/httputils'
-require 'domain_name'
-
-# This class is used to represent an HTTP Cookie.
-class Mechanize::Cookie
-  attr_reader :name
-  attr_accessor :value, :version
-  attr_accessor :domain, :path, :secure
-  attr_accessor :comment, :max_age
-
-  attr_accessor :session
-  attr_accessor :httponly
-
-  attr_accessor :created_at
-  attr_accessor :accessed_at
-
-  # :call-seq:
-  #     new(name, value)
-  #     new(name, value, attr_hash)
-  #     new(attr_hash)
-  #
-  # Creates a cookie object.  For each key of +attr_hash+, the setter
-  # is called if defined.  Each key can be either a symbol or a
-  # string, downcased or not.
-  #
-  # e.g.
-  #     new("uid", "a12345")
-  #     new("uid", "a12345", :domain => 'example.org',
-  #                          :for_domain => true, :expired => Time.now + 7*86400)
-  #     new("name" => "uid", "value" => "a12345", "Domain" => 'www.example.org')
-  #
-  def initialize(*args)
-    @version = 0     # Netscape Cookie
-
-    @domain = @path = @secure = @comment = @max_age =
-      @expires = @comment_url = @discard = @port = @httponly = nil
-
-    @created_at = @accessed_at = Time.now
-    case args.size
-    when 2
-      @name, @value = *args
-      @for_domain = false
-      return
-    when 3
-      @name, @value, attr_hash = *args
-    when 1
-      attr_hash = args.first
-    else
-      raise ArgumentError, "wrong number of arguments (#{args.size} for 1-3)"
-    end
-    for_domain = false
-    attr_hash.each_pair { |key, val|
-      skey = key.to_s.downcase
-      skey.sub!(/[!?]\z/, '')
-      case skey
-      when 'for_domain'
-        for_domain = !!val
-      when 'name'
-        @name = val
-      when 'value'
-        @value = val
+warn 'mechanize/cookie will be deprecated.  Please migrate to the http-cookie APIs.' if $VERBOSE
+
+require 'http/cookie'
+
+class Mechanize
+  module CookieDeprecated
+    def __deprecated__(to = nil)
+      $VERBOSE or return
+      method = caller[0][/([^`]+)(?='$)/]
+      to ||= method
+      case self
+      when Class
+        lname = name[/[^:]+$/]
+        klass = 'Mechanize::%s' % lname
+        this = '%s.%s' % [klass, method]
+        that = 'HTTP::%s.%s' % [lname, to]
       else
-        setter = :"#{skey}="
-        send(setter, val) if respond_to?(setter)
+        lname = self.class.name[/[^:]+$/]
+        klass = 'Mechanize::%s' % lname
+        this = '%s#%s' % [klass, method]
+        that = 'HTTP::%s#%s' % [lname, to]
       end
-    }
-    @for_domain = for_domain
-  end
-
-  # If this flag is true, this cookie will be sent to any host in the
-  # +domain+.  If it is false, this cookie will be sent only to the
-  # host indicated by the +domain+.
-  attr_accessor :for_domain
-  alias for_domain? for_domain
-
-  class << self
-    # Parses a Set-Cookie header line +str+ sent from +uri+ into an
-    # array of Cookie objects.  Note that this array may contain
-    # nil's when some of the cookie-pairs are malformed.
-    def parse(uri, str, log = Mechanize.log)
-      return str.to_s.split(/,(?=[^;,]*=)|,$/).map { |c|
-        cookie_elem = c.split(/;+/)
-        first_elem = cookie_elem.shift
-        first_elem.strip!
-        key, value = first_elem.split(/\=/, 2)
-
-        begin
-          cookie = new(key, value.dup)
-        rescue
-          log.warn("Couldn't parse key/value: #{first_elem}") if log
-          next
-        end
-
-        cookie_elem.each do |pair|
-          pair.strip!
-          key, value = pair.split(/=/, 2)
-          next unless key
-          value = WEBrick::HTTPUtils.dequote(value.strip) if value
-
-
-          case key.downcase
-          when 'domain'
-            next unless value && !value.empty?
-            begin
-              cookie.domain = value
-              cookie.for_domain = true
-            rescue
-              log.warn("Couldn't parse domain: #{value}") if log
-            end
-          when 'path'
-            next unless value && !value.empty?
-            cookie.path = value
-          when 'expires'
-            next unless value && !value.empty?
-            begin
-              cookie.expires = Time::parse(value)
-            rescue
-              log.warn("Couldn't parse expires: #{value}") if log
-            end
-          when 'max-age'
-            next unless value && !value.empty?
-            begin
-              cookie.max_age = Integer(value)
-            rescue
-              log.warn("Couldn't parse max age '#{value}'") if log
-            end
-          when 'comment'
-            next unless value
-            cookie.comment = value
-          when 'version'
-            next unless value
-            begin
-              cookie.version = Integer(value)
-            rescue
-              log.warn("Couldn't parse version '#{value}'") if log
-              cookie.version = nil
-            end
-          when 'httponly'
-            cookie.httponly = true
-          when 'secure'
-            cookie.secure = true
-          end
-        end
-
-        cookie.path    ||= (uri + './').path
-        cookie.secure  ||= false
-        cookie.domain  ||= uri.host
-        cookie.httponly ||= false
-
-        # RFC 6265 4.1.2.2
-        cookie.expires   = Time.now + cookie.max_age if cookie.max_age
-        cookie.session   = !cookie.expires
-
-        # Move this in to the cookie jar
-        yield cookie if block_given?
-
-        cookie
-      }
+      warn '%s: The call of %s needs to be fixed to follow the new API (%s).' % [caller[1], this, that]
     end
+    private :__deprecated__
   end
 
-  alias set_domain domain=
+  module CookieCMethods
+    include CookieDeprecated
 
-  # Sets the domain attribute.  A leading dot in +domain+ implies
-  # turning the +for_domain?+ flag on.
-  def domain=(domain)
-    if DomainName === domain
-      @domain_name = domain
-    else
-      domain.is_a?(String) or
-        (domain.respond_to?(:to_str) && (domain = domain.to_str).is_a?(String)) or
-        raise TypeError, "#{domain.class} is not a String"
-      if domain.start_with?('.')
-        @for_domain = true
-        domain = domain[1..-1]
-      end
-      # Do we really need to support this?
-      if domain.match(/\A([^:]+):[0-9]+\z/)
-        domain = $1
+    def parse(arg1, arg2, arg3 = nil, &block)
+      if arg1.is_a?(URI)
+        __deprecated__
+        return [] if arg2.nil?
+        super(arg2, arg1, { :logger => arg3 })
+      else
+        super
       end
-      @domain_name = DomainName.new(domain)
     end
-    set_domain(@domain_name.hostname)
-  end
-
-  def expires=(t)
-    @expires = t && (t.is_a?(Time) ? t.httpdate : t.to_s)
   end
 
-  def expires
-    @expires && Time.parse(@expires)
-  end
+  module CookieIMethods
+    include CookieDeprecated
 
-  def expired?
-    return false unless expires
-    Time.now > expires
+    def set_domain(domain)
+      __deprecated__ :domain=
+      @domain = domain
+    end
   end
 
-  alias secure? secure
-  alias httponly? httponly
+  Cookie = ::HTTP::Cookie
 
-  def acceptable_from_uri?(uri)
-    host = DomainName.new(uri.host)
+  # Compatibility for Ruby 1.8/1.9
+  unless Cookie.respond_to?(:prepend, true)
+    require 'mechanize/prependable'
 
-    # RFC 6265 5.3
-    # When the user agent "receives a cookie":
-    return host.hostname == domain unless @for_domain
+    class Cookie
+      extend Prependable
 
-    if host.cookie_domain?(@domain_name)
-      true
-    elsif host.hostname == domain
-      @for_domain = false
-      true
-    else
-      false
+      class << self
+        extend Prependable
+      end
     end
   end
 
-  def valid_for_uri?(uri)
-    return false if secure? && uri.scheme != 'https'
-    acceptable_from_uri?(uri) && uri.path.start_with?(path)
-  end
-
-  def to_s
-    "#{@name}=#{@value}"
-  end
-
-  def init_with(coder)
-    yaml_initialize(coder.tag, coder.map)
-  end
+  class Cookie
+    prepend CookieIMethods
 
-  def yaml_initialize(tag, map)
-    @for_domain = true    # for forward compatibility
-    map.each { |key, value|
-      case key
-      when 'domain'
-        self.domain = value # ditto
-      else
-        instance_variable_set(:"@#{key}", value)
-      end
-    }
+    class << self
+      prepend CookieCMethods
+    end
   end
 end

data/lib/mechanize/cookie_jar.rb

@@ -1,220 +1,152 @@
-##
-# This class is used to manage the Cookies that have been returned from
-# any particular website.
-
-class Mechanize::CookieJar
-  include Enumerable
-
-  # add_cookie wants something resembling a URI.
-
-  attr_reader :jar
-
-  def initialize
-    @jar = {}
-  end
-
-  def initialize_copy other # :nodoc:
-    @jar = Marshal.load Marshal.dump other.jar
-  end
-
-  # Add a +cookie+ to the jar if it is considered acceptable from
-  # +uri+.  Return nil if the cookie was not added, otherwise return
-  # the cookie added.
-  def add(uri, cookie)
-    return nil unless cookie.acceptable_from_uri?(uri)
-    add!(cookie)
-    cookie
-  end
-
-  # Add a +cookie+ to the jar and return self.
-  def add!(cookie)
-    normal_domain = cookie.domain.downcase
-
-    @jar[normal_domain] ||= {} unless @jar.has_key?(normal_domain)
-
-    @jar[normal_domain][cookie.path] ||= {}
-    @jar[normal_domain][cookie.path][cookie.name] = cookie
-
-    self
-  end
-  alias << add!
-
-  # Fetch the cookies that should be used for the URI object passed in.
-  def cookies(url)
-    cleanup
-    url.path = '/' if url.path.empty?
-    now = Time.now
-
-    select { |cookie|
-      !cookie.expired? && cookie.valid_for_uri?(url) && (cookie.accessed_at = now)
-    }.sort_by { |cookie|
-      # RFC 6265 5.4
-      # Precedence: 1. longer path  2. older creation
-      [-cookie.path.length, cookie.created_at]
-    }
-  end
-
-  def empty?(url)
-    cookies(url).length > 0 ? false : true
-  end
-
-  def each
-    block_given? or return enum_for(__method__)
-    cleanup
-    @jar.each { |domain, paths|
-      paths.each { |path, hash|
-        hash.each_value { |cookie|
-          yield cookie
+warn 'mechanize/cookie_jar will be deprecated.  Please migrate to the http-cookie APIs.' if $VERBOSE
+
+require 'http/cookie_jar'
+require 'http/cookie_jar/yaml_saver'
+require 'mechanize/cookie'
+
+class Mechanize
+  module CookieJarIMethods
+    include CookieDeprecated
+
+    def add(arg1, arg2 = nil)
+      if arg2
+        __deprecated__ 'add and origin='
+        super arg2.dup.tap { |ncookie|
+          begin
+            ncookie.origin = arg1
+          rescue
+            return nil
+          end
         }
-      }
-    }
-  end
-
-  # call-seq:
-  #   jar.save_as(file, format = :yaml)
-  #   jar.save_as(file, options)
-  #
-  # Save the cookie jar to a file in the format specified and return
-  # self.
-  #
-  # Available option keywords are below:
-  #
-  # * +format+
-  #   [<tt>:yaml</tt>]
-  #     YAML structure (default)
-  #   [<tt>:cookiestxt</tt>]
-  #     Mozilla's cookies.txt format
-  # * +session+
-  #   [+true+]
-  #     Save session cookies as well.
-  #   [+false+]
-  #     Do not save session cookies. (default)
-  def save_as(file, options = nil)
-    if Symbol === options
-      format = options
-      session = false
-    else
-      options ||= {}
-      format = options[:format] || :yaml
-      session = !!options[:session]
+      else
+        super arg1
+      end
     end
 
-    jar = dup
-    jar.cleanup !session
+    # See HTTP::CookieJar#add.
+    def add!(cookie)
+      __deprecated__ :add
+      cookie.domain.nil? and raise NoMethodError, 'raised for compatibility'
+      @store.add(cookie)
+      self
+    end
 
-    open(file, 'w') { |f|
-      case format
-      when :yaml then
-        load_yaml
+    # See HTTP::CookieJar#save.
+    def save_as(filename, *options)
+      __deprecated__ :save
+      save(filename, *options)
+    end
 
-        YAML.dump(jar.jar, f)
-      when :cookiestxt then
-        jar.dump_cookiestxt(f)
-      else
-        raise ArgumentError, "Unknown cookie jar file format"
-      end
-    }
+    # See HTTP::CookieJar#clear.
+    def clear!
+      __deprecated__ :clear
+      clear
+    end
 
-    self
+    # See HTTP::CookieJar#store.
+    def jar
+      __deprecated__ :store
+      @store.instance_variable_get(:@jar)
+    end
   end
 
-  # Load cookie jar from a file in the format specified.
-  #
-  # Available formats:
-  # :yaml  <- YAML structure.
-  # :cookiestxt  <- Mozilla's cookies.txt format
-  def load(file, format = :yaml)
-    @jar = open(file) { |f|
-      case format
-      when :yaml then
-        load_yaml
-
-        YAML.load(f)
-      when :cookiestxt then
-        load_cookiestxt(f)
+  class CookieJar < ::HTTP::CookieJar
+    def save(filename, *options)
+      opthash = {
+        :format => :yaml,
+        :session => false,
+      }
+      case options.size
+      when 0
+      when 1
+        case options = options.first
+        when Symbol
+          opthash[:format] = options
+        else
+          opthash.update(options) if options
+        end
+      when 2
+        opthash[:format], options = options
+        opthash.update(options) if options
       else
-        raise ArgumentError, "Unknown cookie jar file format"
+        raise ArgumentError, 'wrong number of arguments (%d for 1-3)' % (1 + options.size)
       end
-    }
 
-    cleanup
+      return super if opthash[:format] != :yaml
 
-    self
-  end
+      session = opthash[:session]
+      nstore = HashStore.new
 
-  def load_yaml # :nodoc:
-    begin
-      require 'psych'
-    rescue LoadError
-    end
+      each { |cookie|
+        next if !session && cookie.session?
 
-    require 'yaml'
-  end
+        if cookie.max_age
+          cookie = cookie.dup
+          cookie.expires = cookie.expires # convert max_age to expires
+        end
+        nstore.add(cookie)
+      }
 
-  # Clear the cookie jar
-  def clear!
-    @jar = {}
-  end
+      yaml = YAML.dump(nstore.instance_variable_get(:@jar))
 
-  # Read cookies from Mozilla cookies.txt-style IO stream
-  def load_cookiestxt(io)
-    now = Time.now
+      # a gross hack
+      yaml.gsub!(%r{^(    [^ ].*: !ruby/object:)HTTP::Cookie$}) {
+        $1 + 'Mechanize::Cookie'
+      }
+      yaml.gsub!(%r{^(      expires: )(?:|!!null|(.+?)) *$}) {
+        $1 + ($2 ? Time.parse($2).httpdate : '')
+      }
 
-    io.each_line do |line|
-      line.chomp!
-      line.gsub!(/#.+/, '')
-      fields = line.split("\t")
+      open(filename, 'w') { |io|
+        io.write yaml
+      }
 
-      next if fields.length != 7
+      self
+    end
 
-      expires_seconds = fields[4].to_i
-      expires = (expires_seconds == 0) ? nil : Time.at(expires_seconds)
-      next if expires and (expires < now)
+    def load(filename, format = :yaml)
+      return super if format != :yaml
 
-      c = Mechanize::Cookie.new(fields[5], fields[6])
-      c.domain = fields[0]
-      c.for_domain = (fields[1] == "TRUE") # Whether this cookie is for domain
-      c.path = fields[2]               # Path for which the cookie is relevant
-      c.secure = (fields[3] == "TRUE") # Requires a secure connection
-      c.expires = expires             # Time the cookie expires.
-      c.version = 0                   # Conforms to Netscape cookie spec.
+      open(filename) { |io|
+        begin
+          data = YAML.load(io)
+        rescue ArgumentError
+          @logger.warn "unloadable YAML cookie data discarded" if @logger
+          return
+        end
 
-      add!(c)
+        case data
+        when Array
+          # Forward compatibility
+          data.each { |cookie|
+            add(cookie)
+          }
+        when Hash
+          data.each { |domain, paths|
+            paths.each { |path, names|
+              names.each { |cookie_name, cookie|
+                add(cookie)
+              }
+            }
+          }
+        else
+          @logger.warn "incompatible YAML cookie data discarded" if @logger
+          return
+        end
+      }
     end
-
-    @jar
   end
 
-  # Write cookies to Mozilla cookies.txt-style IO stream
-  def dump_cookiestxt(io)
-    io.puts "# HTTP Cookie File"
-    io.puts "# This file was generated by Ruby Mechanize " \
-      "#{Mechanize::VERSION} https://github.com/sparklemotion/mechanize.\n\n"
-    to_a.each do |cookie|
-      io.puts([
-        (cookie.for_domain? ? "." : "") + cookie.domain,
-        cookie.for_domain? ? "TRUE" : "FALSE",
-        cookie.path,
-        cookie.secure ? "TRUE" : "FALSE",
-        cookie.expires.to_i.to_s,
-        cookie.name,
-        cookie.value
-      ].join("\t"))
+  # Compatibility for Ruby 1.8/1.9
+  unless ::HTTP::CookieJar.respond_to?(:prepend, true)
+    require 'mechanize/prependable'
+
+    class ::HTTP::CookieJar
+      extend Prependable
     end
   end
 
-  protected
-
-  # Remove expired cookies
-  def cleanup session = false
-    @jar.each do |domain, paths|
-      paths.each do |path, names|
-        names.each do |cookie_name, cookie|
-          paths[path].delete(cookie_name) if
-            cookie.expired? or (session and cookie.session)
-        end
-      end
-    end
+  class ::HTTP::CookieJar
+    prepend CookieJarIMethods
   end
 end
-