mechanize 2.0.1 → 2.1.pre.1

data/lib/mechanize/content_type_error.rb

@@ -1,14 +1,13 @@
-class Mechanize
-  # =Synopsis
-  # This class contains an error for when a pluggable parser tries to
-  # parse a content type that it does not know how to handle.  For example
-  # if Mechanize::Page were to try to parse a PDF, a ContentTypeError
-  # would be thrown.
-  class ContentTypeError < Mechanize::Error
-    attr_reader :content_type
+##
+# This error is raised when a pluggable parser tries to parse a content type
+# that it does not know how to handle.  For example if Mechanize::Page were to
+# try to parse a PDF, a ContentTypeError would be thrown.
 
-    def initialize(content_type)
-      @content_type = content_type
-    end
+class Mechanize::ContentTypeError < Mechanize::Error
+  attr_reader :content_type
+
+  def initialize(content_type)
+    @content_type = content_type
   end
 end
+

data/lib/mechanize/cookie.rb

@@ -1,77 +1,170 @@
 require 'time'
 require 'webrick/cookie'
+require 'domain_name'
 
 # This class is used to represent an HTTP Cookie.
-class Mechanize::Cookie < WEBrick::Cookie
+class Mechanize::Cookie
+  attr_reader :name
+  attr_accessor :value, :version
+  attr_accessor :domain, :path, :secure
+  attr_accessor :comment, :max_age
 
   attr_accessor :session
 
-  def self.parse(uri, str, log = Mechanize.log)
-    return str.split(/,(?=[^;,]*=)|,$/).map { |c|
-      cookie_elem = c.split(/;+/)
-      first_elem = cookie_elem.shift
-      first_elem.strip!
-      key, value = first_elem.split(/\=/, 2)
-
-      cookie = nil
-      begin
-        cookie = new(key, value.dup)
-      rescue
-        log.warn("Couldn't parse key/value: #{first_elem}") if log
+  attr_accessor :created_at
+  attr_accessor :accessed_at
+
+  # :call-seq:
+  #     new(name, value)
+  #     new(name, value, attr_hash)
+  #     new(attr_hash)
+  #
+  # Creates a cookie object.  For each key of +attr_hash+, the setter
+  # is called if defined.  Each key can be either a symbol or a
+  # string, downcased or not.
+  #
+  # e.g.
+  #     new("uid", "a12345")
+  #     new("uid", "a12345", :domain => 'example.org',
+  #                          :for_domain => true, :expired => Time.now + 7*86400)
+  #     new("name" => "uid", "value" => "a12345", "Domain" => 'www.example.org')
+  #
+  def initialize(*args)
+    @version = 0     # Netscape Cookie
+
+    @domain = @path = @secure = @comment = @max_age =
+      @expires = @comment_url = @discard = @port = nil
+
+    @created_at = @accessed_at = Time.now
+    case args.size
+    when 2
+      @name, @value = *args
+      @for_domain = false
+      return
+    when 3
+      @name, @value, attr_hash = *args
+    when 1
+      attr_hash = args.first
+    else
+      raise ArgumentError, "wrong number of arguments (#{args.size} for 1-3)"
+    end
+    for_domain = false
+    attr_hash.each_pair { |key, val|
+      skey = key.to_s.downcase
+      skey.sub!(/[!?]\z/, '')
+      case skey
+      when 'for_domain'
+        for_domain = !!val
+      when 'name'
+        @name = val
+      when 'value'
+        @value = val
+      else
+        setter = :"#{skey}="
+        send(setter, val) if respond_to?(setter)
       end
+    }
+    @for_domain = for_domain
+  end
 
-      next unless cookie
-
-      cookie_elem.each do |pair|
-        pair.strip!
-        key, value = pair.split(/\=/, 2)
-        next unless key
-        value = WEBrick::HTTPUtils.dequote(value.strip) if value
-
-        case key.downcase
-        when "domain" then
-          value = ".#{value}" unless value =~ /^\./
-          cookie.domain = value
-        when "path" then
-          cookie.path = value
-        when 'expires'
-          if value.empty? then
-            cookie.session = true
-            next
-          end
+  # If this flag is true, this cookie will be sent to any host in the
+  # +domain+.  If it is false, this cookie will be sent only to the
+  # host indicated by the +domain+.
+  attr_accessor :for_domain
+  alias for_domain? for_domain
 
-          begin
-            cookie.expires = Time::parse(value)
-          rescue
-            log.warn("Couldn't parse expires: #{value}") if log
-          end
-        when "max-age" then
-          begin
-            cookie.max_age = Integer(value)
-          rescue
-            log.warn("Couldn't parse max age '#{value}'") if log
-            cookie.max_age = nil
-          end
-        when "comment" then cookie.comment = value
-        when "version" then
-          begin
-            cookie.version = Integer(value)
-          rescue
-            log.warn("Couldn't parse version '#{value}'") if log
-            cookie.version = nil
+  class << self
+    def parse(uri, str, log = Mechanize.log)
+      return str.split(/,(?=[^;,]*=)|,$/).map { |c|
+        cookie_elem = c.split(/;+/)
+        first_elem = cookie_elem.shift
+        first_elem.strip!
+        key, value = first_elem.split(/\=/, 2)
+
+        begin
+          cookie = new(key, value.dup)
+        rescue
+          log.warn("Couldn't parse key/value: #{first_elem}") if log
+          next
+        end
+
+        cookie_elem.each do |pair|
+          pair.strip!
+          key, value = pair.split(/\=/, 2)
+          next unless key
+          value = WEBrick::HTTPUtils.dequote(value.strip) if value
+
+          case key.downcase
+          when 'domain'
+            cookie.domain = value
+            cookie.for_domain = true
+          when 'path'
+            cookie.path = value
+          when 'expires'
+            if value.empty?
+              cookie.session = true
+              next
+            end
+
+            begin
+              cookie.expires = Time::parse(value)
+            rescue
+              log.warn("Couldn't parse expires: #{value}") if log
+            end
+          when 'max-age'
+            begin
+              cookie.max_age = Integer(value)
+            rescue
+              log.warn("Couldn't parse max age '#{value}'") if log
+            end
+          when 'comment'
+            cookie.comment = value
+          when 'version'
+            begin
+              cookie.version = Integer(value)
+            rescue
+              log.warn("Couldn't parse version '#{value}'") if log
+              cookie.version = nil
+            end
+          when 'secure'
+            cookie.secure = true
           end
-        when "secure"  then cookie.secure = true
         end
-      end
 
-      cookie.path    ||= uri.path.to_s.sub(%r%[^/]*$%, '')
-      cookie.secure  ||= false
-      cookie.domain  ||= uri.host
-      # Move this in to the cookie jar
-      yield cookie if block_given?
+        cookie.path    ||= (uri + './').path
+        cookie.secure  ||= false
+        cookie.domain  ||= uri.host
+        # Move this in to the cookie jar
+        yield cookie if block_given?
 
-      cookie
-    }
+        cookie
+      }
+    end
+  end
+
+  alias set_domain domain=
+
+  # Sets the domain attribute.  A leading dot in +domain+ implies
+  # turning the +for_domain?+ flag on.
+  def domain=(domain)
+    if domain.start_with?('.')
+      @for_domain = true
+      domain = domain[1..-1]
+    end
+    # Do we really need to support this?
+    if domain.match(/\A([^:]+):[0-9]+\z/)
+      domain = $1
+    end
+    @domain_name = DomainName.new(domain)
+    set_domain(@domain_name.hostname)
+  end
+
+  def expires=(t)
+    @expires = t && (t.is_a?(Time) ? t.httpdate : t.to_s)
+  end
+
+  def expires
+    @expires && Time.parse(@expires)
   end
 
   def expired?
@@ -79,7 +172,47 @@ class Mechanize::Cookie < WEBrick::Cookie
     Time.now > expires
   end
 
+  alias secure? secure
+
+  def acceptable_from_uri?(uri)
+    host = DomainName.new(uri.host)
+
+    # RFC 6265 5.3
+    # When the user agent "receives a cookie":
+    return host.hostname == domain unless @for_domain
+
+    if host.cookie_domain?(@domain_name)
+      true
+    elsif host.hostname == domain
+      @for_domain = false
+      true
+    else
+      false
+    end
+  end
+
+  def valid_for_uri?(uri)
+    return false if secure? && uri.scheme != 'https'
+    acceptable_from_uri?(uri) && uri.path.start_with?(path)
+  end
+
   def to_s
     "#{@name}=#{@value}"
   end
+
+  def init_with(coder)
+    yaml_initialize(coder.tag, coder.map)
+  end
+
+  def yaml_initialize(tag, map)
+    @for_domain = true    # for forward compatibility
+    map.each { |key, value|
+      case key
+      when 'domain'
+        self.domain = value # ditto
+      else
+        instance_variable_set(:"@#{key}", value)
+      end
+    }
+  end
 end

data/lib/mechanize/cookie_jar.rb

@@ -3,11 +3,10 @@
 # any particular website.
 
 class Mechanize::CookieJar
+  include Enumerable
 
   # add_cookie wants something resembling a URI.
 
-  FakeURI = Struct.new(:host) # :nodoc:
-
   attr_reader :jar
 
   def initialize
@@ -18,10 +17,17 @@ class Mechanize::CookieJar
     @jar = Marshal.load Marshal.dump other.jar
   end
 
-  # Add a cookie to the Jar.
+  # Add a +cookie+ to the jar if it is considered acceptable from
+  # +uri+.  Return nil if the cookie was not added, otherwise return
+  # the cookie added.
   def add(uri, cookie)
-    return unless valid_cookie_for_uri?(uri, cookie)
+    return nil unless cookie.acceptable_from_uri?(uri)
+    add!(cookie)
+    cookie
+  end
 
+  # Add a +cookie+ to the jar and return self.
+  def add!(cookie)
     normal_domain = cookie.domain.downcase
 
     @jar[normal_domain] ||= {} unless @jar.has_key?(normal_domain)
@@ -29,46 +35,39 @@ class Mechanize::CookieJar
     @jar[normal_domain][cookie.path] ||= {}
     @jar[normal_domain][cookie.path][cookie.name] = cookie
 
-    cookie
+    self
   end
+  alias << add!
 
   # Fetch the cookies that should be used for the URI object passed in.
   def cookies(url)
     cleanup
     url.path = '/' if url.path.empty?
+    now = Time.now
 
-    domains = @jar.find_all { |domain, _|
-      cookie_domain = self.class.strip_port(domain)
-      if cookie_domain.start_with?('.')
-        url.host =~ /#{Regexp.escape cookie_domain}$/i
-      else
-        url.host =~ /^#{Regexp.escape cookie_domain}$/i
-      end
+    select { |cookie|
+      !cookie.expired? && cookie.valid_for_uri?(url) && (cookie.accessed_at = now)
+    }.sort_by { |cookie|
+      # RFC 6265 5.4
+      # Precedence: 1. longer path  2. older creation
+      [-cookie.path.length, cookie.created_at]
     }
-
-    return [] unless domains.length > 0
-
-    cookies = domains.map { |_,paths|
-      paths.find_all { |path, _|
-        url.path =~ /^#{Regexp.escape(path)}/
-      }.map { |_,cookie| cookie.values }
-    }.flatten
-
-    cookies.find_all { |cookie| ! cookie.expired? }
   end
 
   def empty?(url)
     cookies(url).length > 0 ? false : true
   end
 
-  def to_a
+  def each
+    block_given? or return enum_for(__method__)
     cleanup
-
-    @jar.map do |domain, paths|
-      paths.map do |path, names|
-        names.values
-      end
-    end.flatten
+    @jar.each { |domain, paths|
+      paths.each { |path, hash|
+        hash.each_value { |cookie|
+          yield cookie
+        }
+      }
+    }
   end
 
   # Save the cookie jar to a file in the format specified.
@@ -151,15 +150,13 @@ class Mechanize::CookieJar
 
       c = Mechanize::Cookie.new(fields[5], fields[6])
       c.domain = fields[0]
-      # Field 1 indicates whether the cookie can be read by other machines at
-      # the same domain.  This is computed by the cookie implementation, based
-      # on the domain value.
+      c.for_domain = (fields[1] == "TRUE") # Whether this cookie is for domain
       c.path = fields[2]               # Path for which the cookie is relevant
       c.secure = (fields[3] == "TRUE") # Requires a secure connection
       c.expires = expires             # Time the cookie expires.
       c.version = 0                   # Conforms to Netscape cookie spec.
 
-      add(FakeURI.new(c.domain), c)
+      add!(c)
     end
 
     @jar
@@ -168,58 +165,18 @@ class Mechanize::CookieJar
   # Write cookies to Mozilla cookies.txt-style IO stream
   def dump_cookiestxt(io)
     to_a.each do |cookie|
-      fields = []
-      fields[0] = cookie.domain
-
-      if cookie.domain =~ /^\./
-        fields[1] = "TRUE"
-      else
-        fields[1] = "FALSE"
-      end
-
-      fields[2] = cookie.path
-
-      if cookie.secure == true
-        fields[3] = "TRUE"
-      else
-        fields[3] = "FALSE"
-      end
-
-      fields[4] = cookie.expires.to_i.to_s
-
-      fields[5] = cookie.name
-      fields[6] = cookie.value
-      io.puts(fields.join("\t"))
+      io.puts([
+        cookie.domain,
+        cookie.for_domain? ? "TRUE" : "FALSE",
+        cookie.path,
+        cookie.secure ? "TRUE" : "FALSE",
+        cookie.expires.to_i.to_s,
+        cookie.name,
+        cookie.value
+      ].join("\t"))
     end
   end
 
-  private
-  # Determine if the cookie's domain and path are valid for
-  # the uri.host based on the rules in RFC 2965
-  def valid_cookie_for_uri?(uri, cookie)
-    cookie_domain = self.class.strip_port(cookie.domain)
-
-    # reject cookies whose domains do not contain an embedded dot
-    # cookies for localhost and .local. are exempt from this rule
-    return false if
-      cookie_domain !~ /.\../ && cookie_domain !~ /(localhost|\.?local)\.?$/
-
-    cookie_domain = if cookie_domain.start_with? '.' then
-                      ".?#{Regexp.escape cookie_domain[1..-1]}"
-                    else
-                      Regexp.escape cookie_domain
-                    end
-
-    # Permitted:     A Set-Cookie for x.foo.com for Domain=.foo.com
-    # Not Permitted: A Set-Cookie for y.x.foo.com for Domain=.foo.com because
-    #                y.x contains a dot
-    # Not Permitted: A Set-Cookie for foo.com for Domain=.bar.com
-    match = uri.host.match(/#{cookie_domain}/i)
-    return false if match.nil? || match.pre_match =~ /.\../
-
-    true
-  end
-
   protected
 
   # Remove expired cookies
@@ -233,9 +190,5 @@ class Mechanize::CookieJar
       end
     end
   end
-
-  def self.strip_port(host)
-    host.gsub(/:[0-9]+$/,'')
-  end
 end