mechanize 1.0.1.beta.20110107104205 → 2.0.pre.1

data/lib/mechanize/content_type_error.rb

@@ -4,7 +4,7 @@ class Mechanize
   # parse a content type that it does not know how to handle.  For example
   # if Mechanize::Page were to try to parse a PDF, a ContentTypeError
   # would be thrown.
-  class ContentTypeError < RuntimeError
+  class ContentTypeError < Mechanize::Error
     attr_reader :content_type
 
     def initialize(content_type)

data/lib/mechanize/cookie.rb

@@ -1,78 +1,85 @@
 require 'time'
 require 'webrick/cookie'
 
-class Mechanize
-  # This class is used to represent an HTTP Cookie.
-  class Cookie < WEBrick::Cookie
-    def self.parse(uri, str, log = Mechanize.log)
-      return str.split(/,(?=[^;,]*=)|,$/).collect { |c|
-        cookie_elem = c.split(/;+/)
-        first_elem = cookie_elem.shift
-        first_elem.strip!
-        key, value = first_elem.split(/\=/, 2)
+# This class is used to represent an HTTP Cookie.
+class Mechanize::Cookie < WEBrick::Cookie
 
-        cookie = nil
-        begin
-          cookie = new(key, value.dup)
-        rescue
-          log.warn("Couldn't parse key/value: #{first_elem}") if log
-        end
-        next unless cookie
+  attr_accessor :session
+
+  def self.parse(uri, str, log = Mechanize.log)
+    return str.split(/,(?=[^;,]*=)|,$/).map { |c|
+      cookie_elem = c.split(/;+/)
+      first_elem = cookie_elem.shift
+      first_elem.strip!
+      key, value = first_elem.split(/\=/, 2)
+
+      cookie = nil
+      begin
+        cookie = new(key, value.dup)
+      rescue
+        log.warn("Couldn't parse key/value: #{first_elem}") if log
+      end
+
+      next unless cookie
+
+      cookie_elem.each do |pair|
+        pair.strip!
+        key, value = pair.split(/\=/, 2)
+        next unless key
+        value = WEBrick::HTTPUtils.dequote(value.strip) if value
+
+        case key.downcase
+        when "domain" then
+          value = ".#{value}" unless value =~ /^\./
+          cookie.domain = value
+        when "path" then
+          cookie.path = value
+        when 'expires'
+          if value.empty? then
+            cookie.session = true
+            next
+          end
 
-        cookie_elem.each do |pair|
-          pair.strip!
-          key, value = pair.split(/\=/, 2)
-          if value
-            value = WEBrick::HTTPUtils.dequote(value.strip)
+          begin
+            cookie.expires = Time::parse(value)
+          rescue
+            log.warn("Couldn't parse expires: #{value}") if log
+          end
+        when "max-age" then
+          begin
+            cookie.max_age = Integer(value)
+          rescue
+            log.warn("Couldn't parse max age '#{value}'") if log
+            cookie.max_age = nil
           end
-          case key.downcase
-          when "domain"  then cookie.domain  = value.sub(/^\./, '')
-          when "path"    then cookie.path    = value
-          when 'expires'
-            begin
-              cookie.expires = Time::parse(value)
-            rescue
-              if log
-                log.warn("Couldn't parse expires: #{value}")
-              end
-            end
-          when "max-age" then
-            begin
-              cookie.max_age = Integer(value)
-            rescue
-              log.warn("Couldn't parse max age '#{value}'") if log
-              cookie.max_age = nil
-            end
-          when "comment" then cookie.comment = value
-          when "version" then
-            begin
-              cookie.version = Integer(value)
-            rescue
-              log.warn("Couldn't parse version '#{value}'") if log
-              cookie.version = nil
-            end
-          when "secure"  then cookie.secure = true
+        when "comment" then cookie.comment = value
+        when "version" then
+          begin
+            cookie.version = Integer(value)
+          rescue
+            log.warn("Couldn't parse version '#{value}'") if log
+            cookie.version = nil
           end
+        when "secure"  then cookie.secure = true
         end
+      end
 
-        cookie.path    ||= uri.path.to_s.sub(%r%[^/]*$%, '')
-        cookie.secure  ||= false
-        cookie.domain  ||= uri.host
-        # Move this in to the cookie jar
-        yield cookie if block_given?
-      }
-    end
+      cookie.path    ||= uri.path.to_s.sub(%r%[^/]*$%, '')
+      cookie.secure  ||= false
+      cookie.domain  ||= uri.host
+      # Move this in to the cookie jar
+      yield cookie if block_given?
 
-    def expired?
-      if expires.nil?
-        false
-      else
-        Time.now > expires
-      end
-    end
+      cookie
+    }
+  end
+
+  def expired?
+    return false unless expires
+    Time.now > expires
+  end
 
-    def to_s
-      "#{@name}=#{@value}"
-    end
+  def to_s
+    "#{@name}=#{@value}"
   end
 end

data/lib/mechanize/cookie_jar.rb

@@ -1,186 +1,235 @@
-require 'yaml'
+##
+# This class is used to manage the Cookies that have been returned from
+# any particular website.
 
-class Mechanize
-  # This class is used to manage the Cookies that have been returned from
-  # any particular website.
-  class CookieJar
-    attr_reader :jar
+class Mechanize::CookieJar
 
-    def initialize
-      @jar = {}
-    end
-
-    # Add a cookie to the Jar.
-    def add(uri, cookie)
-      return unless uri.host =~ /#{CookieJar.strip_port(cookie.domain)}$/i
+  # add_cookie wants something resembling a URI.
 
-      normal_domain = cookie.domain.downcase
+  FakeURI = Struct.new(:host) # :nodoc:
 
-      unless @jar.has_key?(normal_domain)
-        @jar[normal_domain] = Hash.new { |h,k| h[k] = {} }
-      end
+  attr_reader :jar
 
-      @jar[normal_domain][cookie.path] ||= {}
-      @jar[normal_domain][cookie.path][cookie.name] = cookie
-      cleanup
-      cookie
-    end
+  def initialize
+    @jar = {}
+  end
 
-    # Fetch the cookies that should be used for the URI object passed in.
-    def cookies(url)
-      cleanup
-      url.path = '/' if url.path.empty?
+  def initialize_copy other # :nodoc:
+    @jar = Marshal.load Marshal.dump other.jar
+  end
 
-      domains = @jar.find_all { |domain, _|
-        url.host =~ /#{CookieJar.strip_port(domain)}$/i
-      }
+  # Add a cookie to the Jar.
+  def add(uri, cookie)
+    return unless valid_cookie_for_uri?(uri, cookie)
 
-      return [] unless domains.length > 0
+    normal_domain = cookie.domain.downcase
 
-      cookies = domains.map { |_,paths|
-        paths.find_all { |path, _|
-          url.path =~ /^#{Regexp.escape(path)}/
-        }.map { |_,cookie| cookie.values }
-      }.flatten
+    @jar[normal_domain] ||= {} unless @jar.has_key?(normal_domain)
 
-      cookies.find_all { |cookie| ! cookie.expired? }
-    end
+    @jar[normal_domain][cookie.path] ||= {}
+    @jar[normal_domain][cookie.path][cookie.name] = cookie
 
-    def empty?(url)
-      cookies(url).length > 0 ? false : true
-    end
+    cookie
+  end
 
-    def to_a
-      cookies = []
-      @jar.each do |domain, paths|
-        paths.each do |path, names|
-          cookies << names.values
-        end
+  # Fetch the cookies that should be used for the URI object passed in.
+  def cookies(url)
+    cleanup
+    url.path = '/' if url.path.empty?
+
+    domains = @jar.find_all { |domain, _|
+      cookie_domain = self.class.strip_port(domain)
+      if cookie_domain.start_with?('.')
+        url.host =~ /#{Regexp.escape cookie_domain}$/i
+      else
+        url.host =~ /^#{Regexp.escape cookie_domain}$/i
       end
-      cookies.flatten
-    end
+    }
 
-    # Save the cookie jar to a file in the format specified.
-    #
-    # Available formats:
-    # :yaml  <- YAML structure
-    # :cookiestxt  <- Mozilla's cookies.txt format
-    def save_as(file, format = :yaml)
-      ::File.open(file, "w") { |f|
-        case format
-        when :yaml then
-          YAML::dump(@jar, f)
-        when :cookiestxt then
-          dump_cookiestxt(f)
-        else
-          raise "Unknown cookie jar file format"
-        end
-      }
-    end
+    return [] unless domains.length > 0
 
-    # Load cookie jar from a file in the format specified.
-    #
-    # Available formats:
-    # :yaml  <- YAML structure.
-    # :cookiestxt  <- Mozilla's cookies.txt format
-    def load(file, format = :yaml)
-      @jar = ::File.open(file) { |f|
-        case format
-        when :yaml then
-          YAML::load(f)
-        when :cookiestxt then
-          load_cookiestxt(f)
-        else
-          raise "Unknown cookie jar file format"
-        end
-      }
-    end
+    cookies = domains.map { |_,paths|
+      paths.find_all { |path, _|
+        url.path =~ /^#{Regexp.escape(path)}/
+      }.map { |_,cookie| cookie.values }
+    }.flatten
 
-    # Clear the cookie jar
-    def clear!
-      @jar = {}
-    end
+    cookies.find_all { |cookie| ! cookie.expired? }
+  end
 
-    # Read cookies from Mozilla cookies.txt-style IO stream
-    def load_cookiestxt(io)
-      now = Time.now
-      fakeuri = Struct.new(:host)    # add_cookie wants something resembling a URI.
+  def empty?(url)
+    cookies(url).length > 0 ? false : true
+  end
 
-      io.each_line do |line|
-        line.chomp!
-        line.gsub!(/#.+/, '')
-        fields = line.split("\t")
+  def to_a
+    cleanup
 
-        next if fields.length != 7
+    @jar.map do |domain, paths|
+      paths.map do |path, names|
+        names.values
+      end
+    end.flatten
+  end
 
-        expires_seconds = fields[4].to_i
+  # Save the cookie jar to a file in the format specified.
+  #
+  # Available formats:
+  # :yaml  <- YAML structure
+  # :cookiestxt  <- Mozilla's cookies.txt format
+  def save_as(file, format = :yaml)
+    jar = dup
+    jar.cleanup true
+
+    open(file, 'w') { |f|
+      case format
+      when :yaml then
         begin
-          expires = (expires_seconds == 0) ? nil : Time.at(expires_seconds)
-        rescue
-          next
-          # Just in case we ever decide to support DateTime...
-          # expires = DateTime.new(1970,1,1) + ((expires_seconds + 1) / (60*60*24.0))
+          require 'psych'
+        rescue LoadError
         end
-        next if (expires_seconds != 0) && (expires < now)
-
-        c = Mechanize::Cookie.new(fields[5], fields[6])
-        c.domain = fields[0]
-        # Field 1 indicates whether the cookie can be read by other machines at the same domain.
-        # This is computed by the cookie implementation, based on the domain value.
-        c.path = fields[2]               # Path for which the cookie is relevant
-        c.secure = (fields[3] == "TRUE") # Requires a secure connection
-        c.expires = expires             # Time the cookie expires.
-        c.version = 0                   # Conforms to Netscape cookie spec.
-
-        add(fakeuri.new(c.domain), c)
+
+        require 'yaml'
+
+        YAML.dump(jar.jar, f)
+      when :cookiestxt then
+        jar.dump_cookiestxt(f)
+      else
+        raise ArgumentError, "Unknown cookie jar file format"
       end
-      @jar
-    end
+    }
 
-    # Write cookies to Mozilla cookies.txt-style IO stream
-    def dump_cookiestxt(io)
-      to_a.each do |cookie|
-        fields = []
-        fields[0] = cookie.domain
+    self
+  end
 
-        if cookie.domain =~ /^\./
-          fields[1] = "TRUE"
-        else
-          fields[1] = "FALSE"
-        end
+  # Load cookie jar from a file in the format specified.
+  #
+  # Available formats:
+  # :yaml  <- YAML structure.
+  # :cookiestxt  <- Mozilla's cookies.txt format
+  def load(file, format = :yaml)
+    @jar = open(file) { |f|
+      case format
+      when :yaml then
+        YAML::load(f)
+      when :cookiestxt then
+        load_cookiestxt(f)
+      else
+        raise ArgumentError, "Unknown cookie jar file format"
+      end
+    }
 
-        fields[2] = cookie.path
+    cleanup
 
-        if cookie.secure == true
-          fields[3] = "TRUE"
-        else
-          fields[3] = "FALSE"
-        end
+    self
+  end
+
+  # Clear the cookie jar
+  def clear!
+    @jar = {}
+  end
+
+  # Read cookies from Mozilla cookies.txt-style IO stream
+  def load_cookiestxt(io)
+    now = Time.now
+
+    io.each_line do |line|
+      line.chomp!
+      line.gsub!(/#.+/, '')
+      fields = line.split("\t")
+
+      next if fields.length != 7
+
+      expires_seconds = fields[4].to_i
+      expires = (expires_seconds == 0) ? nil : Time.at(expires_seconds)
+      next if expires and (expires < now)
+
+      c = Mechanize::Cookie.new(fields[5], fields[6])
+      c.domain = fields[0]
+      # Field 1 indicates whether the cookie can be read by other machines at
+      # the same domain.  This is computed by the cookie implementation, based
+      # on the domain value.
+      c.path = fields[2]               # Path for which the cookie is relevant
+      c.secure = (fields[3] == "TRUE") # Requires a secure connection
+      c.expires = expires             # Time the cookie expires.
+      c.version = 0                   # Conforms to Netscape cookie spec.
+
+      add(FakeURI.new(c.domain), c)
+    end
+
+    @jar
+  end
 
-        fields[4] = cookie.expires.to_i.to_s
+  # Write cookies to Mozilla cookies.txt-style IO stream
+  def dump_cookiestxt(io)
+    to_a.each do |cookie|
+      fields = []
+      fields[0] = cookie.domain
 
-        fields[5] = cookie.name
-        fields[6] = cookie.value
-        io.puts(fields.join("\t"))
+      if cookie.domain =~ /^\./
+        fields[1] = "TRUE"
+      else
+        fields[1] = "FALSE"
       end
+
+      fields[2] = cookie.path
+
+      if cookie.secure == true
+        fields[3] = "TRUE"
+      else
+        fields[3] = "FALSE"
+      end
+
+      fields[4] = cookie.expires.to_i.to_s
+
+      fields[5] = cookie.name
+      fields[6] = cookie.value
+      io.puts(fields.join("\t"))
     end
+  end
 
-    private
-    # Remove expired cookies
-    def cleanup
-      @jar.each do |domain, paths|
-        paths.each do |path, names|
-          names.each do |cookie_name, cookie|
-            if cookie.expired?
-              paths[path].delete(cookie_name)
-            end
-          end
+  private
+  # Determine if the cookie's domain and path are valid for
+  # the uri.host based on the rules in RFC 2965
+  def valid_cookie_for_uri?(uri, cookie)
+    cookie_domain = self.class.strip_port(cookie.domain)
+
+    # reject cookies whose domains do not contain an embedded dot
+    # cookies for localhost and .local. are exempt from this rule
+    return false if
+      cookie_domain !~ /.\../ && cookie_domain !~ /(localhost|\.?local)\.?$/
+
+    cookie_domain = if cookie_domain.start_with? '.' then
+                      ".?#{Regexp.escape cookie_domain[1..-1]}"
+                    else
+                      Regexp.escape cookie_domain
+                    end
+
+    # Permitted:     A Set-Cookie for x.foo.com for Domain=.foo.com
+    # Not Permitted: A Set-Cookie for y.x.foo.com for Domain=.foo.com because
+    #                y.x contains a dot
+    # Not Permitted: A Set-Cookie for foo.com for Domain=.bar.com
+    match = uri.host.match(/#{cookie_domain}/i)
+    return false if match.nil? || match.pre_match =~ /.\../
+
+    true
+  end
+
+  protected
+
+  # Remove expired cookies
+  def cleanup session = false
+    @jar.each do |domain, paths|
+      paths.each do |path, names|
+        names.each do |cookie_name, cookie|
+          paths[path].delete(cookie_name) if
+            cookie.expired? or (session and cookie.session)
         end
       end
     end
+  end
 
-    def self.strip_port(host)
-      host.gsub(/:[0-9]+$/,'')
-    end
+  def self.strip_port(host)
+    host.gsub(/:[0-9]+$/,'')
   end
 end
+