tenderlove-mechanize 0.9.3.20090623142847 → 0.9.3.20090911221705

data/lib/mechanize/chain/response_reader.rb

@@ -0,0 +1,39 @@
+class Mechanize
+  class Chain
+    class ResponseReader
+      include Mechanize::Handler
+
+      def initialize(response)
+        @response = response
+      end
+
+      def handle(ctx, params)
+        params[:response] = @response
+        body = StringIO.new
+        total = 0
+        @response.read_body { |part|
+          total += part.length
+          body.write(part)
+          Mechanize.log.debug("Read #{total} bytes") if Mechanize.log
+        }
+        body.rewind
+
+        res_klass = Net::HTTPResponse::CODE_TO_OBJ[@response.code.to_s]
+        raise ResponseCodeError.new(@response) unless res_klass
+
+        # Net::HTTP ignores EOFError if Content-length is given, so we emulate it here.
+        unless res_klass <= Net::HTTPRedirection
+          raise EOFError if (!params[:request].is_a?(Net::HTTP::Head)) && @response.content_length() && @response.content_length() != total
+        end
+
+        @response.each_header { |k,v|
+          Mechanize.log.debug("response-header: #{ k } => #{ v }")
+        } if Mechanize.log
+
+        params[:response_body] = body
+        params[:res_klass] = res_klass
+        super
+      end
+    end
+  end
+end

data/lib/mechanize/chain/ssl_resolver.rb

@@ -0,0 +1,40 @@
+class Mechanize
+  class Chain
+    class SSLResolver
+      include Mechanize::Handler
+
+      def initialize(ca_file, verify_callback, cert, key, pass)
+        @ca_file = ca_file
+        @verify_callback = verify_callback
+        @cert = cert
+        @key = key
+        @pass = pass
+      end
+
+      def handle(ctx, params)
+        uri       = params[:uri]
+        http_obj  = params[:connection]
+
+        ssl = nil
+        if http_obj.instance_variable_defined?(:@ssl_context)
+          ssl = http_obj.instance_variable_get(:@ssl_context)
+        end
+
+        if uri.scheme == 'https' && ! http_obj.started? && ! ssl.frozen?
+          http_obj.use_ssl = true
+          http_obj.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          if @ca_file
+            http_obj.ca_file = @ca_file
+            http_obj.verify_mode = OpenSSL::SSL::VERIFY_PEER
+            http_obj.verify_callback = @verify_callback if @verify_callback
+          end
+          if @cert && @key
+            http_obj.cert = OpenSSL::X509::Certificate.new(::File.read(@cert))
+            http_obj.key  = OpenSSL::PKey::RSA.new(::File.read(@key), @pass)
+          end
+        end
+        super
+      end
+    end
+  end
+end

data/lib/mechanize/chain/uri_resolver.rb

@@ -0,0 +1,75 @@
+class Mechanize
+  class Chain
+    class URIResolver
+      include Mechanize::Handler
+
+      def initialize(scheme_handlers)
+        @scheme_handlers = scheme_handlers
+      end
+
+      def handle(ctx, params)
+        raise ArgumentError.new('uri must be specified') unless params[:uri]
+        params[:uri] = params[:uri].dup if params[:uri].is_a?(URI)
+        uri     = params[:uri]
+        referer = params[:referer]
+        unless uri.is_a?(URI)
+          uri = uri.to_s.strip.gsub(/[^#{0.chr}-#{126.chr}]/) { |match|
+            if RUBY_VERSION >= "1.9.0"
+              CGI.escape(match)
+            else
+              sprintf('%%%X', match.unpack($KCODE == 'UTF8' ? 'U' : 'c')[0])
+            end
+          }
+
+          escaped_uri = Util.html_unescape(
+                                           uri.split(/(?:%[0-9A-Fa-f]{2})+|#/).zip(
+                                                                                   uri.scan(/(?:%[0-9A-Fa-f]{2})+|#/)
+                                                                                   ).map { |x,y|
+                                             "#{URI.escape(x)}#{y}"
+                                           }.join('')
+                                           )
+
+          begin
+            uri = URI.parse(escaped_uri)
+          rescue
+            uri = URI.parse(URI.escape(escaped_uri))
+          end
+
+        end
+        uri = @scheme_handlers[
+                               uri.relative? ? 'relative' : uri.scheme.downcase
+                              ].call(uri, params[:referer])
+
+        if params[:referer] && params[:referer].uri
+          if uri.path.length == 0 && uri.relative?
+            uri.path = params[:referer].uri.path
+          end
+        end
+
+        uri.path = '/' if uri.path.length == 0
+
+        if uri.relative?
+          raise 'need absolute URL' unless referer && referer.uri
+          base = nil
+          if referer.respond_to?(:bases) && referer.parser
+            base = referer.bases.last
+          end
+
+          uri = ((base && base.uri && base.uri.absolute?) ?
+                 base.uri :
+                 referer.uri) + uri
+          uri = referer.uri + uri
+          # Strip initial "/.." bits from the path
+          uri.path.sub!(/^(\/\.\.)+(?=\/)/, '')
+        end
+
+        unless ['http', 'https', 'file'].include?(uri.scheme.downcase)
+          raise "unsupported scheme: #{uri.scheme}"
+        end
+        params[:uri] = uri
+
+        super
+      end
+    end
+  end
+end

data/lib/mechanize/content_type_error.rb

@@ -0,0 +1,14 @@
+class Mechanize
+  # =Synopsis
+  # This class contains an error for when a pluggable parser tries to
+  # parse a content type that it does not know how to handle.  For example
+  # if Mechanize::Page were to try to parse a PDF, a ContentTypeError
+  # would be thrown.
+  class ContentTypeError < RuntimeError
+    attr_reader :content_type
+
+    def initialize(content_type)
+      @content_type = content_type
+    end
+  end
+end

data/lib/mechanize/cookie.rb

@@ -0,0 +1,70 @@
+require 'time'
+require 'webrick/cookie'
+
+class Mechanize
+  # This class is used to represent an HTTP Cookie.
+  class Cookie < WEBrick::Cookie
+    def self.parse(uri, str, log = Mechanize.log)
+      return str.split(/,(?=[^;,]*=)|,$/).collect { |c|
+        cookie_elem = c.split(/;+/)
+        first_elem = cookie_elem.shift
+        first_elem.strip!
+        key, value = first_elem.split(/\=/, 2)
+
+        cookie = nil
+        begin
+          cookie = new(key, WEBrick::HTTPUtils.dequote(value))
+        rescue
+          log.warn("Couldn't parse key/value: #{first_elem}") if log
+        end
+        next unless cookie
+
+        cookie_elem.each do |pair|
+          pair.strip!
+          key, value = pair.split(/\=/, 2)
+          if value
+            value = WEBrick::HTTPUtils.dequote(value.strip)
+          end
+          case key.downcase
+          when "domain"  then cookie.domain  = value.sub(/^\./, '')
+          when "path"    then cookie.path    = value
+          when 'expires'
+            begin
+              cookie.expires = Time::parse(value)
+            rescue
+              if log
+                log.warn("Couldn't parse expires: #{value}")
+              end
+            end
+          when "max-age" then
+            begin
+              cookie.max_age = Integer(value)
+            rescue
+              log.warn("Couldn't parse max age '#{value}'") if log
+              cookie.max_age = nil
+            end
+          when "comment" then cookie.comment = value
+          when "version" then
+            begin
+              cookie.version = Integer(value)
+            rescue
+              log.warn("Couldn't parse version '#{value}'") if log
+              cookie.version = nil
+            end
+          when "secure"  then cookie.secure = true
+          end
+        end
+
+        cookie.path    ||= uri.path.to_s.sub(%r%[^/]*$%, '')
+        cookie.secure  ||= false
+        cookie.domain  ||= uri.host
+        # Move this in to the cookie jar
+        yield cookie if block_given?
+      }
+    end
+
+    def to_s
+      "#{@name}=#{@value}"
+    end
+  end
+end

data/lib/mechanize/cookie_jar.rb

@@ -0,0 +1,188 @@
+require 'yaml'
+
+class Mechanize
+  # This class is used to manage the Cookies that have been returned from
+  # any particular website.
+  class CookieJar
+    attr_reader :jar
+
+    def initialize
+      @jar = {}
+    end
+
+    # Add a cookie to the Jar.
+    def add(uri, cookie)
+      return unless uri.host =~ /#{CookieJar.strip_port(cookie.domain)}$/i
+
+      normal_domain = cookie.domain.downcase
+
+      unless @jar.has_key?(normal_domain)
+        @jar[normal_domain] = Hash.new { |h,k| h[k] = {} }
+      end
+
+      @jar[normal_domain][cookie.path][cookie.name] = cookie
+      cleanup
+      cookie
+    end
+
+    # Fetch the cookies that should be used for the URI object passed in.
+    def cookies(url)
+      cleanup
+      url.path = '/' if url.path.empty?
+
+      domains = @jar.find_all { |domain, _|
+        url.host =~ /#{CookieJar.strip_port(domain)}$/i
+      }
+
+      return [] unless domains.length > 0
+
+      cookies = domains.map { |_,paths|
+        paths.find_all { |path, _|
+          url.path =~ /^#{Regexp.escape(path)}/
+        }.map { |_,cookie| cookie.values }
+      }.flatten
+
+      cookies.find_all { |cookie|
+        !cookie.expires || Time.now < cookie.expires
+      }
+    end
+
+    def empty?(url)
+      cookies(url).length > 0 ? false : true
+    end
+
+    def to_a
+      cookies = []
+      @jar.each do |domain, paths|
+        paths.each do |path, names|
+          cookies << names.values
+        end
+      end
+      cookies.flatten
+    end
+
+    # Save the cookie jar to a file in the format specified.
+    #
+    # Available formats:
+    # :yaml  <- YAML structure
+    # :cookiestxt  <- Mozilla's cookies.txt format
+    def save_as(file, format = :yaml)
+      ::File.open(file, "w") { |f|
+        case format
+        when :yaml then
+          YAML::dump(@jar, f)
+        when :cookiestxt then
+          dump_cookiestxt(f)
+        else
+          raise "Unknown cookie jar file format"
+        end
+      }
+    end
+
+    # Load cookie jar from a file in the format specified.
+    #
+    # Available formats:
+    # :yaml  <- YAML structure.
+    # :cookiestxt  <- Mozilla's cookies.txt format
+    def load(file, format = :yaml)
+      @jar = ::File.open(file) { |f|
+        case format
+        when :yaml then
+          YAML::load(f)
+        when :cookiestxt then
+          load_cookiestxt(f)
+        else
+          raise "Unknown cookie jar file format"
+        end
+      }
+    end
+
+    # Clear the cookie jar
+    def clear!
+      @jar = {}
+    end
+
+
+    # Read cookies from Mozilla cookies.txt-style IO stream
+    def load_cookiestxt(io)
+      now = Time.now
+      fakeuri = Struct.new(:host)    # add_cookie wants something resembling a URI.
+
+      io.each_line do |line|
+        line.chomp!
+        line.gsub!(/#.+/, '')
+        fields = line.split("\t")
+
+        next if fields.length != 7
+
+        expires_seconds = fields[4].to_i
+        begin
+          expires = Time.at(expires_seconds)
+        rescue
+          next
+          # Just in case we ever decide to support DateTime...
+          # expires = DateTime.new(1970,1,1) + ((expires_seconds + 1) / (60*60*24.0))
+        end
+        next if expires < now
+
+        c = Mechanize::Cookie.new(fields[5], fields[6])
+        c.domain = fields[0]
+        # Field 1 indicates whether the cookie can be read by other machines at the same domain.
+        # This is computed by the cookie implementation, based on the domain value.
+        c.path = fields[2]               # Path for which the cookie is relevant
+        c.secure = (fields[3] == "TRUE") # Requires a secure connection
+        c.expires = expires             # Time the cookie expires.
+        c.version = 0                   # Conforms to Netscape cookie spec.
+
+        add(fakeuri.new(c.domain), c)
+      end
+      @jar
+    end
+
+    # Write cookies to Mozilla cookies.txt-style IO stream
+    def dump_cookiestxt(io)
+      to_a.each do |cookie|
+        fields = []
+        fields[0] = cookie.domain
+
+        if cookie.domain =~ /^\./
+          fields[1] = "TRUE"
+        else
+          fields[1] = "FALSE"
+        end
+
+        fields[2] = cookie.path
+
+        if cookie.secure == true
+          fields[3] = "TRUE"
+        else
+          fields[3] = "FALSE"
+        end
+
+        fields[4] = cookie.expires.to_i.to_s
+
+        fields[5] = cookie.name
+        fields[6] = cookie.value
+        io.puts(fields.join("\t"))
+      end
+    end
+
+    private
+    # Remove expired cookies
+    def cleanup
+      @jar.each do |domain, paths|
+        paths.each do |path, names|
+          names.each do |cookie_name, cookie|
+            if cookie.expires && Time.now > cookie.expires
+              paths[path].delete(cookie_name)
+            end
+          end
+        end
+      end
+    end
+
+    def self.strip_port(host)
+      host.gsub(/:[0-9]+$/,'')
+    end
+  end
+end

data/lib/mechanize/file.rb

@@ -0,0 +1,71 @@
+class Mechanize
+  # = Synopsis
+  # This is the default (and base) class for the Pluggable Parsers.  If
+  # Mechanize cannot find an appropriate class to use for the content type,
+  # this class will be used.  For example, if you download a JPG, Mechanize
+  # will not know how to parse it, so this class will be instantiated.
+  #
+  # This is a good class to use as the base class for building your own
+  # pluggable parsers.
+  #
+  # == Example
+  #  require 'rubygems'
+  #  require 'mechanize'
+  #
+  #  agent = Mechanize.new
+  #  agent.get('http://example.com/foo.jpg').class  #=> Mechanize::File
+  #
+  class File
+    attr_accessor :uri, :response, :body, :code, :filename
+    alias :header :response
+
+    alias :content :body
+
+    def initialize(uri=nil, response=nil, body=nil, code=nil)
+      @uri, @body, @code = uri, body, code
+      @response = Headers.new
+
+      # Copy the headers in to a hash to prevent memory leaks
+      if response
+        response.each { |k,v|
+          @response[k] = v
+        }
+      end
+
+      @filename = 'index.html'
+
+      # Set the filename
+      if disposition = @response['content-disposition']
+        disposition.split(/;\s*/).each do |pair|
+          k,v = pair.split(/=/, 2)
+            @filename = v if k && k.downcase == 'filename'
+          end
+        else
+          if @uri
+            @filename = @uri.path.split(/\//).last || 'index.html'
+          @filename << ".html" unless @filename =~ /\./
+        end
+      end
+
+      yield self if block_given?
+    end
+
+    # Use this method to save the content of this object to filename
+    def save_as(filename = nil)
+      if filename.nil?
+        filename = @filename
+        number = 1
+        while(::File.exists?(filename))
+          filename = "#{@filename}.#{number}"
+          number += 1
+        end
+      end
+
+      ::File::open(filename, "wb") { |f|
+        f.write body
+      }
+    end
+
+    alias :save :save_as
+  end
+end