tenderlove-mechanize 0.9.3.20090623142847 → 0.9.3.20090911221705

data/lib/mechanize/chain.rb

@@ -0,0 +1,33 @@
+require 'mechanize/chain/handler'
+require 'mechanize/chain/uri_resolver'
+require 'mechanize/chain/parameter_resolver'
+require 'mechanize/chain/request_resolver'
+require 'mechanize/chain/custom_headers'
+require 'mechanize/chain/connection_resolver'
+require 'mechanize/chain/ssl_resolver'
+require 'mechanize/chain/pre_connect_hook'
+require 'mechanize/chain/auth_headers'
+require 'mechanize/chain/header_resolver'
+require 'mechanize/chain/response_body_parser'
+require 'mechanize/chain/response_header_handler'
+require 'mechanize/chain/response_reader'
+require 'mechanize/chain/body_decoding_handler'
+
+class Mechanize
+  class Chain
+    def initialize(list)
+      @list = list
+      @list.each { |l| l.chain = self }
+    end
+
+    def handle(request)
+      @list.first.handle(self, request)
+    end
+
+    def pass(obj, request)
+      next_link = @list[@list.index(obj) + 1]
+      next_link.handle(self, request) if next_link
+    end
+  end
+end
+

data/lib/mechanize/chain/auth_headers.rb

@@ -0,0 +1,78 @@
+class Mechanize
+  class Chain
+    class AuthHeaders
+      include Mechanize::Handler
+
+      @@nonce_count = Hash.new(0)
+      CNONCE = Digest::MD5.hexdigest("%x" % (Time.now.to_i + rand(65535)))
+
+      def initialize(auth_hash, user, password, digest)
+        @auth_hash = auth_hash
+        @user      = user
+        @password  = password
+        @digest    = digest
+      end
+
+      def handle(ctx, params)
+        uri     = params[:uri]
+        request = params[:request]
+
+        if( @auth_hash[uri.host] )
+          case @auth_hash[uri.host]
+          when :basic
+            request.basic_auth(@user, @password)
+          when :iis_digest
+            digest_response = self.gen_auth_header(uri,request, @digest, true)
+            request['Authorization'] = digest_response
+          when :digest
+            if @digest
+              digest_response = self.gen_auth_header(uri,request, @digest)
+              request['Authorization'] = digest_response
+            end
+          end
+        end
+        super
+      end
+
+      def gen_auth_header(uri, request, auth_header, is_IIS = false)
+        auth_header =~ /^(\w+) (.*)/
+
+        params = {}
+        $2.gsub(/(\w+)=("[^"]*"|[^,]*)/) {
+          params[$1] = $2.gsub(/^"/, '').gsub(/"$/, '')
+        }
+
+        @@nonce_count[params['nonce']] += 1
+
+        a_1 = "#{@user}:#{params['realm']}:#{@password}"
+        a_2 = "#{request.method}:#{uri.path}"
+        request_digest = ''
+        request_digest << Digest::MD5.hexdigest(a_1)
+        request_digest << ':' << params['nonce']
+        request_digest << ':' << ('%08x' % @@nonce_count[params['nonce']])
+        request_digest << ':' << CNONCE
+        request_digest << ':' << params['qop']
+        request_digest << ':' << Digest::MD5.hexdigest(a_2)
+
+        header = ''
+        header << "Digest username=\"#{@user}\", "
+        if is_IIS then
+          header << "qop=\"#{params['qop']}\", "
+        else
+          header << "qop=#{params['qop']}, "
+        end
+        header << "uri=\"#{uri.path}\", "
+        header << %w{ algorithm opaque nonce realm }.map { |field|
+          next unless params[field]
+          "#{field}=\"#{params[field]}\""
+        }.compact.join(', ')
+
+        header << ", nc=#{'%08x' % @@nonce_count[params['nonce']]}, "
+        header << "cnonce=\"#{CNONCE}\", "
+        header << "response=\"#{Digest::MD5.hexdigest(request_digest)}\""
+
+        return header
+      end
+    end
+  end
+end

data/lib/mechanize/chain/body_decoding_handler.rb

@@ -0,0 +1,46 @@
+class Mechanize
+  class Chain
+    class BodyDecodingHandler
+      include Mechanize::Handler
+
+      def handle(ctx, options)
+        body = options[:response_body]
+        response = options[:response]
+
+        options[:response_body] =
+          if encoding = response['Content-Encoding']
+            case encoding.downcase
+            when 'gzip'
+              Mechanize.log.debug('gunzip body') if Mechanize.log
+              if response['Content-Length'].to_i > 0 || body.length > 0
+                begin
+                  Zlib::GzipReader.new(body).read
+                rescue Zlib::BufError, Zlib::GzipFile::Error
+                  if Mechanize.log
+                    Mechanize.log.error('Caught a Zlib::BufError')
+                  end
+                  body.rewind
+                  body.read(10)
+                  Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body.read)
+                rescue Zlib::DataError
+                  if Mechanize.log
+                    Mechanize.log.error("Caught a Zlib::DataError, unable to decode page: #{$!.to_s}")
+                  end
+                  ''
+                end
+              else
+                ''
+              end
+            when 'x-gzip'
+              body.read
+            else
+              raise 'Unsupported content encoding'
+            end
+          else
+            body.read
+          end
+        super
+      end
+    end
+  end
+end

data/lib/mechanize/chain/connection_resolver.rb

@@ -0,0 +1,76 @@
+class Mechanize
+  class Chain
+    class ConnectionResolver
+      include Mechanize::Handler
+
+      def initialize( connection_cache,
+                      keep_alive,
+                      proxy_addr,
+                      proxy_port,
+                      proxy_user,
+                      proxy_pass )
+
+        @connection_cache = connection_cache
+        @keep_alive = keep_alive
+        @proxy_addr = proxy_addr
+        @proxy_port = proxy_port
+        @proxy_user = proxy_user
+        @proxy_pass = proxy_pass
+      end
+
+      def handle(ctx, params)
+        uri = params[:uri]
+        http_obj = nil
+
+        case uri.scheme.downcase
+        when 'http', 'https'
+          cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= {
+                         :connection         => nil,
+                         :keep_alive_options => {},
+                       })
+          http_obj = cache_obj[:connection]
+          if http_obj.nil? || ! http_obj.started?
+            http_obj = cache_obj[:connection] =
+              Net::HTTP.new( uri.host,
+                             uri.port,
+                             @proxy_addr,
+                             @proxy_port,
+                             @proxy_user,
+                             @proxy_pass
+                             )
+            cache_obj[:keep_alive_options] = {}
+          end
+
+          # If we're keeping connections alive and the last request time is too
+          # long ago, stop the connection.  Or, if the max requests left is 1,
+          # reset the connection.
+          if @keep_alive && http_obj.started?
+            opts = cache_obj[:keep_alive_options]
+            if((opts[:timeout] &&
+                Time.now.to_i - cache_obj[:last_request_time] > opts[:timeout].to_i) ||
+               opts[:max] && opts[:max].to_i == 1)
+
+              Mechanize.log.debug('Finishing stale connection') if Mechanize.log
+              http_obj.finish
+
+            end
+          end
+
+          cache_obj[:last_request_time] = Time.now.to_i
+        when 'file'
+          http_obj = Object.new
+          class << http_obj
+            def started?; true; end
+            def request(request, *args, &block)
+              response = FileResponse.new(request.uri.path)
+              yield response
+            end
+          end
+        end
+
+        params[:connection] = http_obj
+        super
+      end
+    end
+  end
+end

data/lib/mechanize/chain/custom_headers.rb

@@ -0,0 +1,21 @@
+class Mechanize
+  class Chain
+    class CustomHeaders
+      include Mechanize::Handler
+
+      def handle(ctx, params)
+        request = params[:request]
+        params[:headers].each do |k,v|
+          case k
+          when :etag then request["ETag"] = v
+          when :if_modified_since then request["If-Modified-Since"] = v
+          else
+            raise ArgumentError.new("unknown header symbol #{k}") if k.is_a? Symbol
+            request[k] = v
+          end
+        end
+        super
+      end
+    end
+  end
+end

data/lib/{www/mechanize → mechanize}/chain/handler.rb

@@ -1,4 +1,4 @@
-module WWW
+class Mechanize
   module Handler
     attr_accessor :chain
 

data/lib/mechanize/chain/header_resolver.rb

@@ -0,0 +1,51 @@
+class Mechanize
+  class Chain
+    class HeaderResolver
+      include Mechanize::Handler
+      def initialize(keep_alive, keep_alive_time, cookie_jar, user_agent, headers)
+        @keep_alive = keep_alive
+        @keep_alive_time = keep_alive_time
+        @cookie_jar = cookie_jar
+        @user_agent = user_agent
+        @headers = headers
+      end
+
+      def handle(ctx, params)
+        uri = params[:uri]
+        referer = params[:referer]
+        request = params[:request]
+
+        if @keep_alive
+          request['Connection'] = 'keep-alive'
+          request['Keep-Alive'] = @keep_alive_time.to_s
+        else
+          request['Connection'] = 'close'
+        end
+        request['Accept-Encoding'] = 'gzip,identity'
+        request['Accept-Language'] = 'en-us,en;q=0.5'
+        host = "#{uri.host}#{[80, 443].include?(uri.port.to_i) ? '' : ':' + uri.port.to_s}"
+        request['Host'] = host
+        request['Accept-Charset'] = 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'
+
+        unless @cookie_jar.empty?(uri)
+          cookies = @cookie_jar.cookies(uri)
+          cookie = cookies.length > 0 ? cookies.join("; ") : nil
+          request.add_field('Cookie', cookie)
+        end
+
+        # Add Referer header to request
+        if referer && referer.uri
+          request['Referer'] = referer.uri.to_s
+        end
+
+        # Add User-Agent header to request
+        request['User-Agent'] = @user_agent if @user_agent
+
+        @headers.each do |k,v|
+          request[k] = v
+        end if request
+        super
+      end
+    end
+  end
+end

data/lib/mechanize/chain/parameter_resolver.rb

@@ -0,0 +1,22 @@
+class Mechanize
+  class Chain
+    class ParameterResolver
+      include Mechanize::Handler
+
+      def handle(ctx, params)
+        parameters  = params[:params]
+        uri         = params[:uri]
+        case params[:verb]
+        when :head, :get, :delete, :trace
+          if parameters.length > 0
+            uri.query ||= ''
+            uri.query << '&' if uri.query.length > 0
+            uri.query << Util.build_query_string(parameters)
+          end
+          params[:params] = []
+        end
+        super
+      end
+    end
+  end
+end

data/lib/mechanize/chain/pre_connect_hook.rb

@@ -0,0 +1,20 @@
+class Mechanize
+  class Chain
+    class PreConnectHook
+      include Mechanize::Handler
+
+      attr_accessor :hooks
+      def initialize
+        @hooks = []
+      end
+
+      def handle(ctx, params)
+        @hooks.each { |hook| hook.call(params) }
+        super
+      end
+    end
+
+    class PostConnectHook < PreConnectHook
+    end
+  end
+end

data/lib/mechanize/chain/request_resolver.rb

@@ -0,0 +1,30 @@
+class Mechanize
+  class Chain
+    class RequestResolver
+      include Mechanize::Handler
+
+      def handle(ctx, params)
+        uri = params[:uri]
+        if %w{ http https }.include?(uri.scheme.downcase)
+          klass = Net::HTTP.const_get(params[:verb].to_s.capitalize)
+          params[:request] ||= klass.new(uri.request_uri)
+        end
+
+        if %w{ file }.include?(uri.scheme.downcase)
+          o = Struct.new(:uri).new(uri)
+          class << o
+            def add_field(*args); end
+            alias :[]= :add_field
+            def path
+              uri.path
+            end
+            def each_header; end
+          end
+          params[:request] ||= o
+        end
+
+        super
+      end
+    end
+  end
+end

data/lib/mechanize/chain/response_body_parser.rb

@@ -0,0 +1,38 @@
+class Mechanize
+  class Chain
+    class ResponseBodyParser
+      include Mechanize::Handler
+
+      def initialize(pluggable_parser, watch_for_set)
+        @pluggable_parser = pluggable_parser
+        @watch_for_set = watch_for_set
+      end
+
+      def handle(ctx, params)
+        response = params[:response]
+        response_body = params[:response_body]
+        uri = params[:uri]
+
+        content_type = nil
+        unless response['Content-Type'].nil?
+          data = response['Content-Type'].match(/^([^;]*)/)
+          content_type = data[1].downcase.split(',')[0] unless data.nil?
+        end
+
+        # Find our pluggable parser
+        params[:page] = @pluggable_parser.parser(content_type).new(
+                                                                   uri,
+                                                                   response,
+                                                                   response_body,
+                                                                   response.code
+                                                                   ) { |parser|
+          parser.mech = params[:agent] if parser.respond_to? :mech=
+          if parser.respond_to?(:watch_for_set=) && @watch_for_set
+            parser.watch_for_set = @watch_for_set
+          end
+        }
+        super
+      end
+    end
+  end
+end

data/lib/mechanize/chain/response_header_handler.rb

@@ -0,0 +1,48 @@
+class Mechanize
+  class Chain
+    class ResponseHeaderHandler
+      include Mechanize::Handler
+
+      def initialize(cookie_jar, connection_cache)
+        @cookie_jar = cookie_jar
+        @connection_cache = connection_cache
+      end
+
+      def handle(ctx, params)
+        response = params[:response]
+        uri = params[:uri]
+        page = params[:page]
+        cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= {
+                       :connection         => nil,
+                       :keep_alive_options => {},
+                     })
+
+        # If the server sends back keep alive options, save them
+        if keep_alive_info = response['keep-alive']
+          keep_alive_info.split(/,\s*/).each do |option|
+            k, v = option.split(/\=/)
+            cache_obj[:keep_alive_options] ||= {}
+            cache_obj[:keep_alive_options][k.intern] = v
+          end
+        end
+
+        if page.is_a?(Page) && page.body =~ /Set-Cookie/n
+          page.search('//meta[@http-equiv="Set-Cookie"]').each do |meta|
+            Cookie::parse(uri, meta['content']) { |c|
+              Mechanize.log.debug("saved cookie: #{c}") if Mechanize.log
+              @cookie_jar.add(uri, c)
+            }
+          end
+        end
+
+        (response.get_fields('Set-Cookie')||[]).each do |cookie|
+          Cookie::parse(uri, cookie) { |c|
+            Mechanize.log.debug("saved cookie: #{c}") if Mechanize.log
+            @cookie_jar.add(uri, c)
+          }
+        end
+        super
+      end
+    end
+  end
+end