knu-mechanize 0.9.3.20090623142847

data/lib/www/mechanize/chain.rb

@@ -0,0 +1,34 @@
+require 'www/mechanize/chain/handler'
+require 'www/mechanize/chain/uri_resolver'
+require 'www/mechanize/chain/parameter_resolver'
+require 'www/mechanize/chain/request_resolver'
+require 'www/mechanize/chain/custom_headers'
+require 'www/mechanize/chain/connection_resolver'
+require 'www/mechanize/chain/ssl_resolver'
+require 'www/mechanize/chain/pre_connect_hook'
+require 'www/mechanize/chain/auth_headers'
+require 'www/mechanize/chain/header_resolver'
+require 'www/mechanize/chain/response_body_parser'
+require 'www/mechanize/chain/response_header_handler'
+require 'www/mechanize/chain/response_reader'
+require 'www/mechanize/chain/body_decoding_handler'
+
+module WWW
+  class Mechanize
+    class Chain
+      def initialize(list)
+        @list = list
+        @list.each { |l| l.chain = self }
+      end
+
+      def handle(request)
+        @list.first.handle(self, request)
+      end
+
+      def pass(obj, request)
+        next_link = @list[@list.index(obj) + 1]
+        next_link.handle(self, request) if next_link
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/auth_headers.rb

@@ -0,0 +1,80 @@
+module WWW
+  class Mechanize
+    class Chain
+      class AuthHeaders
+        include WWW::Handler
+
+        @@nonce_count = Hash.new(0)
+        CNONCE = Digest::MD5.hexdigest("%x" % (Time.now.to_i + rand(65535)))
+
+        def initialize(auth_hash, user, password, digest)
+          @auth_hash = auth_hash
+          @user      = user
+          @password  = password
+          @digest    = digest
+        end
+
+        def handle(ctx, params)
+          uri     = params[:uri]
+          request = params[:request]
+
+          if( @auth_hash[uri.host] )
+            case @auth_hash[uri.host]
+            when :basic
+              request.basic_auth(@user, @password)
+            when :iis_digest
+                digest_response = self.gen_auth_header(uri,request, @digest, true)
+                request['Authorization'] = digest_response
+            when :digest
+              if @digest
+                digest_response = self.gen_auth_header(uri,request, @digest)
+                request['Authorization'] = digest_response
+              end
+            end
+          end
+          super
+        end
+
+        def gen_auth_header(uri, request, auth_header, is_IIS = false)
+          auth_header =~ /^(\w+) (.*)/
+
+          params = {}
+          $2.gsub(/(\w+)=("[^"]*"|[^,]*)/) {
+            params[$1] = $2.gsub(/^"/, '').gsub(/"$/, '')
+          }
+  
+          @@nonce_count[params['nonce']] += 1
+
+          a_1 = "#{@user}:#{params['realm']}:#{@password}"
+          a_2 = "#{request.method}:#{uri.path}"
+          request_digest = ''
+          request_digest << Digest::MD5.hexdigest(a_1)
+          request_digest << ':' << params['nonce']
+          request_digest << ':' << ('%08x' % @@nonce_count[params['nonce']])
+          request_digest << ':' << CNONCE
+          request_digest << ':' << params['qop']
+          request_digest << ':' << Digest::MD5.hexdigest(a_2)
+  
+          header = ''
+          header << "Digest username=\"#{@user}\", "
+          if is_IIS then
+            header << "qop=\"#{params['qop']}\", "
+          else
+            header << "qop=#{params['qop']}, "
+          end
+          header << "uri=\"#{uri.path}\", "
+          header << %w{ algorithm opaque nonce realm }.map { |field|
+            next unless params[field]
+            "#{field}=\"#{params[field]}\""
+          }.compact.join(', ')
+
+          header << ", nc=#{'%08x' % @@nonce_count[params['nonce']]}, "
+          header << "cnonce=\"#{CNONCE}\", "
+          header << "response=\"#{Digest::MD5.hexdigest(request_digest)}\""
+  
+          return header
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/body_decoding_handler.rb

@@ -0,0 +1,48 @@
+module WWW
+  class Mechanize
+    class Chain
+      class BodyDecodingHandler
+        include WWW::Handler
+
+        def handle(ctx, options)
+          body = options[:response_body]
+          response = options[:response]
+
+          options[:response_body] = 
+            if encoding = response['Content-Encoding']
+              case encoding.downcase
+              when 'gzip'
+                Mechanize.log.debug('gunzip body') if Mechanize.log
+                if response['Content-Length'].to_i > 0 || body.length > 0
+                  begin
+                    Zlib::GzipReader.new(body).read
+                  rescue Zlib::BufError, Zlib::GzipFile::Error
+                    if Mechanize.log
+                      Mechanize.log.error('Caught a Zlib::BufError')
+                    end
+                    body.rewind
+                    body.read(10)
+                    Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body.read)
+                  rescue Zlib::DataError
+                    if Mechanize.log
+                      Mechanize.log.error("Caught a Zlib::DataError, unable to decode page: #{$!.to_s}")
+                    end
+                    ''
+                  end
+                else
+                  ''
+                end
+              when 'x-gzip'
+                body.read
+              else
+                raise 'Unsupported content encoding'
+              end
+            else
+              body.read
+            end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/connection_resolver.rb

@@ -0,0 +1,78 @@
+module WWW
+  class Mechanize
+    class Chain
+      class ConnectionResolver
+        include WWW::Handler
+
+        def initialize( connection_cache,
+                        keep_alive,
+                        proxy_addr,
+                        proxy_port,
+                        proxy_user,
+                        proxy_pass )
+
+          @connection_cache = connection_cache
+          @keep_alive = keep_alive
+          @proxy_addr = proxy_addr
+          @proxy_port = proxy_port
+          @proxy_user = proxy_user
+          @proxy_pass = proxy_pass
+        end
+
+        def handle(ctx, params)
+          uri = params[:uri]
+          http_obj = nil
+
+          case uri.scheme.downcase
+          when 'http', 'https'
+            cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= {
+              :connection         => nil,
+              :keep_alive_options => {},
+            })
+            http_obj = cache_obj[:connection]
+            if http_obj.nil? || ! http_obj.started?
+              http_obj = cache_obj[:connection] =
+                  Net::HTTP.new( uri.host,
+                          uri.port,
+                          @proxy_addr,
+                          @proxy_port,
+                          @proxy_user,
+                          @proxy_pass
+                        )
+              cache_obj[:keep_alive_options] = {}
+            end
+
+            # If we're keeping connections alive and the last request time is too
+            # long ago, stop the connection.  Or, if the max requests left is 1,
+            # reset the connection.
+            if @keep_alive && http_obj.started?
+              opts = cache_obj[:keep_alive_options]
+              if((opts[:timeout] &&
+                 Time.now.to_i - cache_obj[:last_request_time] > opts[:timeout].to_i) ||
+                  opts[:max] && opts[:max].to_i == 1)
+  
+                Mechanize.log.debug('Finishing stale connection') if Mechanize.log
+                http_obj.finish
+  
+              end
+            end
+
+            cache_obj[:last_request_time] = Time.now.to_i
+          when 'file'
+            http_obj = Object.new
+            class << http_obj
+              def started?; true; end
+              def request(request, *args, &block)
+                response = FileResponse.new(request.uri.path)
+                yield response
+              end
+            end
+          end
+
+          params[:connection] = http_obj
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/custom_headers.rb

@@ -0,0 +1,23 @@
+module WWW
+  class Mechanize
+    class Chain
+      class CustomHeaders
+        include WWW::Handler
+
+        def handle(ctx, params)
+          request = params[:request]
+          params[:headers].each do |k,v|
+            case k
+            when :etag then request["ETag"] = v
+            when :if_modified_since then request["If-Modified-Since"] = v
+            else
+              raise ArgumentError.new("unknown header symbol #{k}") if k.is_a? Symbol
+              request[k] = v
+            end
+          end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/handler.rb

@@ -0,0 +1,9 @@
+module WWW
+  module Handler
+    attr_accessor :chain
+
+    def handle(ctx, request)
+      chain.pass(self, request)
+    end
+  end
+end

data/lib/www/mechanize/chain/header_resolver.rb

@@ -0,0 +1,53 @@
+module WWW
+  class Mechanize
+    class Chain
+      class HeaderResolver
+        include WWW::Handler
+        def initialize(keep_alive, keep_alive_time, cookie_jar, user_agent, headers)
+          @keep_alive = keep_alive
+          @keep_alive_time = keep_alive_time
+          @cookie_jar = cookie_jar
+          @user_agent = user_agent
+          @headers = headers
+        end
+
+        def handle(ctx, params)
+          uri = params[:uri]
+          referer = params[:referer]
+          request = params[:request]
+
+          if @keep_alive
+            request['Connection'] = 'keep-alive'
+            request['Keep-Alive'] = @keep_alive_time.to_s
+          else
+            request['Connection'] = 'close'
+          end
+          request['Accept-Encoding'] = 'gzip,identity'
+          request['Accept-Language'] = 'en-us,en;q=0.5'
+          host = "#{uri.host}#{[80, 443].include?(uri.port.to_i) ? '' : ':' + uri.port.to_s}"
+          request['Host'] = host
+          request['Accept-Charset'] = 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'
+  
+          unless @cookie_jar.empty?(uri)
+            cookies = @cookie_jar.cookies(uri)
+            cookie = cookies.length > 0 ? cookies.join("; ") : nil
+            request.add_field('Cookie', cookie)
+          end
+  
+          # Add Referer header to request
+          if referer && referer.uri
+            request['Referer'] = referer.uri.to_s
+          end
+  
+          # Add User-Agent header to request
+          request['User-Agent'] = @user_agent if @user_agent 
+
+          @headers.each do |k,v|
+            request[k] = v
+          end if request
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/parameter_resolver.rb

@@ -0,0 +1,24 @@
+module WWW
+  class Mechanize
+    class Chain
+      class ParameterResolver
+        include WWW::Handler
+
+        def handle(ctx, params)
+          parameters  = params[:params]
+          uri         = params[:uri]
+          case params[:verb]
+          when :head, :get, :delete, :trace
+            if parameters.length > 0
+              uri.query ||= ''
+              uri.query << '&' if uri.query.length > 0
+              uri.query << Util.build_query_string(parameters)
+            end
+            params[:params] = []
+          end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/pre_connect_hook.rb

@@ -0,0 +1,22 @@
+module WWW
+  class Mechanize
+    class Chain
+      class PreConnectHook
+        include WWW::Handler
+
+        attr_accessor :hooks
+        def initialize
+          @hooks = []
+        end
+
+        def handle(ctx, params)
+          @hooks.each { |hook| hook.call(params) }
+          super
+        end
+      end
+
+      class PostConnectHook < PreConnectHook
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/request_resolver.rb

@@ -0,0 +1,32 @@
+module WWW
+  class Mechanize
+    class Chain
+      class RequestResolver
+        include WWW::Handler
+
+        def handle(ctx, params)
+          uri = params[:uri]
+          if %w{ http https }.include?(uri.scheme.downcase)
+            klass = Net::HTTP.const_get(params[:verb].to_s.capitalize)
+            params[:request] ||= klass.new(uri.request_uri)
+          end
+
+          if %w{ file }.include?(uri.scheme.downcase)
+            o = Struct.new(:uri).new(uri)
+            class << o
+              def add_field(*args); end
+              alias :[]= :add_field
+              def path
+                uri.path
+              end
+              def each_header; end
+            end
+            params[:request] ||= o
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/response_body_parser.rb

@@ -0,0 +1,40 @@
+module WWW
+  class Mechanize
+    class Chain
+      class ResponseBodyParser
+        include WWW::Handler
+
+        def initialize(pluggable_parser, watch_for_set)
+          @pluggable_parser = pluggable_parser
+          @watch_for_set = watch_for_set
+        end
+
+        def handle(ctx, params)
+          response = params[:response]
+          response_body = params[:response_body]
+          uri = params[:uri]
+
+          content_type = nil
+          unless response['Content-Type'].nil?
+            data = response['Content-Type'].match(/^([^;]*)/)
+            content_type = data[1].downcase.split(',')[0] unless data.nil?
+          end
+
+          # Find our pluggable parser
+          params[:page] = @pluggable_parser.parser(content_type).new(
+            uri,
+            response,
+            response_body,
+            response.code
+          ) { |parser|
+            parser.mech = params[:agent] if parser.respond_to? :mech=
+            if parser.respond_to?(:watch_for_set=) && @watch_for_set
+              parser.watch_for_set = @watch_for_set
+            end
+          }
+          super
+        end
+      end
+    end
+  end
+end