tenderlove-mechanize 0.9.3.20090617085936

data/lib/www/mechanize/chain/response_reader.rb

@@ -0,0 +1,41 @@
+module WWW
+  class Mechanize
+    class Chain
+      class ResponseReader
+        include WWW::Handler
+
+        def initialize(response)
+          @response = response
+        end
+
+        def handle(ctx, params)
+          params[:response] = @response
+          body = StringIO.new
+          total = 0
+          @response.read_body { |part|
+            total += part.length
+            body.write(part)
+            Mechanize.log.debug("Read #{total} bytes") if Mechanize.log
+          }
+          body.rewind
+
+          res_klass = Net::HTTPResponse::CODE_TO_OBJ[@response.code.to_s]
+          raise ResponseCodeError.new(@response) unless res_klass
+
+          # Net::HTTP ignores EOFError if Content-length is given, so we emulate it here.
+          unless res_klass <= Net::HTTPRedirection
+            raise EOFError if (!params[:request].is_a?(Net::HTTP::Head)) && @response.content_length() && @response.content_length() != total
+          end
+  
+          @response.each_header { |k,v|
+            Mechanize.log.debug("response-header: #{ k } => #{ v }")
+          } if Mechanize.log
+
+          params[:response_body] = body
+          params[:res_klass] = res_klass
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/ssl_resolver.rb

@@ -0,0 +1,42 @@
+module WWW
+  class Mechanize
+    class Chain
+      class SSLResolver
+        include WWW::Handler
+
+        def initialize(ca_file, verify_callback, cert, key, pass)
+          @ca_file = ca_file
+          @verify_callback = verify_callback
+          @cert = cert
+          @key = key
+          @pass = pass
+        end
+
+        def handle(ctx, params)
+          uri       = params[:uri]
+          http_obj  = params[:connection]
+
+          ssl = nil
+          if http_obj.instance_variable_defined?(:@ssl_context)
+            http_obj.instance_variable_get(:@ssl_context)
+          end
+
+          if uri.scheme == 'https' && ! http_obj.started? && ! ssl.frozen?
+            http_obj.use_ssl = true
+            http_obj.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            if @ca_file
+              http_obj.ca_file = @ca_file
+              http_obj.verify_mode = OpenSSL::SSL::VERIFY_PEER
+              http_obj.verify_callback = @verify_callback if @verify_callback
+            end
+            if @cert && @key
+              http_obj.cert = OpenSSL::X509::Certificate.new(::File.read(@cert))
+              http_obj.key  = OpenSSL::PKey::RSA.new(::File.read(@key), @pass)
+            end
+          end
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain/uri_resolver.rb

@@ -0,0 +1,77 @@
+module WWW
+  class Mechanize
+    class Chain
+      class URIResolver
+        include WWW::Handler
+
+        def initialize(scheme_handlers)
+          @scheme_handlers = scheme_handlers
+        end
+
+        def handle(ctx, params)
+          raise ArgumentError.new('uri must be specified') unless params[:uri]
+          params[:uri] = params[:uri].dup if params[:uri].is_a?(URI)
+          uri     = params[:uri]
+          referer = params[:referer]
+          unless uri.is_a?(URI)
+            uri = uri.to_s.strip.gsub(/[^#{0.chr}-#{126.chr}]/) { |match|
+              if RUBY_VERSION >= "1.9.0"
+                CGI.escape(match)
+              else
+                sprintf('%%%X', match.unpack($KCODE == 'UTF8' ? 'U' : 'c')[0])
+              end
+            }
+
+            escaped_uri = Util.html_unescape(
+              uri.split(/(?:%[0-9A-Fa-f]{2})+|#/).zip(
+                uri.scan(/(?:%[0-9A-Fa-f]{2})+|#/)
+              ).map { |x,y|
+                "#{URI.escape(x)}#{y}"
+              }.join('')
+            )
+
+            begin
+              uri = URI.parse(escaped_uri)
+            rescue
+              uri = URI.parse(URI.escape(escaped_uri))
+            end
+
+          end
+          uri = @scheme_handlers[
+            uri.relative? ? 'relative' : uri.scheme.downcase
+          ].call(uri, params[:referer])
+
+          if params[:referer] && params[:referer].uri
+            if uri.path.length == 0 && uri.relative?
+              uri.path = params[:referer].uri.path
+            end
+          end
+
+          uri.path = '/' if uri.path.length == 0
+
+          if uri.relative?
+            raise 'need absolute URL' unless referer && referer.uri
+            base = nil
+            if referer.respond_to?(:bases) && referer.parser
+              base = referer.bases.last
+            end
+
+            uri = ((base && base.uri && base.uri.absolute?) ?
+                    base.uri :
+                    referer.uri) + uri
+            uri = referer.uri + uri
+            # Strip initial "/.." bits from the path
+            uri.path.sub!(/^(\/\.\.)+(?=\/)/, '')
+          end
+
+          unless ['http', 'https', 'file'].include?(uri.scheme.downcase)
+            raise "unsupported scheme: #{uri.scheme}"
+          end
+          params[:uri] = uri
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize/chain.rb

@@ -0,0 +1,34 @@
+require 'www/mechanize/chain/handler'
+require 'www/mechanize/chain/uri_resolver'
+require 'www/mechanize/chain/parameter_resolver'
+require 'www/mechanize/chain/request_resolver'
+require 'www/mechanize/chain/custom_headers'
+require 'www/mechanize/chain/connection_resolver'
+require 'www/mechanize/chain/ssl_resolver'
+require 'www/mechanize/chain/pre_connect_hook'
+require 'www/mechanize/chain/auth_headers'
+require 'www/mechanize/chain/header_resolver'
+require 'www/mechanize/chain/response_body_parser'
+require 'www/mechanize/chain/response_header_handler'
+require 'www/mechanize/chain/response_reader'
+require 'www/mechanize/chain/body_decoding_handler'
+
+module WWW
+  class Mechanize
+    class Chain
+      def initialize(list)
+        @list = list
+        @list.each { |l| l.chain = self }
+      end
+
+      def handle(request)
+        @list.first.handle(self, request)
+      end
+
+      def pass(obj, request)
+        next_link = @list[@list.index(obj) + 1]
+        next_link.handle(self, request) if next_link
+      end
+    end
+  end
+end

data/lib/www/mechanize/content_type_error.rb

@@ -0,0 +1,16 @@
+module WWW
+  class Mechanize
+    # =Synopsis
+    # This class contains an error for when a pluggable parser tries to
+    # parse a content type that it does not know how to handle.  For example
+    # if WWW::Mechanize::Page were to try to parse a PDF, a ContentTypeError
+    # would be thrown.
+    class ContentTypeError < RuntimeError
+      attr_reader :content_type
+    
+      def initialize(content_type)
+        @content_type = content_type
+      end
+    end
+  end
+end

data/lib/www/mechanize/cookie.rb

@@ -0,0 +1,72 @@
+require 'time'
+require 'webrick/cookie'
+
+module WWW
+  class Mechanize
+  # This class is used to represent an HTTP Cookie.
+    class Cookie < WEBrick::Cookie
+      def self.parse(uri, str, log = Mechanize.log)
+        return str.split(/,(?=[^;,]*=)|,$/).collect { |c|
+          cookie_elem = c.split(/;+/)
+          first_elem = cookie_elem.shift
+          first_elem.strip!
+          key, value = first_elem.split(/=/, 2)
+
+          cookie = nil
+          begin
+            cookie = new(key, WEBrick::HTTPUtils.dequote(value))
+          rescue
+            log.warn("Couldn't parse key/value: #{first_elem}") if log
+          end
+          next unless cookie
+
+          cookie_elem.each{|pair|
+            pair.strip!
+            key, value = pair.split(/=/, 2)
+            if value
+              value = WEBrick::HTTPUtils.dequote(value.strip)
+            end
+            case key.downcase
+            when "domain"  then cookie.domain  = value.sub(/^\./, '')
+            when "path"    then cookie.path    = value
+            when 'expires'
+              begin
+                cookie.expires = Time::parse(value)
+              rescue
+                if log
+                  log.warn("Couldn't parse expires: #{value}")
+                end
+              end
+            when "max-age" then
+              begin
+                cookie.max_age = Integer(value)
+              rescue
+                log.warn("Couldn't parse max age '#{value}'") if log
+                cookie.max_age = nil
+              end
+            when "comment" then cookie.comment = value
+            when "version" then
+              begin
+                cookie.version = Integer(value)
+              rescue
+                log.warn("Couldn't parse version '#{value}'") if log
+                cookie.version = nil
+              end
+            when "secure"  then cookie.secure = true
+            end
+          }
+
+          cookie.path    ||= uri.path.to_s.sub(/[^\/]*$/, '')
+          cookie.secure  ||= false
+          cookie.domain  ||= uri.host
+          # Move this in to the cookie jar
+          yield cookie if block_given?
+        }
+      end
+    
+      def to_s
+        "#{@name}=#{@value}"
+      end
+    end
+  end
+end

data/lib/www/mechanize/cookie_jar.rb

@@ -0,0 +1,191 @@
+require 'yaml'
+
+module WWW
+  class Mechanize
+    # This class is used to manage the Cookies that have been returned from
+    # any particular website.
+    class CookieJar
+      attr_reader :jar
+
+      def initialize
+        @jar = {}
+      end
+
+      # Add a cookie to the Jar.
+      def add(uri, cookie)
+        return unless uri.host =~ /#{CookieJar.strip_port(cookie.domain)}$/i
+
+        normal_domain = cookie.domain.downcase
+
+        unless @jar.has_key?(normal_domain)
+          @jar[normal_domain] = Hash.new { |h,k| h[k] = {} }
+        end
+
+        @jar[normal_domain][cookie.path][cookie.name] = cookie
+        cleanup
+        cookie
+      end
+
+      # Fetch the cookies that should be used for the URI object passed in.
+      def cookies(url)
+        cleanup
+        url.path = '/' if url.path.empty?
+
+        domains = @jar.find_all { |domain, _|
+          url.host =~ /#{CookieJar.strip_port(domain)}$/i
+        }
+
+        return [] unless domains.length > 0
+
+        cookies = domains.map { |_,paths|
+          paths.find_all { |path, _|
+            url.path =~ /^#{Regexp.escape(path)}/
+          }.map { |_,cookie| cookie.values }
+        }.flatten
+
+        cookies.find_all { |cookie|
+          !cookie.expires || Time.now < cookie.expires
+        }
+      end
+
+      def empty?(url)
+        cookies(url).length > 0 ? false : true
+      end
+
+      def to_a
+        cookies = []
+        @jar.each do |domain, paths|
+          paths.each do |path, names|
+            cookies << names.values
+          end
+        end
+        cookies.flatten
+      end
+
+      # Save the cookie jar to a file in the format specified.
+      #
+      # Available formats:
+      # :yaml  <- YAML structure
+      # :cookiestxt  <- Mozilla's cookies.txt format
+      def save_as(file, format = :yaml)
+        ::File.open(file, "w") { |f|
+          case format
+          when :yaml then
+            YAML::dump(@jar, f)
+          when :cookiestxt then
+            dump_cookiestxt(f)
+          else
+            raise "Unknown cookie jar file format"
+          end
+        }
+      end
+
+      # Load cookie jar from a file in the format specified.
+      #
+      # Available formats:
+      # :yaml  <- YAML structure.
+      # :cookiestxt  <- Mozilla's cookies.txt format
+      def load(file, format = :yaml)
+        @jar = ::File.open(file) { |f|
+          case format
+          when :yaml then
+            YAML::load(f)
+          when :cookiestxt then
+            load_cookiestxt(f)
+          else
+            raise "Unknown cookie jar file format"
+          end
+        }
+      end
+
+      # Clear the cookie jar
+      def clear!
+        @jar = {}
+      end
+
+
+      # Read cookies from Mozilla cookies.txt-style IO stream
+      def load_cookiestxt(io)
+        now = Time.now
+        fakeuri = Struct.new(:host)    # add_cookie wants something resembling a URI.
+
+        io.each_line do |line|
+          line.chomp!
+          line.gsub!(/#.+/, '')
+          fields = line.split("\t")
+
+          next if fields.length != 7
+
+          expires_seconds = fields[4].to_i
+          begin
+            expires = Time.at(expires_seconds)
+          rescue
+            next
+            # Just in case we ever decide to support DateTime...
+            # expires = DateTime.new(1970,1,1) + ((expires_seconds + 1) / (60*60*24.0))
+          end
+          next if expires < now
+
+          c = WWW::Mechanize::Cookie.new(fields[5], fields[6])
+          c.domain = fields[0]
+          # Field 1 indicates whether the cookie can be read by other machines at the same domain.
+          # This is computed by the cookie implementation, based on the domain value.
+          c.path = fields[2]               # Path for which the cookie is relevant
+          c.secure = (fields[3] == "TRUE") # Requires a secure connection
+          c.expires = expires             # Time the cookie expires.
+          c.version = 0                   # Conforms to Netscape cookie spec.
+
+          add(fakeuri.new(c.domain), c)
+        end
+        @jar
+      end
+
+      # Write cookies to Mozilla cookies.txt-style IO stream
+      def dump_cookiestxt(io)
+        to_a.each do |cookie|
+          fields = []
+          fields[0] = cookie.domain
+
+          if cookie.domain =~ /^\./
+            fields[1] = "TRUE"
+          else
+            fields[1] = "FALSE"
+          end
+
+          fields[2] = cookie.path
+
+          if cookie.secure == true
+            fields[3] = "TRUE"
+          else
+            fields[3] = "FALSE"
+          end
+
+          fields[4] = cookie.expires.to_i.to_s
+
+          fields[5] = cookie.name
+          fields[6] = cookie.value
+          io.puts(fields.join("\t"))
+        end
+      end
+
+      private
+      # Remove expired cookies
+      def cleanup
+        @jar.each do |domain, paths|
+          paths.each do |path, names|
+            names.each do |cookie_name, cookie|
+              if cookie.expires && Time.now > cookie.expires
+                paths[path].delete(cookie_name)
+              end
+            end
+          end
+        end
+      end
+
+      def self.strip_port(host)
+        host.gsub(/:[0-9]+$/,'')
+      end
+
+    end
+  end
+end

data/lib/www/mechanize/file.rb

@@ -0,0 +1,73 @@
+module WWW
+  class Mechanize
+    # = Synopsis
+    # This is the default (and base) class for the Pluggable Parsers.  If
+    # Mechanize cannot find an appropriate class to use for the content type,
+    # this class will be used.  For example, if you download a JPG, Mechanize
+    # will not know how to parse it, so this class will be instantiated.
+    #
+    # This is a good class to use as the base class for building your own
+    # pluggable parsers.
+    #
+    # == Example
+    #  require 'rubygems'
+    #  require 'mechanize'
+    #
+    #  agent = WWW::Mechanize.new
+    #  agent.get('http://example.com/foo.jpg').class  #=> WWW::Mechanize::File
+    #
+    class File
+      attr_accessor :uri, :response, :body, :code, :filename
+      alias :header :response
+
+      alias :content :body
+
+      def initialize(uri=nil, response=nil, body=nil, code=nil)
+        @uri, @body, @code = uri, body, code
+        @response = Headers.new
+
+        # Copy the headers in to a hash to prevent memory leaks
+        if response
+          response.each { |k,v|
+            @response[k] = v
+          }
+        end
+
+        @filename = 'index.html'
+
+        # Set the filename
+        if disposition = @response['content-disposition']
+          disposition.split(/;\s*/).each do |pair|
+            k,v = pair.split(/=/, 2)
+            @filename = v if k && k.downcase == 'filename'
+          end
+        else
+          if @uri
+            @filename = @uri.path.split(/\//).last || 'index.html'
+            @filename << ".html" unless @filename =~ /\./
+          end
+        end
+
+        yield self if block_given?
+      end
+
+      # Use this method to save the content of this object to filename
+      def save_as(filename = nil)
+        if filename.nil?
+          filename = @filename
+          number = 1
+          while(::File.exists?(filename))
+            filename = "#{@filename}.#{number}"
+            number += 1
+          end
+        end
+
+        ::File::open(filename, "wb") { |f|
+          f.write body
+        }
+      end
+
+      alias :save :save_as
+    end
+  end
+end

data/lib/www/mechanize/file_response.rb

@@ -0,0 +1,62 @@
+module WWW
+  class Mechanize
+    ###
+    # Fake response for dealing with file:/// requests
+    class FileResponse
+      def initialize(file_path)
+        @file_path = file_path
+      end
+
+      def read_body
+        if ::File.exists?(@file_path)
+          if directory?
+            yield dir_body
+          else
+            yield ::File.read(@file_path)
+          end
+        else
+          yield ''
+        end
+      end
+
+      def code
+        ::File.exists?(@file_path) ? 200 : 400
+      end
+
+      def content_length
+        return dir_body.length if directory?
+        ::File.exists?(@file_path) ? ::File.stat(@file_path).size : 0
+      end
+
+      def each_header; end
+
+      def [](key)
+        return nil unless key.downcase == 'content-type'
+        return 'text/html' if directory?
+        return 'text/html' if ['.html', '.xhtml'].any? { |extn|
+          @file_path =~ /#{extn}$/
+        }
+        nil
+      end
+
+      def each
+      end
+
+      def get_fields(key)
+        []
+      end
+
+      private
+      def dir_body
+        '<html><body>' +
+        Dir[::File.join(@file_path, '*')].map { |f|
+          "<a href=\"file://#{f}\">#{::File.basename(f)}</a>"
+        }.join("\n") + '</body></html>'
+      end
+
+      def directory?
+        ::File.directory?(@file_path)
+      end
+    end
+  end
+end

data/lib/www/mechanize/file_saver.rb

@@ -0,0 +1,39 @@
+module WWW
+  class Mechanize
+    # = Synopsis
+    # This is a pluggable parser that automatically saves every file
+    # it encounters.  It saves the files as a tree, reflecting the
+    # host and file path.
+    #
+    # == Example to save all PDF's
+    #  require 'rubygems'
+    #  require 'mechanize'
+    #
+    #  agent = WWW::Mechanize.new
+    #  agent.pluggable_parser.pdf = WWW::Mechanize::FileSaver
+    #  agent.get('http://example.com/foo.pdf')
+    #
+    class FileSaver < File
+      attr_reader :filename
+  
+      def initialize(uri=nil, response=nil, body=nil, code=nil)
+        super(uri, response, body, code)
+        path = uri.path.empty? ? 'index.html' : uri.path.gsub(/^[\/]*/, '')
+        path += 'index.html' if path =~ /\/$/
+  
+        split_path = path.split(/\//)
+        filename = split_path.length > 0 ? split_path.pop : 'index.html'
+        joined_path = split_path.join(::File::SEPARATOR)
+        path = if joined_path.empty?
+          uri.host
+        else
+          "#{uri.host}#{::File::SEPARATOR}#{joined_path}"
+        end
+  
+        @filename = "#{path}#{::File::SEPARATOR}#{filename}"
+        FileUtils.mkdir_p(path)
+        save_as(@filename)
+      end
+    end
+  end
+end

data/lib/www/mechanize/form/button.rb

@@ -0,0 +1,8 @@
+module WWW
+  class Mechanize
+    class Form
+      # This class represents a Submit button in a form.
+      class Button < Field; end 
+    end
+  end
+end

data/lib/www/mechanize/form/check_box.rb

@@ -0,0 +1,13 @@
+module WWW
+  class Mechanize
+    class Form
+      # This class represents a check box found in a Form.  To activate the
+      # CheckBox in the Form, set the checked method to true.
+      class CheckBox < RadioButton
+        def query_value
+          [[@name, @value || "on"]]
+        end
+      end
+    end
+  end
+end