kitamomonga-mechanize 0.9.3.20090724215219

data/lib/mechanize/page/meta.rb

@@ -0,0 +1,50 @@
+class Mechanize
+  class Page < Mechanize::File
+    # This class encapsulates a Meta tag.  Mechanize treats meta tags just
+    # like 'a' tags.  Meta objects will contain links, but most likely will
+    # have no text.
+    class Meta < Link
+
+      # Matches the content attribute of a meta tag.  After the match:
+      #
+      #   $1:: delay
+      #   $3:: url
+      #
+      CONTENT_REGEXP = /^\s*(\d+\.?\d*)(;|;\s*url=\s*['"]?(\S*?)['"]?)?\s*$/i
+
+      class << self
+        # Parses the delay and url from the content attribute of a meta tag.
+        # Parse requires the uri of the current page to infer a url when no
+        # url is specified.  If a block is given, the parsed delay and url
+        # will be passed to it for further processing.
+        #
+        # Returns nil if the delay and url cannot be parsed.
+        #
+        #   # <meta http-equiv="refresh" content="5;url=http://example.com/" />
+        #   uri = URI.parse('http://current.com/')
+        #
+        #   Meta.parse("5;url=http://example.com/", uri)  # => ['5', 'http://example.com/']
+        #   Meta.parse("5;url=", uri)                     # => ['5', 'http://current.com/']
+        #   Meta.parse("5", uri)                          # => ['5', 'http://current.com/']
+        #   Meta.parse("invalid content", uri)            # => nil
+        #
+        def parse(content, uri)
+          if content && content =~ CONTENT_REGEXP
+            delay, url = $1, $3
+
+            url = case url
+                  when nil, "" then uri.to_s
+                  when /^http/i then url
+                  else "http://#{uri.host}#{url}"
+                  end
+
+            block_given? ? yield(delay, url) : [delay, url]
+          else
+            nil
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/mechanize/pluggable_parsers.rb

@@ -0,0 +1,101 @@
+require 'mechanize/file'
+require 'mechanize/file_saver'
+require 'mechanize/page'
+
+class Mechanize
+  # = Synopsis
+  # This class is used to register and maintain pluggable parsers for
+  # Mechanize to use.
+  #
+  # A Pluggable Parser is a parser that Mechanize uses for any particular
+  # content type.  Mechanize will ask PluggableParser for the class it
+  # should initialize given any content type.  This class allows users to
+  # register their own pluggable parsers, or modify existing pluggable
+  # parsers.
+  #
+  # PluggableParser returns a Mechanize::File object for content types
+  # that it does not know how to handle.  Mechanize::File provides
+  # basic functionality for any content type, so it is a good class to
+  # extend when building your own parsers.
+  # == Example
+  # To create your own parser, just create a class that takes four
+  # parameters in the constructor.  Here is an example of registering
+  # a pluggable parser that handles CSV files:
+  #  class CSVParser < Mechanize::File
+  #    attr_reader :csv
+  #    def initialize(uri=nil, response=nil, body=nil, code=nil)
+  #      super(uri, response, body, code)
+  #      @csv = CSV.parse(body)
+  #    end
+  #  end
+  #  agent = Mechanize.new
+  #  agent.pluggable_parser.csv = CSVParser
+  #  agent.get('http://example.com/test.csv')  # => CSVParser
+  # Now any page that returns the content type of 'text/csv' will initialize
+  # a CSVParser and return that object to the caller.
+  #
+  # To register a pluggable parser for a content type that pluggable parser
+  # does not know about, just use the hash syntax:
+  #  agent.pluggable_parser['text/something'] = SomeClass
+  #
+  # To set the default parser, just use the 'defaut' method:
+  #  agent.pluggable_parser.default = SomeClass
+  # Now all unknown content types will be instances of SomeClass.
+  class PluggableParser
+    CONTENT_TYPES = {
+      :html => 'text/html',
+      :wap  => 'application/vnd.wap.xhtml+xml',
+      :xhtml => 'application/xhtml+xml',
+      :pdf  => 'application/pdf',
+      :csv  => 'text/csv',
+      :xml  => 'text/xml',
+    }
+
+    attr_accessor :default
+
+    def initialize
+      @parsers = { CONTENT_TYPES[:html]   => Page,
+        CONTENT_TYPES[:xhtml]  => Page,
+        CONTENT_TYPES[:wap]    => Page,
+      }
+      @default = File
+    end
+
+    def parser(content_type)
+      content_type.nil? ? default : @parsers[content_type] || default
+    end
+
+    def register_parser(content_type, klass)
+      @parsers[content_type] = klass
+    end
+
+    def html=(klass)
+      register_parser(CONTENT_TYPES[:html], klass)
+      register_parser(CONTENT_TYPES[:xhtml], klass)
+    end
+
+    def xhtml=(klass)
+      register_parser(CONTENT_TYPES[:xhtml], klass)
+    end
+
+    def pdf=(klass)
+      register_parser(CONTENT_TYPES[:pdf], klass)
+    end
+
+    def csv=(klass)
+      register_parser(CONTENT_TYPES[:csv], klass)
+    end
+
+    def xml=(klass)
+      register_parser(CONTENT_TYPES[:xml], klass)
+    end
+
+    def [](content_type)
+      @parsers[content_type]
+    end
+
+    def []=(content_type, klass)
+      @parsers[content_type] = klass
+    end
+  end
+end

data/lib/mechanize/redirect_limit_reached_error.rb

@@ -0,0 +1,16 @@
+class Mechanize
+  # Thrown when too many redirects are sent
+  class RedirectLimitReachedError < RuntimeError
+    attr_reader :page, :response_code, :redirects
+    def initialize(page, redirects)
+      @page           = page
+      @redirects      = redirects
+      @response_code  = page.code
+    end
+
+    def to_s
+      "Maximum redirect limit (#{redirects}) reached"
+    end
+    alias :inspect :to_s
+  end
+end

data/lib/mechanize/redirect_not_get_or_head_error.rb

@@ -0,0 +1,18 @@
+class Mechanize
+  # Thrown when a POST, PUT, or DELETE request results in a redirect
+  # see RFC 2616 10.3.2, 10.3.3 http://www.ietf.org/rfc/rfc2616.txt
+  class RedirectNotGetOrHeadError < RuntimeError
+    attr_reader :page, :response_code, :verb, :uri
+    def initialize(page, verb)
+      @page           = page
+      @verb           = verb
+      @uri            = page.uri
+      @response_code  = page.code
+    end
+
+    def to_s
+      "#{@response_code} redirect received after a #{@verb} request"
+    end
+    alias :inspect :to_s
+  end
+end

data/lib/mechanize/response_code_error.rb

@@ -0,0 +1,22 @@
+class Mechanize
+  # =Synopsis
+  # This error is thrown when Mechanize encounters a response code it does
+  # not know how to handle.  Currently, this exception will be thrown
+  # if Mechanize encounters response codes other than 200, 301, or 302.
+  # Any other response code is up to the user to handle.
+  class ResponseCodeError < RuntimeError
+    attr_reader :response_code
+    attr_reader :page
+
+    def initialize(page)
+      @page          = page
+      @response_code = page.code
+    end
+
+    def to_s
+      "#{response_code} => #{Net::HTTPResponse::CODE_TO_OBJ[response_code]}"
+    end
+
+    def inspect; to_s; end
+  end
+end

data/lib/mechanize/unsupported_scheme_error.rb

@@ -0,0 +1,8 @@
+class Mechanize
+  class UnsupportedSchemeError < RuntimeError
+    attr_accessor :scheme
+    def initialize(scheme)
+      @scheme = scheme
+    end
+  end
+end

data/lib/mechanize/util.rb

@@ -0,0 +1,73 @@
+require 'cgi'
+
+class Mechanize
+  class Util
+    CODE_DIC = {
+      :JIS => "ISO-2022-JP",
+      :EUC => "EUC-JP",
+      :SJIS => "SHIFT_JIS",
+      :UTF8 => "UTF-8", :UTF16 => "UTF-16", :UTF32 => "UTF-32"}
+
+    NKF_TO_ICONV = {
+      'ASCII-8BIT' => 'CP1252',
+      'SHIFT_JIS' => 'CP932',
+    }
+
+    class << self
+      def build_query_string(parameters, enc=nil)
+        parameters.map { |k,v|
+          # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
+          [CGI.escape(k.to_s), CGI.escape(v.to_s)].join("=") if k
+        }.compact.join('&')
+      end
+
+      def to_native_charset(s, code=nil)
+        if Mechanize.html_parser == Nokogiri::HTML
+          return unless s
+          code ||= detect_charset(s)
+          Iconv.iconv("UTF-8", code, s).join("")
+        else
+          s
+        end
+      end
+
+      def from_native_charset(s, code)
+        if Mechanize.html_parser == Nokogiri::HTML
+          return unless s
+          Iconv.iconv(code, "UTF-8", s).join("")
+        else
+          return s
+        end
+      end
+
+      def html_unescape(s)
+        return s unless s
+        s.gsub(/&(\w+|#[0-9]+);/) { |match|
+          number = case match
+                   when /&(\w+);/
+                     Mechanize.html_parser::NamedCharacters[$1]
+                   when /&#([0-9]+);/
+                     $1.to_i
+                   end
+
+          number ? ([number].pack('U') rescue match) : match
+        }
+      end
+
+      def detect_charset(src)
+        tmp = NKF.guess(src || "<html></html>")
+        if RUBY_VERSION >= "1.9.0"
+          enc = tmp.to_s.upcase
+        else
+          enc = NKF.constants.find{|c|
+            NKF.const_get(c) == tmp
+          }
+          enc = CODE_DIC[enc.intern]
+        end
+        enc = NKF_TO_ICONV[enc] if NKF_TO_ICONV[enc]
+        enc || "CP1252"
+      end
+
+    end
+  end
+end

data/test/chain/test_argument_validator.rb

@@ -0,0 +1,14 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+class TestURIResolver < Test::Unit::TestCase
+  def test_handle
+    v = Mechanize::Chain.new([
+      Mechanize::Chain::URIResolver.new(Hash.new { |h,k|
+        h[k] = lambda { |u,r| u }
+      })
+    ])
+    assert_raises(ArgumentError) { v.handle({}) }
+    assert_nothing_raised { v.handle({:uri => 'http://google.com/'}) }
+    assert_raises(RuntimeError) { v.handle({:uri => 'google'}) }
+  end
+end

data/test/chain/test_auth_headers.rb

@@ -0,0 +1,25 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+class TestAuthHeaders < Test::Unit::TestCase
+  def test_auth
+    url = URI.parse('http://www.anthonychaves.net/tests.xml')
+    digest = %Q!Digest realm="www.anthonychaves.net", qop="auth", algorithm=MD5, nonce="MTI0NTEyMTYyNjo0ZTY2MjhlZWMyZmM1ZjA0M2Y1Njc1MGU0YTA2MWY5OQ==", opaque="9f455d4e71e8d46a6d3aaef8bf8b0d9e"!
+    v = Mechanize::Chain.new([
+      Mechanize::Chain::AuthHeaders.new({(url.host) => :digest}, "anthony", "password", digest)
+    ])
+    
+    hash = {
+      :request => Net::HTTP::Get.new(url.request_uri),
+      :uri => url
+    }
+    v.handle(hash) 
+    actual_authorization = hash[:request]['Authorization'] 
+    # The chain gave our request an Authorization header with client-generated values and derivatives.
+    # They should be scrubbed before comparing to the expected result because they change
+    # on each invokation
+    actual_authorization.gsub!(/cnonce=\"\w+?\"/, "cnonce=\"scrubbed_cnonce\"").gsub!(/response=\"\w+?\"/, "response=\"scrubbed_response\"")
+    
+    expected_authorization = %Q!Digest username="anthony", qop=auth, uri="/tests.xml", algorithm="MD5", opaque="9f455d4e71e8d46a6d3aaef8bf8b0d9e", nonce="MTI0NTEyMTYyNjo0ZTY2MjhlZWMyZmM1ZjA0M2Y1Njc1MGU0YTA2MWY5OQ==", realm="www.anthonychaves.net", nc=00000001, cnonce="scrubbed_cnonce", response="scrubbed_response"!
+    assert_equal(expected_authorization, actual_authorization)
+  end
+end

data/test/chain/test_custom_headers.rb

@@ -0,0 +1,18 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+class TestCustomHeaders < Test::Unit::TestCase
+  def test_handle
+    v = Mechanize::Chain.new([
+      Mechanize::Chain::CustomHeaders.new
+    ])
+    url = URI.parse('http://tenderlovemaking.com/')
+    hash = {
+      :request => Net::HTTP::Get.new(url.request_uri),
+      :headers => { 'Content-Length' => 300 }
+    }
+    v.handle(hash)
+    headers = hash[:request].to_hash
+    assert(headers.key?('content-length'))
+    assert_equal([300], headers['content-length'])
+  end
+end

data/test/chain/test_header_resolver.rb

@@ -0,0 +1,28 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+class TestHeaderResolver < Test::Unit::TestCase
+  def setup
+    @chain = Mechanize::Chain.new([
+      Mechanize::Chain::HeaderResolver.new(
+        true,
+        300,
+        Mechanize::CookieJar.new,
+        'foobar',
+        {
+          'hello' => 'world',
+          'Content-Type' => 'utf-8'
+        }
+      )
+    ])
+  end
+
+  def test_handle
+    hash = {
+      :request => {},
+      :uri => URI.parse('http://google.com/')
+    }
+    @chain.handle(hash)
+    assert_equal 'world', hash[:request]['hello']
+    assert_equal 'utf-8', hash[:request]['Content-Type']
+  end
+end

data/test/chain/test_parameter_resolver.rb

@@ -0,0 +1,35 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+class TestParameterResolver < Test::Unit::TestCase
+  def test_handle_get
+    v = Mechanize::Chain.new([
+      Mechanize::Chain::ParameterResolver.new
+    ])
+    hash = {
+      :uri    => URI.parse('http://google.com/'),
+      :params => { :q => 'hello' },
+      :verb   => :get
+    }
+    assert_nothing_raised {
+      v.handle(hash)
+    }
+    assert_equal('q=hello', hash[:uri].query)
+    assert_equal([], hash[:params])
+  end
+
+  def test_handle_post
+    v = Mechanize::Chain.new([
+      Mechanize::Chain::ParameterResolver.new
+    ])
+    hash = {
+      :uri    => URI.parse('http://google.com/'),
+      :params => { :q => 'hello' },
+      :verb   => :post
+    }
+    assert_nothing_raised {
+      v.handle(hash)
+    }
+    assert_equal('', hash[:uri].query.to_s)
+    assert_equal({ :q => 'hello' }, hash[:params])
+  end
+end

data/test/chain/test_request_resolver.rb

@@ -0,0 +1,29 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', "helper"))
+
+class TestRequestResolver < Test::Unit::TestCase
+  def test_handle_get
+    v = Mechanize::Chain.new([
+      Mechanize::Chain::RequestResolver.new
+    ])
+    hash = {
+      :uri  => URI.parse('http://google.com'),
+      :verb => :get
+    }
+    assert_nothing_raised { v.handle(hash) }
+    assert_kind_of(Net::HTTP::Get, hash[:request])
+    assert_equal('/', hash[:request].path)
+  end
+
+  def test_handle_post
+    v = Mechanize::Chain.new([
+      Mechanize::Chain::RequestResolver.new
+    ])
+    hash = {
+      :uri  => URI.parse('http://google.com'),
+      :verb => :post
+    }
+    assert_nothing_raised { v.handle(hash) }
+    assert_kind_of(Net::HTTP::Post, hash[:request])
+    assert_equal('/', hash[:request].path)
+  end
+end