mechanize 1.0.1.beta.20110107104205 → 2.0.pre.1

data/lib/mechanize/page/image.rb

@@ -17,8 +17,12 @@ class Mechanize
         case src
         when %r{^https?://}
           src
-        else
-          (page.uri + src).to_s
+        else 
+          if page.bases[0]
+            (page.bases[0].href + src).to_s
+          else
+            (page.uri + src).to_s
+          end
         end
       end
     end

data/lib/mechanize/redirect_limit_reached_error.rb

@@ -1,6 +1,6 @@
 class Mechanize
   # Thrown when too many redirects are sent
-  class RedirectLimitReachedError < RuntimeError
+  class RedirectLimitReachedError < Mechanize::Error
     attr_reader :page, :response_code, :redirects
     def initialize(page, redirects)
       @page           = page

data/lib/mechanize/redirect_not_get_or_head_error.rb

@@ -1,7 +1,7 @@
 class Mechanize
   # Thrown when a POST, PUT, or DELETE request results in a redirect
   # see RFC 2616 10.3.2, 10.3.3 http://www.ietf.org/rfc/rfc2616.txt
-  class RedirectNotGetOrHeadError < RuntimeError
+  class RedirectNotGetOrHeadError < Mechanize::Error
     attr_reader :page, :response_code, :verb, :uri
     def initialize(page, verb)
       @page           = page

data/lib/mechanize/response_code_error.rb

@@ -4,17 +4,17 @@ class Mechanize
   # not know how to handle.  Currently, this exception will be thrown
   # if Mechanize encounters response codes other than 200, 301, or 302.
   # Any other response code is up to the user to handle.
-  class ResponseCodeError < RuntimeError
+  class ResponseCodeError < Mechanize::Error
     attr_reader :response_code
     attr_reader :page
 
     def initialize(page)
       @page          = page
-      @response_code = page.code
+      @response_code = page.code.to_s
     end
 
     def to_s
-      "#{response_code} => #{Net::HTTPResponse::CODE_TO_OBJ[response_code]}"
+      "#{@response_code} => #{Net::HTTPResponse::CODE_TO_OBJ[@response_code]}"
     end
 
     def inspect; to_s; end

data/lib/mechanize/unsupported_scheme_error.rb

@@ -1,5 +1,5 @@
 class Mechanize
-  class UnsupportedSchemeError < RuntimeError
+  class UnsupportedSchemeError < Mechanize::Error
     attr_accessor :scheme
     def initialize(scheme)
       @scheme = scheme

data/lib/mechanize/uri_resolver.rb

@@ -0,0 +1,82 @@
+class Mechanize::URIResolver
+
+  attr_reader :scheme_handlers
+
+  def initialize
+    @scheme_handlers = Hash.new { |h, scheme|
+      h[scheme] = lambda { |link, page|
+        raise Mechanize::UnsupportedSchemeError, scheme
+      }
+    }
+
+    @scheme_handlers['http']      = lambda { |link, page| link }
+    @scheme_handlers['https']     = @scheme_handlers['http']
+    @scheme_handlers['relative']  = @scheme_handlers['http']
+    @scheme_handlers['file']      = @scheme_handlers['http']
+  end
+
+  def resolve uri, referer = nil
+    uri = uri.dup if uri.is_a?(URI)
+
+    unless uri.is_a?(URI)
+      uri = uri.to_s.strip.gsub(/[^#{0.chr}-#{126.chr}]/o) { |match|
+        if RUBY_VERSION >= "1.9.0"
+          Mechanize::Util.uri_escape(match)
+        else
+          sprintf('%%%X', match.unpack($KCODE == 'UTF8' ? 'U' : 'C')[0])
+        end
+      }
+
+      unescaped = uri.split(/(?:%[0-9A-Fa-f]{2})+|#/)
+      escaped   = uri.scan(/(?:%[0-9A-Fa-f]{2})+|#/)
+
+      escaped_uri = Mechanize::Util.html_unescape(
+        unescaped.zip(escaped).map { |x,y|
+          "#{WEBrick::HTTPUtils.escape(x)}#{y}"
+        }.join('')
+      )
+
+      begin
+        uri = URI.parse(escaped_uri)
+      rescue
+        uri = URI.parse(WEBrick::HTTPUtils.escape(escaped_uri))
+      end
+    end
+
+    scheme = uri.relative? ? 'relative' : uri.scheme.downcase
+    uri = @scheme_handlers[scheme].call(uri, referer)
+
+    if referer && referer.uri
+      if uri.path.length == 0 && uri.relative?
+        uri.path = referer.uri.path
+      end
+    end
+
+    uri.path = '/' if uri.path.length == 0
+
+    if uri.relative?
+      raise ArgumentError, "absolute URL needed (not #{uri})" unless
+        referer && referer.uri
+
+      base = nil
+      if referer.respond_to?(:bases) && referer.parser
+        base = referer.bases.last
+      end
+
+      uri = ((base && base.uri && base.uri.absolute?) ?
+             base.uri :
+             referer.uri) + uri
+      uri = referer.uri + uri
+      # Strip initial "/.." bits from the path
+      uri.path.sub!(/^(\/\.\.)+(?=\/)/, '')
+    end
+
+    unless ['http', 'https', 'file'].include?(uri.scheme.downcase)
+      raise ArgumentError, "unsupported scheme: #{uri.scheme}"
+    end
+
+    uri
+  end
+
+end
+

data/lib/mechanize/util.rb

@@ -1,73 +1,89 @@
 require 'cgi'
 
-class Mechanize
-  class Util
-    CODE_DIC = {
-      :JIS => "ISO-2022-JP",
-      :EUC => "EUC-JP",
-      :SJIS => "SHIFT_JIS",
-      :UTF8 => "UTF-8", :UTF16 => "UTF-16", :UTF32 => "UTF-32"}
+class Mechanize::Util
+  CODE_DIC = {
+    :JIS => "ISO-2022-JP",
+    :EUC => "EUC-JP",
+    :SJIS => "SHIFT_JIS",
+    :UTF8 => "UTF-8", :UTF16 => "UTF-16", :UTF32 => "UTF-32"}
 
-    class << self
-      def build_query_string(parameters, enc=nil)
-        parameters.map { |k,v|
-          # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
-          [CGI.escape(k.to_s), CGI.escape(v.to_s)].join("=") if k
-        }.compact.join('&')
-      end
+  def self.build_query_string(parameters, enc=nil)
+    parameters.map { |k,v|
+      # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
+      [CGI.escape(k.to_s), CGI.escape(v.to_s)].join("=") if k
+    }.compact.join('&')
+  end
 
-      def to_native_charset(s, code=nil)
-        if Mechanize.html_parser == Nokogiri::HTML
-          return unless s
-          code ||= detect_charset(s)
-          Iconv.iconv("UTF-8", code, s).join("")
-        else
-          s
-        end
-      end
+  def self.to_native_charset(s, code=nil)
+    if Mechanize.html_parser == Nokogiri::HTML
+      return unless s
+      code ||= detect_charset(s)
+      Iconv.iconv("UTF-8", code, s).join("")
+    else
+      s
+    end
+  end
 
-      def from_native_charset(s, code)
-        return s unless s && code
-        return s unless Mechanize.html_parser == Nokogiri::HTML
+  def self.from_native_charset(s, code)
+    return s unless s && code
+    return s unless Mechanize.html_parser == Nokogiri::HTML
 
-        if RUBY_VERSION < '1.9.2'
-          begin
-            Iconv.iconv(code.to_s, "UTF-8", s).join("")
-          rescue Iconv::InvalidEncoding, Iconv::IllegalSequence
-            s
-          end
-        else
-          s.encode("UTF-8") rescue s
-        end
+    if RUBY_VERSION < '1.9.2'
+      begin
+        Iconv.iconv(code.to_s, "UTF-8", s).join("")
+      rescue Iconv::InvalidEncoding, Iconv::IllegalSequence
+        s
       end
+    else
+      s.encode("UTF-8") rescue s
+    end
+  end
 
-      def html_unescape(s)
-        return s unless s
-        s.gsub(/&(\w+|#[0-9]+);/) { |match|
-          number = case match
-                   when /&(\w+);/
-                     Mechanize.html_parser::NamedCharacters[$1]
-                   when /&#([0-9]+);/
-                     $1.to_i
-                   end
-
-          number ? ([number].pack('U') rescue match) : match
-        }
-      end
+  def self.html_unescape(s)
+    return s unless s
+    s.gsub(/&(\w+|#[0-9]+);/) { |match|
+      number = case match
+               when /&(\w+);/
+                 Mechanize.html_parser::NamedCharacters[$1]
+               when /&#([0-9]+);/
+                 $1.to_i
+               end
 
-      def detect_charset(src)
-        tmp = NKF.guess(src || "<html></html>")
-        if RUBY_VERSION >= "1.9.0"
-          enc = tmp.to_s.upcase
-        else
-          enc = NKF.constants.find{|c|
-            NKF.const_get(c) == tmp
-          }
-          enc = CODE_DIC[enc.intern]
-        end
-        enc || "ISO-8859-1"
-      end
+      number ? ([number].pack('U') rescue match) : match
+    }
+  end
 
+  def self.detect_charset(src)
+    tmp = NKF.guess(src || "<html></html>")
+    if RUBY_VERSION >= "1.9.0"
+      enc = tmp.to_s.upcase
+    else
+      enc = NKF.constants.find{|c|
+        NKF.const_get(c) == tmp
+      }
+      enc = CODE_DIC[enc.intern]
     end
+    enc || "ISO-8859-1"
+  end
+
+  def self.uri_escape str
+    @parser ||= begin
+                  URI::Parser.new
+                rescue NameError
+                  URI
+                end
+
+    @parser.escape str
   end
+
+  def self.uri_unescape str
+    @parser ||= begin
+                  URI::Parser.new
+                rescue NameError
+                  URI
+                end
+
+    @parser.unescape str
+  end
+
 end

data/test/helper.rb

@@ -3,6 +3,8 @@ require 'rubygems'
 require 'mechanize'
 require 'webrick/httputils'
 require 'servlets'
+require 'tmpdir'
+require 'tempfile'
 
 BASE_DIR = File.dirname(__FILE__)
 
@@ -65,7 +67,7 @@ class Net::HTTP
 
     path = '/index.html' if path == '/'
 
-    res = Response.new
+    res = ::Response.new
     res.query_params = url.query
 
     request.query = if 'POST' != request.method && url.query then
@@ -95,14 +97,37 @@ class Net::HTTP
     end
 
     res['Content-Type'] ||= 'text/html'
-    res['Content-Length'] ||= res.body.length.to_s
     res.code ||= "200"
 
+    response_klass = Net::HTTPResponse::CODE_TO_OBJ[res.code.to_s]
+    response = response_klass.new res.http_version, res.code, res.message
+
+    res.header.each do |k,v|
+      v = v.first if v.length == 1
+      response[k] = v
+    end
+
     res.cookies.each do |cookie|
-      res.add_field('Set-Cookie', cookie.to_s)
+      response.add_field 'Set-Cookie', cookie.to_s
+    end
+
+    response['Content-Type'] ||= 'text/html'
+    response['Content-Length'] = res['Content-Length'] || res.body.length.to_s
+
+    io = StringIO.new(res.body)
+    response.instance_variable_set :@socket, io
+    def io.read clen, dest, _
+      dest << string[0, clen]
     end
-    yield res if block_given?
-    res
+
+    body_exist = request.response_body_permitted? &&
+                 response_klass.body_permitted?
+
+    response.instance_variable_set :@body_exist, body_exist
+
+    yield response if block_given?
+
+    response
   end
 end
 
@@ -116,6 +141,7 @@ class Response
   attr_reader :code
   attr_accessor :body, :query, :cookies
   attr_accessor :query_params, :http_version
+  attr_accessor :header
 
   def code=(c)
     @code = c.to_s
@@ -136,4 +162,8 @@ class Response
   def read_body
     yield body
   end
+
+  def message
+    ''
+  end
 end

data/test/htdocs/dir with spaces/foo.html

@@ -0,0 +1 @@
+HELLO

data/test/htdocs/rails_3_encoding_hack_form_test.html

@@ -0,0 +1,27 @@
+<html>
+  <head>
+    <meta name="csrf-param" content="authenticity_token"/> 
+    <meta name="csrf-token" content="+6MKmkYpUcOC7ClPngk3FMTDL1Yc0cU1sS9800eeAPA="/> 
+  </head>
+  <body>
+
+    <form accept-charset="UTF-8" action="/form_post" class="new_user_session" id="new_user_session" method="post">
+      <div style="margin:0;padding:0;display:inline">
+        <input name="utf8" type="hidden" value="&#x2713;" />
+        <input name="authenticity_token" type="hidden" value="+6MKmkYpUcOC7ClPngk3FMTDL1Yc0cU1sS9800eeAPA=" />
+      </div> 
+      <div class='field'> 
+        <label for="user_session_email">Email</label> 
+        <input id="user_session_email" name="user_session[email]" size="30" type="text" /> 
+      </div> 
+      <div class='field'> 
+        <label for="user_session_password">Password</label> 
+        <input id="user_session_password" name="user_session[password]" size="30" type="password" /> 
+      </div> 
+      <div class='buttons'> 
+        <input id="user_session_submit" name="commit" type="submit" value="Login" /> 
+      </div> 
+    </form> 
+
+  </body>
+</html>

data/test/htdocs/tc_base_images.html

@@ -0,0 +1,10 @@
+<html>
+  <head>
+    <base href="http://localhost/">
+  </head>
+  <body>
+    <img src="a.jpg">
+	<img src="b.gif">
+  </body>
+</html>
+

data/test/htdocs/tc_images.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <img src="a.jpg">
+	<img src="b.gif">
+  </body>
+</html>

data/test/htdocs/test_click.html

@@ -0,0 +1,11 @@
+<html>
+  <head><title>Page Title</title></head>
+  <body>
+    <a href="/frame_test.html">This link is not called "A Link"</a>
+    <form method="get" action="/frame_test.html">
+      <input type="text" name="words" value="nil">
+      <input type="submit" value="A Button">
+    </form>
+    <a href="/index.html">A Link</a>
+  </body>
+</html>

data/test/servlets.rb

@@ -9,7 +9,7 @@ class VerbServlet < WEBrick::HTTPServlet::AbstractServlet
   %w(HEAD GET POST PUT DELETE).each do |verb|
     eval(<<-eomethod)
       def do_#{verb}(req, res)
-        res.body = "method: #{verb}"
+        res.header['X-Request-Method'] = #{verb.dump}
       end
     eomethod
   end
@@ -129,7 +129,8 @@ class GzipServlet < WEBrick::HTTPServlet::AbstractServlet
       res['Content-Encoding'] = 'gzip'
       res['Content-Type'] = "text/html"
     else
-      raise 'no gzip'
+      res.code = 400
+      res.body = 'no gzip'
     end
   end
 end