mechanize 0.4.0 → 0.4.1

data/lib/mechanize.rb

@@ -7,7 +7,7 @@
 # Please see the LICENSE file for licensing.
 #
 
-Version = "0.4.0"
+Version = "0.4.1"
 
 # required due to the missing get_fields method in Ruby 1.8.2
 unless RUBY_VERSION > "1.8.2"
@@ -16,379 +16,19 @@ end
 require 'net/http'
 require 'net/https'
 
-require 'web/htmltools/xmltree'   # narf
-require 'mechanize/parsing'
-require 'mechanize/cookie'
 require 'uri'
 require 'logger'
 require 'webrick'
 require 'date'
+require 'web/htmltools/xmltree'   # narf
+require 'mechanize/parsing'
+require 'mechanize/cookie'
+require 'mechanize/form'
+require 'mechanize/form_elements'
+require 'mechanize/page'
+require 'mechanize/page_elements'
 
 module WWW
-
-class Field
-  attr_accessor :name, :value
-
-  def initialize(name, value)
-    @name, @value = name, value
-  end
-
-  # Returns an array of Field objects
-  # TODO: is this correct?
-  def self.extract_all_from(root_node)
-    fields = []
-    root_node.each_recursive {|node|
-      if (node.name.downcase == 'input' and 
-         %w(text password hidden checkbox radio int).include?(node.attributes['type'].downcase)) or
-         %w(textarea option).include?(node.name.downcase)
-        fields << Field.new(node.attributes['name'], node.attributes['value']) 
-      end
-    }
-    return fields
-  end
-end
-
-class FileUpload
-  # value is the file-name, not the file-content
-  attr_accessor :name
-  
-  attr_accessor :file_name, :file_data
-
-  def initialize(name, file_name)
-    @name, @file_name = name, file_name
-    @file_data = nil
-  end
-end
-
-class Button
-  attr_accessor :name, :value
-
-  def initialize(name, value)
-    @name, @value = name, value
-  end
-
-  def add_to_query(query)
-    query[@name] = @value || "" if @name
-  end
-
-  # Returns an array of Button objects
-  def self.extract_all_from(root_node)
-    buttons = []
-    root_node.each_recursive {|node|
-      if node.name.downcase == 'input' and 
-         ['submit'].include?(node.attributes['type'].downcase)
-        buttons << Button.new(node.attributes['name'], node.attributes['value'])
-      end
-    }
-    return buttons
-  end
-end 
-
-class ImageButton < Button
-  attr_accessor :x, :y
-  
-  def add_to_query(query)
-    if @name
-      query[@name] = @value || ""
-      query[@name+".x"] = (@x || "0").to_s
-      query[@name+".y"] = (@y || "0").to_s
-    end
-  end
-end
-
-class RadioButton
-  attr_accessor :name, :value, :checked
-
-  def initialize(name, value, checked)
-    @name, @value, @checked = name, value, checked
-  end
-end
-
-class CheckBox
-  attr_accessor :name, :value, :checked
-
-  def initialize(name, value, checked)
-    @name, @value, @checked = name, value, checked
-  end
-end
-
-class SelectList
-  attr_accessor :name, :value, :options
-
-  def initialize(name, node)
-    @name = name
-    @options = []
-
-    # parse
-    node.each_recursive {|n|
-      if n.name.downcase == 'option'
-        value = n.attributes['value']
-        @options << value 
-        @value = value if n.attributes['selected']
-      end
-    }
-  end
-end
-
-# Class Form does not work in the case there is some invalid (unbalanced) html
-# involved, such as: 
-#
-#   <td>
-#     <form>
-#   </td>
-#   <td>
-#     <input .../>
-#     </form>
-#   </td>
-# 
-# GlobalForm takes two nodes, the node where the form tag is located
-# (form_node), and another node, from which to start looking for form elements
-# (elements_node) like buttons and the like. For class Form both fall together
-# into one and the same node.
-
-class GlobalForm
-  attr_reader :form_node, :elements_node
-  attr_accessor :method, :action, :name
-
-  attr_reader :fields, :buttons, :file_uploads, :radiobuttons, :checkboxes
-
-  def initialize(form_node, elements_node)
-    @form_node, @elements_node = form_node, elements_node
-
-    @method = (@form_node.attributes['method'] || 'GET').upcase
-    @action = @form_node.attributes['action'] 
-    @name = @form_node.attributes['name']
-
-    parse
-  end
-
-  # In the case of malformed HTML, fields of multiple forms might occure in this forms'
-  # field array. If the fields have the same name, posterior fields overwrite former fields.
-  # To avoid this, this method rejects all posterior duplicate fields.
-
-  def uniq_fields!
-    names_in = {}
-    fields.reject! {|f|
-      if names_in.include?(f.name)
-        true
-      else
-        names_in[f.name] = true
-        false
-      end
-    }
-  end
-
-  def build_query
-    query = {}
-
-    fields().each do |f|
-      query[f.name] = f.value || ""
-    end
-
-    checkboxes().each do |f|
-      query[f.name] = f.value || "on" if f.checked
-    end
-
-    radio_groups = {}
-    radiobuttons().each do |f|
-      radio_groups[f.name] ||= []
-      radio_groups[f.name] << f 
-    end
-
-    # take one radio button from each group
-    radio_groups.each_value do |g|
-      checked = g.select {|f| f.checked}
-
-      if checked.size == 1
-        f = checked.first
-        query[f.name] = f.value || ""
-      elsif checked.size > 1 
-        raise "multiple radiobuttons are checked in the same group!" 
-      end
-    end
-
-    query
-  end
-
-  def parse
-    @fields = []
-    @buttons = []
-    @file_uploads = []
-    @radiobuttons = []
-    @checkboxes = []
-
-    @elements_node.each_recursive {|node|
-      case node.name.downcase
-      when 'input'
-        case (node.attributes['type'] || 'text').downcase
-        when 'text', 'password', 'hidden', 'int'
-          @fields << Field.new(node.attributes['name'], node.attributes['value']) 
-        when 'radio'
-          @radiobuttons << RadioButton.new(node.attributes['name'], node.attributes['value'], node.attributes.has_key?('checked'))
-        when 'checkbox'
-          @checkboxes << CheckBox.new(node.attributes['name'], node.attributes['value'], node.attributes.has_key?('checked'))
-        when 'file'
-          @file_uploads << FileUpload.new(node.attributes['name'], node.attributes['value']) 
-        when 'submit'
-          @buttons << Button.new(node.attributes['name'], node.attributes['value'])
-        when 'image'
-          @buttons << ImageButton.new(node.attributes['name'], node.attributes['value'])
-        end
-      when 'textarea'
-        @fields << Field.new(node.attributes['name'], node.all_text)
-      when 'select'
-        @fields << SelectList.new(node.attributes['name'], node)
-      end
-    }
-  end
-
-end
-
-class Form < GlobalForm
-  attr_reader :node
-
-  def initialize(node)
-    @node = node
-    super(@node, @node)
-  end
-end
-
-class Link
-  attr_reader :node
-  attr_reader :href
-  attr_reader :text
-
-  def initialize(node)
-    @node = node
-    @href = node.attributes['href'] 
-    @text = node.all_text
-  end
-end
-
-class Meta < Link
-end
-
-# = Synopsis
-# This class encapsulates a page.
-#
-# == Example
-#  require 'rubygems'
-#  require 'mechanize'
-#  require 'logger'
-#  
-#  class Body
-#    def initialize(node)
-#      puts node.attributes['bgcolor']
-#    end
-#  end
-#  
-#  agent = WWW::Mechanize.new { |a| a.log = Logger.new("mech.log") }
-#  agent.user_agent_alias = 'Mac Safari'
-#  page = agent.get("http://www.google.com/")
-#  page.watch_for_set = { 'body' => Body }
-#  
-#  body = page.watches
-class Page 
-  attr_accessor :uri, :cookies, :response, :body, :code, :watch_for_set
-
-  def initialize(uri=nil, cookies=[], response=nil, body=nil, code=nil)
-    @uri, @cookies, @response, @body, @code = uri, cookies, response, body, code
-  end
-
-  def header
-    @response.header
-  end
-
-  def content_type
-    header['Content-Type']
-  end
-
-  def forms
-    parse_html() unless @forms
-    @forms
-  end
-
-  def links
-    parse_html() unless @links
-    @links
-  end
-
-  def root
-    parse_html() unless @root
-    @root
-  end
-
-  # This method watches out for a particular tag, and will call back to the
-  # class specified for the tag in the watch_for_set method.  See the example
-  # in this class.
-  def watches
-    parse_html() unless @watches 
-    @watches 
-  end
-
-  def meta
-    parse_html() unless @meta 
-    @meta 
-  end
-
-  private
-
-  def parse_html
-    raise "no html" unless content_type() =~ /^text\/html/ 
-
-    # construct parser and feed with HTML
-    parser = HTMLTree::XMLParser.new
-    begin
-      parser.feed(@body)
-    rescue => ex
-      if ex.message =~ /attempted adding second root element to document/ and
-        # Put the whole document inside a single root element, which I simply name
-        # <root>, just to make the parser happy. It's no longer valid HTML, but 
-        # without a single root element, it's not valid HTML as well.
-
-        # TODO: leave a possible doctype definition outside this element.
-        parser = HTMLTree::XMLParser.new
-        parser.feed("<root>" + @body + "</root>")
-      else
-        raise
-      end
-    end
-
-    @root = parser.document
-
-    @forms = []
-    @links = []
-    @meta  = []
-    @watches = {}
-
-    @root.each_recursive {|node|
-      name = node.name.downcase
-
-      case name
-      when 'form'
-        @forms << Form.new(node)
-      when 'a'
-        @links << Link.new(node)
-      when 'meta'
-        equiv   = node.attributes['http-equiv']
-        content = node.attributes['content']
-        if equiv != nil && equiv.downcase == 'refresh'
-          if content != nil && content =~ /^\d+\s*;\s*url\s*=\s*(\S+)/i
-            node.attributes['href'] = $1
-            @meta << Meta.new(node)
-          end
-        end
-      else
-        if @watch_for_set and @watch_for_set.keys.include?( name )
-          @watches[name] = [] unless @watches[name]
-          klass = @watch_for_set[name]
-          @watches[name] << (klass ? klass.new(node) : node)
-        end
-      end
-    }
-  end
-end
-
 class ResponseCodeError < RuntimeError
   attr_reader :response_code
 
@@ -503,7 +143,7 @@ class Mechanize
     uri = to_absolute_uri(URI::escape(form.action))
     case form.method.upcase
     when 'POST'
-      post(uri, query) 
+      post_form(uri, form) 
     when 'GET'
       if uri.query.nil?
         get(uri.to_s + "?" + build_query_string(query))
@@ -542,6 +182,24 @@ class Mechanize
     return uri
   end
 
+  def post_form(url, form)
+    cur_page = current_page() || Page.new
+
+    request_data = [form.request_data]
+
+    # this is called before the request is sent
+    pre_request_hook = proc {|request|
+      log.debug("query: #{ request_data.inspect }")
+      request.add_field('Content-Type', form.enctype)
+      request.add_field('Content-Length', request_data[0].size.to_s)
+    }
+
+    # fetch the page
+    page = fetch_page(to_absolute_uri(url, cur_page), :post, cur_page, pre_request_hook, request_data)
+    add_to_history(page) 
+    page
+  end
+
   # uri is an absolute URI
   def fetch_page(uri, method=:get, cur_page=current_page(), pre_request_hook=nil, request_data=[])
     raise "unsupported scheme" unless ['http', 'https'].include?(uri.scheme)

data/test/htdocs/file_upload.html

@@ -0,0 +1,13 @@
+<html>
+  <head>
+    <title>File Upload Form</title>
+  </head>
+  <body>
+    <h1>File Upload Test</h1>
+    <form enctype="multipart/form-data" action="/file_upload" method="post">
+      Your name: <input type="text" name="name" /><br />
+      File to process: <input name="userfile1" type="file" /><br />
+      <input type="submit" value="Send File" />
+    </form>
+  </body>
+</html>

data/test/htdocs/frame_test.html

@@ -0,0 +1,23 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN"
+   "http://www.w3.org/TR/html4/frameset.dtd">
+<HTML>
+<HEAD>
+<TITLE>A simple frameset document</TITLE>
+</HEAD>
+<FRAMESET cols="20%, 80%">
+  <FRAMESET rows="100, 200">
+  <FRAME name="frame1" src="/google.html">
+  <FRAME name="frame2" src="/form_test.html">
+  </FRAMESET>
+  <FRAME name="frame3" src="/file_upload.html">
+  <NOFRAMES>
+      <P>This frameset document contains:
+      <UL>
+         <LI><A href="/google.html">Some neat contents</A>
+         <LI><A href="/form_test.html">Form Test</A>
+         <LI><A href="/file_upload.html">Some other neat contents</A>
+      </UL>
+  </NOFRAMES>
+</FRAMESET>
+</HTML>
+

data/test/server.rb

@@ -12,6 +12,7 @@ s.mount("/send_cookies", SendCookiesTest)
 s.mount("/form_post", FormTest)
 s.mount("/form post", FormTest)
 s.mount("/response_code", ResponseCodeTest)
+s.mount("/file_upload", FileUploadTest)
 
 trap("INT") { s.stop }
 

data/test/servlets.rb

@@ -2,6 +2,12 @@ require 'webrick'
 require 'logger'
 require 'date'
 
+class FileUploadTest < WEBrick::HTTPServlet::AbstractServlet
+  def do_POST(req, res)
+    res.body = req.body
+  end
+end
+
 class ResponseCodeTest < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res['Content-Type'] = "text/html"

data/test/tc_cookies.rb

@@ -7,11 +7,10 @@ require 'mechanize'
 require 'servlets'
 require 'net/http'
 require 'uri'
+require 'test_includes'
 
 class FormsMechTest < Test::Unit::TestCase
-  def setup
-    @port = 2000
-  end
+  include TestMethods
 
   def test_send_cookies
     agent = WWW::Mechanize.new { |a| a.log = Logger.new(nil) }

data/test/tc_forms.rb

@@ -5,11 +5,10 @@ require 'test/unit'
 require 'rubygems'
 require 'mechanize'
 require 'servlets'
+require 'test_includes'
 
 class FormsMechTest < Test::Unit::TestCase
-  def setup
-    @port = 2000
-  end
+  include TestMethods
 
   def test_post
     agent = WWW::Mechanize.new { |a| a.log = Logger.new(nil) }

data/test/tc_frames.rb

@@ -0,0 +1,22 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'rubygems'
+require 'mechanize'
+require 'test_includes'
+
+class FramesMechTest < Test::Unit::TestCase
+  include TestMethods
+
+  def test_frames
+    agent = WWW::Mechanize.new { |a| a.log = Logger.new(nil) }
+    page = agent.get("http://localhost:#{@port}/frame_test.html")
+    assert_equal(3, page.frames.size)
+    assert_equal("frame1", page.frames[0].name)
+    assert_equal("frame2", page.frames[1].name)
+    assert_equal("frame3", page.frames[2].name)
+    assert_equal("/google.html", page.frames[0].src)
+    assert_equal("/form_test.html", page.frames[1].src)
+    assert_equal("/file_upload.html", page.frames[2].src)
+  end
+end

data/test/tc_links.rb

@@ -3,11 +3,10 @@ $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
 require 'test/unit'
 require 'rubygems'
 require 'mechanize'
+require 'test_includes'
 
 class LinksMechTest < Test::Unit::TestCase
-  def setup
-    @port = 2000
-  end
+  include TestMethods
 
   def test_find_meta
     agent = WWW::Mechanize.new { |a| a.log = Logger.new(nil) }

data/test/tc_mech.rb

@@ -4,11 +4,10 @@ require 'webrick'
 require 'test/unit'
 require 'rubygems'
 require 'mechanize'
+require 'test_includes'
 
 class MechMethodsTest < Test::Unit::TestCase
-  def setup
-    @port = 2000
-  end
+  include TestMethods
 
   def test_history
     agent = WWW::Mechanize.new { |a| a.log = Logger.new(nil) }

data/test/tc_response_code.rb

@@ -3,11 +3,10 @@ $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
 require 'test/unit'
 require 'rubygems'
 require 'mechanize'
+require 'test_includes'
 
 class FormsMechTest < Test::Unit::TestCase
-  def setup
-    @port = 2000
-  end
+  include TestMethods
 
   def test_redirect
     agent = WWW::Mechanize.new { |a| a.log = Logger.new(nil) }

data/test/tc_upload.rb

@@ -0,0 +1,34 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'rubygems'
+require 'mechanize'
+require 'test_includes'
+
+class FormsMechTest < Test::Unit::TestCase
+  include TestMethods
+
+  def test_form_enctype
+    agent = WWW::Mechanize.new { |a| a.log = Logger.new(nil) }
+    page = agent.get("http://localhost:#{@port}/file_upload.html")
+    assert_equal('multipart/form-data', page.forms[0].enctype)
+
+    form = page.forms.first
+    form.file_uploads.first.file_name = "README"
+    form.file_uploads.first.mime_type = "text/plain"
+    form.file_uploads.first.file_data = "Hello World\n\n"
+
+    page = agent.submit(form)
+
+    assert_match(
+      "Content-Disposition: form-data; name=\"userfile1\"; filename=\"README\"",
+      page.body
+    )
+    assert_match(
+      "Content-Disposition: form-data; name=\"name\"",
+      page.body
+    )
+    assert_match('Content-Type: text/plain', page.body)
+    assert_match('Hello World', page.body)
+  end
+end

data/test/test_includes.rb

@@ -0,0 +1,5 @@
+module TestMethods
+  def setup
+    @port = 2000
+  end
+end

data/test/ts_mech.rb

@@ -6,4 +6,6 @@ require 'tc_forms'
 require 'tc_mech'
 require 'tc_links'
 require 'tc_response_code'
+require 'tc_upload'
+require 'tc_forms'