mechanize 0.4.0 → 0.4.1

data/EXAMPLES

@@ -0,0 +1,55 @@
+= WWW::Mechanize examples
+
+== Google
+  require 'rubygems'
+  require 'mechanize'
+  require 'logger'
+  
+  agent = WWW::Mechanize.new { |a| a.log = Logger.new("mech.log") }
+  agent.user_agent_alias = 'Mac Safari'
+  page = agent.get("http://www.google.com/")
+  search_form = page.forms.find { |f| f.name == "f" }
+  search_form.fields.find { |f| f.name == "q" }.value = "Hello"
+  search_results = agent.submit(search_form)
+  puts search_results.body
+
+== Rubyforge
+  require 'mechanize'
+  
+  agent = WWW::Mechanize.new {|a| a.log = Logger.new(STDERR) }
+  page = agent.get('http://rubyforge.org/')
+  link = page.links.find {|l| l.node.text =~ /Log In/ }
+  page = agent.click(link)
+  form = page.forms[1]
+  form.fields.find {|f| f.name == 'form_loginname'}.value = ARGV[0]
+  form.fields.find {|f| f.name == 'form_pw'}.value = ARGV[1]
+  page = agent.submit(form, form.buttons.first)
+  
+  puts page.body
+
+== File Upload
+This example uploads one image as two different images to flickr.
+
+  require 'rubygems'
+  require 'mechanize'
+  
+  agent = WWW::Mechanize.new
+  page = agent.get('http://flickr.com/signin/flickr/')
+  form = page.forms.first
+  form.fields.find { |f| f.name == 'email' }.value = ARGV[0]
+  form.fields.find { |f| f.name == 'password' }.value = ARGV[1]
+  page = agent.submit(form)
+  page = agent.click(page.links.find { |l| l.text == 'Upload' })
+  form = page.forms.first
+  img1 = form.file_uploads.find { |f| f.name == 'file1' }
+  img2 = form.file_uploads.find { |f| f.name == 'file2' }
+  
+  img1.file_name = img2.file_name = ARGV[2]
+  File.open(ARGV[2], "r") { |f|
+    img1.file_data = img2.file_data = f.read 
+  }
+  
+  img1.mime_type = img2.mime_type = 'image/jpeg'
+  
+  agent.submit(form)
+  

data/README

@@ -20,18 +20,9 @@ Note that the files in the net-overrides/ directory are taken from Ruby 1.9.0.
 
   Note that only the ruby-htmltools package bunded with narf works.
 
-== Example
-  require 'rubygems'
-  require 'mechanize'
-  require 'logger'
-  
-  agent = WWW::Mechanize.new { |a| a.log = Logger.new("mech.log") }
-  agent.user_agent_alias = 'Mac Safari'
-  page = agent.get("http://www.google.com/")
-  search_form = page.forms.find { |f| f.name == "f" }
-  search_form.fields.find { |f| f.name == "q" }.value = "Hello"
-  search_results = agent.submit(search_form)
-  puts search_results.body
+== Examples
+
+See the EXAMPLES[link://files/EXAMPLES.html] file
 
 == Authors
 

data/lib/mechanize/cookie.rb

@@ -15,8 +15,7 @@ module WWW
 
       expires_key = cookie.keys.find { |k| k.downcase == "expires" }
       if expires_key
-        @expires = DateTime.parse(cookie.delete(expires_key),
-                                      "%A %d-%b-%y %H:%M:%S %Z")
+        @expires = DateTime.parse(cookie.delete(expires_key))
       end
 
       secure_key = cookie.keys.find { |k| k.downcase == "secure" }

data/lib/mechanize/form.rb

@@ -0,0 +1,182 @@
+module WWW
+# Class Form does not work in the case there is some invalid (unbalanced) html
+# involved, such as: 
+#
+#   <td>
+#     <form>
+#   </td>
+#   <td>
+#     <input .../>
+#     </form>
+#   </td>
+# 
+# GlobalForm takes two nodes, the node where the form tag is located
+# (form_node), and another node, from which to start looking for form elements
+# (elements_node) like buttons and the like. For class Form both fall together
+# into one and the same node.
+  class GlobalForm
+    attr_reader :form_node, :elements_node
+    attr_accessor :method, :action, :name
+  
+    attr_reader :fields, :buttons, :file_uploads, :radiobuttons, :checkboxes
+    attr_reader :enctype
+  
+    def initialize(form_node, elements_node)
+      @form_node, @elements_node = form_node, elements_node
+  
+      @method = (@form_node.attributes['method'] || 'GET').upcase
+      @action = @form_node.attributes['action'] 
+      @name = @form_node.attributes['name']
+      @enctype = @form_node.attributes['enctype'] || 'application/x-www-form-urlencoded'
+  
+      parse
+    end
+  
+    # In the case of malformed HTML, fields of multiple forms might occure in this forms'
+    # field array. If the fields have the same name, posterior fields overwrite former fields.
+    # To avoid this, this method rejects all posterior duplicate fields.
+  
+    def uniq_fields!
+      names_in = {}
+      fields.reject! {|f|
+        if names_in.include?(f.name)
+          true
+        else
+          names_in[f.name] = true
+          false
+        end
+      }
+    end
+  
+    def build_query
+      query = {}
+  
+      fields().each do |f|
+        query[f.name] = f.value || ""
+      end
+  
+      checkboxes().each do |f|
+        query[f.name] = f.value || "on" if f.checked
+      end
+  
+      radio_groups = {}
+      radiobuttons().each do |f|
+        radio_groups[f.name] ||= []
+        radio_groups[f.name] << f 
+      end
+  
+      # take one radio button from each group
+      radio_groups.each_value do |g|
+        checked = g.select {|f| f.checked}
+  
+        if checked.size == 1
+          f = checked.first
+          query[f.name] = f.value || ""
+        elsif checked.size > 1 
+          raise "multiple radiobuttons are checked in the same group!" 
+        end
+      end
+  
+      query
+    end
+  
+    def request_data
+      query_params = build_query()
+      query = nil
+      case @enctype.downcase
+      when 'multipart/form-data'
+        boundary = rand_string(20)
+        @enctype << ", boundary=#{boundary}"
+        params = []
+        query_params.each { |k,v| params << param_to_multipart(k, v) }
+        @file_uploads.each { |f| params << file_to_multipart(f) }
+        query = params.collect { |p| "--#{boundary}\r\n#{p}" }.join('') +
+          "--#{boundary}--\r\n"
+      else
+        query = build_query_string(query_params)
+      end
+  
+      query
+    end
+  
+    def parse
+      @fields = []
+      @buttons = []
+      @file_uploads = []
+      @radiobuttons = []
+      @checkboxes = []
+  
+      @elements_node.each_recursive {|node|
+        case node.name.downcase
+        when 'input'
+          case (node.attributes['type'] || 'text').downcase
+          when 'text', 'password', 'hidden', 'int'
+            @fields << Field.new(node.attributes['name'], node.attributes['value']) 
+          when 'radio'
+            @radiobuttons << RadioButton.new(node.attributes['name'], node.attributes['value'], node.attributes.has_key?('checked'))
+          when 'checkbox'
+            @checkboxes << CheckBox.new(node.attributes['name'], node.attributes['value'], node.attributes.has_key?('checked'))
+          when 'file'
+            @file_uploads << FileUpload.new(node.attributes['name'], node.attributes['value']) 
+          when 'submit'
+            @buttons << Button.new(node.attributes['name'], node.attributes['value'])
+          when 'image'
+            @buttons << ImageButton.new(node.attributes['name'], node.attributes['value'])
+          end
+        when 'textarea'
+          @fields << Field.new(node.attributes['name'], node.all_text)
+        when 'select'
+          @fields << SelectList.new(node.attributes['name'], node)
+        end
+      }
+    end
+  
+    private
+    def rand_string(len = 10)
+      chars = ("a".."z").to_a + ("A".."Z").to_a
+      string = ""
+      1.upto(len) { |i| string << chars[rand(chars.size-1)] }
+      string
+    end
+  
+    def param_to_multipart(name, value)
+      return "Content-Disposition: form-data; name=\"" +
+              "#{WEBrick::HTTPUtils.escape_form(name)}\"\r\n" +
+              "\r\n#{value}\r\n"
+    end
+  
+    def file_to_multipart(file)
+      body =  "Content-Disposition: form-data; name=\"" +
+              "#{WEBrick::HTTPUtils.escape_form(file.name)}\"; " +
+              "filename=\"#{file.file_name}\"\r\n" +
+              "Content-Transfer-Encoding: binary\r\n"
+      if file.mime_type != nil
+        body << "Content-Type: #{file.mime_type}\r\n"
+      end
+  
+      body << "\r\n#{file.file_data}\r\n"
+  
+      body
+    end
+
+    def build_query_string(hash)
+      vals = []
+      hash.each_pair do |k,v|
+        vals << [
+          WEBrick::HTTPUtils.escape_form(k),
+          WEBrick::HTTPUtils.escape_form(v)
+        ].join("=")
+      end
+      vals.join("&")
+    end
+  end
+  
+  class Form < GlobalForm
+    attr_reader :node
+  
+    def initialize(node)
+      @node = node
+      super(@node, @node)
+    end
+  end
+end

data/lib/mechanize/form_elements.rb

@@ -0,0 +1,105 @@
+module WWW
+  class Field
+    attr_accessor :name, :value
+  
+    def initialize(name, value)
+      @name, @value = name, value
+    end
+  
+    # Returns an array of Field objects
+    # TODO: is this correct?
+    def self.extract_all_from(root_node)
+      fields = []
+      root_node.each_recursive {|node|
+        if (node.name.downcase == 'input' and 
+           %w(text password hidden checkbox radio int).include?(node.attributes['type'].downcase)) or
+           %w(textarea option).include?(node.name.downcase)
+          fields << Field.new(node.attributes['name'], node.attributes['value']) 
+        end
+      }
+      return fields
+    end
+  end
+  
+  class FileUpload
+    # value is the file-name, not the file-content
+    attr_accessor :name
+    
+    attr_accessor :file_name, :file_data, :mime_type
+  
+    def initialize(name, file_name)
+      @name, @file_name = name, file_name
+      @file_data = nil
+    end
+  end
+  
+  class Button
+    attr_accessor :name, :value
+  
+    def initialize(name, value)
+      @name, @value = name, value
+    end
+  
+    def add_to_query(query)
+      query[@name] = @value || "" if @name
+    end
+  
+    # Returns an array of Button objects
+    def self.extract_all_from(root_node)
+      buttons = []
+      root_node.each_recursive {|node|
+        if node.name.downcase == 'input' and 
+           ['submit'].include?(node.attributes['type'].downcase)
+          buttons << Button.new(node.attributes['name'], node.attributes['value'])
+        end
+      }
+      return buttons
+    end
+  end 
+  
+  class ImageButton < Button
+    attr_accessor :x, :y
+    
+    def add_to_query(query)
+      if @name
+        query[@name] = @value || ""
+        query[@name+".x"] = (@x || "0").to_s
+        query[@name+".y"] = (@y || "0").to_s
+      end
+    end
+  end
+  
+  class RadioButton
+    attr_accessor :name, :value, :checked
+  
+    def initialize(name, value, checked)
+      @name, @value, @checked = name, value, checked
+    end
+  end
+  
+  class CheckBox
+    attr_accessor :name, :value, :checked
+  
+    def initialize(name, value, checked)
+      @name, @value, @checked = name, value, checked
+    end
+  end
+  
+  class SelectList
+    attr_accessor :name, :value, :options
+  
+    def initialize(name, node)
+      @name = name
+      @options = []
+  
+      # parse
+      node.each_recursive {|n|
+        if n.name.downcase == 'option'
+          value = n.attributes['value']
+          @options << value 
+          @value = value if n.attributes['selected']
+        end
+      }
+    end
+  end
+end

data/lib/mechanize/page.rb

@@ -0,0 +1,130 @@
+module WWW
+# = Synopsis
+# This class encapsulates a page.
+#
+# == Example
+#  require 'rubygems'
+#  require 'mechanize'
+#  require 'logger'
+#  
+#  class Body
+#    def initialize(node)
+#      puts node.attributes['bgcolor']
+#    end
+#  end
+#  
+#  agent = WWW::Mechanize.new { |a| a.log = Logger.new("mech.log") }
+#  agent.user_agent_alias = 'Mac Safari'
+#  page = agent.get("http://www.google.com/")
+#  page.watch_for_set = { 'body' => Body }
+#  
+#  body = page.watches
+  class Page 
+    attr_accessor :uri, :cookies, :response, :body, :code, :watch_for_set
+  
+    def initialize(uri=nil, cookies=[], response=nil, body=nil, code=nil)
+      @uri, @cookies, @response, @body, @code = uri, cookies, response, body, code
+    end
+  
+    def header
+      @response.header
+    end
+  
+    def content_type
+      header['Content-Type']
+    end
+  
+    def forms
+      parse_html() unless @forms
+      @forms
+    end
+  
+    def links
+      parse_html() unless @links
+      @links
+    end
+  
+    def root
+      parse_html() unless @root
+      @root
+    end
+  
+    # This method watches out for a particular tag, and will call back to the
+    # class specified for the tag in the watch_for_set method.  See the example
+    # in this class.
+    def watches
+      parse_html() unless @watches 
+      @watches 
+    end
+  
+    def meta
+      parse_html() unless @meta 
+      @meta 
+    end
+
+    def frames
+      parse_html() unless @frames
+      @frames
+    end
+  
+    private
+  
+    def parse_html
+      raise "no html" unless content_type() =~ /^text\/html/ 
+  
+      # construct parser and feed with HTML
+      parser = HTMLTree::XMLParser.new
+      begin
+        parser.feed(@body)
+      rescue => ex
+        if ex.message =~ /attempted adding second root element to document/ and
+          # Put the whole document inside a single root element, which I simply name
+          # <root>, just to make the parser happy. It's no longer valid HTML, but 
+          # without a single root element, it's not valid HTML as well.
+  
+          # TODO: leave a possible doctype definition outside this element.
+          parser = HTMLTree::XMLParser.new
+          parser.feed("<root>" + @body + "</root>")
+        else
+          raise
+        end
+      end
+  
+      @root = parser.document
+  
+      @forms    = []
+      @links    = []
+      @meta     = []
+      @frames   = []
+      @watches  = {}
+  
+      @root.each_recursive {|node|
+        name = node.name.downcase
+  
+        case name
+        when 'form'
+          @forms << Form.new(node)
+        when 'a'
+          @links << Link.new(node)
+        when 'meta'
+          equiv   = node.attributes['http-equiv']
+          content = node.attributes['content']
+          if equiv != nil && equiv.downcase == 'refresh'
+            if content != nil && content =~ /^\d+\s*;\s*url\s*=\s*(\S+)/i
+              node.attributes['href'] = $1
+              @meta << Meta.new(node)
+            end
+          end
+        when 'frame'
+          @frames << Frame.new(node)
+        else
+          if @watch_for_set and @watch_for_set.keys.include?( name )
+            @watches[name] = [] unless @watches[name]
+            klass = @watch_for_set[name]
+            @watches[name] << (klass ? klass.new(node) : node)
+          end
+        end
+      }
+    end
+  end
+end

data/lib/mechanize/page_elements.rb

@@ -0,0 +1,28 @@
+module WWW
+  class Link
+    attr_reader :node
+    attr_reader :href
+    attr_reader :text
+  
+    def initialize(node)
+      @node = node
+      @href = node.attributes['href'] 
+      @text = node.all_text
+    end
+  end
+  
+  class Meta < Link
+  end
+
+  class Frame
+    attr_reader :node
+    attr_reader :name
+    attr_reader :src
+
+    def initialize(node)
+      @node = node
+      @name = node.attributes['name']
+      @src  = node.attributes['src']
+    end
+  end
+end