RubyGems - tenderlove-mechanize - Versions diffs - 0.9.3.20090623142847 → 0.9.3.20090911221705 - Mend

tenderlove-mechanize 0.9.3.20090623142847 → 0.9.3.20090911221705

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (165) hide show

data/Manifest.txt +55 -48
data/Rakefile +12 -22
data/lib/mechanize.rb +618 -4
data/lib/mechanize/chain.rb +33 -0
data/lib/mechanize/chain/auth_headers.rb +78 -0
data/lib/mechanize/chain/body_decoding_handler.rb +46 -0
data/lib/mechanize/chain/connection_resolver.rb +76 -0
data/lib/mechanize/chain/custom_headers.rb +21 -0
data/lib/{www/mechanize → mechanize}/chain/handler.rb +1 -1
data/lib/mechanize/chain/header_resolver.rb +51 -0
data/lib/mechanize/chain/parameter_resolver.rb +22 -0
data/lib/{www/mechanize → mechanize}/chain/post_connect_hook.rb +0 -0
data/lib/mechanize/chain/pre_connect_hook.rb +20 -0
data/lib/mechanize/chain/request_resolver.rb +30 -0
data/lib/mechanize/chain/response_body_parser.rb +38 -0
data/lib/mechanize/chain/response_header_handler.rb +48 -0
data/lib/mechanize/chain/response_reader.rb +39 -0
data/lib/mechanize/chain/ssl_resolver.rb +40 -0
data/lib/mechanize/chain/uri_resolver.rb +75 -0
data/lib/mechanize/content_type_error.rb +14 -0
data/lib/mechanize/cookie.rb +70 -0
data/lib/mechanize/cookie_jar.rb +188 -0
data/lib/mechanize/file.rb +71 -0
data/lib/mechanize/file_response.rb +60 -0
data/lib/mechanize/file_saver.rb +37 -0
data/lib/mechanize/form.rb +378 -0
data/lib/mechanize/form/button.rb +9 -0
data/lib/mechanize/form/check_box.rb +11 -0
data/lib/mechanize/form/field.rb +30 -0
data/lib/mechanize/form/file_upload.rb +22 -0
data/lib/mechanize/form/image_button.rb +21 -0
data/lib/mechanize/form/multi_select_list.rb +67 -0
data/lib/mechanize/form/option.rb +49 -0
data/lib/mechanize/form/radio_button.rb +49 -0
data/lib/mechanize/form/select_list.rb +43 -0
data/lib/mechanize/headers.rb +11 -0
data/lib/mechanize/history.rb +65 -0
data/lib/mechanize/inspect.rb +88 -0
data/lib/{www/mechanize → mechanize}/monkey_patch.rb +4 -6
data/lib/mechanize/page.rb +206 -0
data/lib/mechanize/page/base.rb +8 -0
data/lib/mechanize/page/frame.rb +20 -0
data/lib/mechanize/page/image.rb +26 -0
data/lib/mechanize/page/label.rb +20 -0
data/lib/mechanize/page/link.rb +48 -0
data/lib/mechanize/page/meta.rb +50 -0
data/lib/mechanize/pluggable_parsers.rb +101 -0
data/lib/mechanize/redirect_limit_reached_error.rb +16 -0
data/lib/mechanize/redirect_not_get_or_head_error.rb +18 -0
data/lib/mechanize/response_code_error.rb +22 -0
data/lib/mechanize/unsupported_scheme_error.rb +8 -0
data/lib/mechanize/util.rb +67 -0
data/mechanize.gemspec +8 -8
data/test/chain/test_argument_validator.rb +2 -2
data/test/chain/test_auth_headers.rb +2 -2
data/test/chain/test_custom_headers.rb +2 -2
data/test/chain/test_header_resolver.rb +3 -3
data/test/chain/test_parameter_resolver.rb +4 -4
data/test/chain/test_request_resolver.rb +4 -4
data/test/chain/test_response_reader.rb +3 -3
data/test/helper.rb +1 -1
data/test/htdocs/tc_bad_charset.html +9 -0
data/test/htdocs/tc_charset.html +6 -0
data/test/htdocs/test_bad_encoding.html +52 -0
data/test/test_authenticate.rb +3 -3
data/test/test_bad_links.rb +1 -1
data/test/test_blank_form.rb +1 -1
data/test/test_checkboxes.rb +1 -1
data/test/test_content_type.rb +2 -2
data/test/test_cookie_class.rb +12 -12
data/test/test_cookie_jar.rb +13 -13
data/test/test_cookies.rb +1 -1
data/test/test_encoded_links.rb +1 -1
data/test/test_errors.rb +2 -2
data/test/test_follow_meta.rb +3 -3
data/test/test_form_action.rb +1 -1
data/test/test_form_as_hash.rb +1 -1
data/test/test_form_button.rb +2 -2
data/test/test_form_no_inputname.rb +1 -1
data/test/test_forms.rb +1 -1
data/test/test_frames.rb +1 -1
data/test/test_get_headers.rb +1 -1
data/test/test_gzipping.rb +2 -2
data/test/test_hash_api.rb +1 -1
data/test/test_history.rb +7 -7
data/test/test_history_added.rb +1 -1
data/test/test_html_unscape_forms.rb +7 -7
data/test/test_if_modified_since.rb +1 -1
data/test/test_keep_alive.rb +1 -1
data/test/test_links.rb +2 -2
data/test/test_mech.rb +2 -2
data/test/test_mechanize_file.rb +7 -7
data/test/test_meta.rb +2 -2
data/test/test_multi_select.rb +1 -1
data/test/test_no_attributes.rb +1 -1
data/test/test_option.rb +1 -1
data/test/test_page.rb +3 -3
data/test/test_pluggable_parser.rb +14 -14
data/test/test_post_form.rb +1 -1
data/test/test_pretty_print.rb +2 -2
data/test/test_radiobutton.rb +1 -1
data/test/test_redirect_limit_reached.rb +1 -3
data/test/test_redirect_verb_handling.rb +1 -3
data/test/test_referer.rb +1 -1
data/test/test_relative_links.rb +1 -1
data/test/test_request.rb +1 -1
data/test/test_response_code.rb +3 -3
data/test/test_save_file.rb +3 -3
data/test/test_scheme.rb +3 -3
data/test/test_select.rb +2 -2
data/test/test_select_all.rb +1 -1
data/test/test_select_none.rb +1 -1
data/test/test_select_noopts.rb +1 -1
data/test/test_set_fields.rb +1 -1
data/test/test_ssl_server.rb +1 -1
data/test/test_subclass.rb +1 -1
data/test/test_textarea.rb +1 -1
data/test/test_upload.rb +1 -1
data/test/test_verbs.rb +1 -1
metadata +61 -56
data/lib/www/mechanize.rb +0 -619
data/lib/www/mechanize/chain.rb +0 -34
data/lib/www/mechanize/chain/auth_headers.rb +0 -80
data/lib/www/mechanize/chain/body_decoding_handler.rb +0 -48
data/lib/www/mechanize/chain/connection_resolver.rb +0 -78
data/lib/www/mechanize/chain/custom_headers.rb +0 -23
data/lib/www/mechanize/chain/header_resolver.rb +0 -53
data/lib/www/mechanize/chain/parameter_resolver.rb +0 -24
data/lib/www/mechanize/chain/pre_connect_hook.rb +0 -22
data/lib/www/mechanize/chain/request_resolver.rb +0 -32
data/lib/www/mechanize/chain/response_body_parser.rb +0 -40
data/lib/www/mechanize/chain/response_header_handler.rb +0 -50
data/lib/www/mechanize/chain/response_reader.rb +0 -41
data/lib/www/mechanize/chain/ssl_resolver.rb +0 -42
data/lib/www/mechanize/chain/uri_resolver.rb +0 -77
data/lib/www/mechanize/content_type_error.rb +0 -16
data/lib/www/mechanize/cookie.rb +0 -72
data/lib/www/mechanize/cookie_jar.rb +0 -191
data/lib/www/mechanize/file.rb +0 -73
data/lib/www/mechanize/file_response.rb +0 -62
data/lib/www/mechanize/file_saver.rb +0 -39
data/lib/www/mechanize/form.rb +0 -360
data/lib/www/mechanize/form/button.rb +0 -8
data/lib/www/mechanize/form/check_box.rb +0 -13
data/lib/www/mechanize/form/field.rb +0 -28
data/lib/www/mechanize/form/file_upload.rb +0 -24
data/lib/www/mechanize/form/image_button.rb +0 -23
data/lib/www/mechanize/form/multi_select_list.rb +0 -69
data/lib/www/mechanize/form/option.rb +0 -51
data/lib/www/mechanize/form/radio_button.rb +0 -38
data/lib/www/mechanize/form/select_list.rb +0 -45
data/lib/www/mechanize/headers.rb +0 -12
data/lib/www/mechanize/history.rb +0 -67
data/lib/www/mechanize/inspect.rb +0 -90
data/lib/www/mechanize/page.rb +0 -181
data/lib/www/mechanize/page/base.rb +0 -10
data/lib/www/mechanize/page/frame.rb +0 -22
data/lib/www/mechanize/page/link.rb +0 -50
data/lib/www/mechanize/page/meta.rb +0 -51
data/lib/www/mechanize/pluggable_parsers.rb +0 -103
data/lib/www/mechanize/redirect_limit_reached_error.rb +0 -18
data/lib/www/mechanize/redirect_not_get_or_head_error.rb +0 -20
data/lib/www/mechanize/response_code_error.rb +0 -25
data/lib/www/mechanize/unsupported_scheme_error.rb +0 -10
data/lib/www/mechanize/util.rb +0 -76

data/lib/mechanize/file_response.rb ADDED Viewed

@@ -0,0 +1,60 @@
+class Mechanize
+  ###
+  # Fake response for dealing with file:/// requests
+  class FileResponse
+    def initialize(file_path)
+      @file_path = file_path
+    end
+    def read_body
+      if ::File.exists?(@file_path)
+        if directory?
+          yield dir_body
+        else
+          yield ::File.read(@file_path)
+        end
+      else
+        yield ''
+      end
+    end
+    def code
+      ::File.exists?(@file_path) ? 200 : 400
+    end
+    def content_length
+      return dir_body.length if directory?
+      ::File.exists?(@file_path) ? ::File.stat(@file_path).size : 0
+    end
+    def each_header; end
+    def [](key)
+      return nil unless key.downcase == 'content-type'
+      return 'text/html' if directory?
+      return 'text/html' if ['.html', '.xhtml'].any? { |extn|
+        @file_path =~ /#{extn}$/
+      }
+      nil
+    end
+    def each
+    end
+    def get_fields(key)
+      []
+    end
+    private
+    def dir_body
+      '<html><body>' +
+        Dir[::File.join(@file_path, '*')].map { |f|
+        "<a href=\"file://#{f}\">#{::File.basename(f)}</a>"
+      }.join("\n") + '</body></html>'
+    end
+    def directory?
+      ::File.directory?(@file_path)
+    end
+  end
+end

data/lib/mechanize/file_saver.rb ADDED Viewed

@@ -0,0 +1,37 @@
+class Mechanize
+  # = Synopsis
+  # This is a pluggable parser that automatically saves every file
+  # it encounters.  It saves the files as a tree, reflecting the
+  # host and file path.
+  #
+  # == Example to save all PDF's
+  #  require 'rubygems'
+  #  require 'mechanize'
+  #
+  #  agent = Mechanize.new
+  #  agent.pluggable_parser.pdf = Mechanize::FileSaver
+  #  agent.get('http://example.com/foo.pdf')
+  #
+  class FileSaver < File
+    attr_reader :filename
+    def initialize(uri=nil, response=nil, body=nil, code=nil)
+      super(uri, response, body, code)
+      path = uri.path.empty? ? 'index.html' : uri.path.gsub(/^[\/]*/, '')
+      path += 'index.html' if path =~ /\/$/
+      split_path = path.split(/\//)
+      filename = split_path.length > 0 ? split_path.pop : 'index.html'
+      joined_path = split_path.join(::File::SEPARATOR)
+      path = if joined_path.empty?
+               uri.host
+             else
+               "#{uri.host}#{::File::SEPARATOR}#{joined_path}"
+             end
+      @filename = "#{path}#{::File::SEPARATOR}#{filename}"
+      FileUtils.mkdir_p(path)
+      save_as(@filename)
+    end
+  end
+end

data/lib/mechanize/form.rb ADDED Viewed

@@ -0,0 +1,378 @@
+require 'mechanize/form/field'
+require 'mechanize/form/file_upload'
+require 'mechanize/form/button'
+require 'mechanize/form/image_button'
+require 'mechanize/form/radio_button'
+require 'mechanize/form/check_box'
+require 'mechanize/form/multi_select_list'
+require 'mechanize/form/select_list'
+require 'mechanize/form/option'
+class Mechanize
+  # =Synopsis
+  # This class encapsulates a form parsed out of an HTML page.  Each type
+  # of input fields available in a form can be accessed through this object.
+  # See GlobalForm for more methods.
+  #
+  # ==Example
+  # Find a form and print out its fields
+  #  form = page.forms.first # => Mechanize::Form
+  #  form.fields.each { |f| puts f.name }
+  # Set the input field 'name' to "Aaron"
+  #  form['name'] = 'Aaron'
+  #  puts form['name']
+  class Form
+    attr_accessor :method, :action, :name
+    attr_reader :fields, :buttons, :file_uploads, :radiobuttons, :checkboxes
+    attr_accessor :enctype
+    alias :elements :fields
+    attr_reader :form_node
+    attr_reader :page
+    def initialize(node, mech=nil, page=nil)
+      @enctype = node['enctype'] || 'application/x-www-form-urlencoded'
+      @form_node        = node
+      @action           = Util.html_unescape(node['action'])
+      @method           = (node['method'] || 'GET').upcase
+      @name             = node['name']
+      @clicked_buttons  = []
+      @page             = page
+      @mech             = mech
+      parse
+    end
+    # Returns whether or not the form contains a field with +field_name+
+    def has_field?(field_name)
+      ! fields.find { |f| f.name.eql? field_name }.nil?
+    end
+    alias :has_key? :has_field?
+    def has_value?(value)
+      ! fields.find { |f| f.value.eql? value }.nil?
+    end
+    def keys; fields.map { |f| f.name }; end
+    def values; fields.map { |f| f.value }; end
+    def submits  ; @submits   ||= buttons.select { |f| f.class == Submit   }; end
+    def resets   ; @resets    ||= buttons.select { |f| f.class == Reset    }; end
+    def texts    ; @texts     ||=  fields.select { |f| f.class == Text     }; end
+    def hiddens  ; @hiddens   ||=  fields.select { |f| f.class == Hidden   }; end
+    def textareas; @textareas ||=  fields.select { |f| f.class == Textarea }; end
+    def submit_button?(button_name) !!  submits.find{|f| f.name == button_name}; end
+    def reset_button?(button_name)  !!   resets.find{|f| f.name == button_name}; end
+    def text_field?(field_name)     !!    texts.find{|f| f.name == field_name}; end
+    def hidden_field?(field_name)   !!  hiddens.find{|f| f.name == field_name}; end
+    def textarea_field?(field_name) !!textareas.find{|f| f.name == field_name}; end
+    # Add a field with +field_name+ and +value+
+    def add_field!(field_name, value = nil)
+      fields << Field.new(field_name, value)
+    end
+    # This method sets multiple fields on the form.  It takes a list of field
+    # name, value pairs.  If there is more than one field found with the
+    # same name, this method will set the first one found.  If you want to
+    # set the value of a duplicate field, use a value which is a Hash with
+    # the key as the index in to the form.  The index
+    # is zero based.  For example, to set the second field named 'foo', you
+    # could do the following:
+    #  form.set_fields( :foo => { 1 => 'bar' } )
+    def set_fields(fields = {})
+      fields.each do |k,v|
+        case v
+        when Hash
+          v.each do |index, value|
+            self.fields_with(:name => k.to_s).[](index).value = value
+          end
+        else
+          value = nil
+          index = 0
+          [v].flatten.each do |val|
+            index = val.to_i unless value.nil?
+            value = val if value.nil?
+          end
+          self.fields_with(:name => k.to_s).[](index).value = value
+        end
+      end
+    end
+    # Fetch the value of the first input field with the name passed in
+    # ==Example
+    # Fetch the value set in the input field 'name'
+    #  puts form['name']
+    def [](field_name)
+      f = field(field_name)
+      f && f.value
+    end
+    # Set the value of the first input field with the name passed in
+    # ==Example
+    # Set the value in the input field 'name' to "Aaron"
+    #  form['name'] = 'Aaron'
+    def []=(field_name, value)
+      f = field(field_name)
+      if f.nil?
+        add_field!(field_name, value)
+      else
+        f.value = value
+      end
+    end
+    # Treat form fields like accessors.
+    def method_missing(id,*args)
+      method = id.to_s.gsub(/=$/, '')
+      if field(method)
+        return field(method).value if args.empty?
+        return field(method).value = args[0]
+      end
+      super
+    end
+    # Submit this form with the button passed in
+    def submit button=nil, headers = {}
+      @mech.submit(self, button, headers)
+    end
+    # Submit form using +button+. Defaults
+    # to the first button.
+    def click_button(button = buttons.first)
+      submit(button)
+    end
+    # This method is sub-method of build_query.
+    # It converts charset of query value of fields into expected one.
+    def proc_query(field)
+      return unless field.query_value
+      field.query_value.map{|(name, val)|
+        [from_native_charset(name), from_native_charset(val.to_s)]
+      }
+    end
+    private :proc_query
+    def from_native_charset(str, enc=nil)
+      if page
+        enc ||= page.encoding
+        Util.from_native_charset(str,enc) rescue str
+      else
+        str
+      end
+    end
+    private :from_native_charset
+    # This method builds an array of arrays that represent the query
+    # parameters to be used with this form.  The return value can then
+    # be used to create a query string for this form.
+    def build_query(buttons = [])
+      query = []
+      fields().each do |f|
+        qval = proc_query(f)
+        query.push(*qval)
+      end
+      checkboxes().each do |f|
+        if f.checked
+          qval = proc_query(f)
+          query.push(*qval)
+        end
+      end
+      radio_groups = {}
+      radiobuttons().each do |f|
+        fname = from_native_charset(f.name)
+        radio_groups[fname] ||= []
+        radio_groups[fname] << f
+      end
+      # take one radio button from each group
+      radio_groups.each_value do |g|
+        checked = g.select {|f| f.checked}
+        if checked.size == 1
+          f = checked.first
+          qval = proc_query(f)
+          query.push(*qval)
+        elsif checked.size > 1
+          raise "multiple radiobuttons are checked in the same group!"
+        end
+      end
+      @clicked_buttons.each { |b|
+        qval = proc_query(b)
+        query.push(*qval)
+      }
+      query
+    end
+    # This method adds a button to the query.  If the form needs to be
+    # submitted with multiple buttons, pass each button to this method.
+    def add_button_to_query(button)
+      @clicked_buttons << button
+    end
+    # This method calculates the request data to be sent back to the server
+    # for this form, depending on if this is a regular post, get, or a
+    # multi-part post,
+    def request_data
+      query_params = build_query()
+      case @enctype.downcase
+      when /^multipart\/form-data/
+        boundary = rand_string(20)
+        @enctype = "multipart/form-data; boundary=#{boundary}"
+        params = []
+        query_params.each { |k,v| params << param_to_multipart(k, v) unless k.nil? }
+        @file_uploads.each { |f| params << file_to_multipart(f) }
+        params.collect { |p| "--#{boundary}\r\n#{p}" }.join('') +
+          "--#{boundary}--\r\n"
+      else
+        Mechanize::Util.build_query_string(query_params)
+      end
+    end
+    # Removes all fields with name +field_name+.
+    def delete_field!(field_name)
+      @fields.delete_if{ |f| f.name == field_name}
+    end
+    { :field        => :fields,
+      :button       => :buttons,
+      :file_upload  => :file_uploads,
+      :radiobutton  => :radiobuttons,
+      :checkbox     => :checkboxes,
+    }.each do |singular,plural|
+      eval(<<-eomethod)
+          def #{plural}_with criteria = {}
+            criteria = {:name => criteria} if String === criteria
+            f = #{plural}.find_all do |thing|
+              criteria.all? { |k,v| v === thing.send(k) }
+            end
+            yield f if block_given?
+            f
+          end
+          def #{singular}_with criteria = {}
+            f = #{plural}_with(criteria).first
+            yield f if block_given?
+            f
+          end
+          alias :#{singular} :#{singular}_with
+        eomethod
+    end
+    private
+    def parse
+      @fields       = []
+      @buttons      = []
+      @file_uploads = []
+      @radiobuttons = []
+      @checkboxes   = []
+      # Find all input tags
+      form_node.search('input').each do |node|
+        type = (node['type'] || 'text').downcase
+        name = node['name']
+        next if name.nil? && !(type == 'submit' || type =='button')
+        case type
+        when 'radio'
+          @radiobuttons << RadioButton.new(node['name'], node['value'], !!node['checked'], self, node)
+        when 'checkbox'
+          @checkboxes << CheckBox.new(node['name'], node['value'], !!node['checked'], self, node)
+        when 'file'
+          @file_uploads << FileUpload.new(node['name'], nil)
+        when 'submit'
+          @buttons << Submit.new(node['name'], node['value'])
+        when 'button'
+          @buttons << Button.new(node['name'], node['value'])
+        when 'reset'
+          @buttons << Reset.new(node['name'], node['value'])
+        when 'image'
+          @buttons << ImageButton.new(node['name'], node['value'])
+        when 'hidden'
+          @fields << Hidden.new(node['name'], node['value'] || '')
+        when 'text'
+          @fields << Text.new(node['name'], node['value'] || '')
+        when 'textarea'
+          @fields << Textarea.new(node['name'], node['value'] || '')
+        else
+          @fields << Field.new(node['name'], node['value'] || '')
+        end
+      end
+      # Find all textarea tags
+      form_node.search('textarea').each do |node|
+        next if node['name'].nil?
+        @fields << Field.new(node['name'], node.inner_text)
+      end
+      # Find all select tags
+      form_node.search('select').each do |node|
+        next if node['name'].nil?
+        if node.has_attribute? 'multiple'
+          @fields << MultiSelectList.new(node['name'], node)
+        else
+          @fields << SelectList.new(node['name'], node)
+        end
+      end
+      # Find all submit button tags
+      # FIXME: what can I do with the reset buttons?
+      form_node.search('button').each do |node|
+        type = (node['type'] || 'submit').downcase
+        next if type == 'reset'
+        @buttons << Button.new(node['name'], node['value'])
+      end
+    end
+    def rand_string(len = 10)
+      chars = ("a".."z").to_a + ("A".."Z").to_a
+      string = ""
+      1.upto(len) { |i| string << chars[rand(chars.size-1)] }
+      string
+    end
+    def mime_value_quote(str)
+      str.gsub(/(["\r\\])/){|s| '\\' + s}
+    end
+    def param_to_multipart(name, value)
+      return "Content-Disposition: form-data; name=\"" +
+        "#{mime_value_quote(name)}\"\r\n" +
+        "\r\n#{value}\r\n"
+    end
+    def file_to_multipart(file)
+      file_name = file.file_name ? ::File.basename(file.file_name) : ''
+      body =  "Content-Disposition: form-data; name=\"" +
+        "#{mime_value_quote(file.name)}\"; " +
+        "filename=\"#{mime_value_quote(file_name)}\"\r\n" +
+        "Content-Transfer-Encoding: binary\r\n"
+      if file.file_data.nil? and ! file.file_name.nil?
+        file.file_data = ::File.open(file.file_name, "rb") { |f| f.read }
+        file.mime_type = WEBrick::HTTPUtils.mime_type(file.file_name,
+                                                      WEBrick::HTTPUtils::DefaultMimeTypes)
+      end
+      if file.mime_type != nil
+        body << "Content-Type: #{file.mime_type}\r\n"
+      end
+      body <<
+        if file.file_data.respond_to? :read
+          "\r\n#{file.file_data.read}\r\n"
+        else
+          "\r\n#{file.file_data}\r\n"
+        end
+      body
+    end
+  end
+end