RubyGems - eric-mechanize - Versions diffs - 0.9.3.20090623142847 - Mend

eric-mechanize 0.9.3.20090623142847

Files changed (173) hide show

data/CHANGELOG.rdoc +504 -0
data/EXAMPLES.rdoc +171 -0
data/FAQ.rdoc +11 -0
data/GUIDE.rdoc +122 -0
data/LICENSE.rdoc +340 -0
data/Manifest.txt +169 -0
data/README.rdoc +60 -0
data/Rakefile +43 -0
data/examples/flickr_upload.rb +23 -0
data/examples/mech-dump.rb +7 -0
data/examples/proxy_req.rb +9 -0
data/examples/rubyforge.rb +21 -0
data/examples/spider.rb +11 -0
data/lib/mechanize.rb +7 -0
data/lib/www/mechanize.rb +619 -0
data/lib/www/mechanize/chain.rb +34 -0
data/lib/www/mechanize/chain/auth_headers.rb +80 -0
data/lib/www/mechanize/chain/body_decoding_handler.rb +48 -0
data/lib/www/mechanize/chain/connection_resolver.rb +78 -0
data/lib/www/mechanize/chain/custom_headers.rb +23 -0
data/lib/www/mechanize/chain/handler.rb +9 -0
data/lib/www/mechanize/chain/header_resolver.rb +53 -0
data/lib/www/mechanize/chain/parameter_resolver.rb +24 -0
data/lib/www/mechanize/chain/post_connect_hook.rb +0 -0
data/lib/www/mechanize/chain/pre_connect_hook.rb +22 -0
data/lib/www/mechanize/chain/request_resolver.rb +32 -0
data/lib/www/mechanize/chain/response_body_parser.rb +40 -0
data/lib/www/mechanize/chain/response_header_handler.rb +50 -0
data/lib/www/mechanize/chain/response_reader.rb +41 -0
data/lib/www/mechanize/chain/ssl_resolver.rb +42 -0
data/lib/www/mechanize/chain/uri_resolver.rb +77 -0
data/lib/www/mechanize/content_type_error.rb +16 -0
data/lib/www/mechanize/cookie.rb +72 -0
data/lib/www/mechanize/cookie_jar.rb +191 -0
data/lib/www/mechanize/file.rb +73 -0
data/lib/www/mechanize/file_response.rb +62 -0
data/lib/www/mechanize/file_saver.rb +39 -0
data/lib/www/mechanize/form.rb +360 -0
data/lib/www/mechanize/form/button.rb +8 -0
data/lib/www/mechanize/form/check_box.rb +13 -0
data/lib/www/mechanize/form/field.rb +28 -0
data/lib/www/mechanize/form/file_upload.rb +24 -0
data/lib/www/mechanize/form/image_button.rb +23 -0
data/lib/www/mechanize/form/multi_select_list.rb +69 -0
data/lib/www/mechanize/form/option.rb +51 -0
data/lib/www/mechanize/form/radio_button.rb +38 -0
data/lib/www/mechanize/form/select_list.rb +45 -0
data/lib/www/mechanize/headers.rb +12 -0
data/lib/www/mechanize/history.rb +67 -0
data/lib/www/mechanize/inspect.rb +90 -0
data/lib/www/mechanize/monkey_patch.rb +37 -0
data/lib/www/mechanize/page.rb +181 -0
data/lib/www/mechanize/page/base.rb +10 -0
data/lib/www/mechanize/page/frame.rb +22 -0
data/lib/www/mechanize/page/link.rb +50 -0
data/lib/www/mechanize/page/meta.rb +51 -0
data/lib/www/mechanize/pluggable_parsers.rb +103 -0
data/lib/www/mechanize/redirect_limit_reached_error.rb +18 -0
data/lib/www/mechanize/redirect_not_get_or_head_error.rb +20 -0
data/lib/www/mechanize/response_code_error.rb +25 -0
data/lib/www/mechanize/unsupported_scheme_error.rb +10 -0
data/lib/www/mechanize/util.rb +76 -0
data/mechanize.gemspec +41 -0
data/test/chain/test_argument_validator.rb +14 -0
data/test/chain/test_auth_headers.rb +25 -0
data/test/chain/test_custom_headers.rb +18 -0
data/test/chain/test_header_resolver.rb +28 -0
data/test/chain/test_parameter_resolver.rb +35 -0
data/test/chain/test_request_resolver.rb +29 -0
data/test/chain/test_response_reader.rb +24 -0
data/test/data/htpasswd +1 -0
data/test/data/server.crt +16 -0
data/test/data/server.csr +12 -0
data/test/data/server.key +15 -0
data/test/data/server.pem +15 -0
data/test/helper.rb +129 -0
data/test/htdocs/alt_text.html +10 -0
data/test/htdocs/bad_form_test.html +9 -0
data/test/htdocs/button.jpg +0 -0
data/test/htdocs/empty_form.html +6 -0
data/test/htdocs/file_upload.html +26 -0
data/test/htdocs/find_link.html +41 -0
data/test/htdocs/form_multi_select.html +16 -0
data/test/htdocs/form_multival.html +37 -0
data/test/htdocs/form_no_action.html +18 -0
data/test/htdocs/form_no_input_name.html +16 -0
data/test/htdocs/form_select.html +16 -0
data/test/htdocs/form_select_all.html +16 -0
data/test/htdocs/form_select_none.html +17 -0
data/test/htdocs/form_select_noopts.html +10 -0
data/test/htdocs/form_set_fields.html +14 -0
data/test/htdocs/form_test.html +188 -0
data/test/htdocs/frame_test.html +30 -0
data/test/htdocs/google.html +13 -0
data/test/htdocs/iframe_test.html +16 -0
data/test/htdocs/index.html +6 -0
data/test/htdocs/link with space.html +5 -0
data/test/htdocs/meta_cookie.html +11 -0
data/test/htdocs/no_title_test.html +6 -0
data/test/htdocs/relative/tc_relative_links.html +21 -0
data/test/htdocs/tc_bad_links.html +5 -0
data/test/htdocs/tc_base_link.html +8 -0
data/test/htdocs/tc_blank_form.html +11 -0
data/test/htdocs/tc_checkboxes.html +19 -0
data/test/htdocs/tc_encoded_links.html +5 -0
data/test/htdocs/tc_follow_meta.html +8 -0
data/test/htdocs/tc_form_action.html +48 -0
data/test/htdocs/tc_links.html +18 -0
data/test/htdocs/tc_no_attributes.html +16 -0
data/test/htdocs/tc_pretty_print.html +17 -0
data/test/htdocs/tc_radiobuttons.html +17 -0
data/test/htdocs/tc_referer.html +10 -0
data/test/htdocs/tc_relative_links.html +19 -0
data/test/htdocs/tc_textarea.html +23 -0
data/test/htdocs/unusual______.html +5 -0
data/test/servlets.rb +365 -0
data/test/ssl_server.rb +48 -0
data/test/test_authenticate.rb +71 -0
data/test/test_bad_links.rb +25 -0
data/test/test_blank_form.rb +16 -0
data/test/test_checkboxes.rb +61 -0
data/test/test_content_type.rb +13 -0
data/test/test_cookie_class.rb +338 -0
data/test/test_cookie_jar.rb +362 -0
data/test/test_cookies.rb +123 -0
data/test/test_encoded_links.rb +20 -0
data/test/test_errors.rb +49 -0
data/test/test_follow_meta.rb +108 -0
data/test/test_form_action.rb +52 -0
data/test/test_form_as_hash.rb +61 -0
data/test/test_form_button.rb +38 -0
data/test/test_form_no_inputname.rb +15 -0
data/test/test_forms.rb +564 -0
data/test/test_frames.rb +25 -0
data/test/test_get_headers.rb +52 -0
data/test/test_gzipping.rb +22 -0
data/test/test_hash_api.rb +45 -0
data/test/test_history.rb +142 -0
data/test/test_history_added.rb +16 -0
data/test/test_html_unscape_forms.rb +39 -0
data/test/test_if_modified_since.rb +20 -0
data/test/test_keep_alive.rb +31 -0
data/test/test_links.rb +120 -0
data/test/test_mech.rb +268 -0
data/test/test_mechanize_file.rb +47 -0
data/test/test_meta.rb +65 -0
data/test/test_multi_select.rb +106 -0
data/test/test_no_attributes.rb +13 -0
data/test/test_option.rb +18 -0
data/test/test_page.rb +124 -0
data/test/test_pluggable_parser.rb +145 -0
data/test/test_post_form.rb +34 -0
data/test/test_pretty_print.rb +22 -0
data/test/test_radiobutton.rb +75 -0
data/test/test_redirect_limit_reached.rb +41 -0
data/test/test_redirect_verb_handling.rb +45 -0
data/test/test_referer.rb +39 -0
data/test/test_relative_links.rb +40 -0
data/test/test_request.rb +13 -0
data/test/test_response_code.rb +52 -0
data/test/test_save_file.rb +48 -0
data/test/test_scheme.rb +48 -0
data/test/test_select.rb +106 -0
data/test/test_select_all.rb +15 -0
data/test/test_select_none.rb +15 -0
data/test/test_select_noopts.rb +16 -0
data/test/test_set_fields.rb +44 -0
data/test/test_ssl_server.rb +20 -0
data/test/test_subclass.rb +14 -0
data/test/test_textarea.rb +45 -0
data/test/test_upload.rb +109 -0
data/test/test_verbs.rb +25 -0
metadata +314 -0

data/lib/www/mechanize/file.rb ADDED Viewed

@@ -0,0 +1,73 @@
+module WWW
+  class Mechanize
+    # = Synopsis
+    # This is the default (and base) class for the Pluggable Parsers.  If
+    # Mechanize cannot find an appropriate class to use for the content type,
+    # this class will be used.  For example, if you download a JPG, Mechanize
+    # will not know how to parse it, so this class will be instantiated.
+    #
+    # This is a good class to use as the base class for building your own
+    # pluggable parsers.
+    #
+    # == Example
+    #  require 'rubygems'
+    #  require 'mechanize'
+    #
+    #  agent = WWW::Mechanize.new
+    #  agent.get('http://example.com/foo.jpg').class  #=> WWW::Mechanize::File
+    #
+    class File
+      attr_accessor :uri, :response, :body, :code, :filename
+      alias :header :response
+      alias :content :body
+      def initialize(uri=nil, response=nil, body=nil, code=nil)
+        @uri, @body, @code = uri, body, code
+        @response = Headers.new
+        # Copy the headers in to a hash to prevent memory leaks
+        if response
+          response.each { |k,v|
+            @response[k] = v
+          }
+        end
+        @filename = 'index.html'
+        # Set the filename
+        if disposition = @response['content-disposition']
+          disposition.split(/;\s*/).each do |pair|
+            k,v = pair.split(/=/, 2)
+            @filename = v if k && k.downcase == 'filename'
+          end
+        else
+          if @uri
+            @filename = @uri.path.split(/\//).last || 'index.html'
+            @filename << ".html" unless @filename =~ /\./
+          end
+        end
+        yield self if block_given?
+      end
+      # Use this method to save the content of this object to filename
+      def save_as(filename = nil)
+        if filename.nil?
+          filename = @filename
+          number = 1
+          while(::File.exists?(filename))
+            filename = "#{@filename}.#{number}"
+            number += 1
+          end
+        end
+        ::File::open(filename, "wb") { |f|
+          f.write body
+        }
+      end
+      alias :save :save_as
+    end
+  end
+end

data/lib/www/mechanize/file_response.rb ADDED Viewed

@@ -0,0 +1,62 @@
+module WWW
+  class Mechanize
+    ###
+    # Fake response for dealing with file:/// requests
+    class FileResponse
+      def initialize(file_path)
+        @file_path = file_path
+      end
+      def read_body
+        if ::File.exists?(@file_path)
+          if directory?
+            yield dir_body
+          else
+            yield ::File.read(@file_path)
+          end
+        else
+          yield ''
+        end
+      end
+      def code
+        ::File.exists?(@file_path) ? 200 : 400
+      end
+      def content_length
+        return dir_body.length if directory?
+        ::File.exists?(@file_path) ? ::File.stat(@file_path).size : 0
+      end
+      def each_header; end
+      def [](key)
+        return nil unless key.downcase == 'content-type'
+        return 'text/html' if directory?
+        return 'text/html' if ['.html', '.xhtml'].any? { |extn|
+          @file_path =~ /#{extn}$/
+        }
+        nil
+      end
+      def each
+      end
+      def get_fields(key)
+        []
+      end
+      private
+      def dir_body
+        '<html><body>' +
+        Dir[::File.join(@file_path, '*')].map { |f|
+          "<a href=\"file://#{f}\">#{::File.basename(f)}</a>"
+        }.join("\n") + '</body></html>'
+      end
+      def directory?
+        ::File.directory?(@file_path)
+      end
+    end
+  end
+end

data/lib/www/mechanize/file_saver.rb ADDED Viewed

@@ -0,0 +1,39 @@
+module WWW
+  class Mechanize
+    # = Synopsis
+    # This is a pluggable parser that automatically saves every file
+    # it encounters.  It saves the files as a tree, reflecting the
+    # host and file path.
+    #
+    # == Example to save all PDF's
+    #  require 'rubygems'
+    #  require 'mechanize'
+    #
+    #  agent = WWW::Mechanize.new
+    #  agent.pluggable_parser.pdf = WWW::Mechanize::FileSaver
+    #  agent.get('http://example.com/foo.pdf')
+    #
+    class FileSaver < File
+      attr_reader :filename
+      def initialize(uri=nil, response=nil, body=nil, code=nil)
+        super(uri, response, body, code)
+        path = uri.path.empty? ? 'index.html' : uri.path.gsub(/^[\/]*/, '')
+        path += 'index.html' if path =~ /\/$/
+        split_path = path.split(/\//)
+        filename = split_path.length > 0 ? split_path.pop : 'index.html'
+        joined_path = split_path.join(::File::SEPARATOR)
+        path = if joined_path.empty?
+          uri.host
+        else
+          "#{uri.host}#{::File::SEPARATOR}#{joined_path}"
+        end
+        @filename = "#{path}#{::File::SEPARATOR}#{filename}"
+        FileUtils.mkdir_p(path)
+        save_as(@filename)
+      end
+    end
+  end
+end

data/lib/www/mechanize/form.rb ADDED Viewed

@@ -0,0 +1,360 @@
+require 'www/mechanize/form/field'
+require 'www/mechanize/form/file_upload'
+require 'www/mechanize/form/button'
+require 'www/mechanize/form/image_button'
+require 'www/mechanize/form/radio_button'
+require 'www/mechanize/form/check_box'
+require 'www/mechanize/form/multi_select_list'
+require 'www/mechanize/form/select_list'
+require 'www/mechanize/form/option'
+module WWW
+  class Mechanize
+    # =Synopsis
+    # This class encapsulates a form parsed out of an HTML page.  Each type
+    # of input fields available in a form can be accessed through this object.
+    # See GlobalForm for more methods.
+    #
+    # ==Example
+    # Find a form and print out its fields
+    #  form = page.forms.first # => WWW::Mechanize::Form
+    #  form.fields.each { |f| puts f.name }
+    # Set the input field 'name' to "Aaron"
+    #  form['name'] = 'Aaron'
+    #  puts form['name']
+    class Form
+      attr_accessor :method, :action, :name
+      attr_reader :fields, :buttons, :file_uploads, :radiobuttons, :checkboxes
+      attr_accessor :enctype
+      alias :elements :fields
+      attr_reader :form_node
+      attr_reader :page
+      def initialize(node, mech=nil, page=nil)
+        @enctype = node['enctype'] || 'application/x-www-form-urlencoded'
+        @form_node        = node
+        @action           = Util.html_unescape(node['action'])
+        @method           = (node['method'] || 'GET').upcase
+        @name             = node['name']
+        @clicked_buttons  = []
+        @page             = page
+        @mech             = mech
+        parse
+      end
+      # Returns whether or not the form contains a field with +field_name+
+      def has_field?(field_name)
+        ! fields.find { |f| f.name.eql? field_name }.nil?
+      end
+      alias :has_key? :has_field?
+      def has_value?(value)
+        ! fields.find { |f| f.value.eql? value }.nil?
+      end
+      def keys; fields.map { |f| f.name }; end
+      def values; fields.map { |f| f.value }; end
+      # Add a field with +field_name+ and +value+
+      def add_field!(field_name, value = nil)
+        fields << Field.new(field_name, value)
+      end
+      # This method sets multiple fields on the form.  It takes a list of field
+      # name, value pairs.  If there is more than one field found with the
+      # same name, this method will set the first one found.  If you want to
+      # set the value of a duplicate field, use a value which is a Hash with
+      # the key as the index in to the form.  The index
+      # is zero based.  For example, to set the second field named 'foo', you
+      # could do the following:
+      #  form.set_fields( :foo => { 1 => 'bar' } )
+      def set_fields(fields = {})
+        fields.each do |k,v|
+          case v
+          when Hash
+            v.each do |index, value|
+              self.fields_with(:name => k.to_s).[](index).value = value
+            end
+          else
+            value = nil
+            index = 0
+            [v].flatten.each do |val|
+              index = val.to_i unless value.nil?
+              value = val if value.nil?
+            end
+            self.fields_with(:name => k.to_s).[](index).value = value
+          end
+        end
+      end
+      # Fetch the value of the first input field with the name passed in
+      # ==Example
+      # Fetch the value set in the input field 'name'
+      #  puts form['name']
+      def [](field_name)
+        f = field(field_name)
+        f && f.value
+      end
+      # Set the value of the first input field with the name passed in
+      # ==Example
+      # Set the value in the input field 'name' to "Aaron"
+      #  form['name'] = 'Aaron'
+      def []=(field_name, value)
+        f = field(field_name)
+        if f.nil?
+          add_field!(field_name, value)
+        else
+          f.value = value
+        end
+      end
+      # Treat form fields like accessors.
+      def method_missing(id,*args)
+        method = id.to_s.gsub(/=$/, '')
+        if field(method)
+          return field(method).value if args.empty?
+          return field(method).value = args[0]
+        end
+        super
+      end
+      # Submit this form with the button passed in
+      def submit button=nil, headers = {}
+        @mech.submit(self, button, headers)
+      end
+      # Submit form using +button+. Defaults
+      # to the first button.
+      def click_button(button = buttons.first)
+        submit(button)
+      end
+      # This method is sub-method of build_query.
+      # It converts charset of query value of fields into expected one.
+      def proc_query(field)
+        return unless field.query_value
+        field.query_value.map{|(name, val)|
+          [from_native_charset(name), from_native_charset(val.to_s)]
+        }
+      end
+      private :proc_query
+      def from_native_charset(str, enc=nil)
+        if page
+          enc ||= page.encoding
+          Util.from_native_charset(str,enc) rescue str
+        else
+          str
+        end
+      end
+      private :from_native_charset
+      # This method builds an array of arrays that represent the query
+      # parameters to be used with this form.  The return value can then
+      # be used to create a query string for this form.
+      def build_query(buttons = [])
+        query = []
+        fields().each do |f|
+          qval = proc_query(f)
+          query.push(*qval)
+        end
+        checkboxes().each do |f|
+          if f.checked
+            qval = proc_query(f)
+            query.push(*qval)
+          end
+        end
+        radio_groups = {}
+        radiobuttons().each do |f|
+          fname = from_native_charset(f.name)
+          radio_groups[fname] ||= []
+          radio_groups[fname] << f
+        end
+        # take one radio button from each group
+        radio_groups.each_value do |g|
+          checked = g.select {|f| f.checked}
+          if checked.size == 1
+            f = checked.first
+            qval = proc_query(f)
+            query.push(*qval)
+          elsif checked.size > 1
+            raise "multiple radiobuttons are checked in the same group!"
+          end
+        end
+        @clicked_buttons.each { |b|
+          qval = proc_query(b)
+          query.push(*qval)
+        }
+        query
+      end
+      # This method adds a button to the query.  If the form needs to be
+      # submitted with multiple buttons, pass each button to this method.
+      def add_button_to_query(button)
+        @clicked_buttons << button
+      end
+      # This method calculates the request data to be sent back to the server
+      # for this form, depending on if this is a regular post, get, or a
+      # multi-part post,
+      def request_data
+        query_params = build_query()
+        case @enctype.downcase
+        when /^multipart\/form-data/
+          boundary = rand_string(20)
+          @enctype = "multipart/form-data; boundary=#{boundary}"
+          params = []
+          query_params.each { |k,v| params << param_to_multipart(k, v) unless k.nil? }
+          @file_uploads.each { |f| params << file_to_multipart(f) }
+          params.collect { |p| "--#{boundary}\r\n#{p}" }.join('') +
+            "--#{boundary}--\r\n"
+        else
+          WWW::Mechanize::Util.build_query_string(query_params)
+        end
+      end
+      # Removes all fields with name +field_name+.
+      def delete_field!(field_name)
+        @fields.delete_if{ |f| f.name == field_name}
+      end
+      { :field        => :fields,
+        :button       => :buttons,
+        :file_upload  => :file_uploads,
+        :radiobutton  => :radiobuttons,
+        :checkbox     => :checkboxes,
+      }.each do |singular,plural|
+        eval(<<-eomethod)
+          def #{plural}_with criteria = {}
+            criteria = {:name => criteria} if String === criteria
+            f = #{plural}.find_all do |thing|
+              criteria.all? { |k,v| v === thing.send(k) }
+            end
+            yield f if block_given?
+            f
+          end
+          def #{singular}_with criteria = {}
+            f = #{plural}_with(criteria).first
+            yield f if block_given?
+            f
+          end
+          alias :#{singular} :#{singular}_with
+        eomethod
+      end
+      private
+      def parse
+        @fields       = []
+        @buttons      = []
+        @file_uploads = []
+        @radiobuttons = []
+        @checkboxes   = []
+        # Find all input tags
+        form_node.search('input').each do |node|
+          type = (node['type'] || 'text').downcase
+          name = node['name']
+          next if name.nil? && !(type == 'submit' || type =='button')
+          case type
+          when 'radio'
+            @radiobuttons << RadioButton.new(node['name'], node['value'], !!node['checked'], self)
+          when 'checkbox'
+            @checkboxes << CheckBox.new(node['name'], node['value'], !!node['checked'], self)
+          when 'file'
+            @file_uploads << FileUpload.new(node['name'], nil)
+          when 'submit'
+            @buttons << Button.new(node['name'], node['value'])
+          when 'button'
+            @buttons << Button.new(node['name'], node['value'])
+          when 'image'
+            @buttons << ImageButton.new(node['name'], node['value'])
+          else
+            @fields << Field.new(node['name'], node['value'] || '')
+          end
+        end
+        # Find all textarea tags
+        form_node.search('textarea').each do |node|
+          next if node['name'].nil?
+          @fields << Field.new(node['name'], node.inner_text)
+        end
+        # Find all select tags
+        form_node.search('select').each do |node|
+          next if node['name'].nil?
+          if node.has_attribute? 'multiple'
+            @fields << MultiSelectList.new(node['name'], node)
+          else
+            @fields << SelectList.new(node['name'], node)
+          end
+        end
+        # Find all submit button tags
+        # FIXME: what can I do with the reset buttons?
+        form_node.search('button').each do |node|
+          type = (node['type'] || 'submit').downcase
+          next if type == 'reset'
+          @buttons << Button.new(node['name'], node['value'])
+        end
+      end
+      def rand_string(len = 10)
+        chars = ("a".."z").to_a + ("A".."Z").to_a
+        string = ""
+        1.upto(len) { |i| string << chars[rand(chars.size-1)] }
+        string
+      end
+      def mime_value_quote(str)
+        str.gsub(/(["\r\\])/){|s| '\\' + s}
+      end
+      def param_to_multipart(name, value)
+        return "Content-Disposition: form-data; name=\"" +
+                "#{mime_value_quote(name)}\"\r\n" +
+                "\r\n#{value}\r\n"
+      end
+      def file_to_multipart(file)
+        file_name = file.file_name ? ::File.basename(file.file_name) : ''
+        body =  "Content-Disposition: form-data; name=\"" +
+                "#{mime_value_quote(file.name)}\"; " +
+                "filename=\"#{mime_value_quote(file_name)}\"\r\n" +
+                "Content-Transfer-Encoding: binary\r\n"
+        if file.file_data.nil? and ! file.file_name.nil?
+          file.file_data = ::File.open(file.file_name, "rb") { |f| f.read }
+          file.mime_type = WEBrick::HTTPUtils.mime_type(file.file_name,
+                                          WEBrick::HTTPUtils::DefaultMimeTypes)
+        end
+        if file.mime_type != nil
+          body << "Content-Type: #{file.mime_type}\r\n"
+        end
+        body <<
+          if file.file_data.respond_to? :read
+            "\r\n#{file.file_data.read}\r\n"
+          else
+            "\r\n#{file.file_data}\r\n"
+          end
+        body
+      end
+    end
+  end
+end