tap-http 0.1.0 → 0.2.0

data/README

@@ -22,7 +22,7 @@ field is :url.
 
   include Tap::Http
   
-  res = Dispatch.submit_request(
+  res = Dispatch.submit(
     :params => {'q' => 'tap-http'},
     :url => 'http://www.google.com/search')
     
@@ -73,12 +73,12 @@ with the http configuration.
       q: tap-http
     request_method: GET
     url: http://www.google.com/search
-    version: "1.1"
+    version: 1.1
   
-Save the file as 'request.yml' and resubmit the form using the Tap::Http::Request
+Save the file as 'request.yml' and resubmit the form using the Tap::Http::Dispatch
 task.
 
-  % rap load requests.yml --:i request --+ dump --no-audit
+  % rap load requests.yml --:i dispatch --+ dump --no-audit
     I[10:51:40]               load request.yml
     I[10:51:40]                GET http://www.google.com/search
     I[10:51:41]                    OK
@@ -98,7 +98,7 @@ configuration could be used to submit the request using Dispatch.
 
 === Bugs/Known Issues
 
-The Tap::Http::Helpers#parse_cgi_request (used in parsing redirected requests
+The Tap::Http::Utils#parse_cgi_request (used in parsing redirected requests
 into a YAML file) is currently untested because I can't figure a way to setup 
 the ENV variables in a standard way.  Of course I could set them up myself, but
 I can't be sure I'm setting up a realistic test environment.  

data/cgi/echo.rb

@@ -10,12 +10,12 @@
 ####################################
 
 require 'cgi'
-require 'tap/http/helpers'
+require 'tap/http/utils'
 
 cgi = CGI.new
 cgi.out("text/plain") do 
   begin
-    request = Tap::Http::Helpers.parse_cgi_request(cgi)
+    request = Tap::Http::Utils.parse_cgi_request(cgi)
     request[:headers].to_yaml + request[:params].to_yaml
   rescue
     "Error: #{$!.message}\n"  +

data/cgi/http_to_yaml.rb

@@ -45,7 +45,7 @@ require 'rubygems'
 require 'cgi'
 require 'yaml'
 require 'net/http'
-require 'tap/http/helpers'
+require 'tap/http/utils'
 
 # included to sort the hash keys
 class Hash
@@ -75,21 +75,21 @@ begin
   #
   
   config = {}
-  Tap::Http::Helpers.parse_cgi_request(cgi).each_pair do |key, value|
+  Tap::Http::Utils.parse_cgi_request(cgi).each_pair do |key, value|
     config[key.to_s] = value
   end
   
   original_action = config['params'].delete("__original_action").to_s
   referer = config['headers']['Referer'].to_s
-  config['url'] = Tap::Http::Helpers.determine_url(original_action, referer)
+  config['url'] = Tap::Http::Utils.determine_url(original_action, referer)
   config['headers']['Host'] = URI.parse(config['url']).host
 
   #
   # format output
   #
   
-  # help = cgi.a(Tap::Http::Helpers::HELP_URL) { "help" }
-  # how_to_get_cookies = cgi.a(Tap::Http::Helpers::COOKIES_HELP_URL) { "how to get cookies" }
+  # help = cgi.a(Tap::Http::Utils::HELP_URL) { "help" }
+  # how_to_get_cookies = cgi.a(Tap::Http::Utils::COOKIES_HELP_URL) { "how to get cookies" }
   # 
   # If you need cookies, see #{how_to_get_cookies} or the #{help}.
   

data/cgi/parse_http.rb

@@ -30,7 +30,7 @@
 
 require 'rubygems'
 require 'cgi'
-require 'tap/http/helpers'
+require 'tap/http/utils'
 
 # included to sort the hash keys
 class Hash
@@ -97,12 +97,12 @@ Connection: keep-alive
     
   else
     config = {}
-    Tap::Http::Helpers.parse_http_request(http_request).each_pair do |key, value|
+    Tap::Http::Utils.parse_http_request(http_request).each_pair do |key, value|
       config[key.to_s] = value
     end
     
-    help = cgi.a(Tap::Http::Helpers::HELP_URL) { "help" }
-    how_to_get_cookies = cgi.a(Tap::Http::Helpers::COOKIES_HELP_URL) { "how to get cookies" }
+    help = cgi.a(Tap::Http::Utils::HELP_URL) { "help" }
+    how_to_get_cookies = cgi.a(Tap::Http::Utils::COOKIES_HELP_URL) { "how to get cookies" }
   
     cgi.out do
       cgi.html do

data/lib/tap/http/dispatch.rb

@@ -1,280 +1,411 @@
-require 'tap/http/helpers'
+require 'tap/http/utils'
 require 'net/http'
+require 'thread'
 
 module Tap
   module Http
-
-    # Dispatch provides methods for constructing and submitting get and post
-    # HTTP requests from a configuration hash.
+    
+    # :startdoc::manifest submits an http request
+    #
+    # Dispatch is a base class for submitting HTTP requests from a request
+    # hash.  Multiple requests may be submitted on individual threads, up 
+    # to a configurable limit.
+    #
+    # Request hashes are like the following:
+    #
+    #   request_method: GET
+    #   url: http://tap.rubyforge.org/
+    #   headers: {}
+    #   params: {}
+    #   version: 1.1
+    #
+    # Missing fields are added from the task configuration. Note that since
+    # Dispatch takes hash inputs, it is often convenient to save requests in
+    # a .yml file and sequence dispatch with load:
     #
-    #   res = Tap::Http::Dispatch.submit_request(
+    #   [requests.yml]
+    #   - url: http://tap.rubyforge.org/
+    #   - url: http://tap.rubyforge.org/about.html
+    #   
+    #   % rap load requests.yml --:i dispatch --+ dump
+    #
+    # :startdoc::manifest-end
+    # === Dispatch Methods
+    #
+    # Dispatch itself provides methods for constructing and submitting get and
+    # post HTTP requests from a request hash.
+    #
+    #   res = Tap::Http::Dispatch.submit(
     #     :url => "http://tap.rubyforge.org",
     #     :version => '1.1',
     #     :request_method => 'GET',
     #     :headers => {},
     #     :params => {}
     #   )
-    #   res.inspect           # => "#<Net::HTTPOK 200 OK readbody=true>"
-    #   res.body =~ /Tap/     # => true
+    #   res.inspect # => "#<Net::HTTPOK 200 OK readbody=true>"
+    #   res.body =~ /Tap/ # => true
     #
     # Headers and parameters take the form:
     #
-    #   { 'single' => 'value', 
-    #     'multiple' => ['value one', 'value two']}
+    #   { 
+    #     'single' => 'value',
+    #     'multiple' => ['value one', 'value two']
+    #   }
     #
-    module Dispatch
-      module_function
-      
-      DEFAULT_CONFIG = {
-        :request_method => 'GET',
-        :version => '1.1',
-        :params => {},
-        :headers => {},
-        :redirection_limit => nil
-      }
-      
-      # Constructs and submits a request to the url using the request configuration.
-      # A url must be specified in the configuration, but other configurations are
-      # optional; if unspecified, the values in DEFAULT_CONFIG will be used.  A
-      # block may be given to receive the Net::HTTP and request just prior to 
-      # submission.
-      #
-      # Returns the response from the submission.  
-      #
-      def submit_request(config)
-        symbolized = DEFAULT_CONFIG.dup
-        config.each_pair do |key, value|
-          symbolized[key.to_sym] = value
+    # To capture request hashes from web forms using Firefox, see the README.
+    class Dispatch < Tap::Task
+      class << self
+        def intern(*args, &block)
+          instance = new(*args)
+          instance.extend Support::Intern(:process_response)
+          instance.process_response_block = block
+          instance
         end
-        config = symbolized
-        
-        request_method = (config[:request_method]).to_s
-        url_or_uri = config[:url]
-        version = config[:version]
-        params = config[:params]
-        headers = headerize_keys(config[:headers])
-        
-        raise ArgumentError, "no url specified" unless url_or_uri
-        uri = url_or_uri.kind_of?(URI) ? url_or_uri : URI.parse(url_or_uri)
-        uri.path = "/" if uri.path.empty?
         
-        # construct the request based on the method
-        request = case request_method
-        when /^get$/i then construct_get(uri, headers, params)
-        when /^post$/i then construct_post(uri, headers, params)
-        else
-          raise ArgumentError, "unsupported request method: #{request_method}" 
-        end     
-        
-        # set the http version
-        version_method = "version_#{version.to_s.gsub(".", "_")}".to_sym
-        if ::Net::HTTP.respond_to?(version_method)
-          ::Net::HTTP.send(version_method)
-        else
-          raise ArgumentError, "unsupported HTTP version: #{version}"
-        end
+        # Constructs and submits an http request to the url using the request hash.
+        # Request hashes are like this:
+        #
+        #   {
+        #     :url => "http://tap.rubyforge.org",
+        #     :version => '1.1',
+        #     :request_method => 'GET',
+        #     :headers => {},
+        #     :params => {}
+        #   }
+        #
+        # If left unspecified, the default configuration values will be used (but 
+        # note that since the default url is nil, a url MUST be specified).
+        # Headers and parameters can use array values to specifiy multiple values 
+        # for the same key. 
+        #
+        # Submit only support get and post request methods; see construct_get and 
+        # construct_post for more details.  A block may be given to receive the 
+        # Net::HTTP and request just prior to submission.  
+        #
+        # Returns the Net::HTTP response.  
+        #
+        def submit(request_hash)
+          url_or_uri = request_hash[:url] || configurations[:url].default
+          headers = request_hash[:headers] || configurations[:headers].default
+          params = request_hash[:params] || configurations[:params].default 
+          request_method = request_hash[:request_method] || configurations[:request_method].default
+          version = request_hash[:version] || configurations[:version].default
+          
+          raise ArgumentError, "no url specified" unless url_or_uri
+          uri = url_or_uri.kind_of?(URI) ? url_or_uri : URI.parse(url_or_uri)
+          uri.path = "/" if uri.path.empty?
+
+          # construct the request based on the method
+          request = case request_method.to_s
+          when /^get$/i then construct_get(uri, headers, params)
+          when /^post$/i then construct_post(uri, headers, params)
+          else
+            raise ArgumentError, "unsupported request method: #{request_method}" 
+          end
+
+          # set the http version
+          version_method = "version_#{version.to_s.gsub(".", "_")}".to_sym
+          if ::Net::HTTP.respond_to?(version_method)
+            ::Net::HTTP.send(version_method)
+          else
+            raise ArgumentError, "unsupported HTTP version: #{version}"
+          end
+
+          # submit the request
+          res = ::Net::HTTP.new(uri.host, uri.port).start do |http| 
+            yield(http, request) if block_given?
+            http.request(request)
+          end
 
-        # submit the request
-        res = ::Net::HTTP.new(uri.host, uri.port).start do |http| 
-          yield(http, request) if block_given?
-          http.request(request)
+          # fetch redirections
+          redirection_limit = request_hash[:redirection_limit]
+          redirection_limit ? fetch_redirection(res, redirection_limit) : res
         end
-        
-        # fetch redirections
-        redirection_limit = config[:redirection_limit]
-        redirection_limit ? fetch_redirection(res, redirection_limit) : res
-      end
 
-      # Constructs a Net::HTTP::Post query, setting headers and parameters.
-      #
-      # ==== Supported Content Types:
-      #
-      # - application/x-www-form-urlencoded (the default)
-      # - multipart/form-data
-      #
-      # The multipart/form-data content type may specify a boundary.  If no 
-      # boundary is specified, a randomly generated boundary will be used
-      # to delimit the parameters.
-      #
-      #   post = construct_post(
-      #     URI.parse('http://some.url/'),
-      #     {:content_type => 'multipart/form-data; boundary=1234'}, 
-      #     {:key => 'value'})
-      #
-      #   post.body
-      #   # => %Q{--1234\r
-      #   # Content-Disposition: form-data; name="key"\r
-      #   # \r
-      #   # value\r
-      #   # --1234--\r
-      #   # }
-      # 
-      # (Note the carriage returns are required in multipart content)
-      #
-      # The content-length header is determined automatically from the
-      # formatted request body; manually specified content-length headers
-      # will be overridden.
-      #
-      def construct_post(uri, headers, params)
-        req = ::Net::HTTP::Post.new( URI.encode("#{uri.path}#{format_query(uri)}") )
-        headers = headerize_keys(headers) 
-        content_type = headers['Content-Type']
+        # Constructs a Net::HTTP::Post query, setting headers and parameters.
+        #
+        # ==== Supported Content Types:
+        #
+        # - application/x-www-form-urlencoded (the default)
+        # - multipart/form-data
+        #
+        # The multipart/form-data content type may specify a boundary.  If no 
+        # boundary is specified, a randomly generated boundary will be used
+        # to delimit the parameters.
+        #
+        #   post = construct_post(
+        #     URI.parse('http://some.url/'),
+        #     {:content_type => 'multipart/form-data; boundary=1234'}, 
+        #     {:key => 'value'})
+        #
+        #   post.body
+        #   # => %Q{--1234\r
+        #   # Content-Disposition: form-data; name="key"\r
+        #   # \r
+        #   # value\r
+        #   # --1234--\r
+        #   # }
+        # 
+        # (Note the carriage returns are required in multipart content)
+        #
+        # The content-length header is determined automatically from the
+        # formatted request body; manually specified content-length headers
+        # will be overridden.
+        #
+        def construct_post(uri, headers, params)
+          req = ::Net::HTTP::Post.new( URI.encode("#{uri.path}#{format_query(uri)}") )
+          headers = headerize_keys(headers)
+          content_type = headers['Content-Type']
 
-        case content_type
-        when nil, /^application\/x-www-form-urlencoded$/i
-          req.body = format_www_form_urlencoded(params)
-          headers['Content-Type'] ||= "application/x-www-form-urlencoded"
-          headers['Content-Length'] = req.body.length
-          
-        when /^multipart\/form-data(;\s*boundary=(.*))?$/i
-          # extract the boundary if it exists
-          boundary = $2 || rand.to_s[2..20]
+          case content_type
+          when nil, /^application\/x-www-form-urlencoded$/i
+            req.body = format_www_form_urlencoded(params)
+            headers['Content-Type'] ||= "application/x-www-form-urlencoded"
+            headers['Content-Length'] = req.body.length
+
+          when /^multipart\/form-data(;\s*boundary=(.*))?$/i
+            # extract the boundary if it exists
+            boundary = $2 || rand.to_s[2..20]
+
+            req.body = format_multipart_form_data(params, boundary)
+            headers['Content-Type'] = "multipart/form-data; boundary=#{boundary}"
+            headers['Content-Length'] = req.body.length
+
+          else
+            raise ArgumentError, "unsupported Content-Type for POST: #{content_type}"
+          end
 
-          req.body = format_multipart_form_data(params, boundary)
-          headers['Content-Type'] = "multipart/form-data; boundary=#{boundary}"
-          headers['Content-Length'] = req.body.length
+          headers.each_pair { |key, value| req[key] = value }
+          req
+        end
 
-        else
-          raise ArgumentError, "unsupported Content-Type for POST: #{content_type}"
+        # Constructs a Net::HTTP::Get query.  All parameters in uri and params are
+        # encoded and added to the request URI.
+        #
+        #   get = construct_get(URI.parse('http://some.url/path'), {}, {:key => 'value'})
+        #   get.path            # => "/path?key=value"
+        #
+        def construct_get(uri, headers, params)
+          req = ::Net::HTTP::Get.new( URI.encode("#{uri.path}#{format_query(uri, params)}") )
+          headerize_keys(headers).each_pair { |key, value| req[key] = value }
+          req
         end
-        
-        headers.each_pair { |key, value| req[key] = value }
-        req
-      end
 
-      # Constructs a Net::HTTP::Get query.  All parameters in uri and params are
-      # encoded and added to the request URI.
-      #
-      #   get = construct_get(URI.parse('http://some.url/path'), {}, {:key => 'value'})
-      #   get.path            # => "/path?key=value"
-      #
-      def construct_get(uri, headers, params)
-        req = ::Net::HTTP::Get.new( URI.encode("#{uri.path}#{format_query(uri, params)}") )
-        headerize_keys(headers).each_pair { |key, value| req[key] = value }
-        req
-      end
-      
-      # Checks the type of the response; if it is a redirection, fetches the
-      # redirection.  Otherwise return the response.
-      # 
-      # Notes:
-      # - Fetch will recurse up to the input redirection limit (default 10)
-      # - Responses that are not Net::HTTPRedirection or Net::HTTPSuccess
-      #   raise an error.
-      def fetch_redirection(res, limit=10)
-        raise 'exceeded the redirection limit' if limit < 1
+        # Checks the type of the response; if it is a redirection, fetches the
+        # redirection.  Otherwise return the response.
+        # 
+        # Notes:
+        # - Fetch will recurse up to the input redirection limit (default 10)
+        # - Responses that are not Net::HTTPRedirection or Net::HTTPSuccess
+        #   raise an error.
+        def fetch_redirection(res, limit=10)
+          raise 'exceeded the redirection limit' if limit < 1
 
-        case res
-        when ::Net::HTTPRedirection 
-          redirect = ::Net::HTTP.get_response( URI.parse(res['location']) )
-          fetch_redirection(redirect, limit - 1)
-        when ::Net::HTTPSuccess 
-          res
-        else 
-          raise StandardError, res.error!
-        end 
-      end
-      
-      # Converts the keys of a hash to headers.  See Helpers#headerize.
-      #
-      #   headerize_keys('some_header' => 'value')   # => {'Some-Header' => 'value'}
-      #
-      def headerize_keys(hash)
-        result = {}
-        hash.each_pair do |key, value|
-          result[Helpers.headerize(key)] = value
+          case res
+          when ::Net::HTTPRedirection 
+            redirect = ::Net::HTTP.get_response( URI.parse(res['location']) )
+            fetch_redirection(redirect, limit - 1)
+          when ::Net::HTTPSuccess 
+            res
+          else 
+            raise StandardError, res.error!
+          end 
         end
-        result
-      end
 
-      # Constructs a URI query string from the uri and the input parameters.
-      # Multiple values for a parameter may be specified using an array.
-      # The query is not encoded, so you may need to URI.encode it later.
-      #
-      #   format_query(URI.parse('http://some.url/path'), {:key => 'value'})
-      #   # => "?key=value"
-      #
-      #   format_query(URI.parse('http://some.url/path?one=1'), {:two => '2'})
-      #   # => "?one=1&two=2"
-      #
-      def format_query(uri, params={})
-        query = []
-        query << uri.query if uri.query    
-        params.each_pair do |key, values|
-          values = [values] unless values.kind_of?(Array)
-          values.each { |value| query << "#{key}=#{value}" }
+        # Constructs a URI query string from the uri and the input parameters.
+        # Multiple values for a parameter may be specified using an array.
+        # The query is not encoded, so you may need to URI.encode it later.
+        #
+        #   format_query(URI.parse('http://some.url/path'), {:key => 'value'})
+        #   # => "?key=value"
+        #
+        #   format_query(URI.parse('http://some.url/path?one=1'), {:two => '2'})
+        #   # => "?one=1&two=2"
+        #
+        def format_query(uri, params={})
+          query = []
+          query << uri.query if uri.query    
+          params.each_pair do |key, values|
+            values = [values] unless values.kind_of?(Array)
+            values.each { |value| query << "#{key}=#{value}" }
+          end
+          "#{query.empty? ? '' : '?'}#{query.join('&')}"
+        end
+
+        # Formats params as 'application/x-www-form-urlencoded' for use as the
+        # body of a post request.  Multiple values for a parameter may be 
+        # specified using an array.  The result is obviously URI encoded.
+        #
+        #   format_www_form_urlencoded(:key => 'value with spaces')
+        #   # => "key=value%20with%20spaces"
+        #
+        def format_www_form_urlencoded(params={})
+          query = []
+          params.each_pair do |key, values|
+            values = [values] unless values.kind_of?(Array)
+            values.each { |value| query << "#{key}=#{value}" }
+          end  
+          URI.encode( query.join('&') )
+        end
+
+        # Formats params as 'multipart/form-data' using the specified boundary,
+        # for use as the body of a post request.  Multiple values for a parameter
+        # may be specified using an array.  All newlines include a carriage
+        # return for proper formatting.
+        #
+        #   format_multipart_form_data(:key => 'value')
+        #   # => %Q{--1234567890\r
+        #   # Content-Disposition: form-data; name="key"\r
+        #   # \r
+        #   # value\r
+        #   # --1234567890--\r
+        #   # }
+        #
+        # To specify a file, use a hash of file-related headers.
+        #
+        #   format_multipart_form_data(:key => {
+        #     'Content-Type' => 'text/plain',
+        #     'Filename' => "path/to/file.txt"}
+        #   )
+        #   # => %Q{--1234567890\r
+        #   # Content-Disposition: form-data; name="key"; filename="path/to/file.txt"\r
+        #   # Content-Type: text/plain\r
+        #   # \r
+        #   # \r
+        #   # --1234567890--\r
+        #   # }
+        #
+        def format_multipart_form_data(params, boundary="1234567890")
+          body = []
+          params.each_pair do |key, values| 
+            values = [values] unless values.kind_of?(Array)
+
+            values.each do |value|
+              body << case value
+              when Hash
+                hash = headerize_keys(value)
+                filename = hash.delete('Filename') || ""
+                content = File.exists?(filename) ? File.read(filename) : ""
+
+                header = "Content-Disposition: form-data; name=\"#{key.to_s}\"; filename=\"#{filename}\"\r\n"
+                hash.each_pair { |key, value| header << "#{key}: #{value}\r\n" }
+                "#{header}\r\n#{content}\r\n"
+              else 
+                %Q{Content-Disposition: form-data; name="#{key.to_s}"\r\n\r\n#{value.to_s}\r\n}
+              end
+            end
+          end
+
+          body.collect {|p| "--#{boundary}\r\n#{p}" }.join('') + "--#{boundary}--\r\n"
+        end
+        
+        protected
+        
+        # Helper to headerize the keys of a hash to headers.  
+        # See Utils#headerize.
+        def headerize_keys(hash) # :nodoc:
+          result = {}
+          hash.each_pair do |key, value|
+            result[Utils.headerize(key)] = value
+          end
+          result
         end
-        "#{query.empty? ? '' : '?'}#{query.join('&')}"
       end
       
-      # Formats params as 'application/x-www-form-urlencoded' for use as the
-      # body of a post request.  Multiple values for a parameter may be 
-      # specified using an array.  The result is obviously URI encoded.
-      #
-      #   format_www_form_urlencoded(:key => 'value with spaces')
-      #   # => "key=value%20with%20spaces"
-      #
-      def format_www_form_urlencoded(params={})
-        query = []
-        params.each_pair do |key, values|
-          values = [values] unless values.kind_of?(Array)
-          values.each { |value| query << "#{key}=#{value}" }
-        end  
-        URI.encode( query.join('&') )
+      config :url, nil                                   # the target url
+      config :headers, {}, &c.hash                       # a hash of request headers
+      config :params, {}, &c.hash                        # a hash of query parameters
+      config :request_method, 'GET'                      # the request method (get or post)
+      config :version, 1.1                               # the HTTP version
+      config :redirection_limit, nil, &c.integer_or_nil  # the redirection limit for the request
+      
+      config :max_threads, 10, &c.integer                # the maximum number of request threads
+      
+      # Prepares the request_hash by symbolizing keys and adding missing 
+      # parameters using the current configuration values.
+      def prepare(request_hash)
+        request_hash.inject(
+          :url => url,
+          :headers => headers,
+          :params => params,
+          :request_method => request_method,
+          :version => version,
+          :redirection_limit => redirection_limit
+        ) do |options, (key, value)|
+          options[(key.to_sym rescue key) || key] = value
+          options
+        end
       end
-
-      # Formats params as 'multipart/form-data' using the specified boundary,
-      # for use as the body of a post request.  Multiple values for a parameter
-      # may be specified using an array.  All newlines include a carriage
-      # return for proper formatting.
-      #
-      #   format_multipart_form_data(:key => 'value')
-      #   # => %Q{--1234567890\r
-      #   # Content-Disposition: form-data; name="key"\r
-      #   # \r
-      #   # value\r
-      #   # --1234567890--\r
-      #   # }
-      #
-      # To specify a file, use a hash of file-related headers.
-      #
-      #   format_multipart_form_data(:key => {
-      #     'Content-Type' => 'text/plain',
-      #     'Filename' => "path/to/file.txt"}
-      #   )
-      #   # => %Q{--1234567890\r
-      #   # Content-Disposition: form-data; name="key"; filename="path/to/file.txt"\r
-      #   # Content-Type: text/plain\r
-      #   # \r
-      #   # \r
-      #   # --1234567890--\r
-      #   # }
-      #
-      def format_multipart_form_data(params, boundary="1234567890")
-        body = []
-        params.each_pair do |key, values| 
-          values = [values] unless values.kind_of?(Array)
-          
-          values.each do |value|
-            body << case value
-            when Hash
-              hash = headerize_keys(value)
-              filename = hash.delete('Filename') || ""
-              content = File.exists?(filename) ? File.read(filename) : ""
-
-              header = "Content-Disposition: form-data; name=\"#{key.to_s}\"; filename=\"#{filename}\"\r\n"
-              hash.each_pair { |key, value| header << "#{key}: #{value}\r\n" }
-              "#{header}\r\n#{content}\r\n"
-            else 
-              %Q{Content-Disposition: form-data; name="#{key.to_s}"\r\n\r\n#{value.to_s}\r\n}
+      
+      def process(*requests)
+        # build a queue of all the requests to be handled
+        queue = Queue.new
+        requests.each_with_index do |request, index|
+          queue.enq [prepare(request), index]
+          index += 1
+        end
+        
+        # submit and retrieve all requests before processing
+        # responses.  this assures responses are processed
+        # in order, in case it matters.
+        lock = Mutex.new
+        responses = []
+        request_threads = Array.new(max_threads) do 
+          Thread.new do
+            begin
+              while !queue.empty?
+                request, index = queue.deq(true)
+                log(request[:request_method], request[:url])
+                
+                res = Dispatch.submit(request)
+                lock.synchronize { responses[index] = res }
+              end
+            rescue(ThreadError)
+              # Catch errors due to the queue being empty.
+              # (this should not occur as the queue is checked)
+              raise $! unless $!.message == 'queue empty'
             end
           end
         end
+        request_threads.each {|thread| thread.join }
 
-        body.collect {|p| "--#{boundary}\r\n#{p}" }.join('') + "--#{boundary}--\r\n"
+        # process responses and collect results
+        errors = []
+        responses = responses.collect do |res|
+          begin
+            process_response(res)
+          rescue(ResponseError)
+            errors << [$!, responses.index(res)]
+            nil
+          end
+        end
+        
+        unless errors.empty?
+          handle_response_errors(responses, errors)
+        end
+        
+        responses
+      end
+      
+      # Hook for processing a response.  By default process_response 
+      # simply logs the response message and returns the response. 
+      def process_response(res)
+        log(nil, res.message)
+        res 
+      end
+      
+      # A hook for handling a batch of response errors, perhaps
+      # doing something meaningful with the successful responses.
+      # By default, concatenates the error messages and raises
+      # a new ResponseError.
+      def handle_response_errors(responses, errors)
+        errors.collect! {|error, n| "request #{n}: #{error.message}"}
+        errors.unshift("Error processing responses:")
+        raise ResponseError, errors.join("\n")
+      end
+      
+      class ResponseError < StandardError
       end
-
     end
   end
-end
-
+end

data/lib/tap/http/{helpers.rb → utils.rb}

@@ -4,7 +4,7 @@ autoload(:StringIO, 'stringio')
 
 module Tap
   module Http
-    module Helpers
+    module Utils
       module_function
     
       # Parses a WEBrick::HTTPRequest from the input socket into a hash that
@@ -163,7 +163,7 @@ module Tap
         
         { :url => File.join("http://", headers['Host'], ENV['PATH_INFO']), 
           :request_method => ENV['REQUEST_METHOD'],
-          :version => ENV['HTTP_VERSION'] =~ /^HTTP\/(.*)$/ ? $1 : ENV['HTTP_VERSION'],
+          :version => ENV['HTTP_VERSION'] =~ /^HTTP\/(.*)$/ ? $1.to_f : ENV['HTTP_VERSION'],
           :headers => headers, 
           :params => params}
       end
@@ -217,7 +217,7 @@ module Tap
       # that accept 'gzip' and 'deflate' content encoding.
       #
       #--
-      # Helpers.inflate(res.body) if res['content-encoding'] == 'gzip'
+      # Utils.inflate(res.body) if res['content-encoding'] == 'gzip'
       # 
       def inflate(str)
         Zlib::GzipReader.new( StringIO.new( str ) ).read

data/lib/tap/test/http_test/requests.rb

@@ -0,0 +1,194 @@
+module Tap
+  module Test
+    module HttpTest
+      
+      # A collection of sample requests used in testing.
+      module Requests
+        GET_REQUESTS = {}
+        POST_REQUESTS = {}
+        
+        def self.add(type, name, request, expected)
+          collection = case type
+          when :get then GET_REQUESTS
+          when :post then POST_REQUESTS
+          end
+          
+          collection["#{type}_#{name}"] = [request.lstrip.gsub(/\n/, "\r\n"), expected]
+        end
+      end
+    end
+  end
+end
+
+#
+# get requests
+#
+
+Tap::Test::HttpTest::Requests.add :get, :basic, %q{
+GET /path HTTP/1.1
+Host: www.example.com
+Keep-Alive: 300
+Connection: keep-alive
+}, {
+:url => "http://www.example.com/path",
+:version => '1.1',
+:request_method => 'GET',
+:headers => {
+  "Host" => "www.example.com", 
+  "Keep-Alive" => "300",
+  "Connection" => 'keep-alive'},
+:params => {}
+}
+
+Tap::Test::HttpTest::Requests.add :get, :header_less, %q{
+GET /path HTTP/1.1
+}, {
+:url => "/path",
+:version => '1.1',
+:request_method => 'GET',
+:headers => {},
+:params => {}
+}
+
+Tap::Test::HttpTest::Requests.add :get, :version_less, %q{
+GET /path
+}, {
+:url => "/path",
+:version => '0.9',
+:request_method => 'GET',
+:headers => {},
+:params => {}
+}
+
+Tap::Test::HttpTest::Requests.add :get, :with_query, %q{
+GET /path?one=value%20one&two=value%20two HTTP/1.1
+}, { 
+:url => "/path",
+:version => '1.1',
+:request_method => 'GET',
+:headers => {},
+:params => {
+  'one' => 'value one', 
+  'two' => 'value two'}
+}
+
+#
+# post requests
+#
+
+Tap::Test::HttpTest::Requests.add :post, :with_multipart_form_data, %q{
+POST /path HTTP/1.1
+Host: www.example.com
+Content-Type: multipart/form-data; boundary=1234567890
+Content-Length: 158
+
+--1234567890
+Content-Disposition: form-data; name="one"
+
+value one
+--1234567890
+Content-Disposition: form-data; name="two"
+
+value two
+--1234567890--
+}, { 
+:url => "http://www.example.com/path",
+:version => '1.1',
+:request_method => 'POST',
+:headers => {
+  "Host" => 'www.example.com',
+  "Content-Type" => "multipart/form-data; boundary=1234567890", 
+  "Content-Length" => "158"},
+:params => {
+  'one' => 'value one',
+  'two' => 'value two'}
+}
+        
+Tap::Test::HttpTest::Requests.add :post, :with_multipart_data_and_multiple_values, %q{
+POST /path HTTP/1.1
+Host: www.example.com
+Content-Type: multipart/form-data; boundary=1234567890
+Content-Length: 158
+
+--1234567890
+Content-Disposition: form-data; name="key"
+
+value one
+--1234567890
+Content-Disposition: form-data; name="key"
+
+value two
+--1234567890--
+}, { 
+:url => "http://www.example.com/path",
+:version => '1.1',
+:request_method => 'POST',
+:headers => {
+  "Host" => 'www.example.com',
+  "Content-Type" => "multipart/form-data; boundary=1234567890", 
+  "Content-Length" => "158"},
+:params => {
+  'key' => ["value one", "value two"]}
+}
+
+Tap::Test::HttpTest::Requests.add :post, :with_file_data, %q{
+POST /path HTTP/1.1
+Host: www.example.com
+Content-Type: multipart/form-data; boundary=1234567890
+Content-Length: 148
+
+--1234567890
+Content-Disposition: form-data; name="key"; filename="file.txt"
+Content-Type: application/octet-stream
+
+value one
+--1234567890--
+}, { 
+:url => "http://www.example.com/path",
+:version => '1.1',
+:request_method => 'POST',
+:headers => {
+  "Host" => 'www.example.com',
+  "Content-Type" => "multipart/form-data; boundary=1234567890", 
+  "Content-Length" => "148"},
+:params => {
+  'key' => {'Filename' => 'file.txt', 'Content-Type' => 'application/octet-stream'}}
+}
+
+Tap::Test::HttpTest::Requests.add :post, :with_mixed_multi_value_file_data, %q{
+POST /path HTTP/1.1
+Host: www.example.com
+Content-Type: multipart/form-data; boundary=1234567890
+Content-Length: 329
+
+--1234567890
+Content-Disposition: form-data; name="key"
+
+one
+--1234567890
+Content-Disposition: form-data; name="key"; filename="one.txt"
+Content-Type: application/octet-stream
+
+value one
+--1234567890
+Content-Disposition: form-data; name="key"; filename="two.txt"
+Content-Type: text/plain
+
+value two
+--1234567890--
+}, { 
+:url => "http://www.example.com/path",
+:version => '1.1',
+:request_method => 'POST',
+:headers => {
+  "Host" => 'www.example.com',
+  "Content-Type" => "multipart/form-data; boundary=1234567890", 
+  "Content-Length" => "329"},
+:params => {
+  'key' => [
+    "one",
+    {'Filename' => 'one.txt', 'Content-Type' => 'application/octet-stream'},
+    {'Filename' => 'two.txt', 'Content-Type' => 'text/plain'}]}
+}
+
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: tap-http
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Simon Chiang
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-11-25 00:00:00 -07:00
+date: 2008-12-03 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -36,14 +36,14 @@ files:
 - cgi/http_to_yaml.rb
 - cgi/parse_http.rb
 - lib/tap/http/dispatch.rb
-- lib/tap/http/helpers.rb
-- lib/tap/http/request.rb
+- lib/tap/http/utils.rb
 - lib/tap/test/http_test.rb
+- lib/tap/test/http_test/requests.rb
 - tap.yml
 - README
 - MIT-LICENSE
 has_rdoc: true
-homepage: http://tap.rubyforge.org/projects/tap-http
+homepage: http://tap.rubyforge.org/tap-http
 post_install_message: 
 rdoc_options: []
 

data/lib/tap/http/request.rb

@@ -1,117 +0,0 @@
-require 'tap/http/dispatch'
-require 'thread'
-
-module Tap
-  module Http
-    
-    # :startdoc::manifest submits an http request
-    #
-    # Request is a base class for submitting HTTP requests from a request
-    # hash.  Multiple requests may be submitted on individual threads, up 
-    # to a configurable limit.
-    #
-    # Configuration hashes are like the following:
-    #
-    #   url: http://tap.rubyforge.org/
-    #   request_method: GET
-    #   headers: {}
-    #   params: {}
-    #
-    # The only required field is the url (by default no headers or params
-    # are specified, and the request method is GET).  Request requires
-    # hash inputs, which can be inconvenient from the command line.  A
-    # good workaround is to save requests in a .yml file and use a simple
-    # workflow:
-    #
-    #   [requests.yml]
-    #   - url: http://tap.rubyforge.org/
-    #   - url: http://tap.rubyforge.org/about.html
-    #   
-    #   % rap load requests.yml --:i request --+ dump
-    #
-    #--
-    # To generate configuration hashes from Firefox, see the redirect_http
-    # ubiquity command.
-    #++
-    class Request < Tap::Task
-      class << self
-        def intern(*args, &block)
-          instance = new(*args)
-          instance.extend Support::Intern(:process_response)
-          instance.process_response_block = block
-          instance
-        end
-      end
-      
-      config :redirection_limit, 10, &c.integer  # the redirection limit for the request
-      config :max_threads, 10, &c.integer        # the maximum number of request threads
-      
-      def process(*requests)
-        # build a queue of all the requests to be handled
-        queue = Queue.new
-        requests.each_with_index do |request, index|
-          request = symbolize_keys(request)
-
-          if request[:url] == nil
-            raise ArgumentError, "no url specified: #{request.inspect}"
-          end
-
-          queue.enq [request, index]
-          index += 1
-        end
-        
-        # submit and retrieve all requests before processing
-        # responses.  this assures responses are processed
-        # in order, in case it matters.
-        lock = Mutex.new
-        responses = []
-        request_threads = Array.new(max_threads) do 
-          Thread.new do
-            begin
-              while !queue.empty?
-                request, index = queue.deq(true)
-              
-                log(request[:request_method], request[:url])
-                if app.verbose
-                  log 'headers', config[:headers].inspect
-                  log 'params', config[:params].inspect
-                end
-                
-                res = Dispatch.submit_request(request)
-                lock.synchronize { responses[index] = res }
-              end
-            rescue(ThreadError)
-              # Catch errors due to the queue being empty.
-              # (this should not occur as the queue is checked)
-              raise $! unless $!.message == 'queue empty'
-            end
-          end
-        end
-        request_threads.each {|thread| thread.join }
-
-        # process responses and collect results
-        responses.collect! do |res|
-          process_response(res)
-        end
-      end
-      
-      # Hook for processing a response.  By default process_response 
-      # simply logs the response message and returns the response. 
-      def process_response(res)
-        log(nil, res.message)
-        res 
-      end
-      
-      protected
-      
-      # Return a new hash with all keys converted to symbols.
-      # Lifted from ActiveSupport
-      def symbolize_keys(hash)
-        hash.inject({}) do |options, (key, value)|
-          options[(key.to_sym rescue key) || key] = value
-          options
-        end
-      end
-    end
-  end
-end