tap-http 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2008, Regents of the University of Colorado.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
+to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or
+substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,25 @@
+= {TapHttp}[http://tap.rubyforge.org/tap-http]
+
+A task library for submitting http requests using {Tap}[http://tap.rubyforge.org].
+
+== Description
+
+* Lighthouse[http://bahuvrihi.lighthouseapp.com/projects/9908-tap-task-application/tickets]
+* Github[http://github.com/bahuvrihi/tap-http/tree/master]
+* {Google Group}[http://groups.google.com/group/ruby-on-tap]
+
+=== Usage
+
+== Installation
+
+TapHttp is available as a gem on RubyForge[http://rubyforge.org/projects/tap].  Use:
+
+  % gem install tap-http
+
+== Info 
+
+Copyright (c) 2006-2008, Regents of the University of Colorado.
+Developer:: {Simon Chiang}[http://bahuvrihi.wordpress.com], {Biomolecular Structure Program}[http://biomol.uchsc.edu/], {Hansen Lab}[http://hsc-proteomics.uchsc.edu/hansenlab/] 
+Support:: CU Denver School of Medicine Deans Academic Enrichment Fund
+Licence:: {MIT-Style}[link:files/MIT-LICENSE.html]
+

data/cgi/echo.rb

@@ -0,0 +1,24 @@
+#!/usr/local/bin/ruby
+
+####################################
+# Echos back the HTTP header and parameters as YAML.  
+#
+# Copyright (c) 2008, Regents of the University of Colorado
+# Developer: Simon Chiang, Biomolecular Structure Program
+# Homepage: http://hsc-proteomics.ucdenver.edu/hansen_lab
+#
+####################################
+
+require 'cgi'
+require 'tap/http/helpers'
+
+cgi = CGI.new
+cgi.out("text/plain") do 
+  begin
+    request = Tap::Http::Helpers.parse_cgi_request(cgi)
+    request[:headers].to_yaml + request[:params].to_yaml
+  rescue
+    "Error: #{$!.message}\n"  +
+    $!.backtrace.join("\n")
+  end
+end

data/cgi/http_to_yaml.rb

@@ -0,0 +1,108 @@
+#!/usr/local/bin/ruby
+
+#################################################
+#
+# Echos back redirected HTTP requests  as YAML, suitable for use with the Tap::Net::Submit 
+# task.  All HTTP parameters and headers are echoed back directly, except for the 
+# '__original_action'  parameter which is used in  conjuction with the 'Referer' header to 
+# reconstruct the original url of the request.  The '__original_action' parameter is not echoed.
+#
+# For example:
+#   __original_action                                         Referer                                       echoed url
+#   http://www.example.com?key=value        (any)                                             http://www.example.com?key=value
+#   /page                                                            http://www.example.com           http://www.example.com/page
+#
+# Simply drop this script into a cgi directory, and send it  a redirected request.  See the 
+# RedirectHTTP Firefox extension for a simple way of redirecting requests to this script.
+#
+# Developer: Simon Chiang, Biomolecular Structure Program 
+# Homepage: http://tap.rubyforge.org
+# Licence: MIT-STYLE
+#
+# Copyright (c) 2008, Regents of the University of Colorado 
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this
+# software and associated documentation files (the "Software"), to deal in the Software
+# without restriction, including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
+# to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+#################################################
+
+require 'rubygems'
+require 'cgi'
+require 'yaml'
+require 'net/http'
+require 'tap/http/helpers'
+
+# included to sort the hash keys
+class Hash
+  def to_yaml( opts = {} )
+    YAML::quick_emit( object_id, opts ) do |out|
+      out.map( taguri, to_yaml_style ) do |map|
+        sorted_keys = keys
+        sorted_keys = begin
+          sorted_keys.sort
+        rescue
+          sorted_keys.sort_by {|k| k.to_s} rescue sorted_keys
+        end
+
+        sorted_keys.each do |k|
+          map.add( k, fetch(k) )
+        end
+      end
+    end
+  end
+end
+
+cgi = CGI.new("html3")
+begin
+  
+  #
+  # gather configs
+  #
+  
+  config = {}
+  Tap::Http::Helpers.parse_cgi_request(cgi).each_pair do |key, value|
+    config[key.to_s] = value
+  end
+  
+  original_action = config['params'].delete("__original_action").to_s
+  referer = config['headers']['Referer'].to_s
+  config['url'] = Tap::Http::Helpers.determine_url(original_action, referer)
+  config['headers']['Host'] = URI.parse(config['url']).host
+
+  #
+  # format output
+  #
+  
+  help = cgi.a(Tap::Http::Helpers::HELP_URL) { "help" }
+  how_to_get_cookies = cgi.a(Tap::Http::Helpers::COOKIES_HELP_URL) { "how to get cookies" }
+  
+  cgi.out('text/plain') do 
+%Q{# Copy and paste into a configuration file. Multiple configs
+# can be added to a single file to perform batch submission.  
+#
+# If you need cookies, see #{how_to_get_cookies} or the #{help}. 
+- #{config.to_yaml[5..-1].gsub(/\n/, "\n  ")}
+}
+  end
+  
+rescue
+  cgi.out("text/plain") do
+    "Error: #{$!.message}\n"  +
+    $!.backtrace.join("\n")
+  end
+end

data/cgi/parse_http.rb

@@ -0,0 +1,129 @@
+#!/usr/local/bin/ruby
+
+#################################################
+#
+# Developer: Simon Chiang, Biomolecular Structure Program 
+# Homepage: http://tap.rubyforge.org
+# Licence: MIT-STYLE
+#
+# Copyright (c) 2008, Regents of the University of Colorado 
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this
+# software and associated documentation files (the "Software"), to deal in the Software
+# without restriction, including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
+# to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+#################################################
+
+require 'rubygems'
+require 'cgi'
+require 'tap/http/helpers'
+
+# included to sort the hash keys
+class Hash
+  def to_yaml( opts = {} )
+    YAML::quick_emit( object_id, opts ) do |out|
+      out.map( taguri, to_yaml_style ) do |map|
+        sorted_keys = keys
+        sorted_keys = begin
+          sorted_keys.sort
+        rescue
+          sorted_keys.sort_by {|k| k.to_s} rescue sorted_keys
+        end
+
+        sorted_keys.each do |k|
+          map.add( k, fetch(k) )
+        end
+      end
+    end
+  end
+end
+
+cgi = CGI.new("html3")
+begin
+  
+  http_request_key = "http_request"
+  http_request = cgi[http_request_key]
+  http_request = http_request.respond_to?(:read) ? http_request.read : http_request.to_s
+
+  case 
+  when http_request.strip.empty?
+  
+    cgi.out do
+      cgi.html do
+        cgi.body do %Q{
+<h1>Parse HTTP Parameters</h1>
+
+<p>Enter an HTTP request, like the ones you can capture using the 
+<a href='https://addons.mozilla.org/en-US/firefox/addon/3829'>LiveHTTPHeaders</a> addon for 
+<a href='http://www.mozilla.com/en-US/firefox/'>Firefox</a>.
+</p>
+
+<form action='' method='post'>
+ <textarea rows='20' cols='60' name='#{http_request_key}'></textarea>
+ <br/>
+ <input type='submit' value='Parse'>
+</form>
+
+<p>Note the request must be properly formated. For example:</p>
+
+<pre>
+GET / HTTP/1.1
+Host: tap.rubyforge.org
+User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12
+Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
+Accept-Language: en-us,en;q=0.5
+Accept-Encoding: gzip,deflate
+Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
+Keep-Alive: 300
+Connection: keep-alive
+</pre>}
+        end
+      end
+    end
+    
+  else
+    config = {}
+    Tap::Http::Helpers.parse_http_request(http_request).each_pair do |key, value|
+      config[key.to_s] = value
+    end
+    
+    help = cgi.a(Tap::Http::Helpers::HELP_URL) { "help" }
+    how_to_get_cookies = cgi.a(Tap::Http::Helpers::COOKIES_HELP_URL) { "how to get cookies" }
+  
+    cgi.out do
+      cgi.html do
+        cgi.body do
+         cgi.pre do %Q{
+# Copy and paste into a configuration file. Multiple configs
+# can be added to a single file to perform batch submission.  
+#
+# If you need cookies, see #{how_to_get_cookies} or the #{help}. 
+- #{config.to_yaml[5..-1].gsub(/\n/, "\n  ")}
+}
+          end
+        end
+      end
+    end    
+    
+  end
+
+rescue
+  cgi.out("text/plain") do
+    "Error: #{$!.message}\n"  +
+    $!.backtrace.join("\n")
+  end
+end

data/lib/tap/http/dispatch.rb

@@ -0,0 +1,220 @@
+require 'tap/http/helpers'
+require 'net/http'
+
+#module Net
+#  class HTTP 
+# attr_reader :socket
+# end
+
+# class BufferedIO
+#include Prosperity::Acts::Monitorable
+
+#private
+
+#def rbuf_fill
+#  tick_monitor
+#  timeout(@read_timeout) {
+#    @rbuf << @io.sysread(1024)
+#  }
+#end
+#end
+#end
+
+module Tap
+  module Http
+
+    # Dispatch provides methods for constructing and submitting get and post
+    # HTTP requests. 
+    module Dispatch
+      REQUEST_KEYS = [:url, :request_method, :headers, :params, :redirection_limit]
+      
+      module_function
+      
+      # Constructs and submits a request to the url using the headers and parameters.
+      # Returns the response from the submission.
+      #
+      #   res = submit_request("http://www.google.com/search", {:request_method => 'get'}, {:q => 'tap rubyforge'})
+      #   # => <Net::HTTPOK 200 OK readbody=true>
+      #
+      # Notes:
+      # - A request method must be specified in the headers; currently only get and
+      #   post are supported.  See construct_post for supported post content types.
+      # - A url or a url (as would result from URI.parse(url)) can be provided to
+      #   submit request.  The Net::HTTP object performing the submission is passed
+      #   to the block, if given, before the request is made.
+      def submit_request(config) # :yields: http
+        url_or_uri = config[:url]
+        params = config[:params] || {}
+        headers = headerize_keys( config[:headers] || {})
+        request_method = (config[:request_method] || 'GET').to_s
+        
+        uri = url_or_uri.kind_of?(URI) ? url_or_uri : URI.parse(url_or_uri)
+        uri.path = "/" if uri.path.empty?
+        
+        # construct the request based on the method
+        request = case request_method
+        when /^get$/i then construct_get(uri, headers, params)
+        when /^post$/i then construct_post(uri, headers, params)
+        else
+          raise ArgumentError.new("Missing or unsupported request_method: #{request_method}") 
+        end     
+        
+        # set the http version
+        # if version = config[:http_version]
+        #   version_method = "version_#{version.to_s.gsub(".", "_")}".to_sym
+        #   if Object::Net::HTTP.respond_to?(version_method)
+        #     Object::Net::HTTP.send(version_method)
+        #   else
+        #     raise ArgumentError.new("unsupported http_version: #{version}")
+        #   end
+        # end  
+
+        # submit the request
+        res = Object::Net::HTTP.new(uri.host, uri.port).start do |http| 
+          yield(http) if block_given?
+          http.request(request)
+        end
+        
+        # fetch redirections
+        redirection_limit = config[:redirection_limit]
+        redirection_limit ? fetch_redirection(res, redirection_limit) : res
+      end
+
+      # Constructs a post query.  The 'Content-Type' header determines the format of the body 
+      # content.  If the content type is 'multipart/form-data', then the parameters will be 
+      # formatted using the boundary in the content-type header, if provided, or a randomly 
+      # generated boundary. 
+      #
+      # Headers for the request are set in this method.  If uri contains query parameters,
+      # they will be included in the request URI.
+      #
+      # Supported content-types:
+      # -  application/x-www-form-urlencoded (the default)
+      # -  multipart/form-data
+      #
+      def construct_post(uri, headers, params)
+        req = Object::Net::HTTP::Post.new( URI.encode("#{uri.path}#{construct_query(uri)}") )
+        headers = headerize_keys(headers) 
+        content_type = headers['Content-Type']
+
+        case content_type
+        when /multipart\/form-data/i
+          # extract the boundary if it exists
+          content_type =~ /boundary=(.*)/i
+          boundary = $1 || rand.to_s[2..20]
+
+          req.body = format_multipart_form_data(boundary, params)
+          headers['Content-Type'] = "multipart/form-data; boundary=#{boundary}"
+          headers['Content-Length'] = req.body.length
+        else
+          req.body = format_www_form_urlencoded(params)
+          headers['Content-Type'] = "application/x-www-form-urlencoded"
+          headers['Content-Length'] = req.body.length
+        end
+
+        headers.each_pair { |key, value| req[key] = value }
+        req
+      end
+
+      # Constructs a get query.  All parameters in uri and params are added to the
+      # request URI.  Headers for the request are set in this method.
+      def construct_get(uri, headers, params)
+        req = Object::Net::HTTP::Get.new( URI.encode("#{uri.path}#{construct_query(uri, params)}") )
+        headerize_keys(headers).each_pair { |key, value| req[key] = value }
+        req
+      end
+      
+      # Checks the type of the response; if it is a redirection, get the redirection, 
+      # otherwise return the response.
+      # 
+      # Notes:
+      # - The redirections will only recurse for the input redirection limit (default 10)
+      # - Responses that are not Net::HTTPRedirection or Net::HTTPSuccess raise an error.
+      def fetch_redirection(res, limit=10)
+        raise ArgumentError, 'Could not follow all the redirections.' if limit == 0
+
+        case res
+        when Object::Net::HTTPRedirection 
+          redirect = Object::Net::HTTP.get_response( URI.parse(res['location']) )
+          fetch_redirection(redirect, limit - 1)
+        when Object::Net::HTTPSuccess then res
+        else raise StandardError, res.error!
+        end 
+      end
+      
+      # Normalizes the header keys to a titleized, dasherized string.
+      #   'some_header' => 'Some-Header'
+      #   :some_header => 'Some-Header'
+      #   'some header' => 'Some-Header'
+      def headerize_keys(headers)
+        result = {}
+        headers.each_pair do |key, value|
+          result[Helpers.headerize(key)] = value
+        end
+        result
+      end
+
+      # Returns a URI query constructed from the query in uri and the input parameters.
+      # The query is not encoded, so you may need to URI.encode it later.
+      def construct_query(uri, params={})
+        query = []
+        params.each_pair do |key, values|
+          values = values.kind_of?(Array) ? values : [values]
+          values.each { |value| query << "#{key}=#{value}" }
+        end
+        query << uri.query if uri.query    
+        "#{query.empty? ? '' : '?'}#{query.join('&')}"
+      end
+
+      def format_www_form_urlencoded(params={})
+        query = []
+        params.each_pair do |key, values|
+          values = values.kind_of?(Array) ? values : [values]
+          values.each { |value| query << "#{key}=#{value}" }
+        end  
+        URI.encode( query.join('&') )
+      end
+
+      # Returns a post body formatted as 'multipart/form-data'.  Special formatting occures 
+      # if value in one of the key-value parameter pairs is an Array or Hash.  
+      #
+      # Array values are treated as a multiple inputs for a single key; each array value
+      # is assigned to the key. Hash values are treated as files, with all file-related 
+      # headers specified in the hash.  
+      #
+      #--
+      # Example:
+      #   "--1234", {:key => 'value'} =>
+      #   --1234
+      #   Content-Disposition: form-data; name="key"
+      #
+      #   value
+      #   --1234--
+      def format_multipart_form_data(boundary, params)
+        body = []
+        params.each_pair do |key, values| 
+          values = values.kind_of?(Array) ? values : [values]
+          
+          values.each do |value|
+            body << case value
+            when Hash
+              hash = headerize_keys(value)
+              filename = hash.delete('Filename') || ""
+              content = File.exists?(filename) ? File.read(filename) : ""
+
+              header = "Content-Disposition: form-data; name=\"#{key.to_s}\"; filename=\"#{filename}\"\r\n"
+              hash.each_pair { |key, value| header << "#{key}: #{value}\r\n" }
+              "#{header}\r\n#{content}\r\n"
+            else 
+              %Q{Content-Disposition: form-data; name="#{key.to_s}"\r\n\r\n#{value.to_s}\r\n}
+            end
+          end
+        end
+
+        body.collect {|p| "--#{boundary}\r\n#{p}" }.join('') + "--#{boundary}--\r\n"
+      end
+
+    end
+  end
+end
+

data/lib/tap/http/helpers.rb

@@ -0,0 +1,230 @@
+autoload(:WEBrick, 'webrick')
+autoload(:Zlib, 'zlib')
+autoload(:StringIO, 'stringio')
+
+module Tap
+  module Http
+    module Helpers
+      module_function
+    
+      # Parses a WEBrick::HTTPRequest from the input socket.
+      #
+      # Notes:
+      # - socket can be any kind of IO (ex File, StringIO)
+      # - the socket should be in a position such that the next line
+      #    is the start of an HTTP request, and the request should
+      #    correctly formatted (see below)
+      #
+      # == WEBrick parsing of HTTP format
+      # WEBrick will parse headers then the body of a request, and
+      # currently (1.8.6) considers an empty line as a break between
+      # them.  In general header parsing is forgiving with end-line 
+      # characters (ie "\r\n" and "\n" are both acceptable) but parsing
+      # of multipart/form data IS NOT.  
+      # 
+      # Multipart/form data REQUIRES that the end-line characters
+      # are "\r\n".  The boundary is always started with "--" and the last
+      # boundary completed with "--".  As always, the content-length
+      # must be correct.
+      #   
+      #   # Notice an empty line between the last header 
+      #   # (in this case 'Content-Length') and the body.
+      #   msg = <<-_end_of_message_
+      #   POST /path HTTP/1.1
+      #   Content-Type: multipart/form-data; boundary=1234567890
+      #   Content-Length: 158
+      #   
+      #   --1234567890
+      #   Content-Disposition: form-data; name="one"
+      #   
+      #   value one
+      #   --1234567890
+      #   Content-Disposition: form-data; name="two"
+      #  
+      #   value two
+      #   --1234567890--
+      #   _end_of_message_
+      # 
+      #   # ensure the end of line characters are correct...
+      #   socket = StringIO.new msg.gsub(/\n/, "\r\n")
+      #
+      #   req = Tap::Net.parse_http_request(socket)
+      #   req.header   # => {"content-type" => ["multipart/form-data; boundary=1234567890"], "content-length" => ["158"]}
+      #   req.query     # => {"one" => "value one", "two" => "value two"}
+      #
+      def parse_http_request(socket, parse_yaml=false)
+        socket = StringIO.new(socket) if socket.kind_of?(String)
+        
+        req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP)
+        req.parse(socket)
+
+        parse_webrick_request(req, parse_yaml)
+      end
+      
+      #
+      # TODO -- test with cookies, HTTPS?
+      #
+      
+      def parse_webrick_request(req, parse_yaml=false)
+        headers = {}
+        req.header.each_pair do |key, values| 
+          headers[headerize(key)] = collect(values) do |value|
+            objectify(value, false)
+          end
+        end
+        
+        params = {}
+        req.query.each_pair do |key, values|
+          params[key] = collect(values.to_ary) do |value|
+            # TODO - special for multipart file data?
+            objectify(value, parse_yaml)
+          end
+        end
+        
+        url = File.join("http://", headers['Host'], req.path_info)
+        
+        { :url => url, 
+          :http_version => req.http_version.to_s,
+          :request_method => req.request_method, 
+          :headers => headers, 
+          :params => params}
+      end
+      
+      def parse_cgi_request(cgi, parse_yaml=false)
+        headers = {}
+        ENV.each_pair do |key, values|
+          key = case key
+          when /^HTTP_(.*)/ then $1
+          when 'CONTENT_TYPE' then key
+          else next
+          end
+          
+          headers[headerize(key)] = collect(values) do |value|
+            objectify(value, false)
+          end
+        end
+        
+        params = {}
+        cgi.params.each_pair do |key, values|
+          params[key] = collect(values) do |value|
+            if value.respond_to?(:read)
+              value = if value.original_filename.empty?
+                value.read
+              else
+                {'Filename' => value.original_filename, 'Content-Type' => value.content_type}
+              end
+            end
+            
+            objectify(value, parse_yaml)
+          end
+        end
+        
+        url = File.join("http://", headers['Host'], ENV['PATH_INFO'])
+        
+        { :url => url, 
+          :http_version => ENV['SERVER_PROTOCOL'],  # right or no?
+          :request_method => ENV['REQUEST_METHOD'], 
+          :headers => headers, 
+          :params => params}
+      end
+      
+      def determine_url(action, referer)
+        base = File.basename(referer)
+        
+        case action
+        when /^https?:/ then action
+        when /\// 
+          # only use host of page_url
+          File.join(base, action)
+        else File.join(base, action)
+        end 
+      end
+      
+      HELP_URL = "http://"
+      COOKIES_HELP_URL = "http://"
+      CGI_VARIABLES = %w{
+        AUTH_TYPE               HTTP_HOST          REMOTE_IDENT
+        CONTENT_LENGTH          HTTP_NEGOTIATE     REMOTE_USER
+        CONTENT_TYPE            HTTP_PRAGMA        REQUEST_METHOD
+        GATEWAY_INTERFACE       HTTP_REFERER       SCRIPT_NAME
+        HTTP_ACCEPT             HTTP_USER_AGENT    SERVER_NAME
+        HTTP_ACCEPT_CHARSET     PATH_INFO          SERVER_PORT
+        HTTP_ACCEPT_ENCODING    PATH_TRANSLATED    SERVER_PROTOCOL
+        HTTP_ACCEPT_LANGUAGE    QUERY_STRING       SERVER_SOFTWARE
+        HTTP_CACHE_CONTROL      REMOTE_ADDR
+        HTTP_FROM               REMOTE_HOST}
+      
+      # Headerizes an underscored string.
+      #
+      #   headerize('SOME_STRING')   # => 'Some-String'
+      #   headerize('some string')   # => 'Some-String'
+      #   headerize('Some-String')   # => 'Some-String'
+      #
+      def headerize(str)
+        str.to_s.gsub(/\s|-/, "_").split("_").collect do |s|
+          s =~ /^(.)(.*)/
+          $1.upcase + $2.downcase
+        end.join("-")
+      end
+      
+      def collect(array)
+        array = [array] unless array.kind_of?(Array)
+        
+        array.collect! do |value|
+          yield(value)
+        end
+        
+        case array.length
+        when 0 then nil
+        when 1 then array.first
+        else array
+        end
+      end
+      
+      def objectify(str, parse_yaml=false)
+        return str unless str.kind_of?(String)
+        
+        case str
+        when /^\d+(\.\d+)?$/ then YAML.load(str)
+        when /^\s*$/ then nil
+        when /^---\s*\n/
+          parse_yaml ? YAML.load(str) : str
+        else str
+        end
+      end
+      
+      # Performs a deep merge of two hashes where hash values for
+      # corresponding keys are merged.
+      def deep_merge(a,b)
+        result = {}
+        a.each_pair do |key, value|
+          result[key] = case value
+          when Hash then value.dup
+          # when Array then value.dup
+          else value
+          end
+        end
+        
+        b.each_pair do |key, value|
+          case value
+          when Hash  then (result[key.to_sym] ||= {}).merge!(value)
+          #when Array then (result[key.to_sym] ||= []).concat(value)
+          else result[key.to_sym] = value
+          end
+        end
+        
+        result
+      end
+      
+      # Inflates (ie unzips) a gzip string, as may be returned by requests
+      # that accept 'gzip' and 'deflate' content encoding.
+      #
+      #--
+      # Helpers.inflate(res.body) if res['content-encoding'] == 'gzip'
+      # 
+      def inflate(str)
+        Zlib::GzipReader.new( StringIO.new( str ) ).read
+      end
+    end
+  end
+end

data/lib/tap/http/request.rb

@@ -0,0 +1,117 @@
+require 'tap/http/dispatch'
+require 'thread'
+
+module Tap
+  module Http
+    
+    # ::manifest submits an http request
+    #
+    # Request is a base class for submitting HTTP requests from a request
+    # hash.  Multiple requests may be submitted on individual threads, up 
+    # to a configurable limit.
+    #
+    # Configuration hashes are like the following:
+    #
+    #   url: http://tap.rubyforge.org/
+    #   request_method: GET
+    #   headers: {}
+    #   params: {}
+    #
+    # The only required field is the url (by default no headers or params
+    # are specified, and the request method is GET).  Request requires
+    # hash inputs, which can be inconvenient from the command line.  A
+    # good workaround is to save requests in a .yml file and use a simple
+    # workflow:
+    #
+    #   [requests.yml]
+    #   - url: http://tap.rubyforge.org/
+    #   - url: http://tap.rubyforge.org/about.html
+    #   
+    #   % rap load requests.yml --:i request --+ dump
+    #
+    #--
+    # To generate configuration hashes from Firefox, see the redirect_http
+    # ubiquity command.
+    #++
+    class Request < Tap::Task
+      class << self
+        def intern(*args, &block)
+          instance = new(*args)
+          instance.extend Support::Intern(:process_response)
+          instance.process_response_block = block
+          instance
+        end
+      end
+      
+      config :redirection_limit, 10, &c.integer  # the redirection limit for the request
+      config :max_threads, 10, &c.integer        # the maximum number of request threads
+      
+      def process(*requests)
+        # build a queue of all the requests to be handled
+        queue = Queue.new
+        requests.each_with_index do |request, index|
+          request = symbolize_keys(request)
+
+          if request[:url] == nil
+            raise ArgumentError, "no url specified: #{request.inspect}"
+          end
+
+          queue.enq [request, index]
+          index += 1
+        end
+        
+        # submit and retrieve all requests before processing
+        # responses.  this assures responses are processed
+        # in order, in case it matters.
+        lock = Mutex.new
+        responses = []
+        request_threads = Array.new(max_threads) do 
+          Thread.new do
+            begin
+              while !queue.empty?
+                request, index = queue.deq(true)
+              
+                log(request[:request_method], request[:url])
+                if app.verbose
+                  log 'headers', config[:headers].inspect
+                  log 'params', config[:params].inspect
+                end
+                
+                res = Dispatch.submit_request(request)
+                lock.synchronize { responses[index] = res }
+              end
+            rescue(ThreadError)
+              # Catch errors due to the queue being empty.
+              # (this should not occur as the queue is checked)
+              raise $! unless $!.message == 'queue empty'
+            end
+          end
+        end
+        request_threads.each {|thread| thread.join }
+
+        # process responses and collect results
+        responses.collect! do |res|
+          process_response(res)
+        end
+      end
+      
+      # Hook for processing a response.  By default process_response 
+      # simply logs the response message and returns the response. 
+      def process_response(res)
+        log(nil, res.message)
+        res 
+      end
+      
+      protected
+      
+      # Return a new hash with all keys converted to symbols.
+      # Lifted from ActiveSupport
+      def symbolize_keys(hash)
+        hash.inject({}) do |options, (key, value)|
+          options[(key.to_sym rescue key) || key] = value
+          options
+        end
+      end
+    end
+  end
+end

data/lib/tap/test/http_test.rb

@@ -0,0 +1,296 @@
+require 'webrick'
+require 'singleton'
+require 'tap/test/subset_test'
+
+module Tap
+  module Test
+    
+    # HttpTest facilitates testing of HTTP requests by initializing a 
+    # Webrick server that echos requests, and providing methods to 
+    # validate echoed requests.
+    # 
+    module HttpTest
+    
+      # The test-specific web server.  All request to the server
+      # are echoed back.
+      class Server
+        include Singleton
+        include WEBrick
+
+        attr_accessor :server_thread, :server
+        
+        def initialize
+          # set the default log level to warn to prevent general access logs, unless otherwise specified
+          log_level = ENV["WEB_LOG_LEVEL"] ? ENV["WEB_LOG_LEVEL"].upcase  : "WARN"
+          logger = Log.new($stderr, Log.const_get( log_level ) )
+
+          self.server = HTTPServer.new(:Port => 2000, 
+            :Logger => logger, 
+            :AccessLog => [          
+              [ logger, AccessLog::COMMON_LOG_FORMAT ],
+              [ logger, AccessLog::REFERER_LOG_FORMAT ]]) 
+              
+          server.mount_proc("/") do |req, res|
+            res.body << req.request_line
+            res.body << req.raw_header.join('')
+            
+            # an extra line must be added to delimit the headers from the body.
+            if req.body
+              res.body << "\r\n"
+              res.body << req.body 
+            end
+            
+            res['Content-Type'] = "text/html"
+          end
+        end
+
+        # Starts the server on a new thread.
+        def start_web_server
+          self.server_thread ||= Thread.new { server.start }
+        end
+      end
+      
+      def self.included(base)
+        base.send(:include, Tap::Test::SubsetTest)
+      end
+      
+      # WEB subset of tests.  Starts HTTPTest::Server if necessary
+      # and yields to the block.
+      def web_test
+        subset_test("WEB", "w") do
+          Server.instance.start_web_server
+          yield
+        end
+      end
+      
+      REQUEST_ATTRIBUTES = %w{
+        request_method  http_version
+
+        host port path
+        script_name path_info 
+
+        header cookies query
+        accept accept_charset
+        accept_encoding accept_language
+
+        user
+        addr peeraddr
+        attributes
+        keep_alive}
+        
+      UNCHECKED_REQUEST_ATTRIBUTES = %w{
+        request_line 
+        unparsed_uri 
+        request_uri 
+        request_time 
+        raw_header 
+        query_string}
+      
+      # Parses expected and actual as an http request (using Tap::Net.parse_http_request)
+      # and asserts that all of the REQUEST_ATTRIBUTES are equal.  See the parse_http_request
+      # documentation for some important notes, particularly involving "\r\n" vs "\n" and
+      # post requests.
+      def assert_request_equal(expected, actual)
+        e = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP)
+        e.parse( StringIO.new(expected) )
+        
+        a = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP)
+        a.parse( StringIO.new(actual) )
+        
+        errors = []
+        REQUEST_ATTRIBUTES.each do |attribute|
+          exp = e.send(attribute)
+          act = a.send(attribute) 
+          next if exp == act 
+          errors << "<#{PP.singleline_pp(exp, '')}> expected for #{attribute} but was:\n<#{PP.singleline_pp(act, '')}>."
+        end
+        
+        if errors.empty?
+          # this rather unecessary assertion is used simply to
+          # make assert_request_equal cause an assertion.
+          assert errors.empty?
+        else
+          flunk errors.join("\n")
+        end
+      end
+      
+      # Convenience method that strips str, strips each line of str, and 
+      # then rejoins them using "\r\n" as is typical of HTTP messages.  
+      #
+      #   strip_align %Q{
+      #     GET /echo HTTP/1.1
+      #     Accept: */*
+      #     Host: localhost:2000}   
+      #
+      #   # => "GET /echo HTTP/1.1\r\nAccept: */*\r\nHost: localhost:2000\r\n"
+      def strip_align(str)
+        str.strip.split(/\r?\n/).collect do |line| 
+          "#{line.strip}\r\n"
+        end.compact.join('')
+      end
+      
+      # Turns a hash of parameters into an encoded HTTP query string.
+      # Multiple values for a given key can be specified by an array.
+      # 
+      #   to_query('key' => 'value', 'array' => ['one', 'two']) # => "array=one&array=two&key=value"
+      #
+      # Note: the order of the parameters in the result is determined 
+      # by hash.each_pair and is thus fairly unpredicatable.
+      def to_query(hash)
+        query = []
+        hash.each_pair do |key,values| 
+          values = values.kind_of?(Array) ? values : [values]
+          values.each { |value| query << "#{key}=#{value}" }
+        end
+        URI.encode(query.join('&'))
+      end
+
+      module RequestLibrary
+        def get_request
+          msg = <<-_end_of_message_
+GET /path?str=value&int=123&yaml=---+%0A-+a%0A-+b%0A-+c%0A&float=1.23 HTTP/1.1
+Host: www.example.com
+Keep-Alive: 300
+Connection: keep-alive
+          _end_of_message_
+        end
+        
+        def _get_request
+          { :url => "http://www.example.com/path",
+            :http_version => '1.1',
+            :request_method => 'GET',
+            :headers => {
+              "Host" => "www.example.com", 
+              "Keep-Alive" => 300,
+              "Connection" => 'keep-alive'},
+            :params => {
+              'str' => 'value', 
+              'int' => 123, 
+              'float' => 1.23, 
+              'yaml' => ['a', 'b', 'c']}
+            }
+        end
+        
+        def get_request_with_multiple_values
+          msg = <<-_end_of_message_
+GET /path?one=value&one=123&one=---+%0A-+a%0A-+b%0A-+c%0A&two=1.23 HTTP/1.1
+Host: www.example.com
+Keep-Alive: 300
+Connection: keep-alive
+          _end_of_message_
+        end
+
+        def _get_request_with_multiple_values
+          { :url => "http://www.example.com/path",
+            :http_version => '1.1',
+            :request_method => 'GET',
+            :headers => {
+              "Host" => "www.example.com", 
+              "Keep-Alive" => 300,
+              "Connection" => 'keep-alive'},
+            :params => {
+              'one' => ['value', 123, ['a', 'b', 'c']], 
+              'two' => 1.23}
+            }
+        end
+        
+        def multipart_request
+          msg = <<-_end_of_message_
+POST /path HTTP/1.1
+Host: www.example.com
+Content-Type: multipart/form-data; boundary=1234567890
+Content-Length: 305
+
+--1234567890
+Content-Disposition: form-data; name="str"
+
+string value
+--1234567890
+Content-Disposition: form-data; name="int"
+
+123
+--1234567890
+Content-Disposition: form-data; name="float"
+
+1.23
+--1234567890
+Content-Disposition: form-data; name="yaml"
+
+--- 
+- a
+- b
+- c
+--1234567890--
+_end_of_message_
+
+          msg.gsub(/\n/, "\r\n")
+        end
+
+        def _multipart_request
+          { :url => "http://www.example.com/path",
+            :http_version => '1.1',
+            :request_method => 'POST',
+            :headers => {
+              "Host" => 'www.example.com',
+              "Content-Type" => "multipart/form-data; boundary=1234567890", 
+              "Content-Length" => 305},
+            :params => {
+              'str' => 'string value', 
+              'int' => 123, 
+              'float' => 1.23, 
+              'yaml' => ['a', 'b', 'c']}
+            }
+        end
+
+        def multipart_request_with_multiple_values
+          msg = <<-_end_of_message_
+POST /path HTTP/1.1
+Host: www.example.com
+Content-Type: multipart/form-data; boundary=1234567890
+Content-Length: 302
+
+--1234567890
+Content-Disposition: form-data; name="one"
+
+string value
+--1234567890
+Content-Disposition: form-data; name="one"
+
+123
+--1234567890
+Content-Disposition: form-data; name="two"
+
+1.23
+--1234567890
+Content-Disposition: form-data; name="one"
+
+--- 
+- a
+- b
+- c
+--1234567890--
+_end_of_message_
+
+          msg.gsub(/\n/, "\r\n")
+        end
+
+        def _multipart_request_with_multiple_values
+          { :url => "http://www.example.com/path",
+            :http_version => '1.1',
+            :request_method => 'POST',
+            :headers => {
+              "Host" => 'www.example.com',
+              "Content-Type" => "multipart/form-data; boundary=1234567890", 
+              "Content-Length" => 302},
+            :params => {
+              'one' => ['string value', 123, ['a', 'b', 'c']], 
+              'two' => 1.23}
+            }
+        end
+      end
+    end
+  end
+end
+
+
+

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification 
+name: tap-http
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Simon Chiang
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-11-18 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: tap
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.11.1
+    version: 
+description: 
+email: simon.a.chiang@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+- MIT-LICENSE
+files: 
+- cgi/echo.rb
+- cgi/http_to_yaml.rb
+- cgi/parse_http.rb
+- lib/tap/http/dispatch.rb
+- lib/tap/http/helpers.rb
+- lib/tap/http/request.rb
+- lib/tap/test/http_test.rb
+- tap.yml
+- README
+- MIT-LICENSE
+has_rdoc: true
+homepage: http://rubyforge.org/projects/tap
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: tap
+rubygems_version: 1.3.0
+signing_key: 
+specification_version: 2
+summary: A task library for submitting http requests using Tap.
+test_files: []
+