brianmario-eventmachine 0.12.2

data/lib/protocols/httpclient.rb

@@ -0,0 +1,264 @@
+# $Id$
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 16 July 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+
+
+
+module EventMachine
+module Protocols
+
+class HttpClient < Connection
+  include EventMachine::Deferrable
+
+
+    MaxPostContentLength = 20 * 1024 * 1024
+
+  # USAGE SAMPLE:
+  #
+  # EventMachine.run {
+  #   http = EventMachine::Protocols::HttpClient.request(
+  #     :host => server,
+  #     :port => 80,
+  #     :request => "/index.html",
+  #     :query_string => "parm1=value1&parm2=value2"
+  #   )
+  #   http.callback {|response|
+  #     puts response[:status]
+  #     puts response[:headers]
+  #     puts response[:content]
+  #   }
+  # }
+  #
+
+  # TODO:
+  # Add streaming so we can support enormous POSTs. Current max is 20meg.
+  # Timeout for connections that run too long or hang somewhere in the middle.
+  # Persistent connections (HTTP/1.1), may need a associated delegate object.
+  # DNS: Some way to cache DNS lookups for hostnames we connect to. Ruby's
+  # DNS lookups are unbelievably slow.
+  # HEAD requests.
+  # Chunked transfer encoding.
+  # Convenience methods for requests. get, post, url, etc.
+  # SSL.
+  # Handle status codes like 304, 100, etc.
+  # Refactor this code so that protocol errors all get handled one way (an exception?),
+  # instead of sprinkling set_deferred_status :failed calls everywhere.
+
+  def self.request( args = {} )
+    args[:port] ||= 80
+    EventMachine.connect( args[:host], args[:port], self ) {|c|
+      # According to the docs, we will get here AFTER post_init is called.
+      c.instance_eval {@args = args}
+    }
+  end
+
+  def post_init
+    @start_time = Time.now
+    @data = ""
+    @read_state = :base
+  end
+
+  # We send the request when we get a connection.
+  # AND, we set an instance variable to indicate we passed through here.
+  # That allows #unbind to know whether there was a successful connection.
+  # NB: This naive technique won't work when we have to support multiple
+  # requests on a single connection.
+  def connection_completed
+    @connected = true
+    send_request @args
+  end
+
+  def send_request args
+    args[:verb] ||= args[:method] # Support :method as an alternative to :verb.
+    args[:verb] ||= :get # IS THIS A GOOD IDEA, to default to GET if nothing was specified?
+
+    verb = args[:verb].to_s.upcase
+    unless ["GET", "POST", "PUT", "DELETE", "HEAD"].include?(verb)
+      set_deferred_status :failed, {:status => 0} # TODO, not signalling the error type
+      return # NOTE THE EARLY RETURN, we're not sending any data.
+    end
+
+    request = args[:request] || "/"
+    unless request[0,1] == "/"
+      request = "/" + request
+    end
+
+    qs = args[:query_string] || ""
+    if qs.length > 0 and qs[0,1] != '?'
+      qs = "?" + qs
+    end
+
+    version = args[:version] || "1.1"
+
+    # Allow an override for the host header if it's not the connect-string.
+    host = args[:host_header] || args[:host] || "_"
+    # For now, ALWAYS tuck in the port string, although we may want to omit it if it's the default.
+    port = args[:port]
+
+    # POST items.
+    postcontenttype = args[:contenttype] || "application/octet-stream"
+    postcontent = args[:content] || ""
+    raise "oversized content in HTTP POST" if postcontent.length > MaxPostContentLength
+
+    # ESSENTIAL for the request's line-endings to be CRLF, not LF. Some servers misbehave otherwise.
+    # TODO: We ASSUME the caller wants to send a 1.1 request. May not be a good assumption.
+    req = [
+      "#{verb} #{request}#{qs} HTTP/#{version}",
+      "Host: #{host}:#{port}",
+      "User-agent: Ruby EventMachine",
+    ]
+
+    if verb == "POST" || verb == "PUT"
+      req << "Content-type: #{postcontenttype}"
+      req << "Content-length: #{postcontent.length}"
+    end
+
+    # TODO, this cookie handler assumes it's getting a single, semicolon-delimited string.
+    # Eventually we will want to deal intelligently with arrays and hashes.
+    if args[:cookie]
+      req << "Cookie: #{args[:cookie]}"
+    end
+
+    # Basic-auth stanza contributed by Mike Murphy.
+    if args[:basic_auth]
+      basic_auth_string = ["#{args[:basic_auth][:username]}:#{args[:basic_auth][:password]}"].pack('m').strip
+      req << "Authorization: Basic #{basic_auth_string}"
+    end 
+
+    req << ""
+    reqstring = req.map {|l| "#{l}\r\n"}.join
+    send_data reqstring
+
+    if verb == "POST" || verb == "PUT"
+      send_data postcontent
+    end
+  end
+
+
+  def receive_data data
+    while data and data.length > 0
+      case @read_state
+      when :base
+        # Perform any per-request initialization here and don't consume any data.
+        @data = ""
+        @headers = []
+        @content_length = nil # not zero
+        @content = ""
+        @status = nil
+        @read_state = :header
+	@connection_close = nil
+      when :header
+        ary = data.split( /\r?\n/m, 2 )
+        if ary.length == 2
+          data = ary.last
+          if ary.first == ""
+	      if (@content_length and @content_length > 0) || @connection_close
+		  @read_state = :content
+	      else
+		  dispatch_response
+		  @read_state = :base
+	      end
+          else
+            @headers << ary.first
+            if @headers.length == 1
+              parse_response_line
+            elsif ary.first =~ /\Acontent-length:\s*/i
+              # Only take the FIRST content-length header that appears,
+              # which we can distinguish because @content_length is nil.
+              # TODO, it's actually a fatal error if there is more than one
+              # content-length header, because the caller is presumptively
+              # a bad guy. (There is an exploit that depends on multiple
+              # content-length headers.)
+              @content_length ||= $'.to_i
+            elsif ary.first =~ /\Aconnection:\s*close/i
+              @connection_close = true
+            end
+          end
+        else
+          @data << data
+          data = ""
+        end
+      when :content
+        # If there was no content-length header, we have to wait until the connection
+        # closes. Everything we get until that point is content.
+        # TODO: Must impose a content-size limit, and also must implement chunking.
+        # Also, must support either temporary files for large content, or calling
+        # a content-consumer block supplied by the user.
+        if @content_length
+          bytes_needed = @content_length - @content.length
+          @content += data[0, bytes_needed]
+          data = data[bytes_needed..-1] || ""
+          if @content_length == @content.length
+            dispatch_response
+            @read_state = :base
+          end
+        else
+          @content << data
+          data = ""
+        end
+      end
+    end
+  end
+
+
+  # We get called here when we have received an HTTP response line.
+  # It's an opportunity to throw an exception or trigger other exceptional
+  # handling.
+  def parse_response_line
+    if @headers.first =~ /\AHTTP\/1\.[01] ([\d]{3})/
+      @status = $1.to_i
+    else
+      set_deferred_status :failed, {
+        :status => 0 # crappy way of signifying an unrecognized response. TODO, find a better way to do this.
+      }
+      close_connection
+    end
+  end
+  private :parse_response_line
+
+  def dispatch_response
+    @read_state = :base
+    set_deferred_status :succeeded, {
+      :content => @content,
+      :headers => @headers,
+      :status => @status
+    }
+    # TODO, we close the connection for now, but this is wrong for persistent clients.
+    close_connection
+  end
+
+  def unbind
+    if !@connected
+      set_deferred_status :failed, {:status => 0} # YECCCCH. Find a better way to signal no-connect/network error.
+    elsif (@read_state == :content and @content_length == nil)
+      dispatch_response
+    end
+  end
+end
+
+
+end
+end
+
+

data/lib/protocols/line_and_text.rb

@@ -0,0 +1,122 @@
+# $Id$
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 15 November 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+# 
+require File.dirname(__FILE__) + '/buftok'
+
+module EventMachine
+  module Protocols
+
+    class LineAndTextProtocol < Connection
+      MaxLineLength = 16*1024
+      MaxBinaryLength = 32*1024*1024
+
+      def initialize *args
+        super
+        lbp_init_line_state
+      end
+      def receive_data data
+        if @lbp_mode == :lines
+          begin
+            @lpb_buffer.extract(data).each do |line| 
+              receive_line(line.chomp) if respond_to?(:receive_line)
+            end
+          rescue Exception
+            receive_error('overlength line') if respond_to?(:receive_error)
+            close_connection
+            return
+          end
+        else
+          if @lbp_binary_limit > 0
+            wanted = @lbp_binary_limit - @lbp_binary_bytes_received
+            chunk = nil
+            if data.length > wanted
+              chunk = data.slice!(0...wanted)
+            else
+              chunk = data
+              data = ""
+            end
+            @lbp_binary_buffer[@lbp_binary_bytes_received...(@lbp_binary_bytes_received+chunk.length)] = chunk
+            @lbp_binary_bytes_received += chunk.length
+            if @lbp_binary_bytes_received == @lbp_binary_limit
+              receive_binary_data(@lbp_binary_buffer) if respond_to?(:receive_binary_data)
+              lbp_init_line_state
+            end
+            receive_data(data) if data.length > 0
+          else
+            receive_binary_data(data) if respond_to?(:receive_binary_data)
+            data = ""
+          end
+        end
+      end
+
+      def unbind
+        if @lbp_mode == :binary and @lbp_binary_limit > 0
+          if respond_to?(:receive_binary_data)
+            receive_binary_data( @lbp_binary_buffer[0...@lbp_binary_bytes_received] )
+          end
+        end
+      end
+
+      # Set up to read the supplied number of binary bytes.
+      # This recycles all the data currently waiting in the line buffer, if any.
+      # If the limit is nil, then ALL subsequent data will be treated as binary
+      # data and passed to the upstream protocol handler as we receive it.
+      # If a limit is given, we'll hold the incoming binary data and not
+      # pass it upstream until we've seen it all, or until there is an unbind
+      # (in which case we'll pass up a partial).
+      # Specifying nil for the limit (the default) means there is no limit.
+      # Specifiyng zero for the limit will cause an immediate transition back to line mode.
+      #
+      def set_binary_mode size = nil
+        if @lbp_mode == :lines
+          if size == 0
+            receive_binary_data("") if respond_to?(:receive_binary_data)
+            # Do no more work here. Stay in line mode and keep consuming data.
+          else
+            @lbp_binary_limit = size.to_i # (nil will be stored as zero)
+            if @lbp_binary_limit > 0
+              raise "Overlength" if @lbp_binary_limit > MaxBinaryLength # arbitrary sanity check
+              @lbp_binary_buffer = "\0" * @lbp_binary_limit
+              @lbp_binary_bytes_received = 0
+            end
+
+            @lbp_mode = :binary
+            receive_data @lpb_buffer.flush
+          end
+        else
+          raise "invalid operation"
+        end
+      end
+
+      #--
+      # For internal use, establish protocol baseline for handling lines.
+      def lbp_init_line_state
+        @lpb_buffer = BufferedTokenizer.new("\n", MaxLineLength)
+        @lbp_mode = :lines
+      end
+      private :lbp_init_line_state
+    end
+  end
+end

data/lib/protocols/linetext2.rb

@@ -0,0 +1,163 @@
+# $Id$
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 15 November 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+# 
+
+
+# In the grand, time-honored tradition of re-inventing the wheel, we offer
+# here YET ANOTHER protocol that handles line-oriented data with interspersed
+# binary text. This one trades away some of the performance optimizations of
+# EventMachine::Protocols::LineAndTextProtocol in order to get better correctness
+# with regard to binary text blocks that can switch back to line mode. It also
+# permits the line-delimiter to change in midstream.
+# This was originally written to support Stomp.
+
+# TODO! We're not enforcing the limits on header lengths and text-lengths.
+# When we get around to that, call #receive_error if the user defined it, otherwise
+# throw exceptions.
+
+module EventMachine
+	module Protocols
+		module LineText2
+			MaxLineLength = 16*1024
+			MaxBinaryLength = 32*1024*1024
+
+			#--
+			# Will be called recursively until there's no data to read.
+			# That way the user-defined handlers we call can modify the
+			# handling characteristics on a per-token basis.
+			#
+			def receive_data data
+				return unless (data and data.length > 0)
+
+				# Do this stuff in lieu of a constructor.
+				@lt2_mode ||= :lines
+				@lt2_delimiter ||= "\n"
+				@lt2_linebuffer ||= []
+
+				if @lt2_mode == :lines
+					if ix = data.index( @lt2_delimiter )
+						@lt2_linebuffer << data[0...ix]
+						ln = @lt2_linebuffer.join
+						@lt2_linebuffer.clear
+						if @lt2_delimiter == "\n"
+							ln.chomp!
+						end
+						receive_line ln
+						receive_data data[(ix+@lt2_delimiter.length)..-1]
+					else
+						@lt2_linebuffer << data
+					end
+				elsif @lt2_mode == :text
+					if @lt2_textsize
+						needed = @lt2_textsize - @lt2_textpos
+						will_take = if data.length > needed
+							needed
+						else
+							data.length
+						end
+
+						@lt2_textbuffer << data[0...will_take]
+						tail = data[will_take..-1]
+
+						@lt2_textpos += will_take
+						if @lt2_textpos >= @lt2_textsize
+							# Reset line mode (the default behavior) BEFORE calling the
+							# receive_binary_data. This makes it possible for user code
+							# to call set_text_mode, enabling chains of text blocks
+							# (which can possibly be of different sizes).
+							set_line_mode
+							receive_binary_data @lt2_textbuffer.join
+							receive_end_of_binary_data
+						end
+
+						receive_data tail
+					else
+						receive_binary_data data
+					end
+				end
+			end
+
+
+			def set_delimiter delim
+				@lt2_delimiter = delim.to_s
+			end
+
+			# Called internally but also exposed to user code, for the case in which
+			# processing of binary data creates a need to transition back to line mode.
+			# We support an optional parameter to "throw back" some data, which might
+			# be an umprocessed chunk of the transmitted binary data, or something else
+			# entirely.
+			def set_line_mode data=""
+				@lt2_mode = :lines
+				(@lt2_linebuffer ||= []).clear
+				receive_data data.to_s
+			end
+
+			def set_text_mode size=nil
+				if size == 0
+					set_line_mode
+				else
+					@lt2_mode = :text
+					(@lt2_textbuffer ||= []).clear
+					@lt2_textsize = size # which can be nil, signifying no limit
+					@lt2_textpos = 0
+				end
+			end
+
+			# Alias for #set_text_mode, added for back-compatibility with LineAndTextProtocol.
+			def set_binary_mode size=nil
+				set_text_mode size
+			end
+
+			# In case of a dropped connection, we'll send a partial buffer to user code
+			# when in sized text mode. User overrides of #receive_binary_data need to
+			# be aware that they may get a short buffer.
+			def unbind
+				if @lt2_mode == :text and @lt2_textpos > 0
+					receive_binary_data @lt2_textbuffer.join
+				end
+			end
+
+			# Stub. Should be subclassed by user code.
+			def receive_line ln
+				# no-op
+			end
+
+			# Stub. Should be subclassed by user code.
+			def receive_binary_data data
+				# no-op
+			end
+
+			# Stub. Should be subclassed by user code.
+			# This is called when transitioning internally from text mode
+			# back to line mode. Useful when client code doesn't want
+			# to keep track of how much data it's received.
+			def receive_end_of_binary_data
+				# no-op
+			end
+		end
+	end
+end
+