crusher-eventmachine 0.12.11

data/lib/em/messages.rb

@@ -0,0 +1,66 @@
+#--
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 16 Jul 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+
+=begin
+
+Message Routing in EventMachine.
+
+The goal here is to enable "routing points," objects that can send and receive
+"messages," which are delimited streams of bytes. The boundaries of a message
+are preserved as it passes through the reactor system.
+
+There will be several module methods defined in EventMachine to create route-point
+objects (which will probably have a base class of EventMachine::MessageRouter
+until someone suggests a better name).
+
+As with I/O objects, routing objects will receive events by having the router
+core call methods on them. And of course user code can and will define handlers
+to deal with events of interest.
+
+The message router base class only really needs a receive_message method. There will
+be an EM module-method to send messages, in addition to the module methods to create
+the various kinds of message receivers.
+
+The simplest kind of message receiver object can receive messages by being named 
+explicitly in a parameter to EM#send_message. More sophisticated receivers can define
+pub-sub selectors and message-queue names. And they can also define channels for
+route-points in other processes or even on other machines.
+
+A message is NOT a marshallable entity. Rather, it's a chunk of flat content more like
+an Erlang message. Initially, all content submitted for transmission as a message will
+have the to_s method called on it. Eventually, we'll be able to transmit certain structured
+data types (XML and YAML documents, Structs within limits) and have them reconstructed
+on the other end.
+
+A fundamental goal of the message-routing capability is to interoperate seamlessly with
+external systems, including non-Ruby systems like ActiveMQ. We will define various protocol
+handlers for things like Stomp and possibly AMQP, but these will be wrapped up and hidden
+from the users of the basic routing capability.
+
+As with Erlang, a critical goal is for programs that are built to use message-passing to work
+WITHOUT CHANGE when the code is re-based on a multi-process system.
+
+=end
+

data/lib/em/process_watch.rb

@@ -0,0 +1,44 @@
+module EventMachine
+
+  # This is subclassed from EventMachine::Connection for use with the process monitoring API. Read the
+  # documentation on the instance methods of this class, and for a full explanation see EventMachine.watch_process.
+  class ProcessWatch < Connection
+    # :stopdoc:
+    Cfork = 'fork'.freeze
+    Cexit = 'exit'.freeze
+    # :startdoc:
+
+    def receive_data(data) # :nodoc:
+      case data
+      when Cfork
+        process_forked
+      when Cexit
+        process_exited
+      end
+    end
+
+    # Returns the pid that EventMachine::watch_process was originally called with.
+    def pid
+      @pid
+    end
+
+    # Should be redefined with the user's custom callback that will be fired when the prcess is forked.
+    #
+    # There is currently not an easy way to get the pid of the forked child.
+    def process_forked
+    end
+
+    # Should be redefined with the user's custom callback that will be fired when the process exits.
+    #
+    # stop_watching is called automatically after this callback
+    def process_exited
+    end
+
+    # Discontinue monitoring of the process.
+    # This will be called automatically when a process dies. User code may call it as well.
+    def stop_watching
+      EventMachine::unwatch_pid(@signature)
+    end
+  end
+
+end

data/lib/em/processes.rb

@@ -0,0 +1,119 @@
+#--
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 13 Dec 07
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-08 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+
+
+module EventMachine
+
+  # EM::DeferrableChildProcess is a sugaring of a common use-case
+  # involving EM::popen.
+  # Call the #open method on EM::DeferrableChildProcess, passing
+  # a command-string. #open immediately returns an EM::Deferrable
+  # object. It also schedules the forking of a child process, which
+  # will execute the command passed to #open.
+  # When the forked child terminates, the Deferrable will be signalled
+  # and execute its callbacks, passing the data that the child process
+  # wrote to stdout.
+  #
+  class DeferrableChildProcess < EventMachine::Connection
+    include EventMachine::Deferrable
+
+    def initialize # :nodoc:
+      super
+      @data = []
+    end
+
+    # Sugars a common use-case involving forked child processes.
+    # #open takes a String argument containing an shell command
+    # string (including arguments if desired). #open immediately
+    # returns an EventMachine::Deferrable object, without blocking.
+    #
+    # It also invokes EventMachine#popen to run the passed-in
+    # command in a forked child process.
+    #
+    # When the forked child terminates, the Deferrable that
+    # #open calls its callbacks, passing the data returned
+    # from the child process.
+    #
+    def self.open cmd
+      EventMachine.popen( cmd, DeferrableChildProcess )
+    end
+
+    def receive_data data # :nodoc:
+      @data << data
+    end
+
+    def unbind # :nodoc:
+      succeed( @data.join )
+    end
+  end
+
+  class SystemCmd < EventMachine::Connection # :nodoc:
+    def initialize cb
+      @cb = cb
+      @output = []
+    end
+    def receive_data data
+      @output << data
+    end
+    def unbind
+      @cb.call @output.join(''), get_status if @cb
+    end
+  end
+
+  # EM::system is a simple wrapper for EM::popen. It is similar to Kernel::system, but requires a
+  # single string argument for the command and performs no shell expansion.
+  #
+  # The block or proc passed to EM::system is called with two arguments: the output generated by the command,
+  # and a Process::Status that contains information about the command's execution.
+  #
+  #  EM.run{
+  #    EM.system('ls'){ |output,status| puts output if status.exitstatus == 0 }
+  #  }
+  #
+  # You can also supply an additional proc to send some data to the process:
+  #
+  #  EM.run{
+  #    EM.system('sh', proc{ |process|
+  #      process.send_data("echo hello\n")
+  #      process.send_data("exit\n")
+  #    }, proc{ |out,status|
+  #      puts(out)
+  #    })
+  #  }
+  #
+  # Like EventMachine.popen, EventMachine.system currently does not work on windows.
+  # It returns the pid of the spawned process.
+  def EventMachine::system cmd, *args, &cb
+    cb ||= args.pop if args.last.is_a? Proc
+    init = args.pop if args.last.is_a? Proc
+
+    # merge remaining arguments into the command
+    cmd = ([cmd] + args.map{|a|a.to_s.dump}).join(' ')
+
+    EM.get_subprocess_pid(EM.popen(cmd, SystemCmd, cb) do |c|
+      init[c] if init
+    end.signature)
+  end
+end

data/lib/em/protocols.rb

@@ -0,0 +1,36 @@
+module EventMachine
+  # This module contains various protocol implementations, including:
+  # - HttpClient and HttpClient2
+  # - Stomp
+  # - Memcache
+  # - SmtpClient and SmtpServer
+  # - SASLauth and SASLauthclient
+  # - LineProtocol, LineAndTextProtocol and LineText2
+  # - HeaderAndContentProtocol
+  # - Postgres3
+  # - ObjectProtocol
+  #
+  # The protocol implementations live in separate files in the protocols/ subdirectory,
+  # but are auto-loaded when they are first referenced in your application.
+  #
+  # EventMachine::Protocols is also aliased to EM::P for easier usage.
+  #
+  module Protocols
+    # TODO : various autotools are completely useless with the lack of naming
+    # convention, we need to correct that!
+    autoload :TcpConnectTester, 'em/protocols/tcptest'
+    autoload :HttpClient, 'em/protocols/httpclient'
+    autoload :HttpClient2, 'em/protocols/httpclient2'
+    autoload :LineAndTextProtocol, 'em/protocols/line_and_text'
+    autoload :HeaderAndContentProtocol, 'em/protocols/header_and_content'
+    autoload :LineText2, 'em/protocols/linetext2'
+    autoload :Stomp, 'em/protocols/stomp'
+    autoload :SmtpClient, 'em/protocols/smtpclient'
+    autoload :SmtpServer, 'em/protocols/smtpserver'
+    autoload :SASLauth, 'em/protocols/saslauth'
+    autoload :Memcache, 'em/protocols/memcache'
+    autoload :Postgres3, 'em/protocols/postgres3'
+    autoload :ObjectProtocol, 'em/protocols/object_protocol'
+    autoload :Socks4, 'em/protocols/socks4'
+  end
+end

data/lib/em/protocols/header_and_content.rb

@@ -0,0 +1,138 @@
+#--
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 15 Nov 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+#
+
+module EventMachine
+  module Protocols
+
+    # === Usage
+    #
+    #  class RequestHandler < EM::P::HeaderAndContentProtocol
+    #    def receive_request headers, content
+    #      p [:request, headers, content]
+    #    end
+    #  end
+    #
+    #  EM.run{
+    #    EM.start_server 'localhost', 80, RequestHandler
+    #  }
+    #
+    #--
+    # Originally, this subclassed LineAndTextProtocol, which in
+    # turn relies on BufferedTokenizer, which doesn't gracefully
+    # handle the transitions between lines and binary text.
+    # Changed 13Sep08 by FCianfrocca.
+    class HeaderAndContentProtocol < Connection
+      include LineText2
+
+      ContentLengthPattern = /Content-length:\s*(\d+)/i
+
+      def initialize *args
+        super
+        init_for_request
+      end
+
+      def receive_line line
+        case @hc_mode
+        when :discard_blanks
+          unless line == ""
+            @hc_mode = :headers
+            receive_line line
+          end
+        when :headers
+          if line == ""
+            raise "unrecognized state" unless @hc_headers.length > 0
+            if respond_to?(:receive_headers)
+              receive_headers @hc_headers
+            end
+            # @hc_content_length will be nil, not 0, if there was no content-length header.
+            if @hc_content_length.to_i > 0
+              set_binary_mode @hc_content_length
+            else
+              dispatch_request
+            end
+          else
+            @hc_headers << line
+            if ContentLengthPattern =~ line
+              # There are some attacks that rely on sending multiple content-length
+              # headers. This is a crude protection, but needs to become tunable.
+              raise "extraneous content-length header" if @hc_content_length
+              @hc_content_length = $1.to_i
+            end
+            if @hc_headers.length == 1 and respond_to?(:receive_first_header_line)
+              receive_first_header_line line
+            end
+          end
+        else
+          raise "internal error, unsupported mode"
+        end
+      end
+
+      def receive_binary_data text
+        @hc_content = text
+        dispatch_request
+      end
+
+      def dispatch_request
+        if respond_to?(:receive_request)
+          receive_request @hc_headers, @hc_content
+        end
+        init_for_request
+      end
+      private :dispatch_request
+
+      def init_for_request
+        @hc_mode = :discard_blanks
+        @hc_headers = []
+        # originally was @hc_headers ||= []; @hc_headers.clear to get a performance
+        # boost, but it's counterproductive because a subclassed handler will have to
+        # call dup to use the header array we pass in receive_headers.
+
+        @hc_content_length = nil
+        @hc_content = ""
+      end
+      private :init_for_request
+
+      # Basically a convenience method. We might create a subclass that does this
+      # automatically. But it's such a performance killer.
+      def headers_2_hash hdrs
+        self.class.headers_2_hash hdrs
+      end
+
+      class << self
+        def headers_2_hash hdrs
+          hash = {}
+          hdrs.each {|h|
+            if /\A([^\s:]+)\s*:\s*/ =~ h
+              tail = $'.dup
+              hash[ $1.downcase.gsub(/-/,"_").intern ] = tail
+            end
+          }
+          hash
+        end
+      end
+
+    end
+  end
+end

data/lib/em/protocols/httpclient.rb

@@ -0,0 +1,276 @@
+#--
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 16 July 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+
+
+
+module EventMachine
+  module Protocols
+
+    # === Usage
+    #
+    #  EventMachine.run {
+    #    http = EventMachine::Protocols::HttpClient.request(
+    #      :host => server,
+    #      :port => 80,
+    #      :request => "/index.html",
+    #      :query_string => "parm1=value1&parm2=value2"
+    #    )
+    #    http.callback {|response|
+    #      puts response[:status]
+    #      puts response[:headers]
+    #      puts response[:content]
+    #    }
+    #  }
+    #--
+    # TODO:
+    # Add streaming so we can support enormous POSTs. Current max is 20meg.
+    # Timeout for connections that run too long or hang somewhere in the middle.
+    # Persistent connections (HTTP/1.1), may need a associated delegate object.
+    # DNS: Some way to cache DNS lookups for hostnames we connect to. Ruby's
+    # DNS lookups are unbelievably slow.
+    # HEAD requests.
+    # Chunked transfer encoding.
+    # Convenience methods for requests. get, post, url, etc.
+    # SSL.
+    # Handle status codes like 304, 100, etc.
+    # Refactor this code so that protocol errors all get handled one way (an exception?),
+    # instead of sprinkling set_deferred_status :failed calls everywhere.
+    class HttpClient < Connection
+      include EventMachine::Deferrable
+
+      MaxPostContentLength = 20 * 1024 * 1024
+
+      # === Arg list
+      # :host => 'ip/dns', :port => fixnum, :verb => 'GET', :request => 'path',
+      # :basic_auth => {:username => '', :password => ''}, :content => 'content',
+      # :contenttype => 'text/plain', :query_string => '', :host_header => '',
+      # :cookie => '', :ssl => false
+      def self.request( args = {} )
+        args[:port] ||= args[:ssl] ? 443 : 80
+        EventMachine.connect( args[:host], args[:port], self, args )
+      end
+
+      def initialize(args)
+        @args = args
+      end
+
+      def post_init
+        start_tls if @args[:ssl]
+        @start_time = Time.now
+        @data = ""
+        @read_state = :base
+      end
+
+      # We send the request when we get a connection.
+      # AND, we set an instance variable to indicate we passed through here.
+      # That allows #unbind to know whether there was a successful connection.
+      # NB: This naive technique won't work when we have to support multiple
+      # requests on a single connection.
+      def connection_completed
+        return if @args[:ssl]
+        @connected = true
+        send_request @args
+      end
+      
+      def ssl_handshake_completed
+        @connected = true
+        send_request @args        
+      end
+
+      def send_request args
+        args[:verb] ||= args[:method] # Support :method as an alternative to :verb.
+        args[:verb] ||= :get # IS THIS A GOOD IDEA, to default to GET if nothing was specified?
+
+        verb = args[:verb].to_s.upcase
+        unless ["GET", "POST", "PUT", "DELETE", "HEAD"].include?(verb)
+          set_deferred_status :failed, {:status => 0} # TODO, not signalling the error type
+          return # NOTE THE EARLY RETURN, we're not sending any data.
+        end
+
+        request = args[:request] || "/"
+        unless request[0,1] == "/"
+          request = "/" + request
+        end
+
+        qs = args[:query_string] || ""
+        if qs.length > 0 and qs[0,1] != '?'
+          qs = "?" + qs
+        end
+
+        version = args[:version] || "1.1"
+
+        # Allow an override for the host header if it's not the connect-string.
+        host = args[:host_header] || args[:host] || "_"
+        # For now, ALWAYS tuck in the port string, although we may want to omit it if it's the default.
+        port = args[:port]
+
+        # POST items.
+        postcontenttype = args[:contenttype] || "application/octet-stream"
+        postcontent = args[:content] || ""
+        raise "oversized content in HTTP POST" if postcontent.length > MaxPostContentLength
+
+        # ESSENTIAL for the request's line-endings to be CRLF, not LF. Some servers misbehave otherwise.
+        # TODO: We ASSUME the caller wants to send a 1.1 request. May not be a good assumption.
+        req = [
+          "#{verb} #{request}#{qs} HTTP/#{version}",
+          "Host: #{host}:#{port}",
+          "User-agent: Ruby EventMachine",
+        ]
+
+          if verb == "POST" || verb == "PUT"
+            req << "Content-type: #{postcontenttype}"
+            req << "Content-length: #{postcontent.length}"
+          end
+
+          # TODO, this cookie handler assumes it's getting a single, semicolon-delimited string.
+          # Eventually we will want to deal intelligently with arrays and hashes.
+          if args[:cookie]
+            req << "Cookie: #{args[:cookie]}"
+          end
+
+          # Allow custom HTTP headers, e.g. SOAPAction
+          args[:custom_headers].each do |k,v|
+            req << "#{k}: #{v}"
+          end if args[:custom_headers]
+
+          # Basic-auth stanza contributed by Matt Murphy.
+          if args[:basic_auth]
+            basic_auth_string = ["#{args[:basic_auth][:username]}:#{args[:basic_auth][:password]}"].pack('m').strip.gsub(/\n/,'')
+            req << "Authorization: Basic #{basic_auth_string}"
+          end
+
+          req << ""
+          reqstring = req.map {|l| "#{l}\r\n"}.join
+          send_data reqstring
+
+          if verb == "POST" || verb == "PUT"
+            send_data postcontent
+          end
+      end
+
+
+      def receive_data data
+        while data and data.length > 0
+          case @read_state
+          when :base
+            # Perform any per-request initialization here and don't consume any data.
+            @data = ""
+            @headers = []
+            @content_length = nil # not zero
+            @content = ""
+            @status = nil
+            @read_state = :header
+            @connection_close = nil
+          when :header
+            ary = data.split( /\r?\n/m, 2 )
+            if ary.length == 2
+              data = ary.last
+              if ary.first == ""
+                if (@content_length and @content_length > 0) || @connection_close
+                  @read_state = :content
+                else
+                  dispatch_response
+                  @read_state = :base
+                end
+              else
+                @headers << ary.first
+                if @headers.length == 1
+                  parse_response_line
+                elsif ary.first =~ /\Acontent-length:\s*/i
+                  # Only take the FIRST content-length header that appears,
+                  # which we can distinguish because @content_length is nil.
+                  # TODO, it's actually a fatal error if there is more than one
+                  # content-length header, because the caller is presumptively
+                  # a bad guy. (There is an exploit that depends on multiple
+                  # content-length headers.)
+                  @content_length ||= $'.to_i
+                elsif ary.first =~ /\Aconnection:\s*close/i
+                  @connection_close = true
+                end
+              end
+            else
+              @data << data
+              data = ""
+            end
+          when :content
+            # If there was no content-length header, we have to wait until the connection
+            # closes. Everything we get until that point is content.
+            # TODO: Must impose a content-size limit, and also must implement chunking.
+            # Also, must support either temporary files for large content, or calling
+            # a content-consumer block supplied by the user.
+            if @content_length
+              bytes_needed = @content_length - @content.length
+              @content += data[0, bytes_needed]
+              data = data[bytes_needed..-1] || ""
+              if @content_length == @content.length
+                dispatch_response
+                @read_state = :base
+              end
+            else
+              @content << data
+              data = ""
+            end
+          end
+        end
+      end
+
+
+      # We get called here when we have received an HTTP response line.
+      # It's an opportunity to throw an exception or trigger other exceptional
+      # handling.
+      def parse_response_line
+        if @headers.first =~ /\AHTTP\/1\.[01] ([\d]{3})/
+          @status = $1.to_i
+        else
+          set_deferred_status :failed, {
+            :status => 0 # crappy way of signifying an unrecognized response. TODO, find a better way to do this.
+          }
+          close_connection
+        end
+      end
+      private :parse_response_line
+
+      def dispatch_response
+        @read_state = :base
+        set_deferred_status :succeeded, {
+          :content => @content,
+          :headers => @headers,
+          :status => @status
+        }
+        # TODO, we close the connection for now, but this is wrong for persistent clients.
+        close_connection
+      end
+
+      def unbind
+        if !@connected
+          set_deferred_status :failed, {:status => 0} # YECCCCH. Find a better way to signal no-connect/network error.
+        elsif (@read_state == :content and @content_length == nil)
+          dispatch_response
+        end
+      end
+    end
+
+  end
+end