eventmachine 0.12.6 → 0.12.8

data/lib/{protocols → em/protocols}/line_and_text.rb

@@ -1,4 +1,4 @@
-# $Id$
+#--
 #
 # Author:: Francis Cianfrocca (gmail: blackhedd)
 # Homepage::  http://rubyeventmachine.com
@@ -23,7 +23,7 @@
 #
 # 
 # 
-require File.dirname(__FILE__) + '/buftok'
+require File.dirname(__FILE__) + '/../buftok'
 
 module EventMachine
   module Protocols

data/lib/em/protocols/linetext2.rb

@@ -0,0 +1,160 @@
+#--
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 15 November 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+
+module EventMachine
+  module Protocols
+    # In the grand, time-honored tradition of re-inventing the wheel, we offer
+    # here YET ANOTHER protocol that handles line-oriented data with interspersed
+    # binary text. This one trades away some of the performance optimizations of
+    # EventMachine::Protocols::LineAndTextProtocol in order to get better correctness
+    # with regard to binary text blocks that can switch back to line mode. It also
+    # permits the line-delimiter to change in midstream.
+    # This was originally written to support Stomp.
+    module LineText2
+      # TODO! We're not enforcing the limits on header lengths and text-lengths.
+      # When we get around to that, call #receive_error if the user defined it, otherwise
+      # throw exceptions.
+
+      MaxLineLength = 16*1024
+      MaxBinaryLength = 32*1024*1024
+
+      #--
+      # Will be called recursively until there's no data to read.
+      # That way the user-defined handlers we call can modify the
+      # handling characteristics on a per-token basis.
+      #
+      def receive_data data
+        return unless (data and data.length > 0)
+
+        # Do this stuff in lieu of a constructor.
+        @lt2_mode ||= :lines
+        @lt2_delimiter ||= "\n"
+        @lt2_linebuffer ||= []
+
+        if @lt2_mode == :lines
+          if ix = data.index( @lt2_delimiter )
+            @lt2_linebuffer << data[0...ix]
+            ln = @lt2_linebuffer.join
+            @lt2_linebuffer.clear
+            if @lt2_delimiter == "\n"
+              ln.chomp!
+            end
+            receive_line ln
+            receive_data data[(ix+@lt2_delimiter.length)..-1]
+          else
+            @lt2_linebuffer << data
+          end
+        elsif @lt2_mode == :text
+          if @lt2_textsize
+            needed = @lt2_textsize - @lt2_textpos
+            will_take = if data.length > needed
+                          needed
+                        else
+                          data.length
+                        end
+
+            @lt2_textbuffer << data[0...will_take]
+            tail = data[will_take..-1]
+
+            @lt2_textpos += will_take
+            if @lt2_textpos >= @lt2_textsize
+              # Reset line mode (the default behavior) BEFORE calling the
+              # receive_binary_data. This makes it possible for user code
+              # to call set_text_mode, enabling chains of text blocks
+              # (which can possibly be of different sizes).
+              set_line_mode
+              receive_binary_data @lt2_textbuffer.join
+              receive_end_of_binary_data
+            end
+
+            receive_data tail
+          else
+            receive_binary_data data
+          end
+        end
+      end
+
+
+      def set_delimiter delim
+        @lt2_delimiter = delim.to_s
+      end
+
+      # Called internally but also exposed to user code, for the case in which
+      # processing of binary data creates a need to transition back to line mode.
+      # We support an optional parameter to "throw back" some data, which might
+      # be an umprocessed chunk of the transmitted binary data, or something else
+      # entirely.
+      def set_line_mode data=""
+        @lt2_mode = :lines
+        (@lt2_linebuffer ||= []).clear
+        receive_data data.to_s
+      end
+
+      def set_text_mode size=nil
+        if size == 0
+          set_line_mode
+        else
+          @lt2_mode = :text
+          (@lt2_textbuffer ||= []).clear
+          @lt2_textsize = size # which can be nil, signifying no limit
+          @lt2_textpos = 0
+        end
+      end
+
+      # Alias for #set_text_mode, added for back-compatibility with LineAndTextProtocol.
+      def set_binary_mode size=nil
+        set_text_mode size
+      end
+
+      # In case of a dropped connection, we'll send a partial buffer to user code
+      # when in sized text mode. User overrides of #receive_binary_data need to
+      # be aware that they may get a short buffer.
+      def unbind
+        if @lt2_mode == :text and @lt2_textpos > 0
+          receive_binary_data @lt2_textbuffer.join
+        end
+      end
+
+      # Stub. Should be subclassed by user code.
+      def receive_line ln
+        # no-op
+      end
+
+      # Stub. Should be subclassed by user code.
+      def receive_binary_data data
+        # no-op
+      end
+
+      # Stub. Should be subclassed by user code.
+      # This is called when transitioning internally from text mode
+      # back to line mode. Useful when client code doesn't want
+      # to keep track of how much data it's received.
+      def receive_end_of_binary_data
+        # no-op
+      end
+    end
+  end
+end
+

data/lib/{protocols → em/protocols}/memcache.rb

@@ -32,6 +32,7 @@ module EventMachine
       ##
       # constants
 
+      # :stopdoc:
       unless defined? Cempty
         Cstored    = 'STORED'.freeze
         Cend       = 'END'.freeze
@@ -42,10 +43,16 @@ module EventMachine
         Cempty     = ''.freeze
         Cdelimiter = "\r\n".freeze
       end
+      # :startdoc:
 
       ##
       # commands
 
+      # Get the value associated with one or multiple keys
+      #
+      #  cache.get(:a){ |v| p v }
+      #  cache.get(:a,:b,:c,:d){ |a,b,c,d| p [a,b,c,d] }
+      #
       def get *keys
         raise ArgumentError unless block_given?
 
@@ -58,6 +65,11 @@ module EventMachine
         }
       end
 
+      # Set the value for a given key
+      #
+      #  cache.set :a, 'hello'
+      #  cache.set(:missing, 'abc'){ puts "stored the value!" }
+      #
       def set key, val, exptime = 0, &cb
         callback{
           val = val.to_s
@@ -68,6 +80,10 @@ module EventMachine
         }
       end
 
+      # Gets multiple values as a hash
+      #
+      #  cache.get_hash(:a, :b, :c, :d){ |h| puts h[:a] }
+      #
       def get_hash *keys
         raise ArgumentError unless block_given?
 
@@ -76,6 +92,11 @@ module EventMachine
         end
       end
 
+      # Delete the value associated with a key
+      #
+      #  cache.del :a
+      #  cache.del(:b){ puts "deleted the value!" }
+      #
       def delete key, expires = 0, &cb
         callback{
           send_data "delete #{key} #{expires}#{cb ? '' : ' noreply'}\r\n"
@@ -84,7 +105,14 @@ module EventMachine
       end
       alias del delete
 
-      def send_cmd cmd, key, flags = 0, exptime = 0, bytes = 0, noreply = false
+      # Connect to a memcached server (must support NOREPLY, memcached >= 1.2.4)
+      def self.connect host = 'localhost', port = 11211
+        EM.connect host, port, self, host, port
+      end
+
+      # :stopdoc:
+
+      def send_cmd cmd, key, flags = 0, exptime = 0, bytes = 0, noreply = false # :nodoc:
         send_data "#{cmd} #{key} #{flags} #{exptime} #{bytes}#{noreply ? ' noreply' : ''}\r\n"
       end
       private :send_cmd
@@ -92,15 +120,12 @@ module EventMachine
       ##
       # errors
 
-      class ParserError < StandardError; end
+      class ParserError < StandardError
+      end
 
       ##
       # em hooks
 
-      def self.connect host = 'localhost', port = 11211
-        EM.connect host, port, self, host, port
-      end
-
       def initialize host, port = 11211
         @host, @port = host, port
       end
@@ -119,6 +144,7 @@ module EventMachine
         # set_line_mode
       end
 
+      #--
       # 19Feb09 Switched to a custom parser, LineText2 is recursive and can cause
       #         stack overflows when there is too much data.
       # include EM::P::LineText2
@@ -136,6 +162,7 @@ module EventMachine
         end
       end
 
+      #--
       # def receive_line line
       def process_cmd line
         case line.strip
@@ -177,6 +204,7 @@ module EventMachine
         end
       end
 
+      #--
       # def receive_binary_data data
       #   @values[@cur_key] = data[0..-3]
       # end
@@ -191,6 +219,8 @@ module EventMachine
           raise 'Unable to connect to memcached server'
         end
       end
+
+      # :startdoc:
     end
   end
 end
@@ -199,7 +229,7 @@ if __FILE__ == $0
   # ruby -I ext:lib -r eventmachine -rubygems lib/protocols/memcache.rb
   require 'em/spec'
 
-  class TestConnection
+  class TestConnection # :nodoc:
     include EM::P::Memcache
     def send_data data
       sent_data << data

data/lib/em/protocols/object_protocol.rb

@@ -0,0 +1,39 @@
+module EventMachine
+  module Protocols
+    # ObjectProtocol allows for easy communication using marshaled ruby objects
+    #
+    #  module RubyServer
+    #    include EM::P::ObjectProtocol
+    #
+    #    def receive_object obj
+    #      send_object({'you said' => obj})
+    #    end
+    #  end
+    #
+    module ObjectProtocol
+      def receive_data data # :nodoc:
+        (@buf ||= '') << data
+
+        while @buf.size >= 4
+          if @buf.size >= 4+(size=@buf.unpack('N').first)
+            @buf.slice!(0,4)
+            receive_object Marshal.load(@buf.slice!(0,size))
+          else
+            break
+          end
+        end
+      end
+
+      # Invoked with ruby objects received over the network
+      def receive_object obj
+        # stub
+      end
+
+      # Sends a ruby object over the network
+      def send_object obj
+        data = Marshal.dump(obj)
+        send_data [data.respond_to?(:bytesize) ? data.bytesize : data.size, data].pack('Na*')
+      end
+    end
+  end
+end

data/lib/em/protocols/postgres3.rb

@@ -0,0 +1,247 @@
+#--
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 15 November 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-08 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+# 
+
+require 'readbytes'
+require 'postgres-pr/message'
+require 'postgres-pr/connection'
+require 'stringio'
+
+class StringIO # :nodoc:
+  # Reads exactly +n+ bytes.
+  #
+  # If the data read is nil an EOFError is raised.
+  #
+  # If the data read is too short a TruncatedDataError is raised and the read
+  # data is obtainable via its #data method.
+  def readbytes(n)
+    str = read(n)
+    if str == nil
+      raise EOFError, "End of file reached"
+    end
+    if str.size < n
+      raise TruncatedDataError.new("data truncated", str) 
+    end
+    str
+  end
+  alias read_exactly_n_bytes readbytes
+end
+
+
+module EventMachine
+  module Protocols
+    # PROVISIONAL IMPLEMENTATION of an evented Postgres client.
+    # This implements version 3 of the Postgres wire protocol, which will work
+    # with any Postgres version from roughly 7.4 onward.
+    #
+    # Objective: we want to access Postgres databases without requiring threads.
+    # Until now this has been a problem because the Postgres client implementations
+    # have all made use of blocking I/O calls, which is incompatible with a
+    # thread-free evented model.
+    #
+    # But rather than re-implement the Postgres Wire3 protocol, we're taking advantage
+    # of the existing postgres-pr library, which was originally written by Michael
+    # Neumann but (at this writing) appears to be no longer maintained. Still, it's
+    # in basically a production-ready state, and the wire protocol isn't that complicated
+    # anyway.
+    #
+    # We need to monkeypatch StringIO because it lacks the #readbytes method needed
+    # by postgres-pr.
+    #
+    # We're tucking in a bunch of require statements that may not be present in garden-variety
+    # EM installations. Until we find a good way to only require these if a program
+    # requires postgres, this file will need to be required explicitly.
+    #
+    # The StringIO monkeypatch is lifted verbatim from the standard library readbytes.rb,
+    # which adds method #readbytes directly to class IO. But StringIO is not a subclass of IO.
+    #
+    # We cloned the handling of postgres messages from lib/postgres-pr/connection.rb
+    # in the postgres-pr library, and modified it for event-handling.
+    #
+    # TODO: The password handling in dispatch_conn_message is totally incomplete.
+    #
+    #
+    # We return Deferrables from the user-level operations surfaced by this interface.
+    # Experimentally, we're using the pattern of always returning a boolean value as the
+    # first argument of a deferrable callback to indicate success or failure. This is
+    # instead of the traditional pattern of calling Deferrable#succeed or #fail, and
+    # requiring the user to define both a callback and an errback function.
+    #
+    # === Usage
+    #  EM.run {
+    #    db = EM.connect_unix_domain( "/tmp/.s.PGSQL.5432", EM::P::Postgres3 )
+    #    db.connect( dbname, username, psw ).callback do |status|
+    #      if status
+    #        db.query( "select * from some_table" ).callback do |status, result, errors|
+    #          if status
+    #            result.rows.each do |row|
+    #              p row
+    #            end
+    #          end
+    #        end
+    #      end
+    #    end
+    #  }
+    class Postgres3 < EventMachine::Connection
+      include PostgresPR
+
+      def initialize
+        @data = ""
+        @params = {}
+      end
+
+      def connect db, user, psw=nil
+        d = EM::DefaultDeferrable.new
+        d.timeout 15
+
+        if @pending_query || @pending_conn
+          d.succeed false, "Operation already in progress"
+        else
+          @pending_conn = d
+          prms = {"user"=>user, "database"=>db}
+          @user = user
+          if psw
+            @password = psw
+            #prms["password"] = psw
+          end
+          send_data PostgresPR::StartupMessage.new( 3 << 16, prms ).dump
+        end
+
+        d
+      end
+
+      def query sql
+        d = EM::DefaultDeferrable.new
+        d.timeout 15
+
+        if @pending_query || @pending_conn
+          d.succeed false, "Operation already in progress"
+        else
+          @r = PostgresPR::Connection::Result.new
+          @e = []
+          @pending_query = d
+          send_data PostgresPR::Query.dump(sql)
+        end
+
+        d
+      end
+
+
+      def receive_data data
+        @data << data
+        while @data.length >= 5
+          pktlen = @data[1...5].unpack("N").first
+          if @data.length >= (1 + pktlen)
+            pkt = @data.slice!(0...(1+pktlen))
+            m = StringIO.open( pkt, "r" ) {|io| PostgresPR::Message.read( io ) }
+            if @pending_conn
+              dispatch_conn_message m
+            elsif @pending_query
+              dispatch_query_message m
+            else
+              raise "Unexpected message from database"
+            end
+          else
+            break # very important, break out of the while
+          end
+        end
+      end
+
+
+      def unbind
+        if o = (@pending_query || @pending_conn)
+          o.succeed false, "lost connection"
+        end
+      end
+
+      # Cloned and modified from the postgres-pr.
+      def dispatch_conn_message msg
+        case msg
+        when AuthentificationClearTextPassword
+          raise ArgumentError, "no password specified" if @password.nil?
+          send_data PasswordMessage.new(@password).dump
+
+        when AuthentificationCryptPassword
+          raise ArgumentError, "no password specified" if @password.nil?
+          send_data PasswordMessage.new(@password.crypt(msg.salt)).dump
+
+        when AuthentificationMD5Password
+          raise ArgumentError, "no password specified" if @password.nil?
+          require 'digest/md5'
+
+          m = Digest::MD5.hexdigest(@password + @user)
+          m = Digest::MD5.hexdigest(m + msg.salt)
+          m = 'md5' + m
+          send_data PasswordMessage.new(m).dump
+
+        when AuthentificationKerberosV4, AuthentificationKerberosV5, AuthentificationSCMCredential
+          raise "unsupported authentification"
+
+        when AuthentificationOk
+        when ErrorResponse
+          raise msg.field_values.join("\t")
+        when NoticeResponse
+          @notice_processor.call(msg) if @notice_processor
+        when ParameterStatus
+          @params[msg.key] = msg.value
+        when BackendKeyData
+          # TODO
+          #p msg
+        when ReadyForQuery
+          # TODO: use transaction status
+          pc,@pending_conn = @pending_conn,nil
+          pc.succeed true
+        else
+          raise "unhandled message type"
+        end
+      end
+
+      # Cloned and modified from the postgres-pr.
+      def dispatch_query_message msg
+        case msg
+        when DataRow
+          @r.rows << msg.columns
+        when CommandComplete
+          @r.cmd_tag = msg.cmd_tag
+        when ReadyForQuery
+          pq,@pending_query = @pending_query,nil
+          pq.succeed true, @r, @e
+        when RowDescription
+          @r.fields = msg.fields
+        when CopyInResponse
+        when CopyOutResponse
+        when EmptyQueryResponse
+        when ErrorResponse
+          # TODO
+          @e << msg
+        when NoticeResponse
+          @notice_processor.call(msg) if @notice_processor
+        else
+          # TODO
+        end
+      end
+    end
+  end
+end