wj_eventmachine 1.3.0.dev.1

data/lib/em/protocols/line_and_text.rb

@@ -0,0 +1,125 @@
+#--
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 15 November 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+# 
+
+module EventMachine
+  module Protocols
+    # A protocol that handles line-oriented data with interspersed binary text.
+    #
+    # This version is optimized for performance. See EventMachine::Protocols::LineText2
+    # for a version which is optimized for correctness with regard to binary text blocks
+    # that can switch back to line mode.
+    class LineAndTextProtocol < Connection
+      MaxBinaryLength = 32*1024*1024
+
+      def initialize *args
+        super
+        lbp_init_line_state
+      end
+
+      def receive_data data
+        if @lbp_mode == :lines
+          begin
+            @lpb_buffer.extract(data).each do |line|
+              receive_line(line.chomp) if respond_to?(:receive_line)
+            end
+          rescue
+            receive_error('overlength line') if respond_to?(:receive_error)
+            close_connection
+            return
+          end
+        else
+          if @lbp_binary_limit > 0
+            wanted = @lbp_binary_limit - @lbp_binary_bytes_received
+            chunk = nil
+            if data.length > wanted
+              chunk = data.slice!(0...wanted)
+            else
+              chunk = data
+              data = ""
+            end
+            @lbp_binary_buffer[@lbp_binary_bytes_received...(@lbp_binary_bytes_received+chunk.length)] = chunk
+            @lbp_binary_bytes_received += chunk.length
+            if @lbp_binary_bytes_received == @lbp_binary_limit
+              receive_binary_data(@lbp_binary_buffer) if respond_to?(:receive_binary_data)
+              lbp_init_line_state
+            end
+            receive_data(data) if data.length > 0
+          else
+            receive_binary_data(data) if respond_to?(:receive_binary_data)
+            data = ""
+          end
+        end
+      end
+
+      def unbind
+        if @lbp_mode == :binary and @lbp_binary_limit > 0
+          if respond_to?(:receive_binary_data)
+            receive_binary_data( @lbp_binary_buffer[0...@lbp_binary_bytes_received] )
+          end
+        end
+      end
+
+      # Set up to read the supplied number of binary bytes.
+      # This recycles all the data currently waiting in the line buffer, if any.
+      # If the limit is nil, then ALL subsequent data will be treated as binary
+      # data and passed to the upstream protocol handler as we receive it.
+      # If a limit is given, we'll hold the incoming binary data and not
+      # pass it upstream until we've seen it all, or until there is an unbind
+      # (in which case we'll pass up a partial).
+      # Specifying nil for the limit (the default) means there is no limit.
+      # Specifiyng zero for the limit will cause an immediate transition back to line mode.
+      #
+      def set_binary_mode size = nil
+        if @lbp_mode == :lines
+          if size == 0
+            receive_binary_data("") if respond_to?(:receive_binary_data)
+            # Do no more work here. Stay in line mode and keep consuming data.
+          else
+            @lbp_binary_limit = size.to_i # (nil will be stored as zero)
+            if @lbp_binary_limit > 0
+              raise "Overlength" if @lbp_binary_limit > MaxBinaryLength # arbitrary sanity check
+              @lbp_binary_buffer = "\0" * @lbp_binary_limit
+              @lbp_binary_bytes_received = 0
+            end
+
+            @lbp_mode = :binary
+            receive_data @lpb_buffer.flush
+          end
+        else
+          raise "invalid operation"
+        end
+      end
+
+      #--
+      # For internal use, establish protocol baseline for handling lines.
+      def lbp_init_line_state
+        @lpb_buffer = BufferedTokenizer.new("\n")
+        @lbp_mode = :lines
+      end
+      private :lbp_init_line_state
+    end
+  end
+end

data/lib/em/protocols/line_protocol.rb

@@ -0,0 +1,33 @@
+module EventMachine
+  module Protocols
+    # LineProtocol will parse out newline terminated strings from a receive_data stream
+    #
+    #  module Server
+    #    include EM::P::LineProtocol
+    #
+    #    def receive_line(line)
+    #      send_data("you said: #{line}")
+    #    end
+    #  end
+    #
+    module LineProtocol
+      # @private
+      def receive_data data
+        (@buf ||= '') << data
+
+        @buf.each_line do |line|
+          if line[-1] == "\n"
+            receive_line(line.chomp)
+          else
+            @buf = line
+          end
+        end
+      end
+
+      # Invoked with lines received over the network
+      def receive_line(line)
+        # stub
+      end
+    end
+  end
+end

data/lib/em/protocols/linetext2.rb

@@ -0,0 +1,179 @@
+#--
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 15 November 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-07 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+
+module EventMachine
+  module Protocols
+    # In the grand, time-honored tradition of re-inventing the wheel, we offer
+    # here YET ANOTHER protocol that handles line-oriented data with interspersed
+    # binary text. This one trades away some of the performance optimizations of
+    # EventMachine::Protocols::LineAndTextProtocol in order to get better correctness
+    # with regard to binary text blocks that can switch back to line mode. It also
+    # permits the line-delimiter to change in midstream.
+    # This was originally written to support Stomp.
+    module LineText2
+      # TODO! We're not enforcing the limits on header lengths and text-lengths.
+      # When we get around to that, call #receive_error if the user defined it, otherwise
+      # throw exceptions.
+
+      MaxBinaryLength = 32*1024*1024
+
+      #--
+      # Will loop internally until there's no data left to read.
+      # That way the user-defined handlers we call can modify the
+      # handling characteristics on a per-token basis.
+      #
+      def receive_data data
+        return unless (data and data.length > 0)
+
+        # Do this stuff in lieu of a constructor.
+        @lt2_mode ||= :lines
+        @lt2_delimiter ||= "\n"
+        @lt2_linebuffer ||= []
+
+        remaining_data = data
+
+        while remaining_data.length > 0
+          if @lt2_mode == :lines
+            delimiter_string = case @lt2_delimiter
+            when Regexp
+              remaining_data.slice(@lt2_delimiter)
+            else
+              @lt2_delimiter
+            end
+            ix = remaining_data.index(delimiter_string) if delimiter_string
+            if ix
+              @lt2_linebuffer << remaining_data[0...ix]
+              ln = @lt2_linebuffer.join
+              @lt2_linebuffer.clear
+              if @lt2_delimiter == "\n"
+                ln.chomp!
+              end
+              receive_line ln
+              remaining_data = remaining_data[(ix+delimiter_string.length)..-1]
+            else
+              @lt2_linebuffer << remaining_data
+              remaining_data = ""
+            end
+          elsif @lt2_mode == :text
+            if @lt2_textsize
+              needed = @lt2_textsize - @lt2_textpos
+              will_take = if remaining_data.length > needed
+                            needed
+                          else
+                            remaining_data.length
+                          end
+
+              @lt2_textbuffer << remaining_data[0...will_take]
+              tail = remaining_data[will_take..-1]
+
+              @lt2_textpos += will_take
+              if @lt2_textpos >= @lt2_textsize
+                # Reset line mode (the default behavior) BEFORE calling the
+                # receive_binary_data. This makes it possible for user code
+                # to call set_text_mode, enabling chains of text blocks
+                # (which can possibly be of different sizes).
+                set_line_mode
+                receive_binary_data @lt2_textbuffer.join
+                receive_end_of_binary_data
+              end
+
+              remaining_data = tail
+            else
+              receive_binary_data remaining_data
+              remaining_data = ""
+            end
+          end
+        end
+      end
+
+      # The line delimiter may be a regular expression or a string.  Anything
+      # passed to set_delimiter other than a regular expression will be
+      # converted to a string.
+      def set_delimiter delim
+        @lt2_delimiter = case delim
+        when Regexp
+          delim
+        else
+          delim.to_s
+        end
+      end
+
+      # Called internally but also exposed to user code, for the case in which
+      # processing of binary data creates a need to transition back to line mode.
+      # We support an optional parameter to "throw back" some data, which might
+      # be an umprocessed chunk of the transmitted binary data, or something else
+      # entirely.
+      def set_line_mode data=""
+        @lt2_mode = :lines
+        (@lt2_linebuffer ||= []).clear
+        receive_data data.to_s
+      end
+
+      def set_text_mode size=nil
+        if size == 0
+          set_line_mode
+        else
+          @lt2_mode = :text
+          (@lt2_textbuffer ||= []).clear
+          @lt2_textsize = size # which can be nil, signifying no limit
+          @lt2_textpos = 0
+        end
+      end
+
+      # Alias for #set_text_mode, added for back-compatibility with LineAndTextProtocol.
+      def set_binary_mode size=nil
+        set_text_mode size
+      end
+
+      # In case of a dropped connection, we'll send a partial buffer to user code
+      # when in sized text mode. User overrides of #receive_binary_data need to
+      # be aware that they may get a short buffer.
+      def unbind
+        @lt2_mode ||= nil
+        if @lt2_mode == :text and @lt2_textpos > 0
+          receive_binary_data @lt2_textbuffer.join
+        end
+      end
+
+      # Stub. Should be subclassed by user code.
+      def receive_line ln
+        # no-op
+      end
+
+      # Stub. Should be subclassed by user code.
+      def receive_binary_data data
+        # no-op
+      end
+
+      # Stub. Should be subclassed by user code.
+      # This is called when transitioning internally from text mode
+      # back to line mode. Useful when client code doesn't want
+      # to keep track of how much data it's received.
+      def receive_end_of_binary_data
+        # no-op
+      end
+    end
+  end
+end

data/lib/em/protocols/memcache.rb

@@ -0,0 +1,331 @@
+module EventMachine
+  module Protocols
+    # Implements the Memcache protocol (http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt).
+    # Requires memcached >= 1.2.4 w/ noreply support
+    #
+    # == Usage example
+    #
+    #   EM.run{
+    #     cache = EM::P::Memcache.connect 'localhost', 11211
+    #
+    #     cache.set :a, 'hello'
+    #     cache.set :b, 'hi'
+    #     cache.set :c, 'how are you?'
+    #     cache.set :d, ''
+    #
+    #     cache.get(:a){ |v| p v }
+    #     cache.get_hash(:a, :b, :c, :d){ |v| p v }
+    #     cache.get(:a,:b,:c,:d){ |a,b,c,d| p [a,b,c,d] }
+    #
+    #     cache.get(:a,:z,:b,:y,:d){ |a,z,b,y,d| p [a,z,b,y,d] }
+    #
+    #     cache.get(:missing){ |m| p [:missing=, m] }
+    #     cache.set(:missing, 'abc'){ p :stored }
+    #     cache.get(:missing){ |m| p [:missing=, m] }
+    #     cache.del(:missing){ p :deleted }
+    #     cache.get(:missing){ |m| p [:missing=, m] }
+    #   }
+    #
+    module Memcache
+      include EM::Deferrable
+
+      ##
+      # constants
+
+      unless defined? Cempty
+        # @private
+        Cstored    = 'STORED'.freeze
+        # @private
+        Cend       = 'END'.freeze
+        # @private
+        Cdeleted   = 'DELETED'.freeze
+        # @private
+        Cunknown   = 'NOT_FOUND'.freeze
+        # @private
+        Cerror     = 'ERROR'.freeze
+
+        # @private
+        Cempty     = ''.freeze
+        # @private
+        Cdelimiter = "\r\n".freeze
+      end
+
+      ##
+      # commands
+
+      # Get the value associated with one or multiple keys
+      #
+      #  cache.get(:a){ |v| p v }
+      #  cache.get(:a,:b,:c,:d){ |a,b,c,d| p [a,b,c,d] }
+      #
+      def get *keys
+        raise ArgumentError unless block_given?
+
+        callback{
+          keys = keys.map{|k| k.to_s.gsub(/\s/,'_') }
+          send_data "get #{keys.join(' ')}\r\n"
+          @get_cbs << [keys, proc{ |values|
+            yield *keys.map{ |k| values[k] }
+          }]
+        }
+      end
+
+      # Set the value for a given key
+      #
+      #  cache.set :a, 'hello'
+      #  cache.set(:missing, 'abc'){ puts "stored the value!" }
+      #
+      def set key, val, exptime = 0, &cb
+        callback{
+          val = val.to_s
+          send_cmd :set, key, 0, exptime, val.respond_to?(:bytesize) ? val.bytesize : val.size, !block_given?
+          send_data val
+          send_data Cdelimiter
+          @set_cbs << cb if cb
+        }
+      end
+
+      # Gets multiple values as a hash
+      #
+      #  cache.get_hash(:a, :b, :c, :d){ |h| puts h[:a] }
+      #
+      def get_hash *keys
+        raise ArgumentError unless block_given?
+
+        get *keys do |*values|
+          yield keys.inject({}){ |hash, k| hash.update k => values[keys.index(k)] }
+        end
+      end
+
+      # Delete the value associated with a key
+      #
+      #  cache.del :a
+      #  cache.del(:b){ puts "deleted the value!" }
+      #
+      def delete key, expires = 0, &cb
+        callback{
+          send_data "delete #{key} #{expires}#{cb ? '' : ' noreply'}\r\n"
+          @del_cbs << cb if cb
+        }
+      end
+      alias del delete
+
+      # Connect to a memcached server (must support NOREPLY, memcached >= 1.2.4)
+      def self.connect host = 'localhost', port = 11211
+        EM.connect host, port, self, host, port
+      end
+
+      def send_cmd cmd, key, flags = 0, exptime = 0, bytes = 0, noreply = false
+        send_data "#{cmd} #{key} #{flags} #{exptime} #{bytes}#{noreply ? ' noreply' : ''}\r\n"
+      end
+      private :send_cmd
+
+      ##
+      # errors
+
+      # @private
+      class ParserError < StandardError
+      end
+
+      ##
+      # em hooks
+
+      # @private
+      def initialize host, port = 11211
+        @host, @port = host, port
+      end
+
+      # @private
+      def connection_completed
+        @get_cbs = []
+        @set_cbs = []
+        @del_cbs = []
+
+        @values = {}
+
+        @reconnecting = false
+        @connected = true
+        succeed
+        # set_delimiter "\r\n"
+        # set_line_mode
+      end
+
+      #--
+      # 19Feb09 Switched to a custom parser, LineText2 is recursive and can cause
+      #         stack overflows when there is too much data.
+      # include EM::P::LineText2
+      # @private
+      def receive_data data
+        (@buffer||='') << data
+
+        while index = @buffer.index(Cdelimiter)
+          begin
+            line = @buffer.slice!(0,index+2)
+            process_cmd line
+          rescue ParserError
+            @buffer[0...0] = line
+            break
+          end
+        end
+      end
+
+      #--
+      # def receive_line line
+      # @private
+      def process_cmd line
+        case line.strip
+        when /^VALUE\s+(.+?)\s+(\d+)\s+(\d+)/ # VALUE <key> <flags> <bytes>
+          bytes = Integer($3)
+          # set_binary_mode bytes+2
+          # @cur_key = $1
+          if @buffer.size >= bytes + 2
+            @values[$1] = @buffer.slice!(0,bytes)
+            @buffer.slice!(0,2) # \r\n
+          else
+            raise ParserError
+          end
+
+        when Cend # END
+          if entry = @get_cbs.shift
+            keys, cb = entry
+            cb.call(@values)
+          end
+          @values = {}
+
+        when Cstored # STORED
+          if cb = @set_cbs.shift
+            cb.call(true)
+          end
+
+        when Cdeleted # DELETED
+          if cb = @del_cbs.shift
+            cb.call(true)
+          end
+
+        when Cunknown # NOT_FOUND
+          if cb = @del_cbs.shift
+            cb.call(false)
+          end
+
+        else
+          p [:MEMCACHE_UNKNOWN, line]
+        end
+      end
+
+      #--
+      # def receive_binary_data data
+      #   @values[@cur_key] = data[0..-3]
+      # end
+
+      # @private
+      def unbind
+        if @connected or @reconnecting
+          EM.add_timer(1){ reconnect @host, @port }
+          @connected = false
+          @reconnecting = true
+          @deferred_status = nil
+        else
+          raise 'Unable to connect to memcached server'
+        end
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  # ruby -I ext:lib -r eventmachine -rubygems lib/protocols/memcache.rb
+  require 'em/spec'
+
+  # @private
+  class TestConnection
+    include EM::P::Memcache
+    def send_data data
+      sent_data << data
+    end
+    def sent_data
+      @sent_data ||= ''
+    end
+
+    def initialize
+      connection_completed
+    end
+  end
+
+  EM.describe EM::Protocols::Memcache do
+
+    before{
+      @c = TestConnection.new
+    }
+
+    should 'send get requests' do
+      @c.get('a'){}
+      @c.sent_data.should == "get a\r\n"
+      done
+    end
+
+    should 'send set requests' do
+      @c.set('a', 1){}
+      @c.sent_data.should == "set a 0 0 1\r\n1\r\n"
+      done
+    end
+
+    should 'use noreply on set without block' do
+      @c.set('a', 1)
+      @c.sent_data.should == "set a 0 0 1 noreply\r\n1\r\n"
+      done
+    end
+
+    should 'send delete requests' do
+      @c.del('a')
+      @c.sent_data.should == "delete a 0 noreply\r\n"
+      done
+    end
+
+    should 'work when get returns no values' do
+      @c.get('a'){ |a|
+        a.should.be.nil
+        done
+      }
+
+      @c.receive_data "END\r\n"
+    end
+
+    should 'invoke block on set' do
+      @c.set('a', 1){
+        done
+      }
+
+      @c.receive_data "STORED\r\n"
+    end
+
+    should 'invoke block on delete' do
+      @c.delete('a'){ |found|
+        found.should.be.false
+      }
+      @c.delete('b'){ |found|
+        found.should.be.true
+        done
+      }
+
+      @c.receive_data "NOT_FOUND\r\n"
+      @c.receive_data "DELETED\r\n"
+    end
+
+    should 'parse split responses' do
+      @c.get('a'){ |a|
+        a.should == 'abc'
+        done
+      }
+
+      @c.receive_data "VAL"
+      @c.receive_data "UE a 0 "
+      @c.receive_data "3\r\n"
+      @c.receive_data "ab"
+      @c.receive_data "c"
+      @c.receive_data "\r\n"
+      @c.receive_data "EN"
+      @c.receive_data "D\r\n"
+    end
+
+  end
+end