eventmachine 0.12.6-java

data/lib/protocols/memcache.rb

@@ -0,0 +1,293 @@
+module EventMachine
+  module Protocols
+    # Implements the Memcache protocol (http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt).
+    # Requires memcached >= 1.2.4 w/ noreply support
+    #
+    # == Usage example
+    #
+    #   EM.run{
+    #     cache = EM::P::Memcache.connect 'localhost', 11211
+    #
+    #     cache.set :a, 'hello'
+    #     cache.set :b, 'hi'
+    #     cache.set :c, 'how are you?'
+    #     cache.set :d, ''
+    #
+    #     cache.get(:a){ |v| p v }
+    #     cache.get_hash(:a, :b, :c, :d){ |v| p v }
+    #     cache.get(:a,:b,:c,:d){ |a,b,c,d| p [a,b,c,d] }
+    #
+    #     cache.get(:a,:z,:b,:y,:d){ |a,z,b,y,d| p [a,z,b,y,d] }
+    #
+    #     cache.get(:missing){ |m| p [:missing=, m] }
+    #     cache.set(:missing, 'abc'){ p :stored }
+    #     cache.get(:missing){ |m| p [:missing=, m] }
+    #     cache.del(:missing){ p :deleted }
+    #     cache.get(:missing){ |m| p [:missing=, m] }
+    #   }
+    #
+    module Memcache
+      include EM::Deferrable
+
+      ##
+      # constants
+
+      unless defined? Cempty
+        Cstored    = 'STORED'.freeze
+        Cend       = 'END'.freeze
+        Cdeleted   = 'DELETED'.freeze
+        Cunknown   = 'NOT_FOUND'.freeze
+        Cerror     = 'ERROR'.freeze
+
+        Cempty     = ''.freeze
+        Cdelimiter = "\r\n".freeze
+      end
+
+      ##
+      # commands
+
+      def get *keys
+        raise ArgumentError unless block_given?
+
+        callback{
+          keys = keys.map{|k| k.to_s.gsub(/\s/,'_') }
+          send_data "get #{keys.join(' ')}\r\n"
+          @get_cbs << [keys, proc{ |values|
+            yield *keys.map{ |k| values[k] }
+          }]
+        }
+      end
+
+      def set key, val, exptime = 0, &cb
+        callback{
+          val = val.to_s
+          send_cmd :set, key, 0, exptime, val.respond_to?(:bytesize) ? val.bytesize : val.size, !block_given?
+          send_data val
+          send_data Cdelimiter
+          @set_cbs << cb if cb
+        }
+      end
+
+      def get_hash *keys
+        raise ArgumentError unless block_given?
+
+        get *keys do |*values|
+          yield keys.inject({}){ |hash, k| hash.update k => values[keys.index(k)] }
+        end
+      end
+
+      def delete key, expires = 0, &cb
+        callback{
+          send_data "delete #{key} #{expires}#{cb ? '' : ' noreply'}\r\n"
+          @del_cbs << cb if cb
+        }
+      end
+      alias del delete
+
+      def send_cmd cmd, key, flags = 0, exptime = 0, bytes = 0, noreply = false
+        send_data "#{cmd} #{key} #{flags} #{exptime} #{bytes}#{noreply ? ' noreply' : ''}\r\n"
+      end
+      private :send_cmd
+
+      ##
+      # errors
+
+      class ParserError < StandardError; end
+
+      ##
+      # em hooks
+
+      def self.connect host = 'localhost', port = 11211
+        EM.connect host, port, self, host, port
+      end
+
+      def initialize host, port = 11211
+        @host, @port = host, port
+      end
+
+      def connection_completed
+        @get_cbs = []
+        @set_cbs = []
+        @del_cbs = []
+
+        @values = {}
+
+        @reconnecting = false
+        @connected = true
+        succeed
+        # set_delimiter "\r\n"
+        # set_line_mode
+      end
+
+      # 19Feb09 Switched to a custom parser, LineText2 is recursive and can cause
+      #         stack overflows when there is too much data.
+      # include EM::P::LineText2
+      def receive_data data
+        (@buffer||='') << data
+
+        while index = @buffer.index(Cdelimiter)
+          begin
+            line = @buffer.slice!(0,index+2)
+            process_cmd line
+          rescue ParserError
+            @buffer[0...0] = line
+            break
+          end
+        end
+      end
+
+      # def receive_line line
+      def process_cmd line
+        case line.strip
+        when /^VALUE\s+(.+?)\s+(\d+)\s+(\d+)/ # VALUE <key> <flags> <bytes>
+          bytes = Integer($3)
+          # set_binary_mode bytes+2
+          # @cur_key = $1
+          if @buffer.size >= bytes + 2
+            @values[$1] = @buffer.slice!(0,bytes)
+            @buffer.slice!(0,2) # \r\n
+          else
+            raise ParserError
+          end
+
+        when Cend # END
+          if entry = @get_cbs.shift
+            keys, cb = entry
+            cb.call(@values)
+          end
+          @values = {}
+
+        when Cstored # STORED
+          if cb = @set_cbs.shift
+            cb.call(true)
+          end
+
+        when Cdeleted # DELETED
+          if cb = @del_cbs.shift
+            cb.call(true)
+          end
+
+        when Cunknown # NOT_FOUND
+          if cb = @del_cbs.shift
+            cb.call(false)
+          end
+
+        else
+          p [:MEMCACHE_UNKNOWN, line]
+        end
+      end
+
+      # def receive_binary_data data
+      #   @values[@cur_key] = data[0..-3]
+      # end
+
+      def unbind
+        if @connected or @reconnecting
+          EM.add_timer(1){ reconnect @host, @port }
+          @connected = false
+          @reconnecting = true
+          @deferred_status = nil
+        else
+          raise 'Unable to connect to memcached server'
+        end
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  # ruby -I ext:lib -r eventmachine -rubygems lib/protocols/memcache.rb
+  require 'em/spec'
+
+  class TestConnection
+    include EM::P::Memcache
+    def send_data data
+      sent_data << data
+    end
+    def sent_data
+      @sent_data ||= ''
+    end
+
+    def initialize
+      connection_completed
+    end
+  end
+
+  EM.describe EM::Protocols::Memcache do
+
+    before{
+      @c = TestConnection.new
+    }
+
+    should 'send get requests' do
+      @c.get('a'){}
+      @c.sent_data.should == "get a\r\n"
+      done
+    end
+
+    should 'send set requests' do
+      @c.set('a', 1){}
+      @c.sent_data.should == "set a 0 0 1\r\n1\r\n"
+      done
+    end
+
+    should 'use noreply on set without block' do
+      @c.set('a', 1)
+      @c.sent_data.should == "set a 0 0 1 noreply\r\n1\r\n"
+      done
+    end
+
+    should 'send delete requests' do
+      @c.del('a')
+      @c.sent_data.should == "delete a 0 noreply\r\n"
+      done
+    end
+
+    should 'work when get returns no values' do
+      @c.get('a'){ |a|
+        a.should.be.nil
+        done
+      }
+
+      @c.receive_data "END\r\n"
+    end
+
+    should 'invoke block on set' do
+      @c.set('a', 1){
+        done
+      }
+
+      @c.receive_data "STORED\r\n"
+    end
+
+    should 'invoke block on delete' do
+      @c.delete('a'){ |found|
+        found.should.be.false
+      }
+      @c.delete('b'){ |found|
+        found.should.be.true
+        done
+      }
+
+      @c.receive_data "NOT_FOUND\r\n"
+      @c.receive_data "DELETED\r\n"
+    end
+
+    should 'parse split responses' do
+      @c.get('a'){ |a|
+        a.should == 'abc'
+        done
+      }
+
+      @c.receive_data "VAL"
+      @c.receive_data "UE a 0 "
+      @c.receive_data "3\r\n"
+      @c.receive_data "ab"
+      @c.receive_data "c"
+      @c.receive_data "\r\n"
+      @c.receive_data "EN"
+      @c.receive_data "D\r\n"
+    end
+
+  end
+end

data/lib/protocols/postgres.rb

@@ -0,0 +1,261 @@
+#
+# $Id$
+#
+# Author:: Francis Cianfrocca (gmail: blackhedd)
+# Homepage::  http://rubyeventmachine.com
+# Date:: 15 November 2006
+# 
+# See EventMachine and EventMachine::Connection for documentation and
+# usage examples.
+#
+#----------------------------------------------------------------------------
+#
+# Copyright (C) 2006-08 by Francis Cianfrocca. All Rights Reserved.
+# Gmail: blackhedd
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of either: 1) the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version; or 2) Ruby's License.
+# 
+# See the file COPYING for complete licensing information.
+#
+#---------------------------------------------------------------------------
+#
+# 
+# 
+
+
+=begin
+PROVISIONAL IMPLEMENTATION of an evented Postgres client.
+This implements version 3 of the Postgres wire protocol, which will work
+with any Postgres version from roughly 7.4 onward.
+
+Until this code is judged ready for prime time, you have to access it by
+explicitly requiring protocols/postgres.
+
+Objective: we want to access Postgres databases without requiring threads.
+Until now this has been a problem because the Postgres client implementations 
+have all made use of blocking I/O calls, which is incompatible with a
+thread-free evented model.
+
+But rather than re-implement the Postgres Wire3 protocol, we're taking advantage
+of the existing postgres-pr library, which was originally written by Michael
+Neumann but (at this writing) appears to be no longer maintained. Still, it's
+in basically a production-ready state, and the wire protocol isn't that complicated
+anyway.
+
+We need to monkeypatch StringIO because it lacks the #readbytes method needed
+by postgres-pr.
+
+We're tucking in a bunch of require statements that may not be present in garden-variety
+EM installations. Until we find a good way to only require these if a program
+requires postgres, this file will need to be required explicitly.
+
+The StringIO monkeypatch is lifted verbatim from the standard library readbytes.rb,
+which adds method #readbytes directly to class IO. But StringIO is not a subclass of IO.
+
+We cloned the handling of postgres messages from lib/postgres-pr/connection.rb
+in the postgres-pr library, and modified it for event-handling.
+
+TODO: The password handling in dispatch_conn_message is totally incomplete.
+
+
+We return Deferrables from the user-level operations surfaced by this interface.
+Experimentally, we're using the pattern of always returning a boolean value as the
+first argument of a deferrable callback to indicate success or failure. This is
+instead of the traditional pattern of calling Deferrable#succeed or #fail, and
+requiring the user to define both a callback and an errback function.
+
+Sample code:
+require 'eventmachine'
+require 'protocols/postgres' # provisionally needed
+
+EM.run {
+	db = EM.connect_unix_domain( "/tmp/.s.PGSQL.5432", EM::P::Postgres3 )
+	db.connect( dbname, username, psw ).callback do |status|
+		if status
+			db.query( "select * from some_table" ).callback do |status, result, errors|
+				if status
+					result.rows.each do |row|
+						p row
+					end
+				end
+			end
+		end
+	end
+}
+
+
+
+=end
+
+
+require 'readbytes'
+require 'postgres-pr/message'
+require 'postgres-pr/connection'
+require 'stringio'
+
+include PostgresPR
+
+class StringIO
+  # Reads exactly +n+ bytes.
+  #
+  # If the data read is nil an EOFError is raised.
+  #
+  # If the data read is too short a TruncatedDataError is raised and the read
+  # data is obtainable via its #data method.
+  def readbytes(n)
+    str = read(n)
+    if str == nil
+      raise EOFError, "End of file reached"
+    end
+    if str.size < n
+      raise TruncatedDataError.new("data truncated", str) 
+    end
+    str
+  end
+end
+
+
+module EventMachine; module Protocols; class Postgres3 < EventMachine::Connection
+
+
+	def initialize
+		@data = ""
+		@params = {}
+	end
+
+	def connect db, user, psw=nil
+		d = EM::DefaultDeferrable.new
+		d.timeout 15
+
+		if @pending_query || @pending_conn
+			d.succeed false, "Operation already in progress"
+		else
+			@pending_conn = d
+			prms = {"user"=>user, "database"=>db}
+			@user = user
+			if psw
+				@password = psw
+				#prms["password"] = psw
+			end
+			send_data PostgresPR::StartupMessage.new( 3 << 16, prms ).dump
+		end
+
+		d
+	end
+
+	def query sql
+		d = EM::DefaultDeferrable.new
+		d.timeout 15
+
+		if @pending_query || @pending_conn
+			d.succeed false, "Operation already in progress"
+		else
+			@r = PostgresPR::Connection::Result.new
+			@e = []
+			@pending_query = d
+			send_data PostgresPR::Query.dump(sql)
+		end
+
+		d
+	end
+
+
+	def receive_data data
+		@data << data
+		while @data.length >= 5
+			pktlen = @data[1...5].unpack("N").first
+			if @data.length >= (1 + pktlen)
+				pkt = @data.slice!(0...(1+pktlen))
+				m = StringIO.open( pkt, "r" ) {|io| PostgresPR::Message.read( io ) }
+				if @pending_conn
+					dispatch_conn_message m
+				elsif @pending_query
+					dispatch_query_message m
+				else
+					raise "Unexpected message from database"
+				end
+			else
+				break # very important, break out of the while
+			end
+		end
+	end
+
+
+	def unbind
+		if o = (@pending_query || @pending_conn)
+			o.succeed false, "lost connection"
+		end
+	end
+
+	# Cloned and modified from the postgres-pr.
+	def dispatch_conn_message msg
+		case msg
+		when AuthentificationClearTextPassword
+			raise ArgumentError, "no password specified" if @password.nil?
+			send_data PasswordMessage.new(@password).dump
+
+		when AuthentificationCryptPassword
+			raise ArgumentError, "no password specified" if @password.nil?
+			send_data PasswordMessage.new(@password.crypt(msg.salt)).dump
+
+		when AuthentificationMD5Password
+			raise ArgumentError, "no password specified" if @password.nil?
+			require 'digest/md5'
+
+			m = Digest::MD5.hexdigest(@password + @user) 
+			m = Digest::MD5.hexdigest(m + msg.salt)
+			m = 'md5' + m
+			send_data PasswordMessage.new(m).dump
+
+		when AuthentificationKerberosV4, AuthentificationKerberosV5, AuthentificationSCMCredential
+			raise "unsupported authentification"
+
+		when AuthentificationOk
+		when ErrorResponse
+			raise msg.field_values.join("\t")
+		when NoticeResponse
+			@notice_processor.call(msg) if @notice_processor
+		when ParameterStatus
+			@params[msg.key] = msg.value
+		when BackendKeyData
+			# TODO
+			#p msg
+		when ReadyForQuery
+			# TODO: use transaction status
+			pc,@pending_conn = @pending_conn,nil
+			pc.succeed true
+		else
+			raise "unhandled message type"
+		end
+	end
+
+	# Cloned and modified from the postgres-pr.
+	def dispatch_query_message msg
+		case msg
+		when DataRow
+			@r.rows << msg.columns
+		when CommandComplete
+			@r.cmd_tag = msg.cmd_tag
+		when ReadyForQuery
+			pq,@pending_query = @pending_query,nil
+			pq.succeed true, @r, @e
+		when RowDescription
+			@r.fields = msg.fields
+		when CopyInResponse
+		when CopyOutResponse
+		when EmptyQueryResponse
+		when ErrorResponse
+			# TODO
+			@e << msg
+		when NoticeResponse
+			@notice_processor.call(msg) if @notice_processor
+		else
+			# TODO
+		end
+	end
+
+end; end; end
+