ruby_skynet 0.1.2 → 0.2.0

data/Gemfile

@@ -6,6 +6,9 @@ end
 
 gem "semantic_logger"
 gem "resilient_socket"
+# Thread Safe Hash and Array
+gem "thread_safe"
+gem "gene_pool"
 # For looking up Service entries in Doozer
 gem "multi_json"
 gem "ruby_protobuf"

data/Gemfile.lock

@@ -9,7 +9,7 @@ GEM
       bson (~> 1.7.0)
     i18n (0.6.1)
     multi_json (1.3.6)
-    resilient_socket (0.2.0)
+    resilient_socket (0.2.1)
       semantic_logger
     ruby_protobuf (0.4.11)
     semantic_logger (0.7.0)

data/lib/ruby_skynet/client.rb

@@ -1,7 +1,4 @@
 require 'bson'
-require 'sync_attr'
-require 'multi_json'
-
 #
 # RubySkynet Client
 #
@@ -13,124 +10,36 @@ module RubySkynet
   class Client
     include SyncAttr
 
-    # Default doozer configuration
-    # To replace this default, set the config as follows:
-    #    RubySkynet::Client.doozer_config = { .... }
-    sync_attr_accessor :doozer_config do
-      {
-        :server                 => '127.0.0.1:8046',
-        :read_timeout           => 5,
-        :connect_timeout        => 3,
-        :connect_retry_interval => 0.1,
-        :connect_retry_count    => 3
-      }
-    end
-
-    # Lazy initialize Doozer Client
-    sync_cattr_reader :doozer do
-      Doozer::Client.new
-    end
-
-    # Create a client connection, call the supplied block and close the connection on
-    # completion of the block
-    #
-    # Example
-    #
-    #  require 'ruby_skynet'
-    #  SemanticLogger.default_level = :trace
-    #  SemanticLogger.appenders << SemanticLogger::Appender::File(STDOUT)
-    #  RubySkynet::Client.connect('TutorialService') do |tutorial_service|
-    #    p tutorial_service.call(:value => 5)
-    #  end
-    def self.connect(service_name, params={})
-      begin
-        client = self.new(service_name, params)
-        yield(client)
-      ensure
-        client.close if client
-      end
-    end
-
     # Returns a new RubySkynet Client for the named service
     #
+    # Calls to an instance of the Client are thread-safe and can be called
+    # concurrently from multiple threads at the same time
+    #
     # Parameters:
     #   :service_name
     #     Name of the service to look for and connect to on Skynet
     #
-    #   :doozer_servers [Array of String]
-    #     Array of URL's of doozer servers to connect to with port numbers
-    #     ['server1:2000', 'server2:2000']
+    #   :version
+    #     Optional version number of the service in Skynet
+    #     Default: '*' being the latest version of the service
     #
-    #     The second server will only be attempted once the first server
-    #     cannot be connected to or has timed out on connect
-    #     A read failure or timeout will not result in switching to the second
-    #     server, only a connection failure or during an automatic reconnect
+    #   :region
+    #     Optional region for this service in Skynet
+    #     Default: 'Development'
     #
-    #   :read_timeout [Float]
-    #     Time in seconds to timeout on read
-    #     Can be overridden by supplying a timeout in the read call
-    #     Default: 60
-    #
-    #   :connect_timeout [Float]
-    #     Time in seconds to timeout when trying to connect to the server
-    #     Default: Half of the :read_timeout ( 30 seconds )
+    # Example
     #
-    #   :connect_retry_count [Fixnum]
-    #     Number of times to retry connecting when a connection fails
-    #     Default: 10
+    #  require 'ruby_skynet'
+    #  SemanticLogger.default_level = :trace
+    #  SemanticLogger.appenders << SemanticLogger::Appender::File(STDOUT)
     #
-    #   :connect_retry_interval [Float]
-    #     Number of seconds between connection retry attempts after the first failed attempt
-    #     Default: 0.5
-    def initialize(service_name, params = {})
+    #  tutorial_service = RubySkynet::Client.new('TutorialService')
+    #  p tutorial_service.call('Add', :value => 5)
+    def initialize(service_name, version='*', region='Development')
       @service_name = service_name
-      @logger = SemanticLogger::Logger.new("#{self.class.name}: #{service_name}")
-
-      # User configurable options
-      params[:read_timeout]           ||= 60
-      params[:connect_timeout]        ||= 30
-      params[:connect_retry_interval] ||= 0.1
-      params[:connect_retry_count]    ||= 5
-
-      # If Server name and port of where Skynet Service is running
-      # is not supplied look for it in Doozer
-      unless params[:server] || params[:servers]
-        params[:server] = self.class.server_for(service_name)
-      end
-
-      # Disable buffering the send since it is a RPC call
-      params[:buffered] = false
-
-      @logger.trace "Socket Connection parameters", params
-
-      # For each new connection perform the Skynet handshake
-      params[:on_connect] = Proc.new do |socket|
-        # Reset user_data on each connection
-        socket.user_data = 0
-
-        # Receive Service Handshake
-        # Registered bool
-        #   Registered indicates the state of this service. If it is false, the connection will
-        #   close immediately and the client should look elsewhere for this service.
-        #
-        # ClientID string
-        #   ClientID is a UUID that is used by the client to identify itself in RPC requests.
-        @logger.debug "Waiting for Service Handshake"
-        service_handshake = self.class.read_bson_document(socket)
-        @logger.trace 'Service Handshake', service_handshake
-
-        # #TODO When a reconnect returns registered == false we need to go back to doozer
-        @registered = service_handshake['registered']
-        @client_id = service_handshake['clientid']
-
-        # Send blank ClientHandshake
-        client_handshake = { 'clientid' => @client_id }
-        @logger.debug "Sending Client Handshake"
-        @logger.trace 'Client Handshake', client_handshake
-        socket.write(BSON.serialize(client_handshake))
-      end
-
-      @socket = ResilientSocket::TCPClient.new(params)
+      @logger       = SemanticLogger::Logger.new("#{self.class.name}: #{service_name}/#{version}/#{region}")
+      @version      = version
+      @region       = region
     end
 
     # Performs a synchronous call to the Skynet Service
@@ -145,149 +54,28 @@ module RubySkynet
     #
     # Raises RubySkynet::ProtocolError
     # Raises RubySkynet::SkynetException
-    def call(method_name, parameters)
+    def call(method_name, parameters, connection_params={})
       # Skynet requires BSON RPC Calls to have the following format:
       # https://github.com/bketelsen/skynet/blob/protocol/protocol.md
       request_id = BSON::ObjectId.new.to_s
       @logger.tagged request_id do
         @logger.benchmark_info "Called Skynet Service: #{@service_name}.#{method_name}" do
-
-          # Resilient Send
-          retry_count = 0
-          @socket.retry_on_connection_failure do |socket|
-            # user_data is maintained per session and a different session could
-            # be supplied with each retry
-            socket.user_data ||= 0
-            header = {
-              'servicemethod' => "#{@service_name}.Forward",
-              'seq'           => socket.user_data,
-            }
-            @logger.debug "Sending Header"
-            @logger.trace 'Header', header
-            socket.write(BSON.serialize(header))
-
-            @logger.trace 'Parameters:', parameters
-
-            # The parameters are placed in the request object in BSON serialized
-            # form
-            request = {
-              'clientid'    => @client_id,
-              'in'          => BSON.serialize(parameters).to_s,
-              'method'      => method_name.to_s,
-              'requestinfo' => {
-                'requestid'     => request_id,
-                # Increment retry count to indicate that the request may have been tried previously
-                # TODO: this should be incremented if request is retried,
-                'retrycount'    => retry_count,
-                # TODO: this should be forwarded along in case of services also
-                # being a client and calling additional services. If empty it will
-                # be stuffed with connecting address
-                'originaddress' => ''
-              }
-            }
-
-            @logger.debug "Sending Request"
-            @logger.trace 'Request', request
-            socket.write(BSON.serialize(request))
-          end
-
-          # Once send is successful it could have been processed, so we can no
-          # longer retry now otherwise we could create a duplicate
-          # retry_count += 1
-
-          # Read header first as a separate BSON document
-          @logger.debug "Reading header from server"
-          header = self.class.read_bson_document(@socket)
-          @logger.debug 'Header', header
-
-          # Read the BSON response document
-          @logger.debug "Reading response from server"
-          response = self.class.read_bson_document(@socket)
-          @logger.trace 'Response', response
-
-          # Ensure the sequence number in the response header matches the
-          # sequence number sent in the request
-          if seq_no = header['seq']
-            raise ProtocolError.new("Incorrect Response received, expected seq=#{@socket.user_data}, received: #{header.inspect}") if seq_no != @socket.user_data
-          else
-            raise ProtocolError.new("Invalid Response header, missing 'seq': #{header.inspect}")
-          end
-
-          # Increment Sequence number only on successful response
-          @socket.user_data += 1
-
-          # If an error is returned from Skynet raise a Skynet exception
-          if error = header['error']
-            raise SkynetException.new(error) if error.to_s.length > 0
-          end
-
-          # If an error is returned from the service raise a Service exception
-          if error = response['error']
-            raise ServiceException.new(error) if error.to_s.length > 0
+          retries = 0
+          # If it cannot connect to a server, try a different server
+          begin
+            Connection.with_connection(Registry.server_for(@service_name, @version, @region), connection_params) do |connection|
+              connection.rpc_call(request_id, @service_name, method_name, parameters)
+            end
+          rescue ResilientSocket::ConnectionFailure => exc
+            if (retries < 3) && exc.cause.is_a?(Errno::ECONNREFUSED)
+              retries += 1
+              retry
+            end
+            # TODO rescue ServiceUnavailable retry x times until the service becomes available
           end
-
-          # Return Value
-          # The return value is inside the response object, it's a byte array of it's own and needs to be deserialized
-          result = BSON.deserialize(response['out'])
-          @logger.trace 'Return Value', result
-          result
         end
       end
     end
 
-    # Returns a BSON document read from the socket.
-    # Returns nil if the operation times out or if a network
-    #         connection failure occurs
-    def self.read_bson_document(socket)
-      bytebuf = BSON::ByteBuffer.new
-      # Read 4 byte size of following BSON document
-      bytes = ''
-      socket.read(4, bytes)
-
-      # Read BSON document
-      sz = bytes.unpack("V")[0]
-      raise "Invalid Data received from server:#{bytes.inspect}" unless sz
-
-      bytebuf.append!(bytes)
-      bytes = ''
-      sz -= 4
-      until bytes.size >= sz
-        buf = ''
-        socket.read(sz, buf)
-        bytes << buf
-      end
-      bytebuf.append!(bytes)
-      return BSON.deserialize(bytebuf)
-    end
-
-    def close()
-      @socket.close
-    end
-
-    ##############################
-    #protected
-
-    # Returns [Array] of the hostname and port pair [String] that implements a particular service
-    # Performs a doozer lookup to find the servers
-    #
-    #   service_name:
-    #   version: Version of service to locate
-    #            Default: Find latest version
-    def self.registered_implementers(service_name, version = '*', region = 'Development')
-      hosts = []
-      doozer.walk("/services/#{service_name}/#{version}/#{region}/*/*").each do |node|
-        entry = MultiJson.load(node.value)
-        hosts << entry if entry['Registered']
-      end
-      hosts
-    end
-
-    # Randomly returns a server that implements the requested service
-    def self.server_for(service_name, version = '*', region = 'Development')
-      hosts = registered_implementers(service_name, version, region)
-      service = hosts[rand(hosts.size)]['Config']['ServiceAddr']
-      "#{service['IPAddress']}:#{service['Port']}"
-    end
-
   end
 end

data/lib/ruby_skynet/connection.rb

@@ -0,0 +1,269 @@
+require 'bson'
+require 'gene_pool'
+require 'thread_safe'
+
+#
+# RubySkynet Connection
+#
+# Handles connecting to Skynet Servers as a host:port pair
+#
+module RubySkynet
+  class Connection
+    include SyncAttr
+
+    # Returns the underlying socket being used by a Connection instance
+    attr_reader :socket
+
+    # Default Pool configuration
+    sync_cattr_accessor :pool_config do
+      {
+        :pool_size    => 30,   # Maximum number of connections to any one server
+        :warn_timeout => 2,    # Log a warning if no connections are available after the :warn_timeout seconds
+        :timeout      => 10,   # Raise a Timeout exception if no connections are available after the :timeout seconds
+        :idle_timeout => 600,  # Renew a connection if it has been idle for this period of time
+      }
+    end
+
+    # Logging instance for the connection pool
+    sync_cattr_reader :logger do
+      SemanticLogger::Logger.new(self)
+    end
+
+    # For each server there is a connection pool keyed on the
+    # server address: 'host:port'
+    @@connection_pools = ThreadSafe::Hash.new
+
+    # Returns a new RubySkynet connection to the server
+    #
+    # Parameters:
+    #   :read_timeout [Float]
+    #     Time in seconds to timeout on read
+    #     Can be overridden by supplying a timeout in the read call
+    #     Default: 60
+    #
+    #   :connect_timeout [Float]
+    #     Time in seconds to timeout when trying to connect to the server
+    #     Default: Half of the :read_timeout ( 30 seconds )
+    #
+    #   :connect_retry_count [Fixnum]
+    #     Number of times to retry connecting when a connection fails
+    #     Default: 10
+    #
+    #   :connect_retry_interval [Float]
+    #     Number of seconds between connection retry attempts after the first failed attempt
+    #     Default: 0.5
+    def initialize(server, params = {})
+      @logger = SemanticLogger::Logger.new("#{self.class.name}: #{server}")
+
+      # User configurable options
+      params[:read_timeout]           ||= 60
+      params[:connect_timeout]        ||= 30
+      params[:connect_retry_interval] ||= 0.1
+      params[:connect_retry_count]    ||= 5
+
+      # Disable send buffering since it is a RPC call
+      params[:buffered] = false
+
+      # For each new connection perform the Skynet handshake
+      params[:on_connect] = Proc.new do |socket|
+        # Reset user_data on each connection
+        socket.user_data = {
+          :seq    => 0,
+          :logger => @logger
+        }
+
+        # Receive Service Handshake
+        # Registered bool
+        #   Registered indicates the state of this service. If it is false, the connection will
+        #   close immediately and the client should look elsewhere for this service.
+        #
+        # ClientID string
+        #   ClientID is a UUID that is used by the client to identify itself in RPC requests.
+        @logger.debug "Waiting for Service Handshake"
+        service_handshake = self.class.read_bson_document(socket)
+        @logger.trace 'Service Handshake', service_handshake
+
+        # #TODO When a reconnect returns registered == false need to throw an exception
+        # so that this host connection is not used
+        registered = service_handshake['registered']
+        client_id = service_handshake['clientid']
+        socket.user_data[:client_id] = client_id
+
+        # Send blank ClientHandshake
+        client_handshake = { 'clientid' => client_id }
+        @logger.debug "Sending Client Handshake"
+        @logger.trace 'Client Handshake', client_handshake
+        socket.write(BSON.serialize(client_handshake))
+      end
+
+      # To prevent strange issues if user incorrectly supplies server names
+      params.delete(:servers)
+      params[:server] = server
+
+      @socket = ResilientSocket::TCPClient.new(params)
+    end
+
+    # Performs a synchronous call to a Skynet server
+    #
+    # Parameters:
+    #   service_name [String|Symbol]:
+    #     Name of the method to pass in the request
+    #   method_name [String|Symbol]:
+    #     Name of the method to pass in the request
+    #   parameters [Hash]:
+    #     Parameters to pass in the request
+    #   idempotent [True|False]:
+    #     If the request can be applied again to the server without changing its state
+    #     Set to true to retry the entire request after the send is successful
+    #
+    # Returns the Hash result returned from the Skynet Service
+    #
+    # Raises RubySkynet::ProtocolError
+    # Raises RubySkynet::SkynetException
+    def rpc_call(request_id, service_name, method_name, parameters, idempotent=false)
+      retry_count = 0
+      header = nil
+      response = nil
+
+      socket.retry_on_connection_failure do |socket|
+        header = {
+          'servicemethod' => "#{service_name}.Forward",
+          'seq'           => socket.user_data[:seq]
+        }
+
+        @logger.debug "Sending Header"
+        @logger.trace 'Header', header
+        socket.write(BSON.serialize(header))
+
+        # The parameters are placed in the request object in BSON serialized form
+        request = {
+          'clientid'    => socket.user_data[:client_id],
+          'in'          => BSON.serialize(parameters).to_s,
+          'method'      => method_name.to_s,
+          'requestinfo' => {
+            'requestid'     => request_id,
+            # Increment retry count to indicate that the request may have been tried previously
+            'retrycount' => retry_count,
+            # TODO: this should be forwarded along in case of services also
+            # being a client and calling additional services. If empty it will
+            # be stuffed with connecting address
+            'originaddress' => ''
+          }
+        }
+
+        @logger.debug "Sending Request"
+        @logger.trace 'Request', request
+        @logger.trace 'Parameters:', parameters
+        socket.write(BSON.serialize(request))
+
+        # Since Send does not affect state on the server we can also retry reads
+        if idempotent
+          @logger.debug "Reading header from server"
+          header = self.class.read_bson_document(socket)
+          @logger.debug 'Response Header', header
+
+          # Read the BSON response document
+          @logger.debug "Reading response from server"
+          response = self.class.read_bson_document(socket)
+          @logger.trace 'Response', response
+        end
+      end
+
+      # Perform the read outside the retry block since a successful write
+      # means that the servers state may have been changed
+      unless idempotent
+        # Read header first as a separate BSON document
+        @logger.debug "Reading header from server"
+        header = self.class.read_bson_document(socket)
+        @logger.debug 'Response Header', header
+
+        # Read the BSON response document
+        @logger.debug "Reading response from server"
+        response = self.class.read_bson_document(socket)
+        @logger.trace 'Response', response
+      end
+
+      # Ensure the sequence number in the response header matches the
+      # sequence number sent in the request
+      seq_no = header['seq']
+      if seq_no != socket.user_data[:seq]
+        raise ProtocolError.new("Incorrect Response received, expected seq=#{socket.user_data[:seq]}, received: #{header.inspect}")
+      end
+
+      # Increment Sequence number only on successful response
+      socket.user_data[:seq] += 1
+
+      # If an error is returned from Skynet raise a Skynet exception
+      error = header['error']
+      raise SkynetException.new(error) if error.to_s.length > 0
+
+      # If an error is returned from the service raise a Service exception
+      error = response['error']
+      raise ServiceException.new(error) if error.to_s.length > 0
+
+      # Return Value
+      # The return value is inside the response object, it's a byte array of it's own and needs to be deserialized
+      result = BSON.deserialize(response['out'])
+      @logger.trace 'Return Value', result
+      result
+    end
+
+    # Execute the supplied block with a connection from the pool
+    def self.with_connection(server, params={}, &block)
+      (@@connection_pools[server] ||= new_connection_pool(server, params)).with_connection(&block)
+    end
+
+    def close
+      @socket.close if @socket
+    end
+
+    ########################
+    protected
+
+    # Returns a BSON document read from the socket.
+    # Returns nil if the operation times out or if a network
+    #         connection failure occurs
+    def self.read_bson_document(socket)
+      bytebuf = BSON::ByteBuffer.new
+      # Read 4 byte size of following BSON document
+      bytes = socket.read(4)
+
+      # Read BSON document
+      sz = bytes.unpack("V")[0]
+      raise "Invalid Data received from server:#{bytes.inspect}" unless sz
+
+      bytebuf.append!(bytes)
+      bytebuf.append!(socket.read(sz - 4))
+      return BSON.deserialize(bytebuf)
+    end
+
+    # Returns a new connection pool for the specified server
+    def self.new_connection_pool(server, params={})
+      # Connection pool configuration options
+      config = pool_config.dup
+
+      # Method to call to close idle connections
+      config[:close_proc] = :close
+      config[:logger]     = logger
+      config[:name]       = "Connection pool for #{server}"
+
+      pool = GenePool.new(pool_config) do
+        new(server, params)
+      end
+
+      # Cleanup corresponding connection pool when a server terminates
+      Registry.on_server_removed(server) do
+        pool = @@connection_pools.delete(server)
+        # Cannot close all the connections since they could still be in use
+        pool.remove_idle(0) if pool
+        #pool.close if pool
+        logger.debug "Connection pool for server:#{server} has been released"
+      end
+
+      pool
+    end
+
+  end
+
+end
+

data/lib/ruby_skynet/doozer/client.rb

@@ -142,14 +142,29 @@ module RubySkynet
         end
       end
 
-      # TODO Implement watching for changes in a separate thread with it's own
-      #      client connection
-      #def watch(path, rev = nil)
-      #  invoke(Request.new(:path => secret, :verb => Request::Verb::WAIT))
-      #end
+      # Wait for changes to the supplied path
+      # Returns the next change to the supplied path
+      def wait(path, rev=current_revision, timeout=-1)
+        invoke(Request.new(:path => path, :rev => rev, :verb => Request::Verb::WAIT), true, timeout)
+      end
+
+      # Watch for any changes to the supplied path, calling the supplied block
+      # for every change
+      # Runs until an exception is thrown
+      #
+      # If a connection error occurs it will create a new connection to doozer
+      # and resubmit the wait. I.e. Will continue from where it left off
+      # without any noticeable effect to the supplied block
+      def watch(path, rev=current_revision)
+        loop do
+          result = wait(path, rev, -1)
+          yield result
+          rev = result.rev + 1
+        end
+      end
 
       #####################
-      # protected
+      #protected
 
       # Call the Doozer server
       #
@@ -158,16 +173,16 @@ module RubySkynet
       #   _only_ if a rev has been supplied
       #
       # When modifier is true
-      def invoke(request, readonly=true)
+      def invoke(request, readonly=true, timeout=nil)
         retry_read = readonly || !request.rev.nil?
         response = nil
         @socket.retry_on_connection_failure do
           send(request)
-          response = read if retry_read
+          response = read(timeout) if retry_read
         end
         # Network error on read must be sent back to caller since we do not
         # know if the modification was made
-        response = read unless retry_read
+        response = read(timeout) unless retry_read
         raise ResponseError.new("#{Response::Err.name_by_value(response.err_code)}: #{response.err_detail}") if response.err_code != 0
         response
       end
@@ -182,19 +197,11 @@ module RubySkynet
       end
 
       # Read the protobuf Response from Doozer
-      def read
+      def read(timeout=nil)
         # First strip the additional header indicating the size of the subsequent response
-        head = @socket.read(4)
+        head = @socket.read(4,nil,timeout)
         length = head.unpack("N")[0]
-
-        # Since can returns upto 'length' bytes we need to make sure it returns
-        # at least 'length' bytes
-        # TODO: Make this a binary buffer
-        data = ''
-        until data.size >= length
-          data << @socket.read(length)
-        end
-        Response.new.parse_from_string(data)
+        Response.new.parse_from_string(@socket.read(length))
       end
 
     end

data/lib/ruby_skynet/exceptions.rb

@@ -1,6 +1,7 @@
 module RubySkynet
   class Exception < ::RuntimeError; end
   class ProtocolError < Exception; end
-  class SkynetException < Exception; end
   class ServiceException < Exception; end
+  class SkynetException < Exception; end
+  class ServiceUnavailable < SkynetException; end
 end

data/lib/ruby_skynet/registry.rb

@@ -0,0 +1,252 @@
+require 'sync_attr'
+require 'multi_json'
+require 'thread_safe'
+require 'gene_pool'
+
+#
+# RubySkynet Registry Client
+#
+# Keeps a local copy of the Skynet Registry
+#
+# Subscribes to Registry changes and the internal copy up to date
+#
+module RubySkynet
+  class Registry
+    include SyncAttr
+
+    # Service Registry has the following format
+    #  Key: [String] 'service_name/version/region'
+    #  Value: [Array<String>] 'host:port', 'host:port'
+    sync_cattr_accessor :service_registry do
+      start_monitoring
+    end
+
+    @@on_server_removed_callbacks = ThreadSafe::Hash.new
+    @@monitor_thread = nil
+
+    DOOZER_SERVICES_PATH = "/services/*/*/*/*/*"
+
+    # Default doozer configuration
+    # To replace this default, set the config as follows:
+    #    RubySkynet::Client.doozer_config = { .... }
+    #
+    #   :servers [Array of String]
+    #     Array of URL's of doozer servers to connect to with port numbers
+    #     ['server1:2000', 'server2:2000']
+    #
+    #     The second server will only be attempted once the first server
+    #     cannot be connected to or has timed out on connect
+    #     A read failure or timeout will not result in switching to the second
+    #     server, only a connection failure or during an automatic reconnect
+    #
+    #   :read_timeout [Float]
+    #     Time in seconds to timeout on read
+    #     Can be overridden by supplying a timeout in the read call
+    #
+    #   :connect_timeout [Float]
+    #     Time in seconds to timeout when trying to connect to the server
+    #
+    #   :connect_retry_count [Fixnum]
+    #     Number of times to retry connecting when a connection fails
+    #
+    #   :connect_retry_interval [Float]
+    #     Number of seconds between connection retry attempts after the first failed attempt
+    sync_cattr_accessor :doozer_config do
+      {
+        :servers                => ['127.0.0.1:8046'],
+        :read_timeout           => 5,
+        :connect_timeout        => 3,
+        :connect_retry_interval => 1,
+        :connect_retry_count    => 300
+      }
+    end
+
+    # Lazy initialize Doozer Client Connection pool
+    sync_cattr_reader :doozer_pool do
+      GenePool.new(
+        :name         =>"Doozer Connection Pool",
+        :pool_size    => 5,
+        :timeout      => 30,
+        :warn_timeout => 5,
+        :idle_timeout => 600,
+        :logger       => logger,
+        :close_proc   => :close
+      ) do
+        Doozer::Client.new(doozer_config)
+      end
+    end
+
+    # Logging instance for this class
+    sync_cattr_reader :logger do
+      SemanticLogger::Logger.new(self, :debug)
+    end
+
+    # Return a server that implements the specified service
+    def self.server_for(service_name, version='*', region='Development')
+      if servers = servers_for(service_name, version, region)
+        # Randomly select one of the servers offering the service
+        servers[rand(servers.size)]
+      else
+        msg = "No servers available for service: #{service_name} with version: #{version} in region: #{region}"
+        logger.warn msg
+        raise ServiceUnavailable.new(msg)
+      end
+    end
+
+    # Returns [Array] of the hostname and port pair [String] that implements a particular service
+    # Performs a doozer lookup to find the servers
+    #
+    #   service_name:
+    #     Name of the service to lookup
+    #   version:
+    #     Version of service to locate
+    #     Default: All versions
+    #   region:
+    #     Region to look for the service in
+    def self.registered_implementers(service_name='*', version='*', region='Development')
+      hosts = []
+      doozer_pool.with_connection do |doozer|
+        doozer.walk("/services/#{service_name}/#{version}/#{region}/*/*").each do |node|
+          entry = MultiJson.load(node.value)
+          hosts << entry if entry['Registered']
+        end
+      end
+      hosts
+    end
+
+    # Returns [Array<String>] a list of servers implementing the requested service
+    def self.servers_for(service_name, version='*', region='Development', remote = false)
+      if remote
+        if version != '*'
+          registered_implementers(service_name, version, region).map do |host|
+            service = host['Config']['ServiceAddr']
+            "#{service['IPAddress']}:#{service['Port']}"
+          end
+        else
+          # Find the highest version of any particular service
+          versions = {}
+          registered_implementers(service_name, version, region).each do |host|
+            service = host['Config']['ServiceAddr']
+            (versions[version.to_i] ||= []) << "#{service['IPAddress']}:#{service['Port']}"
+          end
+          # Return the servers implementing the highest version number
+          versions.sort.last.last
+        end
+      else
+        if version == '*'
+          # Find the highest version for the named service in this region
+          version = -1
+          service_registry.keys.each do |key|
+            if match = key.match(/#{service_name}\/(\d+)\/#{region}/)
+              ver = match[1].to_i
+              version = ver if ver > version
+            end
+          end
+        end
+        service_registry["#{service_name}/#{version}/#{region}"]
+      end
+    end
+
+    # Invokes registered callbacks when a specific server is shutdown or terminates
+    # Not when a server de-registers itself
+    # The callback will only be called once and will need to be re-registered
+    # after being called if future callbacks are required for that server
+    def self.on_server_removed(server, &block)
+      (@@on_server_removed_callbacks[server] ||= ThreadSafe::Array.new) << block
+    end
+
+    ############################
+    #protected
+
+    # Fetch the all registry information from Doozer and set the internal registry
+    # Also starts the monitoring thread to keep the registry up to date
+    def self.start_monitoring
+      registry = ThreadSafe::Hash.new
+      revision = nil
+      doozer_pool.with_connection do |doozer|
+        revision = doozer.current_revision
+        doozer.walk(DOOZER_SERVICES_PATH, revision).each do |node|
+          # path: "/services/TutorialService/1/Development/127.0.0.1/9000"
+          e = node.path.split('/')
+
+          # Key: [String] 'service_name/version/region'
+          key = "#{e[2]}/#{e[3]}/#{e[4]}"
+          server = "#{e[5]}:#{e[6]}"
+
+          if node.value.strip.size > 0
+            entry = MultiJson.load(node.value)
+            if entry['Registered']
+              # Value: [Array<String>] 'host:port', 'host:port'
+              servers = (registry[key] ||= ThreadSafe::Array.new)
+              servers << server unless servers.include?(server)
+              logger.debug "#start_monitoring Add Service: #{key} => #{server}"
+            end
+          end
+        end
+      end
+      # Start monitoring thread to keep the registry up to date
+      @@monitor_thread = Thread.new { self.watch(revision + 1) }
+      registry
+    end
+
+    # Waits for any updates from Doozer and updates the internal service registry
+    def self.watch(revision)
+      logger.info "Start monitoring #{DOOZER_SERVICES_PATH}"
+      # This thread must use its own dedicated doozer connection
+      doozer = Doozer::Client.new(doozer_config)
+      doozer.watch(DOOZER_SERVICES_PATH, revision) do |node|
+        # path: "/services/TutorialService/1/Development/127.0.0.1/9000"
+        e = node.path.split('/')
+
+        # Key: [String] 'service_name/version/region'
+        key = "#{e[2]}/#{e[3]}/#{e[4]}"
+        server = "#{e[5]}:#{e[6]}"
+
+        if node.value.strip.size > 0
+          entry = MultiJson.load(node.value)
+          if entry['Registered']
+            # Value: [Array<String>] 'host:port', 'host:port'
+            servers = (@@service_registry[key] ||= ThreadSafe::Array.new)
+            servers << server unless servers.include?(server)
+            logger.debug "#monitor Add/Update Service: #{key} => #{server}"
+          else
+            logger.debug "#monitor Service deregistered, remove: #{key} => #{server}"
+            if @@service_registry[key]
+              @@service_registry[key].delete(server)
+              @@service_registry.delete(key) if @@service_registry[key].size == 0
+            end
+          end
+        else
+          # Service has stopped and needs to be removed
+          logger.debug "#monitor Service stopped, remove: #{key} => #{server}"
+          if @@service_registry[key]
+            @@service_registry[key].delete(server)
+            @@service_registry.delete(key) if @@service_registry[key].size == 0
+            server_removed(server)
+          end
+        end
+        logger.debug "Updated registry", @@service_registry
+      end
+      logger.info "Stopping monitoring thread normally"
+    rescue Exception => exc
+      logger.error "Exception in monitoring thread", exc
+    ensure
+      logger.info "Stopped monitoring"
+    end
+
+    # Invoke any registered callbacks for the specific server
+    def self.server_removed(server)
+      if callbacks = @@on_server_removed_callbacks.delete(server)
+        callbacks.each do |block|
+          begin
+            logger.info "Calling callback for server: #{server}"
+            block.call(server)
+          rescue Exception => exc
+            logger.error("Exception during a callback for server: #{server}", exc)
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/ruby_skynet/version.rb

@@ -1,3 +1,3 @@
 module RubySkynet #:nodoc
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/lib/ruby_skynet.rb

@@ -7,5 +7,7 @@ module RubySkynet
   module Doozer
     autoload :Client, 'ruby_skynet/doozer/client'
   end
-  autoload :Client, 'ruby_skynet/client'
+  autoload :Registry,   'ruby_skynet/registry'
+  autoload :Connection, 'ruby_skynet/connection'
+  autoload :Client,     'ruby_skynet/client'
 end

data/test/ruby_skynet_client_test.rb

@@ -7,6 +7,7 @@ require 'test/unit'
 require 'shoulda'
 require 'ruby_skynet'
 require 'simple_server'
+require 'multi_json'
 
 SemanticLogger::Logger.default_level = :trace
 SemanticLogger::Logger.appenders << SemanticLogger::Appender::File.new('test.log')
@@ -17,43 +18,57 @@ class RubySkynetClientTest < Test::Unit::TestCase
 
     context "without server" do
       should "raise exception when cannot reach server after 5 retries" do
-        exception = assert_raise ResilientSocket::ConnectionFailure do
-          RubySkynet::Client.new('SomeService',
-            :server                 => 'localhost:3300',
-            :connect_retry_interval => 0.1,
-            :connect_retry_count    => 5)
+        exception = assert_raise RubySkynet::ServiceUnavailable do
+          client = RubySkynet::Client.new('SomeService')
+          client.call(:test, :hello => 'there')
         end
-        assert_match /After 5 connection attempts to host 'localhost:3300': Errno::ECONNREFUSED/, exception.message
+        assert_match /No servers available for service: SomeService with version: \* in region: Development/, exception.message
       end
 
     end
 
     context "with server" do
       setup do
+        @port = 2000
         @read_timeout = 3.0
-        @server = SimpleServer.new(2000)
-        @server_name = 'localhost:2000'
+        @server = SimpleServer.new(@port)
+        @server_name = "localhost:#{@port}"
+
+        # Register service in doozer
+        @service_name = "TestService"
+        @version = 1
+        @region = 'Test'
+        @ip_address = "127.0.0.1"
+        config = {
+          "Config" => {
+            "UUID" => "3978b371-15e9-40f8-9b7b-59ae88d8c7ec",
+            "Name" => @service_name,
+            "Version" => @version.to_s,
+            "Region" => @region,
+            "ServiceAddr" => {
+              "IPAddress" => @ip_address,
+              "Port" => @port,
+              "MaxPort" => @port + 999
+            },
+          },
+          "Registered" => true
+        }
+        RubySkynet::Registry.doozer_pool.with_connection do |doozer|
+          doozer["/services/#{@service_name}/#{@version}/#{@region}/#{@ip_address}/#{@port}"] = MultiJson.encode(config)
+        end
       end
 
       teardown do
         @server.stop if @server
-      end
-
-      context "using blocks" do
-        should "call server" do
-          RubySkynet::Client.connect('TutorialService', :read_timeout => @read_timeout, :server => @server_name) do |tutorial_service|
-            assert_equal 'test1', tutorial_service.call(:test1, 'some' => 'parameters')['result']
-          end
+        # De-register server in doozer
+        RubySkynet::Registry.doozer_pool.with_connection do |doozer|
+          doozer.delete("/services/#{@service_name}/#{@version}/#{@region}/#{@ip_address}/#{@port}") rescue nil
         end
       end
 
       context "with client connection" do
         setup do
-          @client = RubySkynet::Client.new('TutorialService', :read_timeout => @read_timeout, :server => @server_name)
-        end
-
-        def teardown
-          @client.close if @client
+          @client = RubySkynet::Client.new(@service_name, @version, @region)
         end
 
         should "successfully send and receive data" do
@@ -66,7 +81,7 @@ class RubySkynetClientTest < Test::Unit::TestCase
 
           exception = assert_raise ResilientSocket::ReadTimeout do
             # Read 4 bytes from server
-            @client.call('sleep', request)
+            @client.call('sleep', request, :read_timeout => @read_timeout)
           end
           assert_match /Timedout after #{@read_timeout} seconds trying to read/, exception.message
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_skynet
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-15 00:00:00.000000000 Z
+date: 2012-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: semantic_logger
@@ -98,13 +98,16 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- dev.log
 - Gemfile
 - Gemfile.lock
 - lib/ruby_skynet/client.rb
+- lib/ruby_skynet/connection.rb
 - lib/ruby_skynet/doozer/client.rb
 - lib/ruby_skynet/doozer/exceptions.rb
 - lib/ruby_skynet/doozer/msg.pb.rb
 - lib/ruby_skynet/exceptions.rb
+- lib/ruby_skynet/registry.rb
 - lib/ruby_skynet/version.rb
 - lib/ruby_skynet.rb
 - LICENSE.txt
@@ -116,8 +119,6 @@ files:
 - nbproject/project.xml
 - Rakefile
 - README.md
-- ruby_skynet-0.1.0.gem
-- ruby_skynet-0.1.1.gem
 - test/doozer_client_test.rb
 - test/ruby_skynet_client_test.rb
 - test/simple_server.rb
@@ -136,7 +137,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4150278993110980832
+      hash: -4330021218658134009
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: