protobuf 1.1.3 → 1.3.0

data/lib/protobuf/rpc/connectors/zmq.rb

@@ -0,0 +1,73 @@
+require 'protobuf/rpc/connectors/base'
+
+module Protobuf
+  module Rpc
+    module Connectors
+      class Zmq < Base
+        include Protobuf::Rpc::Connectors::Common
+        include Protobuf::Logger::LogMethods
+
+        def send_request
+          check_async
+          setup_connection
+          connect_to_rpc_server
+          post_init
+          read_response
+        ensure
+          @socket.close if @socket
+          @zmq_context.terminate if @zmq_context
+          @zmq_context = nil
+        end
+
+        private
+
+        def check_async
+          if async?
+            log_error { "[client-#{self.class}] Cannot run in async mode" }
+            raise "Cannot use Zmq client in async mode" 
+          end
+        end
+
+        def close_connection
+          return if(@error)
+          zmq_error_check(@socket.close)
+          zmq_error_check(@zmq_context.terminate)
+          log_debug { "[client-#{self.class}] Connector closed"  }
+        end
+
+        def connect_to_rpc_server
+          return if(@error)
+          log_debug { "[client-#{self.class} Establishing connection: #{options[:host]}:#{options[:port]}" }
+          @zmq_context = ::ZMQ::Context.new
+          @socket = @zmq_context.socket(::ZMQ::REQ)
+          zmq_error_check(@socket.connect("tcp://#{options[:host]}:#{options[:port]}"))
+          log_debug { "[client-#{self.class}] Connection established #{options[:host]}:#{options[:port]}" }
+        end
+
+        # Method to determine error state, must be used with Connector api
+        def error?
+          !!@error
+        end
+
+        def read_response
+          return if(@error)
+          @response_data = ''
+          zmq_error_check(@socket.recv_string(@response_data))
+          parse_response
+        end
+
+        def send_data
+          return if(@error)
+          log_debug { "[#{log_signature}] Sending Request: %s" % @request_data }
+          @stats.request_size = @request_data.size
+          zmq_error_check(@socket.send_string(@request_data))
+          log_debug { "[client-#{self.class}] write closed" }
+        end
+
+        def zmq_error_check(return_code)
+          raise "Last API call failed at #{caller(1)}" unless return_code >= 0
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/error.rb

@@ -9,7 +9,7 @@ module Protobuf
       attr_reader :error_type
       
       def initialize message='An unknown RpcError occurred', error_type='RPC_ERROR'
-        @error_type = error_type.is_a?(String) ? Protobuf::Socketrpc::ErrorReason.const_get(error_type) : error_type
+        @error_type = error_type.is_a?(String) ? ::Protobuf::Socketrpc::ErrorReason.const_get(error_type) : error_type
         super message
       end
       
@@ -18,7 +18,6 @@ module Protobuf
         response.error_reason = @error_type
       end
     end
-    
   end
 end
 

data/lib/protobuf/rpc/error/client_error.rb

@@ -4,25 +4,25 @@ module Protobuf
   module Rpc
     
     class InvalidRequestProto < PbError
-      def initialize message='Invalid request type given'
+      def initialize(message='Invalid request type given')
         super message, 'INVALID_REQUEST_PROTO'
       end
     end
     
     class BadResponseProto < PbError
-      def initialize message='Bad response type from server'
+      def initialize(message='Bad response type from server')
         super message, 'BAD_RESPONSE_PROTO'
       end
     end
     
     class UnkownHost < PbError
-      def initialize message='Unknown host or port'
+      def initialize(message='Unknown host or port')
         super message, 'UNKNOWN_HOST'
       end
     end
     
     class IOError < PbError
-      def initialize message='IO Error occurred'
+      def initialize(message='IO Error occurred')
         super message, 'IO_ERROR'
       end
     end

data/lib/protobuf/rpc/server.rb

@@ -1,4 +1,3 @@
-# require 'protobuf'
 require 'protobuf/common/logger'
 require 'protobuf/rpc/rpc.pb'
 require 'protobuf/rpc/buffer'
@@ -7,52 +6,46 @@ require 'protobuf/rpc/stat'
 
 module Protobuf
   module Rpc
-    module Server 
-      
+    module Server
+
       # Invoke the service method dictated by the proto wrapper request object
       def handle_client
-        @stats.request_size = @buffer.size
-        
-        # Setup the initial request and response
-        @request = Protobuf::Socketrpc::Request.new
-        @response = Protobuf::Socketrpc::Response.new
-        
         # Parse the protobuf request from the socket
-        log_debug "[#{log_signature}] Parsing request from client"
+        log_debug { "[#{log_signature}] Parsing request from client" }
         parse_request_from_buffer
-      
+
         # Determine the service class and method name from the request
-        log_debug "[#{log_signature}] Extracting procedure call info from request"
+        log_debug { "[#{log_signature}] Extracting procedure call info from request" }
         parse_service_info
-        
+
         # Call the service method
-        log_debug "[#{log_signature}] Dispatching client request to service"
+        log_debug { "[#{log_signature}] Dispatching client request to service" }
         invoke_rpc_method
       rescue => error
         # Ensure we're handling any errors that try to slip out the back door
-        log_error error.message
-        log_error error.backtrace.join("\n")
+        log_error(error.message)
+        log_error(error.backtrace.join("\n"))
         handle_error(error)
         send_response
       end
 
       # Client error handler. Receives an exception object and writes it into the @response
       def handle_error(error)
-        log_debug "[#{log_signature}] handle_error: %s" % error.inspect
+        log_debug { "[#{log_signature}] handle_error: %s" % error.inspect }
         if error.respond_to?(:to_response)
           error.to_response(@response)
         else
           message = error.respond_to?(:message) ? error.message : error.to_s
           code = error.respond_to?(:code) ? error.code.to_s : "RPC_ERROR"
-          PbError.new(message, code).to_response(@response)
+          ::Protobuf::Rpc::PbError.new(message, code).to_response(@response)
         end
       end
-      
+
       # Assuming all things check out, we can call the service method
       def invoke_rpc_method
         # Get a new instance of the service
         @service = @klass.new
-        
+
         # Define our response callback to perform the "successful" response to our client
         # This decouples the service's rpc method from our response to the client,
         # allowing the service to be the dictator for when the response should be sent back.
@@ -65,30 +58,30 @@ module Protobuf
             send_response
           end
         end
-        
+
         @service.on_rpc_failed do |error|
           unless @did_respond
             handle_error(error)
             send_response
           end
         end
-        
+
         # Call the service method
-        log_debug "[#{log_signature}] Invoking %s#%s with request %s" % [@klass.name, @method, @request.inspect]
+        log_debug { "[#{log_signature}] Invoking %s#%s with request %s" % [@klass.name, @method, @request.inspect] }
         @service.__send__(@method, @request)
       end
 
       def log_signature
         @log_signature ||= "server-#{self.class}"
       end
-      
+
       # Parse the incoming request object into our expected request object
       def parse_request_from_buffer
-        log_debug "[#{log_signature}] parsing request from buffer: %s" % @buffer.data.inspect
-        @request.parse_from_string(@buffer.data)
+        log_debug { "[#{log_signature}] parsing request from buffer: %s" % @request_data }
+        @request.parse_from_string(@request_data)
       rescue => error
-        exc = BadRequestData.new 'Unable to parse request: %s' % error.message
-        log_error exc.message
+        exc = ::Protobuf::Rpc::BadRequestData.new 'Unable to parse request: %s' % error.message
+        log_error { exc.message }
         raise exc
       end
 
@@ -97,18 +90,18 @@ module Protobuf
       # response to the protobuf response wrapper
       def parse_response_from_service(response)
         expected = @klass.rpcs[@klass][@method].response_type
-        
+
         # Cannibalize the response if it's a Hash
         response = expected.new(response) if response.is_a?(Hash)
         actual = response.class
-        log_debug "[#{log_signature}] response (should/actual): %s/%s" % [expected.name, actual.name]
-        
+        log_debug { "[#{log_signature}] response (should/actual): %s/%s" % [expected.name, actual.name] }
+
         # Determine if the service tried to change response types on us
         if expected == actual
           serialize_response(response)
         else
           # response types do not match, throw the appropriate error
-          raise BadResponseProto, 'Response proto changed from %s to %s' % [expected.name, actual.name]
+          raise ::Protobuf::Rpc::BadResponseProto, 'Response proto changed from %s to %s' % [expected.name, actual.name]
         end
       rescue => error
         log_error error.message
@@ -118,13 +111,13 @@ module Protobuf
 
       # Parses and returns the service and method name from the request wrapper proto
       def parse_service_info
-        @klass = Util.constantize(@request.service_name)
-        @method = Util.underscore(@request.method_name).to_sym
+        @klass = ::Protobuf::Util.constantize(@request.service_name)
+        @method = ::Protobuf::Util.underscore(@request.method_name).to_sym
 
         unless @klass.instance_methods.include?(@method)
           raise MethodNotFound, "Service method #{@request.method_name} is not defined by the service"
         end
-        
+
         @stats.service = @klass.name
         @stats.method = @method
       rescue NameError
@@ -134,24 +127,19 @@ module Protobuf
       # Write the response wrapper to the client
       def send_response
         raise 'Response already sent to client' if @did_respond
-        log_debug "[#{log_signature}] Sending response to client: %s" % @response.inspect
-        response_buffer = Protobuf::Rpc::Buffer.new(:write, @response)
-        send_data(response_buffer.write)
-        @stats.response_size = response_buffer.size
+        log_debug { "[#{log_signature}] Sending response to client: %s" % @response.inspect }
+        send_data
         @stats.end
         @stats.log_stats
         @did_respond = true
       end
 
       def serialize_response(response)
-        log_debug "[#{log_signature}] serializing response: %s" % response.inspect
+        log_debug { "[#{log_signature}] serializing response: %s" % response.inspect }
         @response.response_proto = response.serialize_to_string
       rescue
         raise BadResponseProto, $!.message
       end
-      
     end
-
   end
-
 end

data/lib/protobuf/rpc/servers/evented/server.rb

@@ -0,0 +1,43 @@
+require 'protobuf/rpc/server'
+
+module Protobuf
+  module Rpc
+    module Evented
+      class Server < ::EventMachine::Connection
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Logger::LogMethods
+
+        # Initialize a new read buffer for storing client request info
+        def post_init
+          log_debug { '[server] Post init, new read buffer created' }
+          @stats = Protobuf::Rpc::Stat.new(:SERVER, true)
+          @stats.client = ::Socket.unpack_sockaddr_in(get_peername)
+          @request_buffer = Protobuf::Rpc::Buffer.new(:read)
+          @request = ::Protobuf::Socketrpc::Request.new
+          @response = ::Protobuf::Socketrpc::Response.new
+
+          @did_respond = false
+        end
+
+        # Receive a chunk of data, potentially flushed to handle_client
+        def receive_data(data)
+          log_debug { '[server] receive_data: %s' % data }
+
+          @request_buffer << data
+          @request_data = @request_buffer.data
+          @stats.request_size = @request_buffer.size
+
+          handle_client if @request_buffer.flushed?
+        end
+
+        def send_data
+          response_buffer = Protobuf::Rpc::Buffer.new(:write)
+          response_buffer.set_data(@response)
+          @stats.response_size = response_buffer.size
+          log_debug { "[#{log_signature}] sending data: #{response_buffer.inspect}" }
+          super(response_buffer.write)
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/evented_runner.rb

@@ -20,7 +20,7 @@ module Protobuf
 
         # Startup and run the rpc server
         EM.schedule do
-          EventMachine.start_server(server.host, server.port, Protobuf::Rpc::EventedServer) && \
+          EventMachine.start_server(server.host, server.port, Protobuf::Rpc::Evented::Server) && \
             Protobuf::Logger.info('RPC Server listening at %s:%d in %s' % [server.host, server.port, server.env])
         end
 

data/lib/protobuf/rpc/servers/socket/server.rb

@@ -0,0 +1,108 @@
+require 'protobuf/rpc/server'
+require 'protobuf/rpc/servers/socket/worker'
+module Protobuf
+  module Rpc
+    module Socket
+
+      class Server
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Logger::LogMethods
+
+        def self.cleanup?
+          # every 10 connections run a cleanup routine after closing the response
+          @threads.size > (@thread_threshold - 1) && (@threads.size % @thread_threshold) == 0
+        end
+
+        def self.cleanup_threads
+          log_debug { "[#{log_signature}] Thread cleanup - #{@threads.size} - start" }
+
+          @threads = @threads.select do |t|
+            if t[:thread].alive?
+              true
+            else
+              t[:thread].join
+              @working.delete(t[:socket])
+              false
+            end
+          end
+
+          log_debug { "[#{log_signature}] Thread cleanup - #{@threads.size} - complete" }
+        end
+
+        def self.log_signature
+          @log_signature ||= "server-#{self}"
+        end
+
+        def self.new_worker(socket)
+          Thread.new(socket) do |sock|
+            ::Protobuf::Rpc::Socket::Worker.new(sock) do |s|
+              s.close
+            end
+          end
+        end
+
+        def self.run(opts = {})
+          log_debug { "[#{log_signature}] Run" }
+          host = opts.fetch(:host, "127.0.0.1")
+          port = opts.fetch(:port, 9399)
+          backlog = opts.fetch(:backlog, 100)
+          thread_threshold = opts.fetch(:thread_threshold, 100)
+          auto_collect_timeout = opts.fetch(:auto_collect_timeout, 20)
+
+          @threads = []
+          @thread_threshold = thread_threshold
+          @server = ::TCPServer.new(host, port)
+          @server.listen(backlog)
+          @working = []
+          @listen_fds = [@server]
+          @running = true
+
+          while running?
+            log_debug { "[#{log_signature}] Waiting for connections" }
+
+            if ready_cnxns = IO.select(@listen_fds, [], [], auto_collect_timeout)
+              cnxns = ready_cnxns.first
+              cnxns.each do |client|
+                case
+                when !running? then
+                  # no-op
+                when client == @server then
+                  log_debug { "[#{log_signature}] Accepted new connection" }
+                  client, sockaddr = @server.accept
+                  @listen_fds << client
+                else
+                  if !@working.include?(client)
+                    @working << @listen_fds.delete(client)
+                    log_debug { "[#{log_signature}] Working"  }
+                    @threads << { :thread => new_worker(client), :socket => client }
+
+                    cleanup_threads if cleanup?
+                  end
+                end
+              end
+            else
+              # Run a cleanup if select times out while waiting
+              cleanup_threads if @threads.size > 1
+            end
+          end
+
+        rescue Errno::EADDRINUSE
+          raise
+        rescue
+          # Closing the server causes the loop to raise an exception here
+          raise if running?
+        end
+
+        def self.running?
+          @running
+        end
+
+        def self.stop
+          @running = false
+          @server.close if @server
+        end
+      end
+
+    end
+  end
+end