protobuf 2.2.5-java

data/lib/protobuf/rpc/rpc.pb.rb

@@ -0,0 +1,118 @@
+### Generated by rprotoc. DO NOT EDIT!
+### <proto file: rpc.proto>
+# // Copyright (c) 2009 Shardul Deo
+# //
+# // Permission is hereby granted, free of charge, to any person obtaining a copy
+# // of this software and associated documentation files (the "Software"), to deal
+# // in the Software without restriction, including without limitation the rights
+# // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# // copies of the Software, and to permit persons to whom the Software is
+# // furnished to do so, subject to the following conditions:
+# //
+# // The above copyright notice and this permission notice shall be included in
+# // all copies or substantial portions of the Software.
+# //
+# // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# // THE SOFTWARE.
+# 
+# // Author: Shardul Deo
+# //
+# // Protobufs needed for socket rpcs.
+# 
+# package protobuf.socketrpc;
+# 
+# message Request {
+# 
+#   // RPC service full name
+#   required string service_name = 1;
+#   
+#   // RPC method name
+#   required string method_name = 2;
+#   
+#   // RPC request proto
+#   required bytes request_proto = 3;
+# }
+# 
+# message Response {
+# 
+#   // RPC response proto
+#   optional bytes response_proto = 1;
+#   
+#   // Error, if any
+#   optional string error = 2;
+#   
+#   // Was callback invoked
+#   optional bool callback = 3 [default = false];
+#   
+#   // Error Reason
+#   optional ErrorReason error_reason = 4;
+# }
+# 
+# // Possible error reasons
+# // The server-side errors are returned in the response from the server.
+# // The client-side errors are returned by the client-side code when it doesn't 
+# // have a response from the server.
+# enum ErrorReason {
+# 
+#   // Server-side errors
+#   BAD_REQUEST_DATA = 0; // Server received bad request data
+#   BAD_REQUEST_PROTO = 1; // Server received bad request proto
+#   SERVICE_NOT_FOUND = 2; // Service not found on server
+#   METHOD_NOT_FOUND = 3; // Method not found on server
+#   RPC_ERROR = 4; // Rpc threw exception on server
+#   RPC_FAILED = 5; // Rpc failed on server
+#   
+#   // Client-side errors (these are returned by the client-side code)
+#   INVALID_REQUEST_PROTO = 6; // Rpc was called with invalid request proto
+#   BAD_RESPONSE_PROTO = 7; // Server returned a bad response proto
+#   UNKNOWN_HOST = 8; // Could not find supplied host
+#   IO_ERROR = 9; // I/O error while communicating with server
+# }
+require 'protobuf/message'
+
+module Protobuf
+  module Socketrpc
+    ##
+    # Enum Classes
+    #
+    class ErrorReason < ::Protobuf::Enum; end
+    
+    ##
+    # Message Classes
+    #
+    class Request < ::Protobuf::Message; end
+    class Response < ::Protobuf::Message; end
+    
+    ##
+    # Enum Values
+    #
+    ::Protobuf::Socketrpc::ErrorReason.define :BAD_REQUEST_DATA, 0
+    ::Protobuf::Socketrpc::ErrorReason.define :BAD_REQUEST_PROTO, 1
+    ::Protobuf::Socketrpc::ErrorReason.define :SERVICE_NOT_FOUND, 2
+    ::Protobuf::Socketrpc::ErrorReason.define :METHOD_NOT_FOUND, 3
+    ::Protobuf::Socketrpc::ErrorReason.define :RPC_ERROR, 4
+    ::Protobuf::Socketrpc::ErrorReason.define :RPC_FAILED, 5
+    ::Protobuf::Socketrpc::ErrorReason.define :INVALID_REQUEST_PROTO, 6
+    ::Protobuf::Socketrpc::ErrorReason.define :BAD_RESPONSE_PROTO, 7
+    ::Protobuf::Socketrpc::ErrorReason.define :UNKNOWN_HOST, 8
+    ::Protobuf::Socketrpc::ErrorReason.define :IO_ERROR, 9
+    
+    ##
+    # Message Fields
+    #
+    ::Protobuf::Socketrpc::Request.required(::Protobuf::Field::StringField, :service_name, 1)
+    ::Protobuf::Socketrpc::Request.required(::Protobuf::Field::StringField, :method_name, 2)
+    ::Protobuf::Socketrpc::Request.required(::Protobuf::Field::BytesField, :request_proto, 3)
+    
+    ::Protobuf::Socketrpc::Response.optional(::Protobuf::Field::BytesField, :response_proto, 1)
+    ::Protobuf::Socketrpc::Response.optional(::Protobuf::Field::StringField, :error, 2)
+    ::Protobuf::Socketrpc::Response.optional(::Protobuf::Field::BoolField, :callback, 3, :default => false)
+    ::Protobuf::Socketrpc::Response.optional(::Protobuf::Socketrpc::ErrorReason, :error_reason, 4)
+    
+  end
+end

data/lib/protobuf/rpc/server.rb

@@ -0,0 +1,89 @@
+require 'protobuf/logger'
+require 'protobuf/rpc/rpc.pb'
+require 'protobuf/rpc/buffer'
+require 'protobuf/rpc/error'
+require 'protobuf/rpc/stat'
+require 'protobuf/rpc/service_dispatcher'
+
+module Protobuf
+  module Rpc
+    module Server
+
+      def initialize_request!
+        log_debug { sign_message('Post init') }
+        @request = ::Protobuf::Socketrpc::Request.new
+        @response = ::Protobuf::Socketrpc::Response.new
+        @stats = Protobuf::Rpc::Stat.new(:SERVER)
+        set_peer
+      end
+
+      def disable_gc!
+        ::GC.disable if ::Protobuf.gc_pause_server_request?
+      end
+
+      def enable_gc!
+        ::GC.enable && ::GC.start if ::Protobuf.gc_pause_server_request?
+      end
+
+      # no-op, implemented by including class if desired.
+      def set_peer; end
+
+      # Invoke the service method dictated by the proto wrapper request object
+      def handle_client
+        parse_request_from_buffer
+        @dispatcher = ::Protobuf::Rpc::ServiceDispatcher.new(@request)
+        @stats.dispatcher = @dispatcher
+
+        disable_gc!
+        @dispatcher.invoke!
+        if @dispatcher.success?
+          @response.response_proto = @dispatcher.response
+        else
+          handle_error(@dispatcher.error)
+        end
+      rescue => error
+        log_exception(error)
+        handle_error(error)
+      ensure
+        send_response
+      end
+
+      # Client error handler. Receives an exception object and writes it into the @response
+      def handle_error(error)
+        log_debug { sign_message("handle_error: #{error.inspect}") }
+        if error.respond_to?(:to_response)
+          error.to_response(@response)
+        else
+          message = error.respond_to?(:message) ? error.message : error.to_s
+          code = error.respond_to?(:code) ? error.code : 'RPC_ERROR'
+          ::Protobuf::Rpc::PbError.new(message, code).to_response(@response)
+        end
+      end
+
+      def log_signature
+        @_log_signature ||= "[server-#{self.class.name}]"
+      end
+
+      # Parse the incoming request object into our expected request object
+      def parse_request_from_buffer
+        log_debug { sign_message("Parsing request from buffer: #{@request_data}") }
+        @request.parse_from_string(@request_data)
+      rescue => error
+        exc = ::Protobuf::Rpc::BadRequestData.new("Unable to parse request: #{error.message}")
+        log_error { exc.message }
+        raise exc
+      end
+
+      # Write the response wrapper to the client
+      def send_response
+        log_debug { sign_message("Sending response to client: #{@response.inspect}") }
+        send_data
+      ensure
+        @stats.stop
+        log_info { @stats.to_s }
+        enable_gc!
+      end
+    end
+  end
+end
+

data/lib/protobuf/rpc/servers/evented/server.rb

@@ -0,0 +1,41 @@
+require 'protobuf/rpc/server'
+
+module Protobuf
+  module Rpc
+    module Evented
+      class Server < ::EventMachine::Connection
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Logger::LogMethods
+
+        # Initialize a new read buffer for storing client request info
+        def post_init
+          initialize_request!
+          @request_buffer = Protobuf::Rpc::Buffer.new(:read)
+        end
+
+        # Receive a chunk of data, potentially flushed to handle_client
+        def receive_data(data)
+          log_debug { sign_message("receive_data: #{data}") }
+
+          @request_buffer << data
+          @request_data = @request_buffer.data
+          @stats.request_size = @request_buffer.size
+
+          handle_client if @request_buffer.flushed?
+        end
+
+        def send_data
+          response_buffer = Protobuf::Rpc::Buffer.new(:write)
+          response_buffer.set_data(@response)
+          @stats.response_size = response_buffer.size
+          log_debug { sign_message("sending data: #{response_buffer.inspect}") }
+          super(response_buffer.write)
+        end
+
+        def set_peer
+          @stats.client = ::Socket.unpack_sockaddr_in(get_peername)
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/evented_runner.rb

@@ -0,0 +1,21 @@
+module Protobuf
+  module Rpc
+    class EventedRunner
+
+      def self.stop
+        ::EventMachine.stop_event_loop if ::EventMachine.reactor_running?
+      end
+
+      def self.run(options)
+        # Startup and run the rpc server
+        ::EventMachine.schedule do
+          ::EventMachine.start_server(options[:host], options[:port], ::Protobuf::Rpc::Evented::Server)
+        end
+
+        # Join or start the reactor
+				yield if block_given?
+        ::EM.reactor_running? ? ::EM.reactor_thread.join : ::EM.run
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/socket/server.rb

@@ -0,0 +1,111 @@
+require 'protobuf/rpc/server'
+require 'protobuf/rpc/servers/socket/worker'
+
+module Protobuf
+  module Rpc
+    module Socket
+
+      class Server
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Logger::LogMethods
+
+        AUTO_COLLECT_TIMEOUT = 20 # seconds
+
+        def self.cleanup?
+          # every 10 connections run a cleanup routine after closing the response
+          @threads.size > (@threshold - 1) && (@threads.size % @threshold) == 0
+        end
+
+        def self.cleanup_threads
+          log_debug { sign_message("Thread cleanup - #{@threads.size} - start") }
+
+          @threads = @threads.select do |t|
+            if t[:thread].alive?
+              true
+            else
+              t[:thread].join
+              @working.delete(t[:socket])
+              false
+            end
+          end
+
+          log_debug { sign_message("Thread cleanup - #{@threads.size} - complete") }
+        end
+
+        def self.log_signature
+          @_log_signature ||= "server-#{self.class.name}"
+        end
+
+        def self.new_worker(socket)
+          Thread.new(socket) do |sock|
+            ::Protobuf::Rpc::Socket::Worker.new(sock) do |s|
+              s.close
+            end
+          end
+        end
+
+        def self.run(options = {})
+          log_debug { sign_message("Run") }
+          host = options[:host]
+          port = options[:port]
+          backlog = options[:backlog]
+          @threshold = options[:threshold]
+
+          @threads = []
+          @server = ::TCPServer.new(host, port)
+          raise "The server was unable to start properly." if @server.closed?
+
+          @server.listen(backlog)
+          @working = []
+          @listen_fds = [@server]
+          @running = true
+
+          while running?
+            log_debug { sign_message("Waiting for connections") }
+
+            if ready_cnxns = IO.select(@listen_fds, [], [], AUTO_COLLECT_TIMEOUT)
+              cnxns = ready_cnxns.first
+              cnxns.each do |client|
+                case
+                when !running? then
+                  # no-op
+                when client == @server then
+                  log_debug { sign_message("Accepted new connection") }
+                  client, sockaddr = @server.accept
+                  @listen_fds << client
+                else
+                  if !@working.include?(client)
+                    @working << @listen_fds.delete(client)
+                    log_debug { sign_message("Working")  }
+                    @threads << { :thread => new_worker(client), :socket => client }
+
+                    cleanup_threads if cleanup?
+                  end
+                end
+              end
+            else
+              # Run a cleanup if select times out while waiting
+              cleanup_threads if @threads.size > 1
+            end
+          end
+
+        rescue Errno::EADDRINUSE
+          raise
+        rescue
+          # Closing the server causes the loop to raise an exception here
+          raise #if running?
+        end
+
+        def self.running?
+          @running
+        end
+
+        def self.stop
+          @running = false
+          @server.close if @server
+        end
+      end
+
+    end
+  end
+end

data/lib/protobuf/rpc/servers/socket/worker.rb

@@ -0,0 +1,66 @@
+require 'protobuf/rpc/server'
+require 'protobuf/logger'
+
+module Protobuf
+  module Rpc
+    module Socket
+
+      class Worker
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Logger::LogMethods
+
+        def initialize(sock, &complete_cb)
+          @socket = sock
+          initialize_request!
+
+          request_buffer = Protobuf::Rpc::Buffer.new(:read)
+          @complete_cb = complete_cb
+
+          log_debug { sign_message("stats are #{@stats.to_s}") }
+          request_buffer << read_data
+          @request_data = request_buffer.data
+
+          @stats.request_size = request_buffer.size
+
+          log_debug { sign_message("handling request") }
+          handle_client if request_buffer.flushed?
+        end
+
+        def read_data
+          size_io = StringIO.new
+
+          until (size_reader = @socket.getc) == "-"
+            size_io << size_reader
+          end
+          str_size_io = size_io.string
+
+          "#{str_size_io}-#{@socket.read(str_size_io.to_i)}"
+        end
+
+        def send_data
+          raise 'Socket closed unexpectedly' unless socket_writable?
+          response_buffer = Protobuf::Rpc::Buffer.new(:write)
+          response_buffer.set_data(@response)
+          @stats.response_size = response_buffer.size
+          log_debug { sign_message("sending data : #{response_buffer.data}") }
+          @socket.write(response_buffer.write)
+          @socket.flush
+          @complete_cb.call(@socket)
+        end
+
+        def log_signature
+          @_log_signature ||= "server-#{self.class}-#{object_id}"
+        end
+
+        def set_peer
+          @stats.client = ::Socket.unpack_sockaddr_in(@socket.getpeername)
+        end
+
+        def socket_writable?
+          ! @socket.nil? && ! @socket.closed?
+        end
+      end
+
+    end
+  end
+end

data/lib/protobuf/rpc/servers/socket_runner.rb

@@ -0,0 +1,27 @@
+module Protobuf
+  module Rpc
+    class SocketRunner
+
+      def self.stop
+        ::Protobuf::Rpc::Socket::Server.stop
+      end
+
+      def self.run(server)
+        server_config = case
+                        when server.is_a?(OpenStruct) then
+                          server.marshal_dump
+                        when server.is_a?(Hash) then
+                          server
+                        when server.respond_to?(:to_hash) then
+                          server.to_hash
+                        else
+                          raise "Cannot parser Socket Server - server options"
+                        end
+
+				yield if block_given?
+        ::Protobuf::Rpc::Socket::Server.run(server_config)
+      end
+
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq/broker.rb

@@ -0,0 +1,87 @@
+require 'resolv'
+require 'protobuf/rpc/servers/zmq/util'
+
+module Protobuf
+  module Rpc
+    module Zmq
+      class Broker
+        include ::Protobuf::Rpc::Zmq::Util
+        attr_reader :frontend, :backend, :poller, :context
+
+        ##
+        # Constructor
+        #
+        def initialize(options = {})
+          @context = ::ZMQ::Context.new
+          @frontend = setup_frontend(options)
+          @backend = setup_backend(options)
+          @poller = setup_poller
+        end
+
+        ##
+        # Instance Methods
+        #
+        def poll
+          poller.poll(1000)
+          poller.readables.each do |socket|
+            case socket
+            when frontend then
+              move_to(backend, socket)
+            when backend then
+              move_to(frontend, socket)
+            end
+          end
+        end
+
+        def teardown
+          frontend.close
+          backend.close
+          context.terminate
+        end
+
+        private
+
+          def move_to(frontend_or_backend, socket)
+            more_data = true
+
+            while more_data do
+              socket.recv_string(data = "")
+              more_data = socket.more_parts?
+              more_data_flag = (more_data ? ::ZMQ::SNDMORE : 0)
+              frontend_or_backend.send_string(data, more_data_flag)
+            end
+          end
+
+          def setup_backend(options = {})
+            dealer_options = options.merge(:port => options[:port] + 1)
+            host = dealer_options[:host]
+            port = dealer_options[:port]
+
+            zmq_backend = context.socket(::ZMQ::DEALER)
+            zmq_error_check(zmq_backend.bind(bind_address(host, port)))
+            zmq_backend
+          end
+
+          def setup_frontend(options = {})
+            host = options[:host]
+            port = options[:port]
+
+            zmq_frontend = context.socket(::ZMQ::ROUTER)
+            zmq_error_check(zmq_frontend.bind(bind_address(host, port)))
+            zmq_frontend
+          end
+
+          def bind_address(host, port)
+            "tcp://#{resolve_ip(host)}:#{port}"
+          end
+
+          def setup_poller
+            zmq_poller = ::ZMQ::Poller.new
+            zmq_poller.register(frontend, ::ZMQ::POLLIN)
+            zmq_poller.register(backend, ::ZMQ::POLLIN)
+            zmq_poller
+          end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq/server.rb

@@ -0,0 +1,50 @@
+require 'protobuf/rpc/servers/zmq/broker'
+require 'protobuf/rpc/servers/zmq/worker'
+require 'protobuf/rpc/servers/zmq/util'
+
+module Protobuf
+  module Rpc
+    module Zmq
+      class Server
+        include ::Protobuf::Rpc::Zmq::Util
+
+        ##
+        # Class Methods
+        #
+        def self.run(options = {})
+          log_debug { sign_message("initializing broker") }
+          @broker = ::Protobuf::Rpc::Zmq::Broker.new(options)
+          local_worker_threads = options[:threads]
+
+          worker_options = options.merge(:port => options[:port] + 1)
+          log_debug { sign_message("starting server workers") }
+          local_worker_threads.times do
+            @threads << Thread.new { ::Protobuf::Rpc::Zmq::Worker.new(worker_options).run }
+          end
+          @running = true
+
+          log_debug { sign_message("server started") }
+          while self.running? do
+            @broker.poll
+          end
+        ensure
+          @broker.teardown if @broker
+        end
+
+        def self.running?
+          !!@running
+        end
+
+        def self.stop
+          @running = false
+
+          @threads.each do |t|
+            t.join
+          end
+        end
+
+        @threads ||= []
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq/util.rb

@@ -0,0 +1,27 @@
+module Protobuf
+  module Rpc
+    module Zmq
+
+      module Util
+        include ::Protobuf::Logger::LogMethods
+        def self.included(base)
+          base.extend(::Protobuf::Rpc::Zmq::Util)
+        end
+
+        def zmq_error_check(return_code)
+          raise "Last API call failed with \"#{::ZMQ::Util.error_string}\"#{$/}#{$/}#{caller(1)}" unless return_code >= 0
+        end
+
+        def log_signature
+          @_log_signature ||= "server-#{self.class}-#{object_id}"
+        end
+
+        def resolve_ip(hostname)
+          ::Resolv.getaddress(hostname)
+        end
+
+      end
+
+    end
+  end
+end