protobuf 1.0.1 → 1.1.0.beta0

data/lib/protobuf/rpc/servers/evented_server.rb

@@ -0,0 +1,29 @@
+require 'protobuf/rpc/server'
+
+module Protobuf
+  module Rpc
+    class EventedServer < EventMachine::Connection
+      include Protobuf::Rpc::Server
+      include Protobuf::Logger::LogMethods
+      
+      # Initialize a new read buffer for storing client request info
+      def post_init
+        log_debug '[server] Post init, new read buffer created'
+        
+        @stats = Protobuf::Rpc::Stat.new(:SERVER, true)
+        @stats.client = Socket.unpack_sockaddr_in(get_peername)
+        
+        @buffer = Protobuf::Rpc::Buffer.new(:read)
+        @did_respond = false
+      end
+      
+      # Receive a chunk of data, potentially flushed to handle_client
+      def receive_data(data)
+        log_debug '[server] receive_data: %s' % data
+        @buffer << data
+        handle_client if @buffer.flushed?
+      end
+
+    end
+  end
+end

data/lib/protobuf/rpc/servers/socket_runner.rb

@@ -0,0 +1,17 @@
+module Protobuf
+  module Rpc
+    class SocketRunner 
+
+      def self.stop
+        Protobuf::Rpc::SocketServer.stop 
+        Protobuf::Logger.info 'Shutdown complete'
+      end
+
+      def self.run(server)
+        Protobuf::Logger.info "SocketServer Running"
+        Protobuf::Rpc::SocketServer.run(server.host, server.port, server.backlog, server.threshold) if !Protobuf::Rpc::SocketServer.running?
+      end
+    end
+
+  end
+end

data/lib/protobuf/rpc/servers/socket_server.rb

@@ -0,0 +1,145 @@
+require 'protobuf/rpc/server'
+
+module Protobuf
+  module Rpc
+    class SocketServer  
+      include Protobuf::Rpc::Server
+      include Protobuf::Logger::LogMethods
+
+      class << self
+
+        def cleanup? 
+          # every 10 connections run a cleanup routine after closing the response
+          @threads.size > (@thread_threshold - 1) && (@threads.size % @thread_threshold) == 0
+        end
+
+        def cleanup_threads
+          log_debug "[#{log_signature}] Thread cleanup - #{@threads.size} - start"
+
+          @threads = @threads.select do |t| 
+            if t[:thread].alive? 
+              true  
+            else
+              t[:thread].join
+              @working.delete(t[:socket])
+              false
+            end
+          end
+
+          log_debug "[#{log_signature}] Thread cleanup - #{@threads.size} - complete"
+        end
+
+        def log_signature
+          @log_signature ||= "server-#{self}"
+        end
+
+        def new_worker(socket)
+          Thread.new(socket) do |sock|
+            Protobuf::Rpc::SocketServer::Worker.new(sock) do |s|
+              s.close
+            end
+          end
+        end
+
+        # TODO make listen/backlog part of config
+        def run(host = "127.0.0.1", port = 9399, backlog = 100, thread_threshold = 100)
+          log_debug "[#{log_signature}] Run"
+          @running = true
+          @threads = []
+          @thread_threshold = thread_threshold
+          @server = TCPServer.new(host, port)
+          @server.listen(backlog)
+          @working = []
+          @listen_fds = [@server]
+
+          while running?
+            log_debug "[#{log_signature}] Waiting for connections"
+
+            if ready_cnxns = IO.select(@listen_fds, [], [], 20)
+              cnxns = ready_cnxns.first
+              cnxns.each do |client|
+                case 
+                when !running? then
+                  # no-op
+                when client == @server then 
+                  log_debug "[#{log_signature}] Accepted new connection"
+                  client, sockaddr = @server.accept
+                  @listen_fds << client
+                else 
+                  if !@working.include?(client)
+                    @working << @listen_fds.delete(client)
+                    log_debug "[#{log_signature}] Working" 
+                    @threads << { :thread => new_worker(client), 
+                                  :socket => client }
+
+                    cleanup_threads if cleanup?
+                  end
+                end
+              end
+            else
+              # Run a cleanup if select times out while waiting
+              cleanup_threads if @threads.size > 1
+            end
+          end
+
+        rescue 
+          # Closing the server causes the loop to raise an exception here
+          raise if running?
+        end
+
+        def running?
+          @running
+        end
+
+        def stop 
+          @running = false
+          @server.close
+        end
+
+      end
+
+      class Worker 
+        include Protobuf::Rpc::Server
+        include Protobuf::Logger::LogMethods
+
+        def initialize(sock, &complete_cb)
+          @did_response = false
+          @socket = sock
+          @request = Protobuf::Socketrpc::Request.new
+          @response = Protobuf::Socketrpc::Response.new
+          @buffer = Protobuf::Rpc::Buffer.new(:read)
+          @stats = Protobuf::Rpc::Stat.new(:SERVER, true)
+          @complete_cb = complete_cb
+          log_debug "[#{log_signature}] Post init, new read buffer created"
+
+          @stats.client = Socket.unpack_sockaddr_in(@socket.getpeername)
+          @buffer << read_data 
+          log_debug "[#{log_signature}] handling request"
+          handle_client if @buffer.flushed?
+        end
+
+        def log_signature
+          @log_signature ||= "server-#{self.class}-#{object_id}"
+        end
+
+        def read_data
+          size_io = StringIO.new
+
+          while((size_reader = @socket.getc) != "-")
+            size_io << size_reader
+          end
+          str_size_io = size_io.string
+
+          "#{str_size_io}-#{@socket.read(str_size_io.to_i)}"
+        end
+
+        def send_data(data)
+          log_debug "[#{log_signature}] sending data : %s" % data
+          @socket.write(data)
+          @socket.flush
+          @complete_cb.call(@socket)
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/service.rb

@@ -4,10 +4,9 @@ require 'protobuf/rpc/error'
 
 module Protobuf
   module Rpc
-    
     # Object to encapsulate the request/response types for a given service method
     # 
-    RpcMethod = Struct.new "RpcMethod", :service, :method, :request_type, :response_type
+    RpcMethod = Struct.new("RpcMethod", :service, :method, :request_type, :response_type)
     
     class Service
       include Protobuf::Logger::LogMethods
@@ -27,34 +26,32 @@ module Protobuf
         
         # You MUST add the method name to this list if you are adding
         # instance methods below, otherwise stuff will definitely break
-        NON_RPC_METHODS = %w( rpcs call_rpc on_rpc_failed rpc_failed request response method_missing async_responder on_send_response send_response  )
+        NON_RPC_METHODS = %w( rpcs call_rpc on_rpc_failed rpc_failed request response method_missing async_responder on_send_response send_response log_signature )
         
         # Override methods being added to the class
         # If the method isn't already a private instance method, or it doesn't start with rpc_, 
         # or it isn't in the reserved method list (NON_RPC_METHODS),
         # We want to remap the method such that we can wrap it in before and after behavior,
         # most notably calling call_rpc against the method. See call_rpc for more info.
-        def method_added old
+        def method_added(old)
           new_method = :"rpc_#{old}"
           return if private_instance_methods.include?(new_method) or old =~ /^rpc_/ or NON_RPC_METHODS.include?(old.to_s)
           
           alias_method new_method, old
           private new_method
           
-          begin
-            define_method(old) do |pb_request|
-              call_rpc old.to_sym, pb_request
-            end
-          rescue ArgumentError => e
-            # Wrap a known issue where an instance method was defined in the class without
-            # it being ignored with NON_RPC_METHODS. 
-            raise ArgumentError, "#{e.message} (Note: This could mean that you need to add the method #{old} to the NON_RPC_METHODS list)"
+          define_method(old) do |pb_request|
+            call_rpc(old.to_sym, pb_request)
           end
+        rescue ArgumentError => e
+          # Wrap a known issue where an instance method was defined in the class without
+          # it being ignored with NON_RPC_METHODS. 
+          raise ArgumentError, "#{e.message} (Note: This could mean that you need to add the method #{old} to the NON_RPC_METHODS list)"
         end
       
         # Generated service classes should call this method on themselves to add rpc methods
         # to the stack with a given request and response type
-        def rpc method, request_type, response_type
+        def rpc(method, request_type, response_type)
           rpcs[self] ||= {}
           rpcs[self][method] = RpcMethod.new self, method, request_type, response_type
         end
@@ -68,7 +65,7 @@ module Protobuf
         # See Client#initialize and ClientConnection::DEFAULT_OPTIONS
         # for all available options.
         #
-        def client options={}
+        def client(options={})
           configure
           Client.new({
             :service => self,
@@ -83,17 +80,17 @@ module Protobuf
         # so that any Clients using the Service.client sugar
         # will not have to configure the location each time.
         # 
-        def configure config={}
+        def configure(config={})
           locations[self] ||= {}
-          locations[self][:host] = config[:host] if config.key? :host
-          locations[self][:port] = config[:port] if config.key? :port
+          locations[self][:host] = config[:host] if config.key?(:host)
+          locations[self][:port] = config[:port] if config.key?(:port)
         end
         
         # Shorthand call to configure, passing a string formatted as hostname:port
         # e.g. 127.0.0.1:9933
         # e.g. localhost:0
         #
-        def located_at location
+        def located_at(location)
           return if location.nil? or location.downcase.strip !~ /[a-z0-9.]+:\d+/
           host, port = location.downcase.strip.split ':'
           configure :host => host, :port => port.to_i
@@ -117,6 +114,10 @@ module Protobuf
         end
         
       end
+
+      def log_signature
+        @log_signature ||= "service-#{self.class}"
+      end
       
       # If a method comes through that hasn't been found, and it
       # is defined in the rpcs method list, we know that the rpc
@@ -129,7 +130,7 @@ module Protobuf
           log_error exc.message
           raise exc
         else
-          log_error '-------------- [service] %s#%s not rpc method, passing to super' % [self.class.name, m.to_s]
+          log_error "-------------- [#{log_signature}] %s#%s not rpc method, passing to super" % [self.class.name, m.to_s]
           super m, params
         end
       end
@@ -141,7 +142,7 @@ module Protobuf
       
       # Callback register for the server when a service
       # method calls rpc_failed. Called by Service#rpc_failed.
-      def on_rpc_failed &rpc_failure_cb
+      def on_rpc_failed(&rpc_failure_cb)
         @rpc_failure_cb = rpc_failure_cb
       end
       
@@ -149,14 +150,11 @@ module Protobuf
       # NOTE: This shortcuts the @async_responder paradigm. There is
       # not any way to get around this currently (and I'm not sure you should want to).
       #
-      def rpc_failed message="RPC Failed while executing service method #{@current_method}"
-        if @rpc_failure_cb.nil?
-          exc = RuntimeError.new 'Unable to invoke rpc_failed, no failure callback is setup.' 
-          log_error exc.message
-          raise exc
-        end
+      def rpc_failed(message="RPC Failed while executing service method #{@current_method}")
+        error_message = 'Unable to invoke rpc_failed, no failure callback is setup.' 
+        log_and_raise_error(error_message) if @rpc_failure_cb.nil?
         error = message.is_a?(String) ? RpcFailed.new(message) : message
-        log_warn '[service] RPC Failed: %s' % error.message
+        log_warn "[#{log_signature}] RPC Failed: %s" % error.message
         @rpc_failure_cb.call(error)
       end
       
@@ -164,7 +162,7 @@ module Protobuf
       # when it is appropriate to generate a response to the client.
       # Used in conjunciton with Service#send_response.
       # 
-      def on_send_response &responder
+      def on_send_response(&responder)
         @responder = responder
       end
       
@@ -175,15 +173,17 @@ module Protobuf
       # will timeout since no data will be sent.
       #
       def send_response
-        if @responder.nil?
-          exc = RuntimeError.new "Unable to send response, responder is nil. It appears you aren't inside of an RPC request/response cycle."
-          log_error exc.message
-          raise exc
-        end
-        @responder.call @response
+        error_message = "Unable to send response, responder is nil. It appears you aren't inside of an RPC request/response cycle."
+        log_and_raise_error(error_message) if @responder.nil?
+        @responder.call(@response)
       end
   
     private
+
+      def log_and_raise_error(error_message)
+        log_error(error_message)
+        raise error_message
+      end
       
       # Call the rpc method that was previously privatized.
       # call_rpc allows us to wrap the normal method call with 
@@ -202,38 +202,36 @@ module Protobuf
       # by calling self.send_response without any arguments. The rpc
       # server is setup to handle synchronous and asynchronous responses.
       #
-      def call_rpc method, pb_request
+      def call_rpc(method, pb_request)
         @current_method = method
         
         # Allows the service to set whether or not
         # it would like to asynchronously respond to the connected client(s)
         @async_responder = false
         
-        begin
-          # Setup the request
-          @request = rpcs[method].request_type.new
-          @request.parse_from_string pb_request.request_proto
-        rescue
-          exc = BadRequestProto.new 'Unable to parse request: %s' % $!.message
-          log_error exc.message
-          log_error $!.backtrace.join("\n")
-          raise exc
-        end
-        
+        # Setup the request
+        @request = rpcs[method].request_type.new
+        @request.parse_from_string(pb_request.request_proto)
+      rescue
+        exc = BadRequestProto.new 'Unable to parse request: %s' % $!.message
+        log_error exc.message
+        log_error $!.backtrace.join("\n")
+        raise exc
+      else   # when no Exception was thrown
         # Setup the response
         @response = rpcs[method].response_type.new
 
-        log_debug '[service] calling service method %s#%s' % [self.class.name, method]
+        log_debug "[#{log_signature}] calling service method %s#%s" % [self.class, method]
         # Call the aliased rpc method (e.g. :rpc_find for :find)
         __send__("rpc_#{method}".to_sym)
-        log_debug '[service] completed service method %s#%s' % [self.class.name, method]
+        log_debug "[#{log_signature}] completed service method %s#%s" % [self.class, method]
         
         # Pass the populated response back to the server
         # Note this will only get called if the rpc method didn't explode (by design)
         if @async_responder
-          log_debug '[service] async request, not sending response (yet)'
+          log_debug "[#{log_signature}] async request, not sending response (yet)"
         else
-          log_debug '[service] trigger server send_response'
+          log_debug "[#{log_signature}] trigger server send_response"
           send_response
         end
       end
@@ -241,4 +239,5 @@ module Protobuf
     end
     
   end
-end
+
+end

data/lib/protobuf/rpc/stat.rb

@@ -51,12 +51,12 @@ module Protobuf
       end
       
       def log_stats
-        Protobuf::Logger.info to_s
+        Protobuf::Logger.info(self.to_s)
       end
       
       def to_s
         [
-          @type == :SERVER ? '[SRV]' : '[CLT]',
+          @type == :SERVER ? "[SRV-#{self.class}]" : "[CLT-#{self.class}]",
           rpc,
           elapsed_time,
           sizes,

data/lib/protobuf/version.rb

@@ -1,3 +1,3 @@
 module Protobuf
-  VERSION = '1.0.1'
+  VERSION = '1.1.0.beta0'
 end

data/protobuf.gemspec

@@ -1,4 +1,4 @@
-# -*- encoding: utf-8 -*-
+# encoding: UTF-8
 $:.push File.expand_path("./lib", File.dirname(__FILE__))
 require "protobuf/version"
 
@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   s.date          = %q{2011-12-07}
 
   s.authors       = ['BJ Neilsen', 'Brandon Dewitt']
-  s.email         = ["bj.neilsen@gmail.com", "brandonsdewitt@gmail.com"]
+  s.email         = ["bj.neilsen@gmail.com", "brandonsdewitt+protobuf@gmail.com"]
   s.homepage      = %q{https://github.com/localshred/protobuf}
   s.summary       = 'Ruby implementation for Protocol Buffers. Works with other protobuf rpc implementations (e.g. Java, Python, C++).'
   s.description   = s.summary + "\n\nThis gem has diverged from https://github.com/macks/ruby-protobuf. All credit for serialization and rprotoc work most certainly goes to the original authors. All RPC implementation code (client/server/service) was written and is maintained by this author. Attempts to reconcile the original codebase with the current RPC implementation went unsuccessful."
@@ -19,7 +19,12 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   
   s.add_dependency 'eventmachine', '~> 0.12.10'
-  
+  s.add_dependency 'eventually', '~> 0.1.0'
+  s.add_dependency 'json_pure', '~> 1.6.4'
+
   s.add_development_dependency 'rake', '~> 0.8.7'
-  s.add_development_dependency 'rspec', '~> 2.7.0'
+  s.add_development_dependency 'rspec', '~> 2.8.0'
+  s.add_development_dependency 'yard', '~> 0.7.4'
+  s.add_development_dependency 'redcarpet', '~> 1.17.2'
+  s.add_development_dependency 'simplecov', '~> 0.5.4'
 end