protobuf 1.1.3 → 1.3.0

data/lib/protobuf/rpc/servers/socket/worker.rb

@@ -0,0 +1,59 @@
+module Protobuf
+  module Rpc
+    module Socket
+
+      class Worker
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Logger::LogMethods
+
+        def initialize(sock, &complete_cb)
+          @did_response = false
+          @socket = sock
+          @request = Protobuf::Socketrpc::Request.new
+          @response = Protobuf::Socketrpc::Response.new
+          request_buffer = Protobuf::Rpc::Buffer.new(:read)
+          @stats = Protobuf::Rpc::Stat.new(:SERVER, true)
+          @complete_cb = complete_cb
+          log_debug { "[#{log_signature}] Post init, new read buffer created" }
+
+          @stats.client = ::Socket.unpack_sockaddr_in(@socket.getpeername)
+          log_debug { "stats are #{@stats.to_s}" }
+          request_buffer << read_data
+          @request_data = request_buffer.data
+
+          @stats.request_size = request_buffer.size
+
+          log_debug { "[#{log_signature}] handling request" }
+          handle_client if request_buffer.flushed?
+        end
+
+        def log_signature
+          @log_signature ||= "server-#{self.class}-#{object_id}"
+        end
+
+        def read_data
+          size_io = StringIO.new
+
+          while((size_reader = @socket.getc) != "-")
+            size_io << size_reader
+          end
+          str_size_io = size_io.string
+
+          "#{str_size_io}-#{@socket.read(str_size_io.to_i)}"
+        end
+
+        def send_data
+          raise 'Socket closed unexpectedly' if(@socket.nil? || @socket.closed?)
+          response_buffer = Protobuf::Rpc::Buffer.new(:write)
+          response_buffer.set_data(@response)
+          @stats.response_size = response_buffer.size
+          log_debug { "[#{log_signature}] sending data : %s" % response_buffer.data }
+          @socket.write(response_buffer.write)
+          @socket.flush
+          @complete_cb.call(@socket)
+        end
+      end
+
+    end
+  end
+end

data/lib/protobuf/rpc/servers/socket_runner.rb

@@ -3,12 +3,12 @@ module Protobuf
     class SocketRunner 
 
       def self.stop
-        Protobuf::Rpc::SocketServer.stop 
+        Protobuf::Rpc::Socket::Server.stop 
         Protobuf::Logger.info 'Shutdown complete'
       end
 
       def self.run(server)
-        Protobuf::Logger.info "SocketServer Running"
+        Protobuf::Logger.info "Socket::Server Running"
         server_config = case 
                         when server.is_a?(OpenStruct) then 
                           server.marshal_dump
@@ -20,7 +20,7 @@ module Protobuf
                           raise "Cannot parser Socket Server - server options"
                         end
 
-        Protobuf::Rpc::SocketServer.run(server_config)
+        Protobuf::Rpc::Socket::Server.run(server_config)
       end
     end
 

data/lib/protobuf/rpc/servers/zmq/broker.rb

@@ -0,0 +1,85 @@
+require 'resolv'
+require 'protobuf/rpc/servers/zmq/util'
+
+module Protobuf
+  module Rpc
+    module Zmq
+      class Broker
+        include ::Protobuf::Rpc::Zmq::Util
+        attr_reader :frontend, :backend, :poller, :context
+
+        ## 
+        # Constructor
+        #
+        def initialize(opts={})
+          @context = ::ZMQ::Context.new
+          @frontend = setup_frontend(opts)
+          @backend = setup_backend(opts)
+          @poller = setup_poller
+        end
+
+        ##
+        # Instance Methods
+        #
+        def poll
+          poller.poll(1000)
+          poller.readables.each do |socket|
+            case socket
+            when frontend then
+              move_to(backend, socket)
+            when backend then
+              move_to(frontend, socket)
+            end
+          end
+        end
+
+        def teardown
+          frontend.close
+          backend.close
+          context.terminate
+        end
+
+        private
+
+          def move_to(frontend_or_backend, socket)
+            more_data = true
+
+            while more_data do
+              socket.recv_string(data = "")
+              more_data = socket.more_parts?
+              more_data_flag = (more_data ? ::ZMQ::SNDMORE : 0)
+              frontend_or_backend.send_string(data, more_data_flag)
+            end
+          end
+
+          def setup_backend(opts={})
+            dealer_options = opts.merge(:port => opts.fetch(:port, 9399) + 1)
+            host = dealer_options.fetch(:host, "127.0.0.1")
+            port = dealer_options.fetch(:port, 9400)
+            protocol = dealer_options.fetch(:protocol, "tcp")
+
+            zmq_backend = context.socket(::ZMQ::DEALER)
+            zmq_error_check(zmq_backend.bind("#{protocol}://#{resolve_ip(host)}:#{port}"))
+            zmq_backend
+          end
+
+          def setup_frontend(opts={})
+            host = opts.fetch(:host, "127.0.0.1")
+            port = opts.fetch(:port, 9399)
+            protocol = opts.fetch(:protocol, "tcp")
+
+            zmq_frontend = context.socket(::ZMQ::ROUTER)
+            zmq_error_check(zmq_frontend.bind("#{protocol}://#{resolve_ip(host)}:#{port}"))
+            zmq_frontend
+          end
+
+          def setup_poller
+            zmq_poller = ::ZMQ::Poller.new
+            zmq_poller.register(frontend, ::ZMQ::POLLIN)
+            zmq_poller.register(backend, ::ZMQ::POLLIN)
+            zmq_poller
+          end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq/server.rb

@@ -0,0 +1,50 @@
+require 'protobuf/rpc/servers/zmq/broker'
+require 'protobuf/rpc/servers/zmq/worker'
+require 'protobuf/rpc/servers/zmq/util'
+
+module Protobuf
+  module Rpc
+    module Zmq
+      class Server
+        include ::Protobuf::Rpc::Zmq::Util
+
+        ##
+        # Class Methods
+        #
+        def self.run(opts = {})
+          log_debug { "[#{log_signature}] initializing broker" }
+          @broker = ::Protobuf::Rpc::Zmq::Broker.new(opts)
+          local_worker_threads = opts.fetch(:threads, 5)
+
+          worker_options = opts.merge(:port => opts.fetch(:port, 9399) + 1)
+          log_debug { "[#{log_signature}] starting server workers" }
+          local_worker_threads.times do
+            @threads << Thread.new { ::Protobuf::Rpc::Zmq::Worker.new(worker_options).run }
+          end
+          @running = true
+
+          log_debug { "[#{log_signature}] server started" }
+          while self.running? do
+            @broker.poll
+          end
+        ensure
+          @broker.teardown if @broker
+        end
+
+        def self.running?
+          !!@running
+        end
+
+        def self.stop
+          @running = false
+
+          @threads.each do |t|
+            t.join
+          end
+        end
+
+        @threads ||= []
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq/util.rb

@@ -0,0 +1,27 @@
+module Protobuf
+  module Rpc
+    module Zmq
+
+      module Util
+        include ::Protobuf::Logger::LogMethods
+        def self.included(base)
+          base.extend(::Protobuf::Rpc::Zmq::Util)
+        end
+
+        def zmq_error_check(return_code)
+          raise "Last API call failed with \"#{::ZMQ::Util.error_string}\"#{$/}#{$/}#{caller(1)}" unless return_code >= 0
+        end
+
+        def log_signature
+          @log_signature ||= "server-#{self.class}-#{object_id}"
+        end
+
+        def resolve_ip(hostname)
+          ::Resolv.getaddress(hostname)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq/worker.rb

@@ -0,0 +1,72 @@
+require 'protobuf/rpc/server'
+require 'protobuf/rpc/servers/zmq/util'
+module Protobuf
+  module Rpc
+    module Zmq
+
+      class Worker
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Rpc::Zmq::Util
+
+        ##
+        # Constructor
+        #
+        def initialize(opts={})
+
+          @options = opts
+          host = @options.fetch(:host, "127.0.0.1")
+          port = @options.fetch(:port, 9400)
+          protocol = @options.fetch(:protocol, "tcp")
+
+          @zmq_context = ::ZMQ::Context.new
+          @socket = @zmq_context.socket(::ZMQ::REP)
+          zmq_error_check(@socket.connect("#{protocol}://#{resolve_ip(host)}:#{port}"))
+
+          @poller = ::ZMQ::Poller.new
+          @poller.register(@socket, ::ZMQ::POLLIN)
+        end
+
+        ##
+        # Instance Methods
+        #
+        def handle_request(socket)
+          @request_data = ''
+          zmq_error_check(socket.recv_string(@request_data))
+          log_debug { "[#{log_signature}] handling request" } if(!@request_data.nil?)
+        end
+
+        def initialize_buffers
+          @did_respond = false
+          @request = ::Protobuf::Socketrpc::Request.new
+          @response = ::Protobuf::Socketrpc::Response.new
+          @stats = ::Protobuf::Rpc::Stat.new(:SERVER, true)
+          log_debug { "[#{log_signature}] Post init" }
+        end
+
+        def run
+          while ::Protobuf::Rpc::Zmq::Server.running? do
+            # poll for 1_000 milliseconds then continue looping
+            # This lets us see whether we need to die
+            @poller.poll(1_000)
+            @poller.readables.each do |socket|
+              initialize_buffers
+              handle_request(socket)
+              handle_client unless(@request_data.nil?)
+            end
+          end
+        ensure
+          @socket.close
+          @zmq_context.terminate
+        end
+
+        def send_data
+          response_data = @response.is_a?(Protobuf::Message) ? @response.serialize_to_string : @response.to_s
+          @stats.response_size = response_data.size
+          zmq_error_check(@socket.send_string(response_data))
+          @did_respond = true
+        end
+      end
+
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq_runner.rb

@@ -0,0 +1,26 @@
+module Protobuf
+  module Rpc
+    class ZmqRunner
+
+      def self.stop
+        Protobuf::Rpc::Zmq::Server.stop 
+        Protobuf::Logger.info('Shutdown complete')
+      end
+
+      def self.run(server)
+        Protobuf::Logger.info "ZmqServer Running"
+        server_config = case 
+                        when server.is_a?(OpenStruct) then 
+                          server.marshal_dump
+                        when server.respond_to?(:to_hash) then
+                          server.to_hash
+                        else
+                          raise "Cannot parser Zmq Server - server options"
+                        end
+
+        Protobuf::Rpc::Zmq::Server.run(server_config)
+      end
+    end
+
+  end
+end

data/lib/protobuf/rpc/service.rb

@@ -130,7 +130,7 @@ module Protobuf
           log_error exc.message
           raise exc
         else
-          log_error "-------------- [#{log_signature}] %s#%s not rpc method, passing to super" % [self.class.name, m.to_s]
+          log_error { "-------------- [#{log_signature}] %s#%s not rpc method, passing to super" % [self.class.name, m.to_s] }
           super m, params
         end
       end
@@ -221,17 +221,17 @@ module Protobuf
         # Setup the response
         @response = rpcs[method].response_type.new
 
-        log_debug "[#{log_signature}] calling service method %s#%s" % [self.class, method]
+        log_debug { "[#{log_signature}] calling service method %s#%s" % [self.class, method] }
         # Call the aliased rpc method (e.g. :rpc_find for :find)
         __send__("rpc_#{method}".to_sym)
-        log_debug "[#{log_signature}] completed service method %s#%s" % [self.class, method]
+        log_debug { "[#{log_signature}] completed service method %s#%s" % [self.class, method] }
         
         # Pass the populated response back to the server
         # Note this will only get called if the rpc method didn't explode (by design)
         if @async_responder
-          log_debug "[#{log_signature}] async request, not sending response (yet)"
+          log_debug { "[#{log_signature}] async request, not sending response (yet)" }
         else
-          log_debug "[#{log_signature}] trigger server send_response"
+          log_debug { "[#{log_signature}] trigger server send_response" }
           send_response
         end
       end

data/lib/protobuf/version.rb

@@ -1,3 +1,3 @@
 module Protobuf
-  VERSION = '1.1.3'
+  VERSION = '1.3.0'
 end

data/protobuf.gemspec

@@ -12,19 +12,21 @@ Gem::Specification.new do |s|
   s.homepage      = %q{https://github.com/localshred/protobuf}
   s.summary       = 'Ruby implementation for Protocol Buffers. Works with other protobuf rpc implementations (e.g. Java, Python, C++).'
   s.description   = s.summary + "\n\nThis gem has diverged from https://github.com/macks/ruby-protobuf. All credit for serialization and rprotoc work most certainly goes to the original authors. All RPC implementation code (client/server/service) was written and is maintained by this author. Attempts to reconcile the original codebase with the current RPC implementation went unsuccessful."
-  
+
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
-  
-  s.add_dependency 'eventmachine', '~> 0.12.10'
-  s.add_dependency 'eventually', '~> 0.1.0'
-  s.add_dependency 'json_pure', '~> 1.6.4'
 
-  s.add_development_dependency 'rake', '~> 0.8.7'
-  s.add_development_dependency 'rspec', '~> 2.8.0'
-  s.add_development_dependency 'yard', '~> 0.7.4'
-  s.add_development_dependency 'redcarpet', '~> 1.17.2'
-  s.add_development_dependency 'simplecov', '~> 0.5.4'
+  s.add_dependency 'eventmachine'
+  s.add_dependency 'eventually'
+  s.add_dependency 'json_pure'
+  s.add_dependency 'ffi-rzmq'
+
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'pry-nav'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'yard'
+  s.add_development_dependency 'simplecov'
 end

data/spec/benchmark/tasks.rb

@@ -2,6 +2,13 @@ require 'benchmark'
 require 'helper/all'
 require 'proto/test_service_impl'
 
+# Including a way to turn on debug logger for spec runs
+if ENV["DEBUG"]
+  puts 'debugging'
+  debug_log = File.expand_path('../debug_bench.log', File.dirname(__FILE__) )
+  Protobuf::Logger.configure(:file => debug_log, :level => ::Logger::DEBUG)
+end
+
 namespace :benchmark do
   include SilentConstants
 
@@ -37,7 +44,7 @@ namespace :benchmark do
     EM.stop if EM.reactor_running?
 
     EventMachine.fiber_run do
-      StubServer.new(:server => Protobuf::Rpc::SocketServer, :port => 9399) do |server| 
+      StubServer.new(:server => Protobuf::Rpc::Socket::Server, :port => 9399) do |server| 
         client = Spec::Proto::TestService.client(:async => false, :port => 9399)
 
         benchmark_wrapper(global_bench) do |bench|
@@ -54,7 +61,7 @@ namespace :benchmark do
   def sock_client_sock_server(number_tests, test_length, global_bench = nil)
     EM.stop if EM.reactor_running?
 
-    StubServer.new(:server => Protobuf::Rpc::SocketServer, :port => 9399) do |server| 
+    StubServer.new(:server => Protobuf::Rpc::Socket::Server, :port => 9399) do |server| 
       with_constants "Protobuf::ClientType" => "Socket" do
         client = Spec::Proto::TestService.client(:async => false, :port => 9399)
 
@@ -88,6 +95,30 @@ namespace :benchmark do
     Thread.kill(em_thread) if em_thread
   end
 
+  def zmq_client_zmq_server(number_tests, test_length, global_bench = nil)
+    require 'ffi-rzmq'
+    require 'protobuf/rpc/connectors/zmq'
+    require 'protobuf/rpc/servers/zmq/server'
+    StubServer.new(:port => 9399, :server => Protobuf::Rpc::Zmq::Server) do |server| 
+      with_constants "Protobuf::ClientType" => "Zmq" do
+        client = Spec::Proto::TestService.client(:async => false, :port => 9399)
+
+        benchmark_wrapper(global_bench) do |bench|
+          bench.report("ZS / ZC") do 
+            (1..number_tests.to_i).each { client.find(:name => "Test Name" * test_length.to_i, :active => true) }
+          end
+        end
+      end
+      server.stop
+    end
+  end
+
+  desc "benchmark ZMQ client with ZMQ server"
+  task :zmq_client_zmq_server, [:number, :length] do |t, args|
+    args.with_defaults(:number => 1000, :length => 100)
+    zmq_client_zmq_server(args[:number], args[:length])
+  end
+
   desc "benchmark EventMachine client with EventMachine server"
   task :em_client_em_server, [:number, :length] do |t, args|
     args.with_defaults(:number => 1000, :length => 100)
@@ -117,10 +148,11 @@ namespace :benchmark do
     args.with_defaults(:number => 1000, :length => 100)
 
     Benchmark.bm(10) do |bench|
-      em_client_em_server(args[:number], args[:length], bench)
-      em_client_sock_server(args[:number], args[:length], bench)
+      # zmq_client_zmq_server(args[:number], args[:length], bench)
+      #      em_client_em_server(args[:number], args[:length], bench)
+      #      em_client_sock_server(args[:number], args[:length], bench)
       sock_client_sock_server(args[:number], args[:length], bench)
-      sock_client_em_server(args[:number], args[:length], bench)
+      #      sock_client_em_server(args[:number], args[:length], bench)
     end
   end
 end