protobuf 2.7.11-java → 2.8.0.beta1-java

data/lib/protobuf/rpc/servers/zmq/util.rb

@@ -1,29 +1,41 @@
+require 'resolv'
+
 module Protobuf
   module Rpc
     module Zmq
 
-      WORKER_READY_MESSAGE = "WORKER_READY"
+      WORKER_READY_MESSAGE = "\1"
 
       module Util
         include ::Protobuf::Logger::LogMethods
+
         def self.included(base)
           base.extend(::Protobuf::Rpc::Zmq::Util)
         end
 
-        def zmq_error_check(return_code)
-          raise "Last API call failed with \"#{::ZMQ::Util.error_string}\"#{$/}#{$/}#{caller(1)}" unless return_code >= 0
+        def zmq_error_check(return_code, source = nil)
+          unless ::ZMQ::Util.resultcode_ok?(return_code)
+            raise <<-ERROR
+            Last ZMQ API call #{source ? "to #{source}" : ""} failed with "#{::ZMQ::Util.error_string}".
+
+            #{caller(1).join($/)}
+            ERROR
+          end
         end
 
         def log_signature
-          @_log_signature ||= "server-#{self.class}-#{object_id}"
+          unless @_log_signature
+            name = (self.class == Class ? self.name : self.class.name)
+            @_log_signature = "[server-#{name}-#{object_id}]"
+          end
+
+          @_log_signature
         end
 
         def resolve_ip(hostname)
           ::Resolv.getaddress(hostname)
         end
-
       end
-
     end
   end
 end

data/lib/protobuf/rpc/servers/zmq/worker.rb

@@ -1,9 +1,9 @@
 require 'protobuf/rpc/server'
 require 'protobuf/rpc/servers/zmq/util'
+
 module Protobuf
   module Rpc
     module Zmq
-
       class Worker
         include ::Protobuf::Rpc::Server
         include ::Protobuf::Rpc::Zmq::Util
@@ -11,63 +11,126 @@ module Protobuf
         ##
         # Constructor
         #
-        def initialize(options = {})
-          host = options[:host]
-          port = options[:worker_port]
-
-          @zmq_context = ::ZMQ::Context.new
-          @socket = @zmq_context.socket(::ZMQ::REQ)
-          zmq_error_check(@socket.connect("tcp://#{resolve_ip(host)}:#{port}"))
-
-          @poller = ::ZMQ::Poller.new
-          @poller.register(@socket, ::ZMQ::POLLIN)
-
-          # Send request to broker telling it we are ready
-          zmq_error_check(@socket.send_string(::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE))
+        def initialize(server)
+          @server = server
+          init_zmq_context
+          init_backend_socket
+          init_shutdown_socket
+        rescue
+          teardown
+          raise
         end
 
         ##
         # Instance Methods
         #
-        def handle_request(socket)
-          message_array = []
-          zmq_error_check(socket.recv_strings(message_array))
+        def alive?
+          @thread.try(:alive?) || false
+        end
 
-          @request_data = message_array[2]
-          @client_address = message_array[0]
-          log_debug { sign_message("handling request") } unless @request_data.nil?
+        def join
+          @thread.try(:join)
+        end
+
+        def process_request
+          @client_address, empty, @request_data = read_from_backend
+
+          unless @request_data.nil?
+            log_debug { sign_message("handling request") }
+            handle_client
+          end
+        end
+
+        def read_from_backend
+          [].tap do |frames|
+            zmq_error_check(@backend_socket.recv_strings(frames))
+          end
         end
 
         def run
-          while ::Protobuf::Rpc::Zmq::Server.running? do
-            # poll for 1_000 milliseconds then continue looping
-            # This lets us see whether we need to die
-            @poller.poll(1_000)
-            @poller.readables.each do |socket|
-              initialize_request!
-              handle_request(socket)
-              handle_client unless @request_data.nil?
+          poller = ::ZMQ::Poller.new
+          poller.register_readable(@backend_socket)
+          poller.register_readable(@shutdown_socket)
+
+          # Send request to broker telling it we are ready
+          write_to_backend([::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE])
+
+          catch(:shutdown) do
+            while poller.poll > 0
+              poller.readables.each do |readable|
+                case readable
+                when @backend_socket
+                  initialize_request!
+                  process_request
+                when @shutdown_socket
+                  throw :shutdown
+                end
+              end
             end
           end
         ensure
-          @socket.close
-          @zmq_context.terminate
+          teardown
         end
 
         def send_data
-          response_data = @response.to_s # to_s is aliases as serialize_to_string in Message
+          data = @response.serialize_to_string
 
-          response_message_set = [
-            @client_address, # client uuid address
-            "",
-            response_data
-          ]
+          @stats.response_size = data.size
 
-          @stats.response_size = response_data.size
-          zmq_error_check(@socket.send_strings(response_message_set))
+          write_to_backend([@client_address, "", data])
+        end
+
+        def shutdown_uri
+          "inproc://#{object_id}"
+        end
+
+        def signal_shutdown
+          socket = @zmq_context.socket ZMQ::PAIR
+          zmq_error_check(socket.connect(shutdown_uri))
+          zmq_error_check(socket.send_string("."))
+          zmq_error_check(socket.close)
+        end
+
+        def start
+          @thread = Thread.new do
+            begin
+              self.run
+            rescue => e
+              message = "Worker failed: #{e.inspect}\n #{e.backtrace.join($/)}"
+              $stderr.puts(message)
+              log_error { message }
+            end
+          end
+
+          self
+        end
+
+        def teardown
+          @backend_socket.try(:close)
+          @shutdown_socket.try(:close)
+          @zmq_context.try(:terminate)
         end
-      end
 
+        def write_to_backend(frames)
+          zmq_error_check(@backend_socket.send_strings(frames))
+        end
+
+        private
+
+        def init_zmq_context
+          @zmq_context = ZMQ::Context.new
+        end
+
+        def init_backend_socket
+          @backend_socket = @zmq_context.socket(ZMQ::REQ)
+          zmq_error_check(@backend_socket.connect(@server.backend_uri))
+        end
+
+        def init_shutdown_socket
+          @shutdown_socket = @zmq_context.socket(ZMQ::PAIR)
+          zmq_error_check(@shutdown_socket.bind(shutdown_uri))
+        end
+      end
     end
   end
 end

data/lib/protobuf/rpc/servers/zmq_runner.rb

@@ -1,35 +1,46 @@
+require 'ostruct'
+
 module Protobuf
   module Rpc
     class ZmqRunner
       include ::Protobuf::Logger::LogMethods
 
-      def self.register_signals
-        trap(:TTIN) do 
-          log_info { "TTIN received: Starting new worker" }
-          ::Protobuf::Rpc::Zmq::Server.start_worker
-          log_info { "Worker count : #{::Protobuf::Rpc::Zmq::Server.threads.size}" }
-        end
-      end
+      def initialize(options)
+        @options = case
+                   when options.is_a?(OpenStruct) then
+                     options.marshal_dump
+                   when options.respond_to?(:to_hash) then
+                     options.to_hash
+                   else
+                     raise "Cannot parser Zmq Server - server options"
+                   end
 
-      def self.run(server)
-        server_config = case
-                        when server.is_a?(OpenStruct) then
-                          server.marshal_dump
-                        when server.respond_to?(:to_hash) then
-                          server.to_hash
-                        else
-                          raise "Cannot parser Zmq Server - server options"
-                        end
+      end
 
-				yield if block_given?
+      def run
+        @server = ::Protobuf::Rpc::Zmq::Server.new(@options)
+        register_signals
+        yield if block_given?
+        @server.run
+      end
 
-        ::Protobuf::Rpc::Zmq::Server.run(server_config)
+      def running?
+        @server.try :running?
       end
 
-      def self.stop
-        ::Protobuf::Rpc::Zmq::Server.stop
+      def stop
+        @server.try :stop
       end
 
+      private
+
+      def register_signals
+        trap(:TTIN) do
+          log_info { "TTIN received: Starting new worker" }
+          @server.start_worker
+          log_info { "Worker count : #{::Protobuf::Rpc::Zmq::Server.threads.size}" }
+        end
+      end
     end
   end
 end

data/lib/protobuf/rpc/service.rb

@@ -96,6 +96,18 @@ module Protobuf
         rpcs.key?(name)
       end
 
+      # An array of defined service classes that contain implementation
+      # code
+      def self.implemented_services
+        classes = (self.subclasses || []).select do |subclass|
+          subclass.rpcs.any? do |(name, method)|
+            subclass.method_defined? name
+          end
+        end
+
+        classes.map &:name
+      end
+
       ##
       # Instance Methods
       #
@@ -122,6 +134,18 @@ module Protobuf
         @_response ||= response_type.new
       end
 
+      # Request object for this rpc cycle. Not assignable.
+      #
+      def request
+        @_request ||= if @_request_bytes.present?
+            request_type.new.parse_from_string(@_request_bytes)
+          else
+            request_type.new
+          end
+      rescue => e
+        raise BadRequestProto, "Unable to parse request: #{e.message}"
+      end
+
       # Convenience method to get back to class method.
       #
       def rpc_method?(name)
@@ -149,18 +173,6 @@ module Protobuf
         @_response_type ||= rpcs[@method_name].response_type
       end
 
-      # Request object for this rpc cycle. Not assignable.
-      #
-      def request
-        @_request ||= if @_request_bytes.present?
-            request_type.new.parse_from_string(@_request_bytes)
-          else
-            request_type.new
-          end
-      rescue => e
-        raise BadRequestProto, "Unable to parse request: #{e.message}"
-      end
-
       def request_type
         @_request_type ||= rpcs[@method_name].request_type
       end

data/lib/protobuf/rpc/service_directory.rb

@@ -0,0 +1,206 @@
+require 'delegate'
+require 'singleton'
+require 'socket'
+require 'thread'
+require 'timeout'
+
+require 'protobuf/rpc/dynamic_discovery.pb'
+
+module Protobuf
+  module Rpc
+    class ServiceDirectory
+      include ::Singleton
+      include ::Protobuf::Logger::LogMethods
+
+      DEFAULT_ADDRESS = "255.255.255.255"
+      DEFAULT_PORT = 53000
+      DEFAULT_TIMEOUT = 1
+
+      class Listing < Delegator
+        attr_reader :expires_at
+
+        def initialize(server)
+          @server = server
+          @expires_at = Time.now.to_i + ttl
+        end
+
+        def current?
+          !expired?
+        end
+
+        def expired?
+          Time.now.to_i >= @expires_at
+        end
+
+        def ttl
+          [super.to_i, 3].max
+        end
+
+        def __getobj__
+          @server
+        end
+      end
+
+      # Class Methods
+      #
+      class << self
+        attr_writer :address, :port
+      end
+
+      def self.address
+        @address ||= DEFAULT_ADDRESS
+      end
+
+      def self.port
+        @port ||= DEFAULT_PORT
+      end
+
+      def self.start
+        yield(self) if block_given?
+        self.instance.start
+      end
+
+      def self.stop
+        self.instance.stop
+      end
+
+      # Instance Methods
+      #
+      def initialize
+        @listings = {}
+        @mutex = Mutex.new
+      end
+
+      def add_listing_for(server)
+        if server && server.uuid
+
+          log_debug do
+            action = @listings[server.uuid] ? "Updating" : "Adding";
+            sign_message("#{action} server: #{server.inspect}")
+          end
+
+          @mutex.synchronize do
+            @listings[server.uuid] = Listing.new(server)
+          end
+
+        else
+          log_info { sign_message("Cannot add server without uuid: #{server.inspect}") }
+        end
+      end
+
+      def lookup(service)
+        @mutex.synchronize do
+          listings = @listings.values.select do |listing|
+            listing.services.any? do |listed_service|
+              listing.current? && listed_service == service.to_s
+            end
+          end
+
+          listings.sample
+        end
+      end
+
+      def remove_expired_listings
+        @mutex.synchronize do
+          @listings.delete_if do |uuid, listing|
+            listing.expired?
+          end
+        end
+      end
+
+      def remove_listing_for(server)
+        if server && server.uuid
+          log_debug { sign_message("Removing server: #{server.inspect}") }
+
+          @mutex.synchronize do
+            @listings.delete(server.uuid)
+          end
+
+        else
+          log_info { sign_message("Cannot remove server without uuid: #{server.inspect}") }
+        end
+      end
+
+      def restart
+        stop
+        start
+      end
+
+      def running?
+        !!@thread.try(:alive?)
+      end
+
+      def start
+        unless running?
+          init_socket
+          log_info { sign_message("listening to udp://#{self.class.address}:#{self.class.port}") }
+          @thread = Thread.new { self.send(:run) }
+        end
+
+        self
+      end
+
+      def stop
+        log_info { sign_message("Stopping directory") }
+
+        @mutex.synchronize do
+          @thread.try(:kill)
+          @thread = nil
+          @listings = {}
+        end
+
+        @socket.try(:close)
+        @socket = nil
+      end
+
+      def wait_for(service, timeout = DEFAULT_TIMEOUT)
+        log_debug { sign_message("waiting for #{service}") }
+        Timeout.timeout(timeout) do
+          sleep(timeout / 10.0) until listing = lookup(service)
+          listing
+        end
+      rescue
+        log_info { sign_message("no listing found for #{service}") }
+        nil
+      end
+
+      private
+
+      def init_socket
+        @socket = UDPSocket.new
+        @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEADDR, true)
+        @socket.bind(self.class.address, self.class.port.to_i)
+      end
+
+      def process_beacon(beacon)
+        case beacon.beacon_type
+        when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
+          add_listing_for(beacon.server)
+        when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
+          remove_listing_for(beacon.server)
+        end
+      end
+
+      def run
+        loop do
+          process_beacon(wait_for_beacon)
+          remove_expired_listings
+        end
+      rescue => e
+        log_debug { sign_message("error: (#{e.class}) #{e.message}") }
+        retry
+      end
+
+      def wait_for_beacon
+        data, addr = @socket.recvfrom(2048)
+
+        ::Protobuf::Rpc::DynamicDiscovery::Beacon.new.tap do |beacon|
+          beacon.parse_from_string(data) rescue nil
+
+          # Favor the address captured by the socket
+          beacon.try(:server).try(:address=, addr[3])
+        end
+      end
+    end
+  end
+end