protobuf 2.7.12 → 2.8.0.beta1

data/lib/protobuf/rpc/servers/evented_runner.rb

@@ -2,22 +2,27 @@ module Protobuf
   module Rpc
     class EventedRunner
 
-      def self.register_signals
-        # Noop
+      def initialize(options)
+        @options = options
       end
 
-      def self.run(options)
+      def run
         # Startup and run the rpc server
         ::EventMachine.schedule do
-          ::EventMachine.start_server(options[:host], options[:port], ::Protobuf::Rpc::Evented::Server)
+          ::EventMachine.start_server(
+            @options[:host],
+            @options[:port],
+            ::Protobuf::Rpc::Evented::Server
+          )
         end
 
         # Join or start the reactor
-				yield if block_given?
+        yield if block_given?
+
         ::EM.reactor_running? ? ::EM.reactor_thread.join : ::EM.run
       end
 
-      def self.stop
+      def stop
         ::EventMachine.stop_event_loop if ::EventMachine.reactor_running?
       end
 

data/lib/protobuf/rpc/servers/socket/server.rb

@@ -11,12 +11,16 @@ module Protobuf
 
         AUTO_COLLECT_TIMEOUT = 5 # seconds
 
-        def self.cleanup?
+        def initialize(options)
+          @options = options
+        end
+
+        def cleanup?
           # every 10 connections run a cleanup routine after closing the response
           @threads.size > (@threshold - 1) && (@threads.size % @threshold) == 0
         end
 
-        def self.cleanup_threads
+        def cleanup_threads
           log_debug { sign_message("Thread cleanup - #{@threads.size} - start") }
 
           @threads = @threads.select do |t|
@@ -32,11 +36,11 @@ module Protobuf
           log_debug { sign_message("Thread cleanup - #{@threads.size} - complete") }
         end
 
-        def self.log_signature
+        def log_signature
           @_log_signature ||= "server-#{self.class.name}"
         end
 
-        def self.new_worker(socket)
+        def new_worker(socket)
           Thread.new(socket) do |sock|
             ::Protobuf::Rpc::Socket::Worker.new(sock) do |s|
               s.close
@@ -44,12 +48,12 @@ module Protobuf
           end
         end
 
-        def self.run(options = {})
+        def run
           log_debug { sign_message("Run") }
-          host = options[:host]
-          port = options[:port]
-          backlog = options[:backlog]
-          @threshold = options[:threshold]
+          host = @options[:host]
+          port = @options[:port]
+          backlog = @options[:backlog]
+          @threshold = @options[:threshold]
 
           @threads = []
           @server = ::TCPServer.new(host, port)
@@ -62,8 +66,9 @@ module Protobuf
 
           while running?
             log_debug { sign_message("Waiting for connections") }
+            ready_cnxns = IO.select(@listen_fds, [], [], AUTO_COLLECT_TIMEOUT) rescue nil
 
-            if ready_cnxns = IO.select(@listen_fds, [], [], AUTO_COLLECT_TIMEOUT)
+            if ready_cnxns
               cnxns = ready_cnxns.first
               cnxns.each do |client|
                 case
@@ -96,16 +101,15 @@ module Protobuf
           raise #if running?
         end
 
-        def self.running?
-          @running
+        def running?
+          !!@running
         end
 
-        def self.stop
+        def stop
           @running = false
-          @server.close if @server
+          @server.try(:close)
         end
       end
-
     end
   end
 end

data/lib/protobuf/rpc/servers/socket_runner.rb

@@ -2,30 +2,33 @@ module Protobuf
   module Rpc
     class SocketRunner
 
-      def self.register_signals
-        # noop
-      end
+      def initialize(options)
+        @options = case
+                   when options.is_a?(OpenStruct) then
+                     options.marshal_dump
+                   when options.is_a?(Hash) then
+                     options
+                   when options.respond_to?(:to_hash) then
+                     options.to_hash
+                   else
+                     raise "Cannot parser Socket Server - server options"
+                   end
 
-      def self.run(server)
-        server_config = case
-                        when server.is_a?(OpenStruct) then
-                          server.marshal_dump
-                        when server.is_a?(Hash) then
-                          server
-                        when server.respond_to?(:to_hash) then
-                          server.to_hash
-                        else
-                          raise "Cannot parser Socket Server - server options"
-                        end
+        @server = ::Protobuf::Rpc::Socket::Server.new(@options)
+      end
 
-				yield if block_given?
-        ::Protobuf::Rpc::Socket::Server.run(server_config)
+      def run
+        yield if block_given?
+        @server.run
       end
 
-      def self.stop
-        ::Protobuf::Rpc::Socket::Server.stop
+      def running?
+        @server.running?
       end
 
+      def stop
+        @server.stop
+      end
     end
   end
 end

data/lib/protobuf/rpc/servers/zmq/broker.rb

@@ -1,128 +1,138 @@
-require 'resolv'
-require 'protobuf/rpc/servers/zmq/util'
 
 module Protobuf
   module Rpc
     module Zmq
       class Broker
         include ::Protobuf::Rpc::Zmq::Util
-        attr_reader :frontend, :backend, :poller, :context, :available_workers, :options, :expected_worker_count
-
-        ##
-        # Constructor
-        #
-        def initialize(options = {})
-          @available_workers = []
-          @options = options.dup
-          @expected_worker_count = @options[:threads]
-          @context = ::ZMQ::Context.new
-          @poller = ::ZMQ::Poller.new
-          setup_backend
-        end
-
-        ##
-        # Instance Methods
-        #
-        def poll
-          if frontend.nil?
-            if local_workers_have_started?
-              # only open the front end when the workers are done booting
-              log_info { "Starting frontend socket in broker, all workers ready!" }
-              setup_frontend
-            end
-          else
-            # Start checking the poller after startup
-            if available_workers.size > 0
-              poller.register(frontend, ::ZMQ::POLLIN) if poller.size < 2
-            else
-              poller.delete(frontend)
-            end
-          end
 
-          poller.poll(1000)
-          poller.readables.each do |socket|
-            case socket
-            when backend then
-              move_to_frontend(socket)
-            when frontend then
-              move_to_backend(socket)
+        def initialize(server)
+          @server = server
+
+          init_zmq_context
+          init_backend_socket
+          init_frontend_socket
+          init_shutdown_socket
+          init_poller
+        rescue
+          teardown
+          raise
+        end
+
+        def join
+          @thread.try(:join)
+        end
+
+        def run
+          @idle_workers = []
+
+          catch(:shutdown) do
+            while @poller.poll > 0
+              @poller.readables.each do |readable|
+                case readable
+                when @frontend_socket
+                  process_frontend
+                when @backend_socket
+                  process_backend
+                when @shutdown_socket
+                  throw :shutdown
+                end
+              end
             end
           end
+        ensure
+          teardown
         end
 
-        def setup_backend
-          host = options[:host]
-          port = options[:worker_port]
+        def start
+          log_debug { sign_message("starting broker") }
 
-          zmq_backend = context.socket(::ZMQ::ROUTER)
-          zmq_error_check(zmq_backend.bind(bind_address(host, port)))
+          @thread = Thread.new { self.run }
 
-          @backend = zmq_backend
-          @poller.register(@backend, ::ZMQ::POLLIN)
+          self
         end
 
-        def setup_frontend
-          host = options[:host]
-          port = options[:port]
-
-          zmq_frontend = context.socket(::ZMQ::ROUTER)
-          zmq_error_check(zmq_frontend.bind(bind_address(host, port)))
-
-          @frontend = zmq_frontend
-          @poller.register(@frontend, ::ZMQ::POLLIN)
+        def shutdown_uri
+          "inproc://#{object_id}"
         end
 
-        def teardown
-          frontend.try(:close)
-          backend.try(:close)
-          context.try(:terminate)
+        def signal_shutdown
+          socket = @zmq_context.socket(ZMQ::PAIR)
+          zmq_error_check(socket.connect(shutdown_uri))
+          zmq_error_check(socket.send_string ".")
+          zmq_error_check(socket.close)
         end
 
         private
 
-          def local_workers_have_started?
-            @local_workers_have_started ||= available_workers.size >= expected_worker_count
-          end
+        def init_backend_socket
+          @backend_socket = @zmq_context.socket(ZMQ::ROUTER)
+          zmq_error_check(@backend_socket.bind(@server.backend_uri))
+        end
 
-          def move_to_backend(socket)
-            message_array = []
-            zmq_error_check(socket.recv_strings(message_array))
+        def init_frontend_socket
+          @frontend_socket = @zmq_context.socket(ZMQ::ROUTER)
+          zmq_error_check(@frontend_socket.bind(@server.frontend_uri))
+        end
 
-            backend_message_set = [
-              available_workers.shift, # Worker UUID for router
-              "",
-              message_array[0], # Client UUID for return value
-              "",
-              message_array[2] # Client Message payload (request)
-            ]
+        def init_poller
+          @poller = ZMQ::Poller.new
+          @poller.register_readable(@frontend_socket)
+          @poller.register_readable(@backend_socket)
+          @poller.register_readable(@shutdown_socket)
+        end
 
-            zmq_error_check(backend.send_strings(backend_message_set))
-          end
+        def init_shutdown_socket
+          @shutdown_socket = @zmq_context.socket(ZMQ::PAIR)
+          zmq_error_check(@shutdown_socket.bind(shutdown_uri))
+        end
 
-          def move_to_frontend(socket)
-            message_array = []
-            zmq_error_check(socket.recv_strings(message_array))
+        def init_zmq_context
+          @zmq_context = ZMQ::Context.new
+        end
 
-            # Push UUID of socket on the available workers queue
-            available_workers << message_array[0]
+        def process_backend
+          worker, ignore, *frames = read_from_backend
 
-            # messages should be [ "uuid of socket", "", "READY_MESSAGE || uuid of client socket"]
-            if message_array[2] == ::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE
-              log_info { "Worker #{available_workers.size} of #{expected_worker_count} ready!" }
-            else
-              frontend_message_set = [
-                message_array[2], # client UUID
-                "",
-                message_array[4] # Reply payload
-              ]
+          @idle_workers << worker
 
-              zmq_error_check(frontend.send_strings(frontend_message_set))
-            end
+          unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
+            write_to_frontend(frames)
+          end
+        end
+
+        def process_frontend
+          if @idle_workers.any?
+            frames = read_from_frontend
+            write_to_backend([@idle_workers.shift, ""] + frames)
           end
+        end
+
+        def read_from_backend
+          [].tap do |frames|
+            zmq_error_check(@backend_socket.recv_strings(frames))
+          end
+        end
 
-          def bind_address(host, port)
-            "tcp://#{resolve_ip(host)}:#{port}"
+        def read_from_frontend
+          [].tap do |frames|
+            zmq_error_check(@frontend_socket.recv_strings(frames))
           end
+        end
+
+        def teardown
+          @frontend_socket.try(:close)
+          @backend_socket.try(:close)
+          @shutdown_socket.try(:close)
+          @zmq_context.try(:terminate)
+        end
+
+        def write_to_backend(frames)
+          zmq_error_check(@backend_socket.send_strings(frames))
+        end
+
+        def write_to_frontend(frames)
+          zmq_error_check(@frontend_socket.send_strings(frames))
+        end
       end
     end
   end

data/lib/protobuf/rpc/servers/zmq/server.rb

@@ -1,6 +1,8 @@
-require 'protobuf/rpc/servers/zmq/broker'
-require 'protobuf/rpc/servers/zmq/worker'
 require 'protobuf/rpc/servers/zmq/util'
+require 'protobuf/rpc/servers/zmq/worker'
+require 'protobuf/rpc/servers/zmq/broker'
+require 'protobuf/rpc/dynamic_discovery.pb'
+require 'securerandom'
 
 module Protobuf
   module Rpc
@@ -8,69 +10,287 @@ module Protobuf
       class Server
         include ::Protobuf::Rpc::Zmq::Util
 
-        ##
-        # Class Methods
-        #
-        def self.run(options = {})
-          @options = options
+        DEFAULT_OPTIONS = {
+          :beacon_interval => 5,
+          :broadcast_beacons => false
+        }
 
-          unless options[:workers_only]
-            log_debug { sign_message("initializing broker") }
-            @broker = ::Protobuf::Rpc::Zmq::Broker.new(options)
-          end
+        attr_accessor :options
 
-          local_worker_threads = options[:threads]
-          log_debug { sign_message("starting server workers") }
+        def initialize(options)
+          @options = DEFAULT_OPTIONS.merge(options)
+          @workers = []
 
-          @running = true
-          local_worker_threads.times do
-            self.start_worker
+          init_zmq_context
+          init_beacon_socket if broadcast_beacons?
+          init_shutdown_socket
+        rescue
+          teardown
+          raise
+        end
+
+        def backend_ip
+          frontend_ip
+        end
+
+        def backend_port
+          options[:worker_port] || frontend_port + 1
+        end
+
+        def backend_uri
+          "tcp://#{backend_ip}:#{backend_port}"
+        end
+
+        def beacon_interval
+          [options[:beacon_interval].to_i, 1].max
+        end
+
+        def beacon_ip
+          unless @beacon_ip
+            unless address = options[:beacon_address]
+              address = ::Protobuf::Rpc::ServiceDirectory.address
+            end
+
+            @beacon_ip = resolve_ip(address)
           end
 
-          log_debug { sign_message("server started") }
-          while self.running? do
-            if options[:workers_only]
-              sleep 5
-              Thread.pass
-            else
-              @broker.poll
+          @beacon_ip
+        end
+
+        def beacon_port
+          unless @beacon_port
+            unless port = options[:beacon_port]
+              port = ::Protobuf::Rpc::ServiceDirectory.port
             end
+
+            @beacon_port = port.to_i
+          end
+
+          @beacon_port
+        end
+
+        def beacon_uri
+          "udp://#{beacon_ip}:#{beacon_port}"
+        end
+
+        def broadcast_beacons?
+          !brokerless? && options[:broadcast_beacons]
+        end
+
+        def broadcast_flatline
+          flatline = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
+            :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE,
+            :server => self.to_proto
+          )
+
+          @beacon_socket.send flatline.serialize_to_string, 0
+        end
+
+        def broadcast_heartbeat
+          @last_beacon = Time.now.to_i
+
+          heartbeat = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
+            :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT,
+            :server => self.to_proto
+          )
+
+          @beacon_socket.send(heartbeat.serialize_to_string, 0)
+
+          log_debug { sign_message("sent heartbeat to #{beacon_uri}") }
+        end
+
+        def broadcast_heartbeat?
+          Time.now.to_i >= next_beacon && broadcast_beacons?
+        end
+
+        def brokerless?
+          !!options[:workers_only]
+        end
+
+        def frontend_ip
+          @frontend_ip ||= resolve_ip(options[:host])
+        end
+
+        def frontend_port
+          options[:port]
+        end
+
+        def frontend_uri
+          "tcp://#{frontend_ip}:#{frontend_port}"
+        end
+
+        def maintenance_timeout
+          1_000 * (next_maintenance - Time.now.to_i)
+        end
+
+        def next_maintenance
+          cycles = [next_reaping]
+          cycles << next_beacon if broadcast_beacons?
+
+          cycles.min
+        end
+
+        def minimum_timeout
+          100
+        end
+
+        def next_beacon
+          if @last_beacon.nil?
+            0
+          else
+            @last_beacon + beacon_interval
           end
+        end
+
+        def next_reaping
+          if @last_reaping.nil?
+            0
+          else
+            @last_reaping + reaping_interval
+          end
+        end
+
+        def reap_dead_workers
+          @last_reaping = Time.now.to_i
+
+          @workers.keep_if do |worker|
+            worker.alive? or worker.join && false
+          end
+        end
+
+        def reap_dead_workers?
+          Time.now.to_i >= next_reaping
+        end
+
+        def reaping_interval
+          5
+        end
+
+        def run
+          @running = true
+
+          start_broker unless brokerless?
+          start_missing_workers
+          wait_for_shutdown_signal
+          broadcast_flatline if broadcast_beacons?
+          stop_workers
+          stop_broker unless brokerless?
         ensure
-          @broker.teardown if @broker
+          @running = false
+          teardown
         end
 
-        def self.running?
+        def running?
           !!@running
         end
 
-        def self.start_worker
-          @threads << Thread.new(@options) { |options|
-            begin
-              ::Protobuf::Rpc::Zmq::Worker.new(options).run
-            rescue => e
-              message =  "Worker Failed, spawning new worker: #{e.inspect}\n #{e.backtrace.join($/)}"
-              $stderr.puts message
-              log_error { message }
+        def shutdown_uri
+          "inproc://#{object_id}"
+        end
 
-              retry if ::Protobuf::Rpc::Zmq::Server.running?
-            end
-          }
+        def signal_shutdown
+          socket = @zmq_context.socket ZMQ::PAIR
+          zmq_error_check(socket.connect shutdown_uri)
+          zmq_error_check(socket.send_string ".")
+          zmq_error_check(socket.close)
         end
 
-        def self.stop
-          @running = false
+        def start_broker
+          @broker = ::Protobuf::Rpc::Zmq::Broker.new(self).start
+        end
+
+        def start_missing_workers
+          missing_workers = total_workers - @workers.size
+
+          if missing_workers > 0
+            missing_workers.times { start_worker }
+            log_debug { sign_message("#{total_workers} workers started") }
+          end
+        end
 
-          @threads.each do |t|
-            t.join(5) || t.kill
+        def start_worker
+          @workers << ::Protobuf::Rpc::Zmq::Worker.new(self).start
+        end
+
+        def stop
+          signal_shutdown
+        end
+
+        def stop_broker
+          @broker.signal_shutdown
+          @broker.join
+        end
+
+        def stop_workers
+          @workers.each(&:signal_shutdown)
+          Thread.pass until reap_dead_workers.empty?
+        end
+
+        def teardown
+          @shutdown_socket.try(:close)
+          @beacon_socket.try(:close)
+          @zmq_context.try(:terminate)
+          @last_reaping = @last_beacon = @timeout = nil
+        end
+
+        def total_workers
+          @total_workers ||= [@options[:threads].to_i, 1].max
+        end
+
+        def timeout
+          if @timeout.nil?
+            @timeout = 0
+          else
+            @timeout = [minimum_timeout, maintenance_timeout].max
+          end
+        end
+
+        def to_proto
+          @proto ||= ::Protobuf::Rpc::DynamicDiscovery::Server.new(
+            :uuid => uuid,
+            :address => frontend_ip,
+            :port => frontend_port.to_s,
+            :ttl => (beacon_interval * 1.5).ceil,
+            :services => ::Protobuf::Rpc::Service.implemented_services
+          )
+        end
+
+        def uuid
+          @uuid ||= SecureRandom.uuid
+        end
+
+        def wait_for_shutdown_signal
+          poller = ZMQ::Poller.new
+          poller.register_readable(@shutdown_socket)
+
+          # If the poller returns 1, a shutdown signal has been received.
+          # If the poller returns -1, something went wrong.
+          while poller.poll(timeout) === 0
+            if reap_dead_workers?
+              reap_dead_workers
+              start_missing_workers
+            end
+
+            broadcast_heartbeat if broadcast_heartbeat?
           end
         end
 
-        def self.threads
-          @threads
+        private
+
+        def init_beacon_socket
+          @beacon_socket = UDPSocket.new
+          @beacon_socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_BROADCAST, true)
+          @beacon_socket.connect(beacon_ip, beacon_port)
+        end
+
+        def init_shutdown_socket
+          @shutdown_socket = @zmq_context.socket(ZMQ::PAIR)
+          zmq_error_check(@shutdown_socket.bind shutdown_uri)
         end
 
-        @threads ||= []
+        def init_zmq_context
+          @zmq_context = ZMQ::Context.new
+        end
       end
     end
   end