prepor-protobuf 3.5.0

data/lib/protobuf/rpc/servers/zmq_runner.rb

@@ -0,0 +1,62 @@
+require 'ostruct'
+require 'thread'
+
+module Protobuf
+  module Rpc
+    class ZmqRunner
+      include ::Protobuf::Logging
+
+      def initialize(options)
+        @options = case
+                   when options.is_a?(OpenStruct) then
+                     options.marshal_dump
+                   when options.respond_to?(:to_hash) then
+                     options.to_hash
+                   else
+                     fail "Cannot parser Zmq Server - server options"
+                   end
+      end
+
+      def run
+        @server = ::Protobuf::Rpc::Zmq::Server.new(@options)
+        register_signals
+        @server.run do
+          yield if block_given?
+        end
+      end
+
+      def running?
+        @server.try :running?
+      end
+
+      def stop
+        @server.try :stop
+      end
+
+      private
+
+      def register_signals
+        trap(:TRAP) do
+          ::Thread.list.each do |thread|
+            logger.info do
+              <<-THREAD_TRACE
+                #{thread.inspect}:
+                  #{thread.backtrace.try(:join, $INPUT_RECORD_SEPARATOR)}"
+              THREAD_TRACE
+            end
+          end
+        end
+
+        trap(:TTIN) do
+          @server.add_worker
+          logger.info { "Increased worker size to: #{@server.total_workers}" }
+        end
+
+        trap(:TTOU) do
+          logger.info { "Current worker size: #{@server.workers.size}" }
+          logger.info { "Current busy worker size: #{@server.busy_worker_count}" }
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/service.rb

@@ -0,0 +1,179 @@
+require 'protobuf/logging'
+require 'protobuf/rpc/client'
+require 'protobuf/rpc/error'
+require 'protobuf/rpc/service_filters'
+
+module Protobuf
+  module Rpc
+    # Object to encapsulate the request/response types for a given service method
+    #
+    RpcMethod = Struct.new("RpcMethod", :method, :request_type, :response_type)
+
+    class Service
+      include ::Protobuf::Logging
+      include ::Protobuf::Rpc::ServiceFilters
+
+      DEFAULT_HOST = '127.0.0.1'.freeze
+      DEFAULT_PORT = 9399
+
+      attr_reader :env, :request
+
+      ##
+      # Constructor!
+      #
+      # Initialize a service with the rpc endpoint name and the bytes
+      # for the request.
+      def initialize(env)
+        @env = env.dup # Dup the env so it doesn't change out from under us
+        @request = env.request
+      end
+
+      ##
+      # Class Methods
+      #
+      # Create a new client for the given service.
+      # See Client#initialize and ClientConnection::DEFAULT_OPTIONS
+      # for all available options.
+      #
+      def self.client(options = {})
+        ::Protobuf::Rpc::Client.new({ :service => self,
+                                      :host => host,
+                                      :port => port }.merge(options))
+      end
+
+      # Allows service-level configuration of location.
+      # Useful for system-startup configuration of a service
+      # so that any Clients using the Service.client sugar
+      # will not have to configure the location each time.
+      #
+      def self.configure(config = {})
+        self.host = config[:host] if config.key?(:host)
+        self.port = config[:port] if config.key?(:port)
+      end
+
+      # The host location of the service.
+      #
+      def self.host
+        @host ||= DEFAULT_HOST
+      end
+
+      # The host location setter.
+      #
+      class << self
+        attr_writer :host
+      end
+
+      # An array of defined service classes that contain implementation
+      # code
+      def self.implemented_services
+        classes = (subclasses || []).select do |subclass|
+          subclass.rpcs.any? do |(name, _)|
+            subclass.method_defined? name
+          end
+        end
+
+        classes.map(&:name)
+      end
+
+      # Shorthand call to configure, passing a string formatted as hostname:port
+      # e.g. 127.0.0.1:9933
+      # e.g. localhost:0
+      #
+      def self.located_at(location)
+        return if location.nil? || location.downcase.strip !~ /.+:\d+/
+        host, port = location.downcase.strip.split ':'
+        configure(:host => host, :port => port.to_i)
+      end
+
+      # The port of the service on the destination server.
+      #
+      def self.port
+        @port ||= DEFAULT_PORT
+      end
+
+      # The port location setter.
+      #
+      class << self
+        attr_writer :port
+      end
+
+      # Define an rpc method with the given request and response types.
+      # This methods is only used by the generated service definitions
+      # and not useful for user code.
+      #
+      def self.rpc(method, request_type, response_type)
+        rpcs[method] = RpcMethod.new(method, request_type, response_type)
+      end
+
+      # Hash containing the set of methods defined via `rpc`.
+      #
+      def self.rpcs
+        @rpcs ||= {}
+      end
+
+      # Check if the given method name is a known rpc endpoint.
+      #
+      def self.rpc_method?(name)
+        rpcs.key?(name)
+      end
+
+      ##
+      # Instance Methods
+      #
+      # Get a callable object that will be used by the dispatcher
+      # to invoke the specified rpc method. Facilitates callback dispatch.
+      # The returned lambda is expected to be called at a later time (which
+      # is why we wrap the method call).
+      #
+      def callable_rpc_method(method_name)
+        -> { run_filters(method_name) }
+      end
+
+      # Response object for this rpc cycle. Not assignable.
+      #
+      def response
+        @response ||= response_type.new
+      end
+
+      # Convenience method to get back to class method.
+      #
+      def rpc_method?(name)
+        self.class.rpc_method?(name)
+      end
+
+      # Convenience method to get back to class rpcs hash.
+      #
+      def rpcs
+        self.class.rpcs
+      end
+
+      private
+
+      def request_type
+        @request_type ||= env.request_type
+      end
+
+      # Sugar to make an rpc method feel like a controller method.
+      # If this method is not called, the response will be the memoized
+      # object returned by the response reader.
+      #
+      def respond_with(candidate)
+        @response = candidate
+      end
+      alias_method :return_from_whence_you_came, :respond_with
+
+      def response_type
+        @response_type ||= env.response_type
+      end
+
+      # Automatically fail a service method.
+      #
+      def rpc_failed(message)
+        message = message.message if message.respond_to?(:message)
+        fail RpcFailed, message
+      end
+    end
+
+    ActiveSupport.run_load_hooks(:protobuf_rpc_service, Service)
+  end
+end

data/lib/protobuf/rpc/service_directory.rb

@@ -0,0 +1,253 @@
+require 'delegate'
+require 'singleton'
+require 'socket'
+require 'set'
+require 'thread'
+require 'timeout'
+
+require 'protobuf/rpc/dynamic_discovery.pb'
+
+module Protobuf
+  module Rpc
+    class ServiceDirectory
+      include ::Singleton
+      include ::Protobuf::Logging
+
+      DEFAULT_ADDRESS = "0.0.0.0"
+      DEFAULT_PORT = 53000
+      DEFAULT_TIMEOUT = 1
+
+      class Listing < SimpleDelegator
+        attr_reader :expires_at
+
+        def initialize(server)
+          update(server)
+        end
+
+        def current?
+          !expired?
+        end
+
+        def eql?(other)
+          uuid.eql?(other.uuid)
+        end
+
+        def expired?
+          Time.now.to_i >= @expires_at
+        end
+
+        def hash
+          uuid.hash
+        end
+
+        def ttl
+          [super.to_i, 1].max
+        end
+
+        def update(server)
+          __setobj__(server)
+          @expires_at = Time.now.to_i + ttl
+        end
+      end
+
+      # Class Methods
+      #
+      class << self
+        attr_writer :address, :port
+      end
+
+      def self.address
+        @address ||= DEFAULT_ADDRESS
+      end
+
+      def self.port
+        @port ||= DEFAULT_PORT
+      end
+
+      def self.start
+        yield(self) if block_given?
+        instance.start
+      end
+
+      def self.stop
+        instance.stop
+      end
+
+      #
+      # Instance Methods
+      #
+      def initialize
+        reset
+      end
+
+      def all_listings_for(service)
+        if running? && @listings_by_service.key?(service.to_s)
+          start_listener_thread if listener_dead?
+          @listings_by_service[service.to_s].entries.shuffle
+        else
+          []
+        end
+      end
+
+      def each_listing(&block)
+        start_listener_thread if listener_dead?
+        @listings_by_uuid.each_value(&block)
+      end
+
+      def lookup(service)
+        return unless running?
+        start_listener_thread if listener_dead?
+        return unless @listings_by_service.key?(service.to_s)
+        @listings_by_service[service.to_s].entries.sample
+      end
+
+      def listener_dead?
+        @thread.nil? || !@thread.alive?
+      end
+
+      def restart
+        stop
+        start
+      end
+
+      def running?
+        !!@running
+      end
+
+      def start
+        unless running?
+          init_socket
+          logger.info { sign_message("listening to udp://#{self.class.address}:#{self.class.port}") }
+          @running = true
+        end
+
+        start_listener_thread if listener_dead?
+        self
+      end
+
+      def start_listener_thread
+        return if @thread.try(:alive?)
+        @thread = Thread.new { send(:run) }
+      end
+
+      def stop
+        logger.info { sign_message("Stopping directory") }
+
+        @running = false
+        @thread.try(:kill).try(:join)
+        @socket.try(:close)
+
+        reset
+      end
+
+      private
+
+      def add_or_update_listing(uuid, server)
+        listing = @listings_by_uuid[uuid]
+
+        if listing
+          action = :updated
+          listing.update(server)
+        else
+          action = :added
+          listing = Listing.new(server)
+          @listings_by_uuid[uuid] = listing
+        end
+
+        listing.services.each do |service|
+          @listings_by_service[service] << listing
+        end
+
+        trigger(action, listing)
+        logger.debug { sign_message("#{action} server: #{server.inspect}") }
+      end
+
+      def init_socket
+        @socket = UDPSocket.new
+        @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEADDR, true)
+
+        if defined?(::Socket::SO_REUSEPORT)
+          @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, true)
+        end
+
+        @socket.bind(self.class.address, self.class.port.to_i)
+      end
+
+      def process_beacon(beacon)
+        server = beacon.server
+        uuid = server.try(:uuid)
+
+        if server && uuid
+          case beacon.beacon_type
+          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
+            add_or_update_listing(uuid, server)
+          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
+            remove_listing(uuid)
+          end
+        else
+          logger.info { sign_message("Ignoring incomplete beacon: #{beacon.inspect}") }
+        end
+      end
+
+      def read_beacon
+        data, addr = @socket.recvfrom(2048)
+
+        beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.decode(data)
+
+        # Favor the address captured by the socket
+        beacon.try(:server).try(:address=, addr[3])
+
+        beacon
+      end
+
+      def remove_expired_listings
+        logger.debug { sign_message("Removing expired listings") }
+        @listings_by_uuid.each do |uuid, listing|
+          remove_listing(uuid) if listing.expired?
+        end
+      end
+
+      def remove_listing(uuid)
+        listing = @listings_by_uuid[uuid] || return
+
+        logger.debug { sign_message("Removing listing: #{listing.inspect}") }
+
+        @listings_by_service.each_value do |listings|
+          listings.delete(listing)
+        end
+
+        trigger(:removed, @listings_by_uuid.delete(uuid))
+      end
+
+      def reset
+        @thread = nil
+        @socket = nil
+        @listings_by_uuid = {}
+        @listings_by_service = Hash.new { |h, k| h[k] = Set.new }
+      end
+
+      def run
+        sweep_interval = 5 # sweep expired listings every 5 seconds
+        next_sweep = Time.now.to_i + sweep_interval
+
+        loop do
+          timeout = [next_sweep - Time.now.to_i, 0.1].max
+          readable = IO.select([@socket], nil, nil, timeout)
+          process_beacon(read_beacon) if readable
+
+          if Time.now.to_i >= next_sweep
+            remove_expired_listings
+            next_sweep = Time.now.to_i + sweep_interval
+          end
+        end
+      rescue => e
+        logger.debug { sign_message("ERROR: (#{e.class}) #{e.message}\n#{e.backtrace.join("\n")}") }
+        retry
+      end
+
+      def trigger(action, listing)
+        ::ActiveSupport::Notifications.instrument("directory.listing.#{action}", :listing => listing)
+      end
+    end
+  end
+end