protobuf-cucumber 3.10.4

data/lib/protobuf/rpc/servers/zmq/util.rb

@@ -0,0 +1,48 @@
+require 'resolv'
+
+module Protobuf
+  module Rpc
+    module Zmq
+
+      ADDRESS_MATCH = /\A\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\z/
+      WORKER_READY_MESSAGE = "\1".freeze
+      CHECK_AVAILABLE_MESSAGE = "\3".freeze
+      NO_WORKERS_AVAILABLE = "\4".freeze
+      WORKERS_AVAILABLE = "\5".freeze
+      EMPTY_STRING = "".freeze
+
+      module Util
+        include ::Protobuf::Logging
+
+        def self.included(base)
+          base.extend(::Protobuf::Rpc::Zmq::Util)
+        end
+
+        def zmq_error_check(return_code, source = nil)
+          return if ::ZMQ::Util.resultcode_ok?(return_code)
+
+          fail <<-ERROR
+          Last ZMQ API call #{source ? "to #{source}" : ''} failed with "#{::ZMQ::Util.error_string}".
+
+          #{caller(1).join($INPUT_RECORD_SEPARATOR)}
+          ERROR
+        end
+
+        def log_signature
+          unless @_log_signature
+            name = (self.class == Class ? self.name : self.class.name)
+            @_log_signature = "[server-#{name}-#{object_id}]"
+          end
+
+          @_log_signature
+        end
+
+        def resolve_ip(hostname)
+          ::Resolv.getaddresses(hostname).find do |address|
+            address =~ ADDRESS_MATCH
+          end
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq/worker.rb

@@ -0,0 +1,105 @@
+require 'protobuf/rpc/server'
+require 'protobuf/rpc/servers/zmq/util'
+require 'thread'
+
+module Protobuf
+  module Rpc
+    module Zmq
+      class Worker
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Rpc::Zmq::Util
+
+        ##
+        # Constructor
+        #
+        def initialize(server, broker)
+          @server = server
+          @broker = broker
+
+          init_zmq_context
+          init_backend_socket
+        rescue
+          teardown
+          raise
+        end
+
+        ##
+        # Instance Methods
+        #
+        def process_request
+          client_address, _, data = read_from_backend
+          return unless data
+
+          gc_pause do
+            encoded_response = handle_request(data)
+            write_to_backend([client_address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, encoded_response])
+          end
+        end
+
+        def run
+          poller = ::ZMQ::Poller.new
+          poller.register_readable(@backend_socket)
+          poller.register_readable(@shutdown_socket)
+
+          # Send request to broker telling it we are ready
+          write_to_backend([::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE])
+
+          loop do
+            rc = poller.poll(500)
+
+            if rc == 0 && !running? # rubocop:disable Style/GuardClause
+              break # The server was shutdown and no requests are pending
+            elsif rc == -1
+              break # Something went wrong
+            elsif rc > 0
+              ::Thread.current[:busy] = true
+              process_request
+              ::Thread.current[:busy] = false
+            end
+          end
+        ensure
+          teardown
+        end
+
+        def running?
+          @broker.running? && @server.running?
+        end
+
+        private
+
+        def init_zmq_context
+          @zmq_context =
+            if inproc?
+              @server.zmq_context
+            else
+              ZMQ::Context.new
+            end
+        end
+
+        def init_backend_socket
+          @backend_socket = @zmq_context.socket(ZMQ::REQ)
+          zmq_error_check(@backend_socket.connect(@server.backend_uri))
+        end
+
+        def inproc?
+          !!@server.try(:inproc?)
+        end
+
+        def read_from_backend
+          frames = []
+          zmq_error_check(@backend_socket.recv_strings(frames))
+          frames
+        end
+
+        def teardown
+          @backend_socket.try(:close)
+          @zmq_context.try(:terminate) unless inproc?
+        end
+
+        def write_to_backend(frames)
+          zmq_error_check(@backend_socket.send_strings(frames))
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq_runner.rb

@@ -0,0 +1,70 @@
+require 'ostruct'
+require 'thread'
+
+module Protobuf
+  module Rpc
+    class ZmqRunner
+      include ::Protobuf::Logging
+
+      def initialize(options)
+        @options = case
+                   when options.is_a?(OpenStruct) then
+                     options.marshal_dump
+                   when options.respond_to?(:to_hash) then
+                     options.to_hash.symbolize_keys
+                   else
+                     fail "Cannot parser Zmq Server - server options"
+                   end
+      end
+
+      def run
+        @server = ::Protobuf::Rpc::Zmq::Server.new(@options)
+        register_signals
+        @server.run do
+          yield if block_given?
+        end
+      end
+
+      def running?
+        @server.try :running?
+      end
+
+      def stop
+        @server.try :stop
+      end
+
+      private
+
+      def register_signals
+        trap(:TRAP) do
+          ::Thread.list.each do |thread|
+            logger.info do
+              <<-THREAD_TRACE
+                #{thread.inspect}:
+                  #{thread.backtrace.try(:join, $INPUT_RECORD_SEPARATOR)}"
+              THREAD_TRACE
+            end
+          end
+        end
+
+        trap(:TTIN) do
+          @server.add_worker
+          logger.info { "Increased worker size to: #{@server.total_workers}" }
+        end
+
+        trap(:TTOU) do
+          logger.info { "Current worker size: #{@server.workers.size}" }
+          logger.info { "Current busy worker size: #{@server.busy_worker_count}" }
+        end
+      end
+    end
+  end
+end
+
+module Protobuf
+  module Rpc
+    module Servers # bad file namespacing
+      ZmqRunner = ::Protobuf::Rpc::ZmqRunner
+    end
+  end
+end

data/lib/protobuf/rpc/service.rb

@@ -0,0 +1,172 @@
+require 'protobuf/logging'
+require 'protobuf/message'
+require 'protobuf/rpc/client'
+require 'protobuf/rpc/error'
+require 'protobuf/rpc/rpc_method'
+require 'protobuf/rpc/service_filters'
+
+module Protobuf
+  module Rpc
+    # Object to encapsulate the request/response types for a given service method
+
+    class Service
+      include ::Protobuf::Logging
+      include ::Protobuf::Rpc::ServiceFilters
+      ::Protobuf::Optionable.inject(self) { ::Google::Protobuf::ServiceOptions }
+
+      DEFAULT_HOST = '127.0.0.1'.freeze
+      DEFAULT_PORT = 9399
+
+      attr_reader :env, :request
+
+      ##
+      # Constructor!
+      #
+      # Initialize a service with the rpc endpoint name and the bytes
+      # for the request.
+      def initialize(env)
+        @env = env.dup # Dup the env so it doesn't change out from under us
+        @request = env.request
+      end
+
+      ##
+      # Class Methods
+      #
+      # Create a new client for the given service.
+      # See Client#initialize and ClientConnection::DEFAULT_OPTIONS
+      # for all available options.
+      #
+      def self.client(options = {})
+        ::Protobuf::Rpc::Client.new({ :service => self,
+                                      :host => host,
+                                      :port => port }.merge(options))
+      end
+
+      # Allows service-level configuration of location.
+      # Useful for system-startup configuration of a service
+      # so that any Clients using the Service.client sugar
+      # will not have to configure the location each time.
+      #
+      def self.configure(config = {})
+        self.host = config[:host] if config.key?(:host)
+        self.port = config[:port] if config.key?(:port)
+      end
+
+      # The host location of the service.
+      #
+      def self.host
+        @host ||= DEFAULT_HOST
+      end
+
+      # The host location setter.
+      #
+      class << self
+        attr_writer :host
+      end
+
+      # An array of defined service classes that contain implementation
+      # code
+      def self.implemented_services
+        classes = (subclasses || []).select do |subclass|
+          subclass.rpcs.any? do |(name, _)|
+            subclass.method_defined? name
+          end
+        end
+
+        classes.map(&:name)
+      end
+
+      # Shorthand call to configure, passing a string formatted as hostname:port
+      # e.g. 127.0.0.1:9933
+      # e.g. localhost:0
+      #
+      def self.located_at(location)
+        return if location.nil? || location.downcase.strip !~ /.+:\d+/
+        host, port = location.downcase.strip.split ':'
+        configure(:host => host, :port => port.to_i)
+      end
+
+      # The port of the service on the destination server.
+      #
+      def self.port
+        @port ||= DEFAULT_PORT
+      end
+
+      # The port location setter.
+      #
+      class << self
+        attr_writer :port
+      end
+
+      # Define an rpc method with the given request and response types.
+      # This methods is only used by the generated service definitions
+      # and not useful for user code.
+      #
+      def self.rpc(method, request_type, response_type, &options_block)
+        rpcs[method] = RpcMethod.new(method, request_type, response_type, &options_block)
+      end
+
+      # Hash containing the set of methods defined via `rpc`.
+      #
+      def self.rpcs
+        @rpcs ||= {}
+      end
+
+      # Check if the given method name is a known rpc endpoint.
+      #
+      def self.rpc_method?(name)
+        rpcs.key?(name)
+      end
+
+      def call(method_name)
+        run_filters(method_name)
+      end
+
+      # Response object for this rpc cycle. Not assignable.
+      #
+      def response
+        @response ||= response_type.new
+      end
+
+      # Convenience method to get back to class method.
+      #
+      def rpc_method?(name)
+        self.class.rpc_method?(name)
+      end
+
+      # Convenience method to get back to class rpcs hash.
+      #
+      def rpcs
+        self.class.rpcs
+      end
+
+      private
+
+      def request_type
+        @request_type ||= env.request_type
+      end
+
+      # Sugar to make an rpc method feel like a controller method.
+      # If this method is not called, the response will be the memoized
+      # object returned by the response reader.
+      #
+      def respond_with(candidate)
+        @response = candidate
+      end
+      alias :return_from_whence_you_came respond_with
+
+      def response_type
+        @response_type ||= env.response_type
+      end
+
+      # Automatically fail a service method.
+      #
+      def rpc_failed(message)
+        message = message.message if message.respond_to?(:message)
+        fail RpcFailed, message
+      end
+    end
+
+    ActiveSupport.run_load_hooks(:protobuf_rpc_service, Service)
+  end
+end

data/lib/protobuf/rpc/service_directory.rb

@@ -0,0 +1,261 @@
+require 'delegate'
+require 'singleton'
+require 'socket'
+require 'set'
+require 'thread'
+require 'timeout'
+
+require 'protobuf/rpc/dynamic_discovery.pb'
+
+module Protobuf
+  module Rpc
+    def self.service_directory
+      @service_directory ||= ::Protobuf::Rpc::ServiceDirectory.instance
+    end
+
+    def self.service_directory=(directory)
+      @service_directory = directory
+    end
+
+    class ServiceDirectory
+      include ::Singleton
+      include ::Protobuf::Logging
+
+      DEFAULT_ADDRESS = '0.0.0.0'.freeze
+      DEFAULT_PORT = 53000
+      DEFAULT_TIMEOUT = 1
+
+      class Listing < SimpleDelegator
+        attr_reader :expires_at
+
+        def initialize(server)
+          update(server)
+        end
+
+        def current?
+          !expired?
+        end
+
+        def eql?(other)
+          uuid.eql?(other.uuid)
+        end
+
+        def expired?
+          Time.now.to_i >= @expires_at
+        end
+
+        def hash
+          uuid.hash
+        end
+
+        def ttl
+          [super.to_i, 1].max
+        end
+
+        def update(server)
+          __setobj__(server)
+          @expires_at = Time.now.to_i + ttl
+        end
+      end
+
+      # Class Methods
+      #
+      class << self
+        attr_writer :address, :port
+      end
+
+      def self.address
+        @address ||= DEFAULT_ADDRESS
+      end
+
+      def self.port
+        @port ||= DEFAULT_PORT
+      end
+
+      def self.start
+        yield(self) if block_given?
+        instance.start
+      end
+
+      def self.stop
+        instance.stop
+      end
+
+      #
+      # Instance Methods
+      #
+      def initialize
+        reset
+      end
+
+      def all_listings_for(service)
+        if running? && @listings_by_service.key?(service.to_s)
+          start_listener_thread if listener_dead?
+          @listings_by_service[service.to_s].entries.shuffle
+        else
+          []
+        end
+      end
+
+      def each_listing(&block)
+        start_listener_thread if listener_dead?
+        @listings_by_uuid.each_value(&block)
+      end
+
+      def lookup(service)
+        return unless running?
+        start_listener_thread if listener_dead?
+        return unless @listings_by_service.key?(service.to_s)
+        @listings_by_service[service.to_s].entries.sample
+      end
+
+      def listener_dead?
+        @thread.nil? || !@thread.alive?
+      end
+
+      def restart
+        stop
+        start
+      end
+
+      def running?
+        !!@running
+      end
+
+      def start
+        unless running?
+          init_socket
+          logger.info { sign_message("listening to udp://#{self.class.address}:#{self.class.port}") }
+          @running = true
+        end
+
+        start_listener_thread if listener_dead?
+        self
+      end
+
+      def start_listener_thread
+        return if @thread.try(:alive?)
+        @thread = Thread.new { send(:run) }
+      end
+
+      def stop
+        logger.info { sign_message("Stopping directory") }
+
+        @running = false
+        @thread.try(:kill).try(:join)
+        @socket.try(:close)
+
+        reset
+      end
+
+      private
+
+      def add_or_update_listing(uuid, server)
+        listing = @listings_by_uuid[uuid]
+
+        if listing
+          action = :updated
+          listing.update(server)
+        else
+          action = :added
+          listing = Listing.new(server)
+          @listings_by_uuid[uuid] = listing
+        end
+
+        listing.services.each do |service|
+          @listings_by_service[service] << listing
+        end
+
+        trigger(action, listing)
+        logger.debug { sign_message("#{action} server: #{server.inspect}") }
+      end
+
+      def init_socket
+        @socket = UDPSocket.new
+        @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEADDR, true)
+
+        if defined?(::Socket::SO_REUSEPORT)
+          @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, true)
+        end
+
+        @socket.bind(self.class.address, self.class.port.to_i)
+      end
+
+      def process_beacon(beacon)
+        server = beacon.server
+        uuid = server.try(:uuid)
+
+        if server && uuid
+          case beacon.beacon_type
+          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
+            add_or_update_listing(uuid, server)
+          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
+            remove_listing(uuid)
+          end
+        else
+          logger.info { sign_message("Ignoring incomplete beacon: #{beacon.inspect}") }
+        end
+      end
+
+      def read_beacon
+        data, addr = @socket.recvfrom(2048)
+
+        beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.decode(data)
+
+        # Favor the address captured by the socket
+        beacon.try(:server).try(:address=, addr[3])
+
+        beacon
+      end
+
+      def remove_expired_listings
+        logger.debug { sign_message("Removing expired listings") }
+        @listings_by_uuid.each do |uuid, listing|
+          remove_listing(uuid) if listing.expired?
+        end
+      end
+
+      def remove_listing(uuid)
+        listing = @listings_by_uuid[uuid] || return
+
+        logger.debug { sign_message("Removing listing: #{listing.inspect}") }
+
+        @listings_by_service.each_value do |listings|
+          listings.delete(listing)
+        end
+
+        trigger(:removed, @listings_by_uuid.delete(uuid))
+      end
+
+      def reset
+        @thread = nil
+        @socket = nil
+        @listings_by_uuid = {}
+        @listings_by_service = Hash.new { |h, k| h[k] = Set.new }
+      end
+
+      def run
+        sweep_interval = 5 # sweep expired listings every 5 seconds
+        next_sweep = Time.now.to_i + sweep_interval
+
+        loop do
+          timeout = [next_sweep - Time.now.to_i, 0.1].max
+          readable = IO.select([@socket], nil, nil, timeout)
+          process_beacon(read_beacon) if readable
+
+          if Time.now.to_i >= next_sweep
+            remove_expired_listings
+            next_sweep = Time.now.to_i + sweep_interval
+          end
+        end
+      rescue => e
+        logger.debug { sign_message("ERROR: (#{e.class}) #{e.message}\n#{e.backtrace.join("\n")}") }
+        retry
+      end
+
+      def trigger(action, listing)
+        ::ActiveSupport::Notifications.instrument("directory.listing.#{action}", :listing => listing)
+      end
+    end
+  end
+end