receptor_controller-client 0.0.2

data/lib/receptor_controller/client/directive_blocking.rb

@@ -0,0 +1,87 @@
+require "receptor_controller/client/directive"
+
+module ReceptorController
+  # Blocking directive for requests through POST /job
+  # Successful POST causes locking current thread until response from Kafka comes
+  #
+  # Raises kind of ReceptorController::Client::Error in case of problems/timeout
+  class Client::DirectiveBlocking < Client::Directive
+    def initialize(name:, account:, node_id:, payload:, client:, log_message_common: nil)
+      super
+      self.response_lock = Mutex.new
+      self.response_waiting = ConditionVariable.new
+      self.response_data = nil
+      self.response_exception = nil
+    end
+
+    def call(body = default_body)
+      @url = JSON.parse(body[:payload])['url']
+      response = connection.post(config.job_path, body.to_json)
+
+      msg_id = JSON.parse(response.body)['id']
+
+      logger.debug("Receptor response: registering message #{msg_id}".tap { |msg| msg << " req: #{body["href_slug"]}" if ENV["LOG_ALL_RECEPTOR_MESSAGES"]&.to_i != 0 })
+      # registers message id for kafka responses
+      response_worker.register_message(msg_id, self)
+      wait_for_response(msg_id)
+    rescue Faraday::Error => e
+      msg = receptor_log_msg("Directive #{name} failed (#{log_message_common}) [MSG: #{msg_id}]", account, node_id, e)
+      raise ReceptorController::Client::ControllerResponseError.new(msg)
+    end
+
+    def wait_for_response(_msg_id)
+      response_lock.synchronize do
+        response_waiting.wait(response_lock)
+
+        raise response_exception if response_failed?
+
+        response_data.dup
+      end
+    end
+
+    # TODO: Review when future plugins with more "response" messages come
+    def response_success(msg_id, message_type, response)
+      response_lock.synchronize do
+        if message_type == MESSAGE_TYPE_RESPONSE
+          self.response_data = response
+        elsif message_type == MESSAGE_TYPE_EOF
+          response_waiting.signal
+        else
+          self.response_exception = ReceptorController::Client::UnknownResponseTypeError.new("#{log_message_common}[MSG: #{msg_id}]")
+          response_waiting.signal
+        end
+      end
+    end
+
+    def response_error(msg_id, response_code, err_message)
+      response_lock.synchronize do
+        self.response_data = nil
+        self.response_exception = ReceptorController::Client::ResponseError.new("#{err_message} (code: #{response_code}) (#{log_message_common}) [MSG: #{msg_id}]")
+        response_waiting.signal
+      end
+    end
+
+    def response_timeout(msg_id)
+      response_lock.synchronize do
+        self.response_data = nil
+        self.response_exception = ReceptorController::Client::ResponseTimeoutError.new("Timeout (#{log_message_common}) [MSG: #{msg_id}]")
+        response_waiting.signal
+      end
+    end
+
+    private
+
+    attr_accessor :response_data, :response_exception, :response_lock, :response_waiting
+
+    def connection
+      @connection ||= Faraday.new(config.controller_url, :headers => client.headers) do |c|
+        c.use(Faraday::Response::RaiseError)
+        c.adapter(Faraday.default_adapter)
+      end
+    end
+
+    def response_failed?
+      response_exception.present?
+    end
+  end
+end

data/lib/receptor_controller/client/directive_non_blocking.rb

@@ -0,0 +1,144 @@
+require "concurrent"
+require "receptor_controller/client/directive"
+
+module ReceptorController
+  # Non-blocking directive for requests through POST /job
+  # Directive's call returns either message ID or nil
+  #
+  # Callback blocks can be specified for handling responses
+  # @example:
+  #         receiver = <object with methods below>
+  #         directive
+  #           .on_success do |msg_id, response|
+  #             receiver.process_response(msg_id, response)
+  #           end
+  #           .on_error do |msg_id, code, response|
+  #             receiver.process_error(msg_id, code, response)
+  #           end
+  #           .on_timeout do |msg_id|
+  #             receiver.process_timeout(msg_id)
+  #           end
+  #           .on_eof do |msg_id|
+  #             receiver.process_eof(msg_id)
+  #           end
+  #           .on_eof do |msg_id|
+  #             logger.debug("[#{msg_id}] EOF message received")
+  #           end
+  #
+  #         directive.call
+  class Client::DirectiveNonBlocking < Client::Directive
+    def initialize(name:, account:, node_id:, payload:, client:, log_message_common: nil)
+      super
+
+      @success_callbacks = []
+      @eof_callbacks     = []
+      @timeout_callbacks = []
+      @error_callbacks   = []
+
+      @responses_count = Concurrent::AtomicFixnum.new
+      @eof_lock        = Mutex.new
+      @eof_wait        = ConditionVariable.new
+    end
+
+    # Entrypoint for request
+    def call(body = default_body)
+      response = Faraday.post(config.job_url, body.to_json, client.headers)
+      if response.success?
+        msg_id = JSON.parse(response.body)['id']
+
+        # registers message id for kafka responses
+        response_worker.register_message(msg_id, self)
+
+        msg_id
+      else
+        logger.error(receptor_log_msg("Directive #{name} failed (#{log_message_common}): HTTP #{response.status}", account, node_id))
+        nil
+      end
+    rescue Faraday::Error => e
+      logger.error(receptor_log_msg("Directive #{name} failed (#{log_message_common}). POST /job error", account, node_id, e))
+      nil
+    rescue => e
+      logger.error(receptor_log_msg("Directive #{name} failed (#{log_message_common})", account, node_id, e))
+      nil
+    end
+
+    def on_success(&block)
+      @success_callbacks << block if block_given?
+      self
+    end
+
+    def on_eof(&block)
+      @eof_callbacks << block if block_given?
+      self
+    end
+
+    def on_timeout(&block)
+      @timeout_callbacks << block if block_given?
+      self
+    end
+
+    def on_error(&block)
+      @error_callbacks << block if block_given?
+      self
+    end
+
+    # Handles successful responses in Threads
+    # EOF processing waits until all response threads are finished
+    def response_success(msg_id, message_type, response)
+      if message_type == MESSAGE_TYPE_EOF
+        eof_thread do
+          @eof_callbacks.each { |block| block.call(msg_id) }
+        end
+      else
+        response_thread do
+          @success_callbacks.each { |block| block.call(msg_id, response) }
+        end
+      end
+    end
+
+    # Handles error responses in Threads
+    # EOF processing waits until all threads are finished
+    def response_error(msg_id, response_code, response)
+      response_thread do
+        @error_callbacks.each { |block| block.call(msg_id, response_code, response) }
+      end
+    end
+
+    # Error state: Any response wasn't received in `Configuration.response_timeout`
+    def response_timeout(msg_id)
+      response_thread do
+        @timeout_callbacks.each { |block| block.call(msg_id) }
+      end
+    end
+
+    private
+
+    # Responses are processed in threads to be able to call subrequests
+    # EOF response is blocked by thread-safe counter
+    def response_thread
+      @responses_count.increment
+
+      Thread.new do
+        yield
+      ensure
+        @responses_count.decrement
+        @eof_lock.synchronize do
+          @eof_wait.signal if @responses_count.value == 0
+        end
+      end
+    end
+
+    # Messages in kafka are received serialized, EOF is always last
+    # => @responses_count has to be always positive
+    #    until all responses are processed
+    def eof_thread
+      Thread.new do
+        @eof_lock.synchronize do
+          @eof_wait.wait(@eof_lock) if @responses_count.value > 0
+        end
+
+        yield
+      end
+    end
+  end
+end

data/lib/receptor_controller/client/exception.rb

@@ -0,0 +1,10 @@
+module ReceptorController
+  class Client
+    class Error < StandardError; end
+
+    class ControllerResponseError < Error; end
+    class ResponseTimeoutError < Error; end
+    class ResponseError < Error; end
+    class UnknownResponseTypeError < Error; end
+  end
+end

data/lib/receptor_controller/client/response_worker.rb

@@ -0,0 +1,214 @@
+require 'base64'
+require "concurrent"
+require 'stringio'
+require 'zlib'
+
+module ReceptorController
+  # ResponseWorker is listening on Kafka topic platform.receptor-controller.responses (@see Configuration.queue_topic)
+  # It asynchronously receives responses requested by POST /job to receptor controller.
+  # Request and response is paired by message ID (response of POST /job and 'in_response_to' value in kafka response here)
+  #
+  # Successful responses are at least two:
+  # * 1+ of 'response' type, containing data
+  # * 1 of 'eof' type, signalizing end of transmission
+  #
+  # Registered messages without response are removed after timeout (Configuration.response_timeout)
+  #
+  # All type of responses/timeout can be sent to registered callbacks (@see :register_message)
+  #
+  # Use "start" and "stop" methods to start/stop listening on Kafka
+  class Client::ResponseWorker
+    attr_reader :started
+    alias started? started
+
+    attr_accessor :received_messages
+
+    def initialize(config, logger)
+      self.config              = config
+      self.lock                = Mutex.new
+      self.timeout_lock        = Mutex.new
+      self.logger              = logger
+      self.registered_messages = Concurrent::Map.new
+      self.received_messages   = Concurrent::Array.new
+      self.started             = Concurrent::AtomicBoolean.new(false)
+      self.workers             = {}
+    end
+
+    # Start listening on Kafka
+    def start
+      lock.synchronize do
+        return if started.value
+
+        started.value         = true
+        workers[:maintenance] = Thread.new { check_timeouts while started.value }
+        workers[:listener]    = Thread.new { listen while started.value }
+      end
+    end
+
+    # Stop listener
+    def stop
+      lock.synchronize do
+        return unless started.value
+
+        started.value = false
+        workers[:listener]&.terminate
+        workers[:maintenance]&.join
+      end
+    end
+
+    # Registers message_id received by request,
+    # Defines response and timeout callback methods
+    #
+    # @param msg_id [String] UUID
+    # @param receiver [Object] any object implementing callbacks
+    # @param response_callback [Symbol] name of receiver's method processing responses
+    # @param timeout_callback [Symbol] name of receiver's method processing timeout [optional]
+    # @param error_callback [Symbol] name of receiver's method processing errors [optional]
+    def register_message(msg_id, receiver, response_callback: :response_success, timeout_callback: :response_timeout, error_callback: :response_error)
+      registered_messages[msg_id] = {:receiver          => receiver,
+                                     :response_callback => response_callback,
+                                     :timeout_callback  => timeout_callback,
+                                     :error_callback    => error_callback,
+                                     :last_checked_at   => Time.now.utc}
+    end
+
+    private
+
+    attr_accessor :config, :lock, :logger, :registered_messages, :timeout_lock, :workers
+    attr_writer :started
+
+    def listen
+      # Open a connection to the messaging service
+      client = ManageIQ::Messaging::Client.open(default_messaging_opts)
+
+      logger.info("Receptor Response worker started...")
+      client.subscribe_topic(queue_opts) do |message|
+        process_message(message)
+      end
+    rescue => err
+      logger.error("Exception in kafka listener: #{err}\n#{err.backtrace.join("\n")}")
+    ensure
+      client&.close
+    end
+
+    def process_message(message)
+      response = JSON.parse(message.payload)
+
+      if (message_id = response['in_response_to'])
+        logger.debug("Receptor response: Received message_id: #{message_id}")
+        if (callbacks = registered_messages[message_id]).present?
+          # Reset last_checked_at to avoid timeout in multi-response messages
+          reset_last_checked_at(callbacks)
+
+          if response['code'] == 0
+            #
+            # Response OK
+            #
+            message_type = response['message_type'] # "response" (with data) or "eof" (without data)
+            registered_messages.delete(message_id) if message_type == 'eof'
+
+            payload = response['payload']
+            payload = unpack_payload(payload) if message_type == 'response' && payload.kind_of?(String)
+
+            logger.debug("Receptor response: OK | message #{message_id} (#{payload})")
+
+            callbacks[:receiver].send(callbacks[:response_callback], message_id, message_type, payload)
+          else
+            #
+            # Response Error
+            #
+            registered_messages.delete(message_id)
+
+            logger.error("Receptor response: ERROR | message #{message_id} (#{response})")
+
+            callbacks[:receiver].send(callbacks[:error_callback], message_id, response['code'], response['payload'])
+          end
+        elsif ENV["LOG_ALL_RECEPTOR_MESSAGES"]&.to_i != 0
+          # noop, it's not error if not registered, can be processed by another pod
+          logger.debug("Receptor response unhandled: #{message_id} (#{response['code']})")
+        end
+      else
+        logger.error("Receptor response: Message id (in_response_to) not received! #{response}")
+      end
+    rescue JSON::ParserError => e
+      logger.error("Receptor response: Failed to parse Kafka response (#{e.message})\n#{message.payload}")
+    rescue => e
+      logger.error("Receptor response: #{e}\n#{e.backtrace.join("\n")}")
+    ensure
+      message.ack unless config.queue_auto_ack
+    end
+
+    def check_timeouts(threshold = config.response_timeout)
+      expired = []
+      #
+      # STEP 1 Collect expired messages
+      #
+      registered_messages.each_pair do |message_id, callbacks|
+        timeout_lock.synchronize do
+          if callbacks[:last_checked_at] < Time.now.utc - threshold
+            expired << message_id
+          end
+        end
+      end
+
+      #
+      # STEP 2 Remove expired messages, send timeout callbacks
+      #
+      expired.each do |message_id|
+        callbacks = registered_messages.delete(message_id)
+        if callbacks[:receiver].respond_to?(callbacks[:timeout_callback])
+          callbacks[:receiver].send(callbacks[:timeout_callback], message_id)
+        end
+      end
+
+      sleep(config.response_timeout_poll_time)
+    rescue => err
+      logger.error("Exception in maintenance worker: #{err}\n#{err.backtrace.join("\n")}")
+    end
+
+    # GZIP recognition
+    # https://tools.ietf.org/html/rfc1952#page-5
+    def gzipped?(data)
+      sign = data.to_s.bytes[0..1]
+
+      sign[0] == '0x1f'.hex && sign[1] == '0x8b'.hex
+    end
+
+    # Tries to decompress String response
+    # If not a gzip, it's a String error from receptor node
+    def unpack_payload(data)
+      decoded = Base64.decode64(data)
+      if gzipped?(decoded)
+        gz = Zlib::GzipReader.new(StringIO.new(decoded))
+        JSON.parse(gz.read)
+      else
+        data
+      end
+    end
+
+    # Reset last_checked_at to avoid timeout in multi-response messages
+    def reset_last_checked_at(callbacks)
+      timeout_lock.synchronize do
+        callbacks[:last_checked_at] = Time.now.utc
+      end
+    end
+
+    # No persist_ref here, because all instances (pods) needs to receive kafka message
+    def queue_opts
+      opts               = {:service  => config.queue_topic,
+                            :auto_ack => config.queue_auto_ack}
+      opts[:max_bytes]   = config.queue_max_bytes if config.queue_max_bytes
+      opts[:persist_ref] = config.queue_persist_ref if config.queue_persist_ref
+      opts
+    end
+
+    def default_messaging_opts
+      {
+        :host       => config.queue_host,
+        :port       => config.queue_port,
+        :protocol   => :Kafka,
+        :client_ref => "receptor_client-responses-#{Time.now.to_i}", # A reference string to identify the client
+      }
+    end
+  end
+end