deimos-temp-fork 0.0.1

data/lib/deimos/config/phobos_config.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/array'
+
+module Deimos
+  # Module to handle phobos.yml as well as outputting the configuration to save
+  # to Phobos itself.
+  module PhobosConfig
+    extend ActiveSupport::Concern
+
+    # @return [Hash]
+    def to_h
+      (FIELDS + [:handler]).map { |f|
+        val = self.send(f)
+        if f == :backoff && val
+          [:backoff, _backoff(val)]
+        elsif val.present?
+          [f, val]
+        end
+      }.to_h
+    end
+
+    # :nodoc:
+    def reset!
+      super
+      Phobos.configure(self.phobos_config)
+    end
+
+    # Create a hash representing the config that Phobos expects.
+    # @return [Hash]
+    def phobos_config
+      p_config = {
+        logger: Logger.new(STDOUT),
+        custom_logger: self.phobos_logger,
+        custom_kafka_logger: self.kafka.logger,
+        kafka: {
+          client_id: self.kafka.client_id,
+          connect_timeout: self.kafka.connect_timeout,
+          socket_timeout: self.kafka.socket_timeout,
+          ssl_verify_hostname: self.kafka.ssl.verify_hostname,
+          seed_brokers: Array.wrap(self.kafka.seed_brokers)
+        },
+        producer: {
+          ack_timeout: self.producers.ack_timeout,
+          required_acks: self.producers.required_acks,
+          max_retries: self.producers.max_retries,
+          retry_backoff: self.producers.retry_backoff,
+          max_buffer_size: self.producers.max_buffer_size,
+          max_buffer_bytesize: self.producers.max_buffer_bytesize,
+          compression_codec: self.producers.compression_codec,
+          compression_threshold: self.producers.compression_threshold,
+          max_queue_size: self.producers.max_queue_size,
+          delivery_threshold: self.producers.delivery_threshold,
+          delivery_interval: self.producers.delivery_interval
+        },
+        consumer: {
+          session_timeout: self.consumers.session_timeout,
+          offset_commit_interval: self.consumers.offset_commit_interval,
+          offset_commit_threshold: self.consumers.offset_commit_threshold,
+          heartbeat_interval: self.consumers.heartbeat_interval
+        },
+        backoff: _backoff(self.consumers.backoff.to_a)
+      }
+
+      p_config[:listeners] = self.consumer_objects.map do |consumer|
+        next nil if consumer.disabled
+
+        hash = consumer.to_h.reject do |k, _|
+          %i(class_name schema namespace key_config backoff disabled).include?(k)
+        end
+        hash = hash.map { |k, v| [k, v.is_a?(Symbol) ? v.to_s : v] }.to_h
+        hash[:handler] = consumer.class_name
+        if consumer.backoff
+          hash[:backoff] = _backoff(consumer.backoff.to_a)
+        end
+        hash
+      end
+      p_config[:listeners].compact!
+
+      if self.kafka.ssl.enabled
+        %w(ca_cert client_cert client_cert_key).each do |key|
+          next if self.kafka.ssl.send(key).blank?
+
+          p_config[:kafka]["ssl_#{key}".to_sym] = ssl_var_contents(self.kafka.ssl.send(key))
+        end
+      end
+      p_config
+    end
+
+    # @param key [String]
+    # @return [String]
+    def ssl_var_contents(key)
+      File.exist?(key) ? File.read(key) : key
+    end
+
+    # Legacy method to parse Phobos config file
+    def phobos_config_file=(file)
+      pconfig = YAML.load(ERB.new(File.read(File.expand_path(file))).result). # rubocop:disable Security/YAMLLoad
+        with_indifferent_access
+      self.logger&.warn('phobos.yml is deprecated - use direct configuration instead.')
+      pconfig[:kafka].each do |k, v|
+        if k.starts_with?('ssl')
+          k = k.sub('ssl_', '')
+          self.kafka.ssl.send("#{k}=", v)
+        else
+          self.kafka.send("#{k}=", v)
+        end
+      end
+      pconfig[:producer].each do |k, v|
+        self.producers.send("#{k}=", v)
+      end
+      pconfig[:consumer].each do |k, v|
+        self.consumers.send("#{k}=", v)
+      end
+      self.consumers.backoff = pconfig[:backoff][:min_ms]..pconfig[:backoff][:max_ms]
+      pconfig[:listeners].each do |listener_hash|
+        self.consumer do
+          listener_hash.each do |k, v|
+            k = 'class_name' if k == 'handler'
+            send(k, v)
+          end
+        end
+      end
+    end
+
+  private
+
+    # @param values [Array<Integer>]
+    # @return [Hash<Integer>]
+    def _backoff(values)
+      {
+        min_ms: values[0],
+        max_ms: values[-1]
+      }
+    end
+  end
+end

data/lib/deimos/consume/batch_consumption.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Consume
+    # Helper methods used by batch consumers, i.e. those with "inline_batch"
+    # delivery. Payloads are decoded then consumers are invoked with arrays
+    # of messages to be handled at once
+    module BatchConsumption
+      extend ActiveSupport::Concern
+      include Phobos::BatchHandler
+
+      # :nodoc:
+      def around_consume_batch(batch, metadata)
+        payloads = []
+        benchmark = Benchmark.measure do
+          if self.class.config[:key_configured]
+            metadata[:keys] = batch.map do |message|
+              decode_key(message.key)
+            end
+          end
+          metadata[:first_offset] = batch.first&.offset
+
+          payloads = batch.map do |message|
+            message.payload ? self.class.decoder.decode(message.payload) : nil
+          end
+          _received_batch(payloads, metadata)
+          _with_span do
+            yield payloads, metadata
+          end
+        end
+        _handle_batch_success(benchmark.real, payloads, metadata)
+      rescue StandardError => e
+        _handle_batch_error(e, payloads, metadata)
+      end
+
+      # Consume a batch of incoming messages.
+      # @param _payloads [Array<Phobos::BatchMessage>]
+      # @param _metadata [Hash]
+      def consume_batch(_payloads, _metadata)
+        raise NotImplementedError
+      end
+
+    protected
+
+      def _received_batch(payloads, metadata)
+        Deimos.config.logger.info(
+          message: 'Got Kafka batch event',
+          message_ids: _payload_identifiers(payloads, metadata),
+          metadata: metadata.except(:keys)
+        )
+        Deimos.config.logger.debug(
+          message: 'Kafka batch event payloads',
+          payloads: payloads
+        )
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:batch_received
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.metrics&.increment(
+          'handler',
+          by: metadata['batch_size'],
+          tags: %W(
+            status:received
+            topic:#{metadata[:topic]}
+          ))
+        if payloads.present?
+          payloads.each { |payload| _report_time_delayed(payload, metadata) }
+        end
+      end
+
+      # @param exception [Throwable]
+      # @param payloads [Array<Hash>]
+      # @param metadata [Hash]
+      def _handle_batch_error(exception, payloads, metadata)
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:batch_error
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.logger.warn(
+          message: 'Error consuming message batch',
+          handler: self.class.name,
+          metadata: metadata.except(:keys),
+          message_ids: _payload_identifiers(payloads, metadata),
+          error_message: exception.message,
+          error: exception.backtrace
+        )
+        _error(exception, payloads, metadata)
+      end
+
+      # @param time_taken [Float]
+      # @param payloads [Array<Hash>]
+      # @param metadata [Hash]
+      def _handle_batch_success(time_taken, payloads, metadata)
+        Deimos.config.metrics&.histogram('handler',
+                                         time_taken,
+                                         tags: %W(
+                                           time:consume_batch
+                                           topic:#{metadata[:topic]}
+                                         ))
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:batch_success
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.metrics&.increment(
+          'handler',
+          by: metadata['batch_size'],
+          tags: %W(
+            status:success
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.logger.info(
+          message: 'Finished processing Kafka batch event',
+          message_ids: _payload_identifiers(payloads, metadata),
+          time_elapsed: time_taken,
+          metadata: metadata.except(:keys)
+        )
+      end
+
+      # Get payload identifiers (key and message_id if present) for logging.
+      # @param payloads [Array<Hash>]
+      # @param metadata [Hash]
+      # @return [Array<Array>] the identifiers.
+      def _payload_identifiers(payloads, metadata)
+        message_ids = payloads&.map do |payload|
+          if payload.is_a?(Hash) && payload.key?('message_id')
+            payload['message_id']
+          end
+        end
+
+        # Payloads may be nil if preprocessing failed
+        messages = payloads || metadata[:keys] || []
+
+        messages.zip(metadata[:keys] || [], message_ids || []).map do |_, k, m_id|
+          ids = {}
+
+          ids[:key] = k if k.present?
+          ids[:message_id] = m_id if m_id.present?
+
+          ids
+        end
+      end
+    end
+  end
+end

data/lib/deimos/consume/message_consumption.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Consume
+    # Methods used by message-by-message (non-batch) consumers. These consumers
+    # are invoked for every individual message.
+    module MessageConsumption
+      extend ActiveSupport::Concern
+      include Phobos::Handler
+
+      # :nodoc:
+      def around_consume(payload, metadata)
+        decoded_payload = payload.nil? ? nil : payload.dup
+        new_metadata = metadata.dup
+        benchmark = Benchmark.measure do
+          _with_span do
+            new_metadata[:key] = decode_key(metadata[:key]) if self.class.config[:key_configured]
+            decoded_payload = payload ? self.class.decoder.decode(payload) : nil
+            _received_message(decoded_payload, new_metadata)
+            yield decoded_payload, new_metadata
+          end
+        end
+        _handle_success(benchmark.real, decoded_payload, new_metadata)
+      rescue StandardError => e
+        _handle_error(e, decoded_payload, new_metadata)
+      end
+
+      # Consume incoming messages.
+      # @param _payload [String]
+      # @param _metadata [Hash]
+      def consume(_payload, _metadata)
+        raise NotImplementedError
+      end
+
+    private
+
+      def _received_message(payload, metadata)
+        Deimos.config.logger.info(
+          message: 'Got Kafka event',
+          payload: payload,
+          metadata: metadata
+        )
+        Deimos.config.metrics&.increment('handler', tags: %W(
+                                           status:received
+                                           topic:#{metadata[:topic]}
+                                         ))
+        _report_time_delayed(payload, metadata)
+      end
+
+      # @param exception [Throwable]
+      # @param payload [Hash]
+      # @param metadata [Hash]
+      def _handle_error(exception, payload, metadata)
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:error
+            topic:#{metadata[:topic]}
+          )
+        )
+        Deimos.config.logger.warn(
+          message: 'Error consuming message',
+          handler: self.class.name,
+          metadata: metadata,
+          data: payload,
+          error_message: exception.message,
+          error: exception.backtrace
+        )
+
+        _error(exception, payload, metadata)
+      end
+
+      # @param time_taken [Float]
+      # @param payload [Hash]
+      # @param metadata [Hash]
+      def _handle_success(time_taken, payload, metadata)
+        Deimos.config.metrics&.histogram('handler', time_taken, tags: %W(
+                                           time:consume
+                                           topic:#{metadata[:topic]}
+                                         ))
+        Deimos.config.metrics&.increment('handler', tags: %W(
+                                           status:success
+                                           topic:#{metadata[:topic]}
+                                         ))
+        Deimos.config.logger.info(
+          message: 'Finished processing Kafka event',
+          payload: payload,
+          time_elapsed: time_taken,
+          metadata: metadata
+        )
+      end
+    end
+  end
+end

data/lib/deimos/consumer.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'deimos/consume/batch_consumption'
+require 'deimos/consume/message_consumption'
+
+# Class to consume messages coming from a Kafka topic
+# Note: According to the docs, instances of your handler will be created
+# for every incoming message/batch. This class should be lightweight.
+module Deimos
+  # Basic consumer class. Inherit from this class and override either consume
+  # or consume_batch, depending on the delivery mode of your listener.
+  # `consume` -> use `delivery :message` or `delivery :batch`
+  # `consume_batch` -> use `delivery :inline_batch`
+  class Consumer
+    include Consume::MessageConsumption
+    include Consume::BatchConsumption
+    include SharedConfig
+
+    class << self
+      # @return [Deimos::SchemaBackends::Base]
+      def decoder
+        @decoder ||= Deimos.schema_backend(schema: config[:schema],
+                                           namespace: config[:namespace])
+      end
+
+      # @return [Deimos::SchemaBackends::Base]
+      def key_decoder
+        @key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
+                                               namespace: config[:namespace])
+      end
+    end
+
+    # Helper method to decode an encoded key.
+    # @param key [String]
+    # @return [Object] the decoded key.
+    def decode_key(key)
+      return nil if key.nil?
+
+      config = self.class.config
+      unless config[:key_configured]
+        raise 'No key config given - if you are not decoding keys, please use '\
+          '`key_config plain: true`'
+      end
+
+      if config[:key_field]
+        self.class.decoder.decode_key(key, config[:key_field])
+      elsif config[:key_schema]
+        self.class.key_decoder.decode(key, schema: config[:key_schema])
+      else # no encoding
+        key
+      end
+    end
+
+  private
+
+    def _with_span
+      @span = Deimos.config.tracer&.start(
+        'deimos-consumer',
+        resource: self.class.name.gsub('::', '-')
+      )
+      yield
+    ensure
+      Deimos.config.tracer&.finish(@span)
+    end
+
+    def _report_time_delayed(payload, metadata)
+      return if payload.nil? || payload['timestamp'].blank?
+
+      begin
+        time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
+      rescue ArgumentError
+        Deimos.config.logger.info(
+          message: "Error parsing timestamp! #{payload['timestamp']}"
+        )
+        return
+      end
+      Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
+                                         time:time_delayed
+                                         topic:#{metadata[:topic]}
+                                       ))
+    end
+
+    # Overrideable method to determine if a given error should be considered
+    # "fatal" and always be reraised.
+    # @param _error [Exception]
+    # @param _payload [Hash]
+    # @param _metadata [Hash]
+    # @return [Boolean]
+    def fatal_error?(_error, _payload, _metadata)
+      false
+    end
+
+    # @param exception [Exception]
+    # @param payload [Hash]
+    # @param metadata [Hash]
+    def _error(exception, payload, metadata)
+      Deimos.config.tracer&.set_error(@span, exception)
+
+      raise if Deimos.config.consumers.reraise_errors ||
+               Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
+               fatal_error?(exception, payload, metadata)
+    end
+  end
+end