deimos-ruby 1.7.0.pre.beta1 → 1.8.0.pre.beta1

data/lib/deimos/consume/batch_consumption.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Consume
+    # Helper methods used by batch consumers, i.e. those with "inline_batch"
+    # delivery. Payloads are decoded then consumers are invoked with arrays
+    # of messages to be handled at once
+    module BatchConsumption
+      include Phobos::BatchHandler
+
+      # :nodoc:
+      def around_consume_batch(batch, metadata)
+        payloads = []
+        benchmark = Benchmark.measure do
+          if self.class.config[:key_configured]
+            metadata[:keys] = batch.map do |message|
+              decode_key(message.key)
+            end
+          end
+
+          payloads = batch.map do |message|
+            message.payload ? self.class.decoder.decode(message.payload) : nil
+          end
+          _received_batch(payloads, metadata)
+          _with_span do
+            yield payloads, metadata
+          end
+        end
+        _handle_batch_success(benchmark.real, payloads, metadata)
+      rescue StandardError => e
+        _handle_batch_error(e, payloads, metadata)
+      end
+
+      # Consume a batch of incoming messages.
+      # @param _payloads [Array<Phobos::BatchMessage>]
+      # @param _metadata [Hash]
+      def consume_batch(_payloads, _metadata)
+        raise NotImplementedError
+      end
+
+    protected
+
+      def _received_batch(payloads, metadata)
+        Deimos.config.logger.info(
+          message: 'Got Kafka batch event',
+          message_ids: _payload_identifiers(payloads, metadata),
+          metadata: metadata.except(:keys)
+        )
+        Deimos.config.logger.debug(
+          message: 'Kafka batch event payloads',
+          payloads: payloads
+        )
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:batch_received
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.metrics&.increment(
+          'handler',
+          by: metadata['batch_size'],
+          tags: %W(
+            status:received
+            topic:#{metadata[:topic]}
+          ))
+        if payloads.present?
+          payloads.each { |payload| _report_time_delayed(payload, metadata) }
+        end
+      end
+
+      # @param exception [Throwable]
+      # @param payloads [Array<Hash>]
+      # @param metadata [Hash]
+      def _handle_batch_error(exception, payloads, metadata)
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:batch_error
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.logger.warn(
+          message: 'Error consuming message batch',
+          handler: self.class.name,
+          metadata: metadata.except(:keys),
+          message_ids: _payload_identifiers(payloads, metadata),
+          error_message: exception.message,
+          error: exception.backtrace
+        )
+        _error(exception, payloads, metadata)
+      end
+
+      # @param time_taken [Float]
+      # @param payloads [Array<Hash>]
+      # @param metadata [Hash]
+      def _handle_batch_success(time_taken, payloads, metadata)
+        Deimos.config.metrics&.histogram('handler',
+                                         time_taken,
+                                         tags: %W(
+                                           time:consume_batch
+                                           topic:#{metadata[:topic]}
+                                         ))
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:batch_success
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.metrics&.increment(
+          'handler',
+          by: metadata['batch_size'],
+          tags: %W(
+            status:success
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.logger.info(
+          message: 'Finished processing Kafka batch event',
+          message_ids: _payload_identifiers(payloads, metadata),
+          time_elapsed: time_taken,
+          metadata: metadata.except(:keys)
+        )
+      end
+
+      # Get payload identifiers (key and message_id if present) for logging.
+      # @param payloads [Array<Hash>]
+      # @param metadata [Hash]
+      # @return [Array<Array>] the identifiers.
+      def _payload_identifiers(payloads, metadata)
+        message_ids = payloads&.map do |payload|
+          if payload.is_a?(Hash) && payload.key?('message_id')
+            payload['message_id']
+          end
+        end
+
+        # Payloads may be nil if preprocessing failed
+        messages = payloads || metadata[:keys] || []
+
+        messages.zip(metadata[:keys] || [], message_ids || []).map do |_, k, m_id|
+          ids = {}
+
+          ids[:key] = k if k.present?
+          ids[:message_id] = m_id if m_id.present?
+
+          ids
+        end
+      end
+    end
+  end
+end

data/lib/deimos/consume/message_consumption.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Consume
+    # Methods used by message-by-message (non-batch) consumers. These consumers
+    # are invoked for every individual message.
+    module MessageConsumption
+      include Phobos::Handler
+
+      # :nodoc:
+      def around_consume(payload, metadata)
+        decoded_payload = payload.dup
+        new_metadata = metadata.dup
+        benchmark = Benchmark.measure do
+          _with_span do
+            new_metadata[:key] = decode_key(metadata[:key]) if self.class.config[:key_configured]
+            decoded_payload = payload ? self.class.decoder.decode(payload) : nil
+            _received_message(decoded_payload, new_metadata)
+            yield decoded_payload, new_metadata
+          end
+        end
+        _handle_success(benchmark.real, decoded_payload, new_metadata)
+      rescue StandardError => e
+        _handle_error(e, decoded_payload, new_metadata)
+      end
+
+      # Consume incoming messages.
+      # @param _payload [String]
+      # @param _metadata [Hash]
+      def consume(_payload, _metadata)
+        raise NotImplementedError
+      end
+
+    private
+
+      def _received_message(payload, metadata)
+        Deimos.config.logger.info(
+          message: 'Got Kafka event',
+          payload: payload,
+          metadata: metadata
+        )
+        Deimos.config.metrics&.increment('handler', tags: %W(
+                                           status:received
+                                           topic:#{metadata[:topic]}
+                                         ))
+        _report_time_delayed(payload, metadata)
+      end
+
+      # @param exception [Throwable]
+      # @param payload [Hash]
+      # @param metadata [Hash]
+      def _handle_error(exception, payload, metadata)
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:error
+            topic:#{metadata[:topic]}
+          )
+        )
+        Deimos.config.logger.warn(
+          message: 'Error consuming message',
+          handler: self.class.name,
+          metadata: metadata,
+          data: payload,
+          error_message: exception.message,
+          error: exception.backtrace
+        )
+
+        _error(exception, payload, metadata)
+      end
+
+      # @param time_taken [Float]
+      # @param payload [Hash]
+      # @param metadata [Hash]
+      def _handle_success(time_taken, payload, metadata)
+        Deimos.config.metrics&.histogram('handler', time_taken, tags: %W(
+                                           time:consume
+                                           topic:#{metadata[:topic]}
+                                         ))
+        Deimos.config.metrics&.increment('handler', tags: %W(
+                                           status:success
+                                           topic:#{metadata[:topic]}
+                                         ))
+        Deimos.config.logger.info(
+          message: 'Finished processing Kafka event',
+          payload: payload,
+          time_elapsed: time_taken,
+          metadata: metadata
+        )
+      end
+    end
+  end
+end

data/lib/deimos/consumer.rb

@@ -1,97 +1,104 @@
 # frozen_string_literal: true
 
-require 'deimos/base_consumer'
-require 'deimos/shared_config'
-require 'phobos/handler'
-require 'active_support/all'
+require 'deimos/consume/batch_consumption'
+require 'deimos/consume/message_consumption'
 
-# Class to consume messages coming from the pipeline topic
+# Class to consume messages coming from a Kafka topic
 # Note: According to the docs, instances of your handler will be created
-# for every incoming message. This class should be lightweight.
+# for every incoming message/batch. This class should be lightweight.
 module Deimos
-  # Parent consumer class.
-  class Consumer < BaseConsumer
-    include Phobos::Handler
+  # Basic consumer class. Inherit from this class and override either consume
+  # or consume_batch, depending on the delivery mode of your listener.
+  # `consume` -> use `delivery :message` or `delivery :batch`
+  # `consume_batch` -> use `delivery :inline_batch`
+  class Consumer
+    include Consume::MessageConsumption
+    include Consume::BatchConsumption
+    include SharedConfig
 
-    # :nodoc:
-    def around_consume(payload, metadata)
-      decoded_payload = payload.dup
-      new_metadata = metadata.dup
-      benchmark = Benchmark.measure do
-        _with_span do
-          new_metadata[:key] = decode_key(metadata[:key]) if self.class.config[:key_configured]
-          decoded_payload = payload ? self.class.decoder.decode(payload) : nil
-          _received_message(decoded_payload, new_metadata)
-          yield decoded_payload, new_metadata
-        end
+    class << self
+      # @return [Deimos::SchemaBackends::Base]
+      def decoder
+        @decoder ||= Deimos.schema_backend(schema: config[:schema],
+                                           namespace: config[:namespace])
+      end
+
+      # @return [Deimos::SchemaBackends::Base]
+      def key_decoder
+        @key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
+                                               namespace: config[:namespace])
       end
-      _handle_success(benchmark.real, decoded_payload, new_metadata)
-    rescue StandardError => e
-      _handle_error(e, decoded_payload, new_metadata)
     end
 
-    # Consume incoming messages.
-    # @param _payload [String]
-    # @param _metadata [Hash]
-    def consume(_payload, _metadata)
-      raise NotImplementedError
+    # Helper method to decode an encoded key.
+    # @param key [String]
+    # @return [Object] the decoded key.
+    def decode_key(key)
+      return nil if key.nil?
+
+      config = self.class.config
+      unless config[:key_configured]
+        raise 'No key config given - if you are not decoding keys, please use '\
+          '`key_config plain: true`'
+      end
+
+      if config[:key_field]
+        self.class.decoder.decode_key(key, config[:key_field])
+      elsif config[:key_schema]
+        self.class.key_decoder.decode(key, schema: config[:key_schema])
+      else # no encoding
+        key
+      end
     end
 
   private
 
-    def _received_message(payload, metadata)
-      Deimos.config.logger.info(
-        message: 'Got Kafka event',
-        payload: payload,
-        metadata: metadata
+    def _with_span
+      @span = Deimos.config.tracer&.start(
+        'deimos-consumer',
+        resource: self.class.name.gsub('::', '-')
       )
-      Deimos.config.metrics&.increment('handler', tags: %W(
-                                         status:received
+      yield
+    ensure
+      Deimos.config.tracer&.finish(@span)
+    end
+
+    def _report_time_delayed(payload, metadata)
+      return if payload.nil? || payload['timestamp'].blank?
+
+      begin
+        time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
+      rescue ArgumentError
+        Deimos.config.logger.info(
+          message: "Error parsing timestamp! #{payload['timestamp']}"
+        )
+        return
+      end
+      Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
+                                         time:time_delayed
                                          topic:#{metadata[:topic]}
                                        ))
-      _report_time_delayed(payload, metadata)
     end
 
-    # @param exception [Throwable]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    def _handle_error(exception, payload, metadata)
-      Deimos.config.metrics&.increment(
-        'handler',
-        tags: %W(
-          status:error
-          topic:#{metadata[:topic]}
-        )
-      )
-      Deimos.config.logger.warn(
-        message: 'Error consuming message',
-        handler: self.class.name,
-        metadata: metadata,
-        data: payload,
-        error_message: exception.message,
-        error: exception.backtrace
-      )
-      super
+    # Overrideable method to determine if a given error should be considered
+    # "fatal" and always be reraised.
+    # @param _error [Exception]
+    # @param _payload [Hash]
+    # @param _metadata [Hash]
+    # @return [Boolean]
+    def fatal_error?(_error, _payload, _metadata)
+      false
     end
 
-    # @param time_taken [Float]
+    # @param exception [Exception]
     # @param payload [Hash]
     # @param metadata [Hash]
-    def _handle_success(time_taken, payload, metadata)
-      Deimos.config.metrics&.histogram('handler', time_taken, tags: %W(
-                                         time:consume
-                                         topic:#{metadata[:topic]}
-                                       ))
-      Deimos.config.metrics&.increment('handler', tags: %W(
-                                         status:success
-                                         topic:#{metadata[:topic]}
-                                       ))
-      Deimos.config.logger.info(
-        message: 'Finished processing Kafka event',
-        payload: payload,
-        time_elapsed: time_taken,
-        metadata: metadata
-      )
+    def _error(exception, payload, metadata)
+      Deimos.config.tracer&.set_error(@span, exception)
+
+      raise if Deimos.config.consumers.reraise_errors ||
+               Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
+               fatal_error?(exception, payload, metadata)
     end
   end
 end

data/lib/deimos/kafka_message.rb

@@ -42,7 +42,7 @@ module Deimos
       messages.map do |m|
         {
           key: m.key.present? ? decoder&.decode_key(m.key) || m.key : nil,
-          payload: decoder&.decoder&.decode(self.message) || self.message
+          payload: decoder&.decoder&.decode(m.message) || m.message
         }
       end
     end

data/lib/deimos/message.rb

@@ -10,7 +10,7 @@ module Deimos
     # @param producer [Class]
     def initialize(payload, producer, topic: nil, key: nil, partition_key: nil)
       @payload = payload&.with_indifferent_access
-      @producer_name = producer.name
+      @producer_name = producer&.name
       @topic = topic
       @key = key
       @partition_key = partition_key
@@ -70,5 +70,10 @@ module Deimos
     def ==(other)
       self.to_h == other.to_h
     end
+
+    # @return [Boolean] True if this message is a tombstone
+    def tombstone?
+      payload.nil?
+    end
   end
 end

data/lib/deimos/utils/db_poller.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require 'deimos/poll_info'
-require 'deimos/utils/executor'
-require 'deimos/utils/signal_handler'
+require 'sigurd/executor'
+require 'sigurd/signal_handler'
 
 module Deimos
   module Utils
@@ -22,10 +22,10 @@ module Deimos
         pollers = Deimos.config.db_poller_objects.map do |poller_config|
           self.new(poller_config)
         end
-        executor = Deimos::Utils::Executor.new(pollers,
-                                               sleep_seconds: 5,
-                                               logger: Deimos.config.logger)
-        signal_handler = Deimos::Utils::SignalHandler.new(executor)
+        executor = Sigurd::Executor.new(pollers,
+                                        sleep_seconds: 5,
+                                        logger: Deimos.config.logger)
+        signal_handler = Sigurd::SignalHandler.new(executor)
         signal_handler.run!
       end