deimos-ruby 1.6.3 → 1.8.1.pre.beta1

data/lib/deimos/active_record_producer.rb

@@ -59,6 +59,29 @@ module Deimos
           k.to_sym != :payload_key && !fields.map(&:name).include?(k)
         end
       end
+
+      # Query to use when polling the database with the DbPoller. Add
+      # includes, joins, or wheres as necessary, or replace entirely.
+      # @param time_from [Time] the time to start the query from.
+      # @param time_to [Time] the time to end the query.
+      # @param column_name [Symbol] the column name to look for.
+      # @param min_id [Numeric] the minimum ID (i.e. all IDs must be greater
+      # than this value).
+      # @return [ActiveRecord::Relation]
+      def poll_query(time_from:, time_to:, column_name: :updated_at, min_id:)
+        klass = config[:record_class]
+        table = ActiveRecord::Base.connection.quote_table_name(klass.table_name)
+        column = ActiveRecord::Base.connection.quote_column_name(column_name)
+        primary = ActiveRecord::Base.connection.quote_column_name(klass.primary_key)
+        klass.where(
+          "((#{table}.#{column} = ? AND #{table}.#{primary} > ?) \
+           OR #{table}.#{column} > ?) AND #{table}.#{column} <= ?",
+          time_from,
+          min_id,
+          time_from,
+          time_to
+        )
+      end
     end
   end
 end

data/lib/deimos/batch_consumer.rb

@@ -1,145 +1,7 @@
 # frozen_string_literal: true
 
-require 'deimos/base_consumer'
-require 'phobos/batch_handler'
-
 module Deimos
-  # Class to consume batches of messages in a topic
-  # Note: According to the docs, instances of your handler will be created
-  # for every incoming batch of messages. This class should be lightweight.
-  class BatchConsumer < BaseConsumer
-    include Phobos::BatchHandler
-
-    # :nodoc:
-    def around_consume_batch(batch, metadata)
-      payloads = []
-      benchmark = Benchmark.measure do
-        if self.class.config[:key_configured]
-          metadata[:keys] = batch.map do |message|
-            decode_key(message.key)
-          end
-        end
-
-        payloads = batch.map do |message|
-          message.payload ? self.class.decoder.decode(message.payload) : nil
-        end
-        _received_batch(payloads, metadata)
-        _with_error_span(payloads, metadata) do
-          yield payloads, metadata
-        end
-      end
-      _handle_success(benchmark.real, payloads, metadata)
-    end
-
-    # Consume a batch of incoming messages.
-    # @param _payloads [Array<Phobos::BatchMessage>]
-    # @param _metadata [Hash]
-    def consume_batch(_payloads, _metadata)
-      raise NotImplementedError
-    end
-
-  protected
-
-    def _received_batch(payloads, metadata)
-      Deimos.config.logger.info(
-        message: 'Got Kafka batch event',
-        message_ids: _payload_identifiers(payloads, metadata),
-        metadata: metadata.except(:keys)
-      )
-      Deimos.config.logger.debug(
-        message: 'Kafka batch event payloads',
-        payloads: payloads
-      )
-      Deimos.config.metrics&.increment(
-        'handler',
-        tags: %W(
-          status:batch_received
-          topic:#{metadata[:topic]}
-        ))
-      Deimos.config.metrics&.increment(
-        'handler',
-        by: metadata['batch_size'],
-        tags: %W(
-          status:received
-          topic:#{metadata[:topic]}
-        ))
-      if payloads.present?
-        payloads.each { |payload| _report_time_delayed(payload, metadata) }
-      end
-    end
-
-    # @param exception [Throwable]
-    # @param payloads [Array<Hash>]
-    # @param metadata [Hash]
-    def _handle_error(exception, payloads, metadata)
-      Deimos.config.metrics&.increment(
-        'handler',
-        tags: %W(
-          status:batch_error
-          topic:#{metadata[:topic]}
-        ))
-      Deimos.config.logger.warn(
-        message: 'Error consuming message batch',
-        handler: self.class.name,
-        metadata: metadata.except(:keys),
-        message_ids: _payload_identifiers(payloads, metadata),
-        error_message: exception.message,
-        error: exception.backtrace
-      )
-      super
-    end
-
-    # @param time_taken [Float]
-    # @param payloads [Array<Hash>]
-    # @param metadata [Hash]
-    def _handle_success(time_taken, payloads, metadata)
-      Deimos.config.metrics&.histogram('handler', time_taken, tags: %W(
-                                         time:consume_batch
-                                         topic:#{metadata[:topic]}
-                                       ))
-      Deimos.config.metrics&.increment(
-        'handler',
-        tags: %W(
-          status:batch_success
-          topic:#{metadata[:topic]}
-        ))
-      Deimos.config.metrics&.increment(
-        'handler',
-        by: metadata['batch_size'],
-        tags: %W(
-          status:success
-          topic:#{metadata[:topic]}
-        ))
-      Deimos.config.logger.info(
-        message: 'Finished processing Kafka batch event',
-        message_ids: _payload_identifiers(payloads, metadata),
-        time_elapsed: time_taken,
-        metadata: metadata.except(:keys)
-      )
-    end
-
-    # Get payload identifiers (key and message_id if present) for logging.
-    # @param payloads [Array<Hash>]
-    # @param metadata [Hash]
-    # @return [Hash] the identifiers.
-    def _payload_identifiers(payloads, metadata)
-      message_ids = payloads&.map do |payload|
-        if payload.is_a?(Hash) && payload.key?('message_id')
-          payload['message_id']
-        end
-      end
-
-      # Payloads may be nil if preprocessing failed
-      messages = payloads || metadata[:keys] || []
-
-      messages.zip(metadata[:keys] || [], message_ids || []).map do |_, k, m_id|
-        ids = {}
-
-        ids[:key] = k if k.present?
-        ids[:message_id] = m_id if m_id.present?
-
-        ids
-      end
-    end
+  # @deprecated Use Deimos::Consumer with `delivery: inline_batch` configured instead
+  class BatchConsumer < Consumer
   end
 end

data/lib/deimos/config/configuration.rb

@@ -47,17 +47,15 @@ module Deimos
       handler_class = listener.handler.constantize
       delivery = listener.delivery
 
-      # Validate that Deimos consumers use proper delivery configs
-      if handler_class < Deimos::BatchConsumer
-        unless delivery == 'inline_batch'
-          raise "BatchConsumer #{listener.handler} must have delivery set to"\
-            ' `inline_batch`'
-        end
-      elsif handler_class < Deimos::Consumer
-        if delivery.present? && !%w(message batch).include?(delivery)
-          raise "Non-batch Consumer #{listener.handler} must have delivery"\
-            ' set to `message` or `batch`'
+      next unless handler_class < Deimos::Consumer
+
+      # Validate that each consumer implements the correct method for its type
+      if delivery == 'inline_batch'
+        if handler_class.instance_method(:consume_batch).owner == Deimos::Consume::BatchConsumption
+          raise "BatchConsumer #{listener.handler} does not implement `consume_batch`"
         end
+      elsif handler_class.instance_method(:consume).owner == Deimos::Consume::MessageConsumption
+        raise "Non-batch Consumer #{listener.handler} does not implement `consume`"
       end
     end
   end
@@ -340,6 +338,26 @@ module Deimos
       setting :heartbeat_interval
     end
 
+    setting_object :db_poller do
+      # Producer class to use for the poller.
+      setting :producer_class
+      # How often to run the poller, in seconds. If the poll takes longer than this
+      # time, it will run again immediately and the timeout
+      # will be pushed to the next e.g. 1 minute.
+      setting :run_every, 60
+      # Column to use to find updates. Must have an index on it.
+      setting :timestamp_column, :updated_at
+      # Amount of time, in seconds, to wait before catching updates, to allow transactions
+      # to complete but still pick up the right records.
+      setting :delay_time, 2
+      # If true, dump the full table rather than incremental changes. Should
+      # only be used for very small tables.
+      setting :full_table, false
+      # If false, start from the current time instead of the beginning of time
+      # if this is the first time running the poller.
+      setting :start_from_beginning, true
+    end
+
     deprecate 'kafka_logger', 'kafka.logger'
     deprecate 'reraise_consumer_errors', 'consumers.reraise_errors'
     deprecate 'schema_registry_url', 'schema.registry_url'

data/lib/deimos/consume/batch_consumption.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Consume
+    # Helper methods used by batch consumers, i.e. those with "inline_batch"
+    # delivery. Payloads are decoded then consumers are invoked with arrays
+    # of messages to be handled at once
+    module BatchConsumption
+      extend ActiveSupport::Concern
+      include Phobos::BatchHandler
+
+      # :nodoc:
+      def around_consume_batch(batch, metadata)
+        payloads = []
+        benchmark = Benchmark.measure do
+          if self.class.config[:key_configured]
+            metadata[:keys] = batch.map do |message|
+              decode_key(message.key)
+            end
+          end
+          metadata[:first_offset] = batch.first&.offset
+
+          payloads = batch.map do |message|
+            message.payload ? self.class.decoder.decode(message.payload) : nil
+          end
+          _received_batch(payloads, metadata)
+          _with_span do
+            yield payloads, metadata
+          end
+        end
+        _handle_batch_success(benchmark.real, payloads, metadata)
+      rescue StandardError => e
+        _handle_batch_error(e, payloads, metadata)
+      end
+
+      # Consume a batch of incoming messages.
+      # @param _payloads [Array<Phobos::BatchMessage>]
+      # @param _metadata [Hash]
+      def consume_batch(_payloads, _metadata)
+        raise NotImplementedError
+      end
+
+    protected
+
+      def _received_batch(payloads, metadata)
+        Deimos.config.logger.info(
+          message: 'Got Kafka batch event',
+          message_ids: _payload_identifiers(payloads, metadata),
+          metadata: metadata.except(:keys)
+        )
+        Deimos.config.logger.debug(
+          message: 'Kafka batch event payloads',
+          payloads: payloads
+        )
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:batch_received
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.metrics&.increment(
+          'handler',
+          by: metadata['batch_size'],
+          tags: %W(
+            status:received
+            topic:#{metadata[:topic]}
+          ))
+        if payloads.present?
+          payloads.each { |payload| _report_time_delayed(payload, metadata) }
+        end
+      end
+
+      # @param exception [Throwable]
+      # @param payloads [Array<Hash>]
+      # @param metadata [Hash]
+      def _handle_batch_error(exception, payloads, metadata)
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:batch_error
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.logger.warn(
+          message: 'Error consuming message batch',
+          handler: self.class.name,
+          metadata: metadata.except(:keys),
+          message_ids: _payload_identifiers(payloads, metadata),
+          error_message: exception.message,
+          error: exception.backtrace
+        )
+        _error(exception, payloads, metadata)
+      end
+
+      # @param time_taken [Float]
+      # @param payloads [Array<Hash>]
+      # @param metadata [Hash]
+      def _handle_batch_success(time_taken, payloads, metadata)
+        Deimos.config.metrics&.histogram('handler',
+                                         time_taken,
+                                         tags: %W(
+                                           time:consume_batch
+                                           topic:#{metadata[:topic]}
+                                         ))
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:batch_success
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.metrics&.increment(
+          'handler',
+          by: metadata['batch_size'],
+          tags: %W(
+            status:success
+            topic:#{metadata[:topic]}
+          ))
+        Deimos.config.logger.info(
+          message: 'Finished processing Kafka batch event',
+          message_ids: _payload_identifiers(payloads, metadata),
+          time_elapsed: time_taken,
+          metadata: metadata.except(:keys)
+        )
+      end
+
+      # Get payload identifiers (key and message_id if present) for logging.
+      # @param payloads [Array<Hash>]
+      # @param metadata [Hash]
+      # @return [Array<Array>] the identifiers.
+      def _payload_identifiers(payloads, metadata)
+        message_ids = payloads&.map do |payload|
+          if payload.is_a?(Hash) && payload.key?('message_id')
+            payload['message_id']
+          end
+        end
+
+        # Payloads may be nil if preprocessing failed
+        messages = payloads || metadata[:keys] || []
+
+        messages.zip(metadata[:keys] || [], message_ids || []).map do |_, k, m_id|
+          ids = {}
+
+          ids[:key] = k if k.present?
+          ids[:message_id] = m_id if m_id.present?
+
+          ids
+        end
+      end
+    end
+  end
+end

data/lib/deimos/consume/message_consumption.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Deimos
+  module Consume
+    # Methods used by message-by-message (non-batch) consumers. These consumers
+    # are invoked for every individual message.
+    module MessageConsumption
+      extend ActiveSupport::Concern
+      include Phobos::Handler
+
+      # :nodoc:
+      def around_consume(payload, metadata)
+        decoded_payload = payload.dup
+        new_metadata = metadata.dup
+        benchmark = Benchmark.measure do
+          _with_span do
+            new_metadata[:key] = decode_key(metadata[:key]) if self.class.config[:key_configured]
+            decoded_payload = payload ? self.class.decoder.decode(payload) : nil
+            _received_message(decoded_payload, new_metadata)
+            yield decoded_payload, new_metadata
+          end
+        end
+        _handle_success(benchmark.real, decoded_payload, new_metadata)
+      rescue StandardError => e
+        _handle_error(e, decoded_payload, new_metadata)
+      end
+
+      # Consume incoming messages.
+      # @param _payload [String]
+      # @param _metadata [Hash]
+      def consume(_payload, _metadata)
+        raise NotImplementedError
+      end
+
+    private
+
+      def _received_message(payload, metadata)
+        Deimos.config.logger.info(
+          message: 'Got Kafka event',
+          payload: payload,
+          metadata: metadata
+        )
+        Deimos.config.metrics&.increment('handler', tags: %W(
+                                           status:received
+                                           topic:#{metadata[:topic]}
+                                         ))
+        _report_time_delayed(payload, metadata)
+      end
+
+      # @param exception [Throwable]
+      # @param payload [Hash]
+      # @param metadata [Hash]
+      def _handle_error(exception, payload, metadata)
+        Deimos.config.metrics&.increment(
+          'handler',
+          tags: %W(
+            status:error
+            topic:#{metadata[:topic]}
+          )
+        )
+        Deimos.config.logger.warn(
+          message: 'Error consuming message',
+          handler: self.class.name,
+          metadata: metadata,
+          data: payload,
+          error_message: exception.message,
+          error: exception.backtrace
+        )
+
+        _error(exception, payload, metadata)
+      end
+
+      # @param time_taken [Float]
+      # @param payload [Hash]
+      # @param metadata [Hash]
+      def _handle_success(time_taken, payload, metadata)
+        Deimos.config.metrics&.histogram('handler', time_taken, tags: %W(
+                                           time:consume
+                                           topic:#{metadata[:topic]}
+                                         ))
+        Deimos.config.metrics&.increment('handler', tags: %W(
+                                           status:success
+                                           topic:#{metadata[:topic]}
+                                         ))
+        Deimos.config.logger.info(
+          message: 'Finished processing Kafka event',
+          payload: payload,
+          time_elapsed: time_taken,
+          metadata: metadata
+        )
+      end
+    end
+  end
+end