deimos-ruby 1.7.0.pre.beta1 → 1.8.0.pre.beta1

data/lib/deimos/active_record_consume/batch_slicer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Deimos
+  module ActiveRecordConsume
+    # Helper class for breaking down batches into independent groups for
+    # processing
+    class BatchSlicer
+      # Split the batch into a series of independent slices. Each slice contains
+      # messages that can be processed in any order (i.e. they have distinct
+      # keys). Messages with the same key will be separated into different
+      # slices that maintain the correct order.
+      # E.g. Given messages A1, A2, B1, C1, C2, C3, they will be sliced as:
+      # [[A1, B1, C1], [A2, C2], [C3]]
+      def self.slice(messages)
+        ops = messages.group_by(&:key)
+
+        # Find maximum depth
+        depth = ops.values.map(&:length).max || 0
+
+        # Generate slices for each depth
+        depth.times.map do |i|
+          ops.values.map { |arr| arr.dig(i) }.compact
+        end
+      end
+    end
+  end
+end

data/lib/deimos/active_record_consume/message_consumption.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Deimos
+  module ActiveRecordConsume
+    # Methods for consuming individual messages and saving them to the database
+    # as ActiveRecord instances.
+    module MessageConsumption
+      # Find the record specified by the given payload and key.
+      # Default is to use the primary key column and the value of the first
+      # field in the key.
+      # @param klass [Class < ActiveRecord::Base]
+      # @param _payload [Hash]
+      # @param key [Object]
+      # @return [ActiveRecord::Base]
+      def fetch_record(klass, _payload, key)
+        klass.unscoped.where(klass.primary_key => key).first
+      end
+
+      # Assign a key to a new record.
+      # @param record [ActiveRecord::Base]
+      # @param _payload [Hash]
+      # @param key [Object]
+      def assign_key(record, _payload, key)
+        record[record.class.primary_key] = key
+      end
+
+      # :nodoc:
+      def consume(payload, metadata)
+        key = metadata.with_indifferent_access[:key]
+        klass = self.class.config[:record_class]
+        record = fetch_record(klass, (payload || {}).with_indifferent_access, key)
+        if payload.nil?
+          destroy_record(record)
+          return
+        end
+        if record.blank?
+          record = klass.new
+          assign_key(record, payload, key)
+        end
+        attrs = record_attributes(payload.with_indifferent_access, key)
+        # don't use attributes= - bypass Rails < 5 attr_protected
+        attrs.each do |k, v|
+          record.send("#{k}=", v)
+        end
+        record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
+        record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
+        record.save!
+      end
+
+      # Destroy a record that received a null payload. Override if you need
+      # to do something other than a straight destroy (e.g. mark as archived).
+      # @param record [ActiveRecord::Base]
+      def destroy_record(record)
+        record&.destroy
+      end
+    end
+  end
+end

data/lib/deimos/active_record_consume/schema_model_converter.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Deimos
+  module ActiveRecordConsume
+    # Convert a message with a schema to an ActiveRecord model
+    class SchemaModelConverter
+      # Create new converter
+      # @param decoder [SchemaBackends::Base] Incoming message schema.
+      # @param klass [ActiveRecord::Base] Model to map to.
+      def initialize(decoder, klass)
+        @decoder = decoder
+        @klass = klass
+      end
+
+      # Convert a message from a decoded hash to a set of ActiveRecord
+      # attributes. Attributes that don't exist in the model will be ignored.
+      # @param payload [Hash] Decoded message payload.
+      # @return [Hash] Model attributes.
+      def convert(payload)
+        attributes = {}
+        @decoder.schema_fields.each do |field|
+          column = @klass.columns.find { |c| c.name == field.name }
+          next if column.nil?
+          next if %w(updated_at created_at).include?(field.name)
+
+          attributes[field.name] = _coerce_field(column, payload[field.name])
+        end
+        attributes
+      end
+
+    private
+
+      # @param column [ActiveRecord::ConnectionAdapters::Column]
+      # @param val [Object]
+      def _coerce_field(column, val)
+        return nil if val.nil?
+
+        if column.type == :datetime
+          int_val = begin
+                      val.is_a?(Integer) ? val : (val.is_a?(String) && Integer(val))
+                    rescue StandardError
+                      nil
+                    end
+
+          return Time.zone.at(int_val) if int_val
+        end
+
+        val
+      end
+    end
+  end
+end

data/lib/deimos/active_record_consumer.rb

@@ -1,101 +1,59 @@
 # frozen_string_literal: true
 
+require 'deimos/active_record_consume/batch_consumption'
+require 'deimos/active_record_consume/message_consumption'
+require 'deimos/active_record_consume/schema_model_converter'
 require 'deimos/consumer'
 
 module Deimos
-  # Consumer that automatically saves the payload into the database.
+  # Basic ActiveRecord consumer class. Consumes messages and upserts them to
+  # the database. For tombstones (null payloads), deletes corresponding
+  # records from the database. Can operate in either message-by-message mode
+  # or in batch mode.
+  #
+  # In batch mode, ActiveRecord callbacks will be skipped and messages will
+  # be batched to minimize database calls.
+
+  # To configure batch vs. message mode, change the delivery mode of your
+  # Phobos listener.
+  # Message-by-message -> use `delivery: message` or `delivery: batch`
+  # Batch -> use `delivery: inline_batch`
   class ActiveRecordConsumer < Consumer
+    include ActiveRecordConsume::MessageConsumption
+    include ActiveRecordConsume::BatchConsumption
+
     class << self
       # param klass [Class < ActiveRecord::Base] the class used to save to the
       # database.
       def record_class(klass)
         config[:record_class] = klass
       end
-    end
 
-    # Find the record specified by the given payload and key.
-    # Default is to use the primary key column and the value of the first
-    # field in the key.
-    # @param klass [Class < ActiveRecord::Base]
-    # @param _payload [Hash]
-    # @param key [Object]
-    # @return [ActiveRecord::Base]
-    def fetch_record(klass, _payload, key)
-      klass.unscoped.where(klass.primary_key => key).first
+      # param val [Boolean] Turn pre-compaction of the batch on or off. If true,
+      # only the last message for each unique key in a batch is processed.
+      def compacted(val)
+        config[:compacted] = val
+      end
     end
 
-    # Assign a key to a new record.
-    # @param record [ActiveRecord::Base]
-    # @param _payload [Hash]
-    # @param key [Object]
-    def assign_key(record, _payload, key)
-      record[record.class.primary_key] = key
-    end
+    # Setup
+    def initialize
+      @klass = self.class.config[:record_class]
+      @converter = ActiveRecordConsume::SchemaModelConverter.new(self.class.decoder, @klass)
 
-    # :nodoc:
-    def consume(payload, metadata)
-      key = metadata.with_indifferent_access[:key]
-      klass = self.class.config[:record_class]
-      record = fetch_record(klass, (payload || {}).with_indifferent_access, key)
-      if payload.nil?
-        destroy_record(record)
-        return
+      if self.class.config[:key_schema]
+        @key_converter = ActiveRecordConsume::SchemaModelConverter.new(self.class.key_decoder, @klass)
       end
-      if record.blank?
-        record = klass.new
-        assign_key(record, payload, key)
-      end
-      attrs = record_attributes(payload.with_indifferent_access)
-      # don't use attributes= - bypass Rails < 5 attr_protected
-      attrs.each do |k, v|
-        record.send("#{k}=", v)
-      end
-      record.created_at ||= Time.zone.now if record.respond_to?(:created_at)
-      record.updated_at = Time.zone.now if record.respond_to?(:updated_at)
-      record.save!
-    end
 
-    # Destroy a record that received a null payload. Override if you need
-    # to do something other than a straight destroy (e.g. mark as archived).
-    # @param record [ActiveRecord::Base]
-    def destroy_record(record)
-      record&.destroy
+      @compacted = self.class.config[:compacted] != false
     end
 
     # Override this method (with `super`) if you want to add/change the default
     # attributes set to the new/existing record.
     # @param payload [Hash]
-    def record_attributes(payload)
-      klass = self.class.config[:record_class]
-      attributes = {}
-      self.class.decoder.schema_fields.each do |field|
-        column = klass.columns.find { |c| c.name == field.name }
-        next if column.nil?
-        next if %w(updated_at created_at).include?(field.name)
-
-        attributes[field.name] = _coerce_field(column, payload[field.name])
-      end
-      attributes
-    end
-
-  private
-
-    # @param column [ActiveRecord::ConnectionAdapters::Column]
-    # @param val [Object]
-    def _coerce_field(column, val)
-      return nil if val.nil?
-
-      if column.type == :datetime
-        int_val = begin
-                    val.is_a?(Integer) ? val : (val.is_a?(String) && Integer(val))
-                  rescue StandardError
-                    nil
-                  end
-
-        return Time.zone.at(int_val) if int_val
-      end
-
-      val
+    # @param _key [String]
+    def record_attributes(payload, _key=nil)
+      @converter.convert(payload)
     end
   end
 end

data/lib/deimos/batch_consumer.rb

@@ -1,147 +1,7 @@
 # frozen_string_literal: true
 
-require 'deimos/base_consumer'
-require 'phobos/batch_handler'
-
 module Deimos
-  # Class to consume batches of messages in a topic
-  # Note: According to the docs, instances of your handler will be created
-  # for every incoming batch of messages. This class should be lightweight.
-  class BatchConsumer < BaseConsumer
-    include Phobos::BatchHandler
-
-    # :nodoc:
-    def around_consume_batch(batch, metadata)
-      payloads = []
-      benchmark = Benchmark.measure do
-        if self.class.config[:key_configured]
-          metadata[:keys] = batch.map do |message|
-            decode_key(message.key)
-          end
-        end
-
-        payloads = batch.map do |message|
-          message.payload ? self.class.decoder.decode(message.payload) : nil
-        end
-        _received_batch(payloads, metadata)
-        _with_span do
-          yield payloads, metadata
-        end
-      end
-      _handle_success(benchmark.real, payloads, metadata)
-    rescue StandardError => e
-      _handle_error(e, payloads, metadata)
-    end
-
-    # Consume a batch of incoming messages.
-    # @param _payloads [Array<Phobos::BatchMessage>]
-    # @param _metadata [Hash]
-    def consume_batch(_payloads, _metadata)
-      raise NotImplementedError
-    end
-
-  protected
-
-    def _received_batch(payloads, metadata)
-      Deimos.config.logger.info(
-        message: 'Got Kafka batch event',
-        message_ids: _payload_identifiers(payloads, metadata),
-        metadata: metadata.except(:keys)
-      )
-      Deimos.config.logger.debug(
-        message: 'Kafka batch event payloads',
-        payloads: payloads
-      )
-      Deimos.config.metrics&.increment(
-        'handler',
-        tags: %W(
-          status:batch_received
-          topic:#{metadata[:topic]}
-        ))
-      Deimos.config.metrics&.increment(
-        'handler',
-        by: metadata['batch_size'],
-        tags: %W(
-          status:received
-          topic:#{metadata[:topic]}
-        ))
-      if payloads.present?
-        payloads.each { |payload| _report_time_delayed(payload, metadata) }
-      end
-    end
-
-    # @param exception [Throwable]
-    # @param payloads [Array<Hash>]
-    # @param metadata [Hash]
-    def _handle_error(exception, payloads, metadata)
-      Deimos.config.metrics&.increment(
-        'handler',
-        tags: %W(
-          status:batch_error
-          topic:#{metadata[:topic]}
-        ))
-      Deimos.config.logger.warn(
-        message: 'Error consuming message batch',
-        handler: self.class.name,
-        metadata: metadata.except(:keys),
-        message_ids: _payload_identifiers(payloads, metadata),
-        error_message: exception.message,
-        error: exception.backtrace
-      )
-      super
-    end
-
-    # @param time_taken [Float]
-    # @param payloads [Array<Hash>]
-    # @param metadata [Hash]
-    def _handle_success(time_taken, payloads, metadata)
-      Deimos.config.metrics&.histogram('handler', time_taken, tags: %W(
-                                         time:consume_batch
-                                         topic:#{metadata[:topic]}
-                                       ))
-      Deimos.config.metrics&.increment(
-        'handler',
-        tags: %W(
-          status:batch_success
-          topic:#{metadata[:topic]}
-        ))
-      Deimos.config.metrics&.increment(
-        'handler',
-        by: metadata['batch_size'],
-        tags: %W(
-          status:success
-          topic:#{metadata[:topic]}
-        ))
-      Deimos.config.logger.info(
-        message: 'Finished processing Kafka batch event',
-        message_ids: _payload_identifiers(payloads, metadata),
-        time_elapsed: time_taken,
-        metadata: metadata.except(:keys)
-      )
-    end
-
-    # Get payload identifiers (key and message_id if present) for logging.
-    # @param payloads [Array<Hash>]
-    # @param metadata [Hash]
-    # @return [Hash] the identifiers.
-    def _payload_identifiers(payloads, metadata)
-      message_ids = payloads&.map do |payload|
-        if payload.is_a?(Hash) && payload.key?('message_id')
-          payload['message_id']
-        end
-      end
-
-      # Payloads may be nil if preprocessing failed
-      messages = payloads || metadata[:keys] || []
-
-      messages.zip(metadata[:keys] || [], message_ids || []).map do |_, k, m_id|
-        ids = {}
-
-        ids[:key] = k if k.present?
-        ids[:message_id] = m_id if m_id.present?
-
-        ids
-      end
-    end
+  # @deprecated Use Deimos::Consumer with `delivery: inline_batch` configured instead
+  class BatchConsumer < Consumer
   end
 end

data/lib/deimos/config/configuration.rb

@@ -47,17 +47,15 @@ module Deimos
       handler_class = listener.handler.constantize
       delivery = listener.delivery
 
-      # Validate that Deimos consumers use proper delivery configs
-      if handler_class < Deimos::BatchConsumer
-        unless delivery == 'inline_batch'
-          raise "BatchConsumer #{listener.handler} must have delivery set to"\
-            ' `inline_batch`'
-        end
-      elsif handler_class < Deimos::Consumer
-        if delivery.present? && !%w(message batch).include?(delivery)
-          raise "Non-batch Consumer #{listener.handler} must have delivery"\
-            ' set to `message` or `batch`'
+      next unless handler_class < Deimos::Consumer
+
+      # Validate that each consumer implements the correct method for its type
+      if delivery == 'inline_batch'
+        if handler_class.instance_method(:consume_batch).owner == Deimos::Consume::BatchConsumption
+          raise "BatchConsumer #{listener.handler} does not implement `consume_batch`"
         end
+      elsif handler_class.instance_method(:consume).owner == Deimos::Consume::MessageConsumption
+        raise "Non-batch Consumer #{listener.handler} does not implement `consume`"
       end
     end
   end