deimos-ruby 1.22.5 → 1.23.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 997678ce4ed796037a0f554a7d0897a9ab58f810f437573da7684a6c23fa5d40
-  data.tar.gz: e13fc14fb0bf985a02c38ff628a527198aa1e3bbf1eaab52510ea63832a03f37
+  metadata.gz: 78613a211afa5b7bc3f065691e02c1417fb5e91568105bf81c812aabdf2480c4
+  data.tar.gz: fe640113779b11d13be54be53d434f2865ced8965dafd7493e44c679bb00d704
 SHA512:
-  metadata.gz: 90062c59b953e4fff9b1f5ddb0c5107e08f98028a7e221c78637bd9cb57c8d13c081a7417dedad57274e4b2df470a58d31ed5a1a1fd985f853ecca9ab53527fc
-  data.tar.gz: fa07d49754b91fdfc48fd6edb46a1906f057ddbd9d0b11d0a66d14382ae1907a0ff7fbd2526e4653e6fc74ed53e78db78203fc043621111193fca370e50da777
+  metadata.gz: ef4de06e3c106abd55703bf5dafa5fa19fa3425b8aca0081704ae1464e1993f94eb9e418b42f609a4b2771133df770b2139a94fddfbfd55ea2c803706124ba6c
+  data.tar.gz: '028fcf4753d456e552af8ad3dd725f0f4448848f88c8dcee542d503f6ddba7daf1aa8e46d4512e43faadee2f7717e910692479e322fd740f1bdf808a2864a74e'

data/.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
       BUNDLE_WITHOUT: development:test
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Ruby 2.7
         uses: ruby/setup-ruby@v1
@@ -39,7 +39,7 @@ jobs:
         ruby: [ '2.6', '2.7', '3.0', '3.1' ]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}

data/CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 1.23.0 - 2024-01-09
+
+- Fix: Fixed handler metric for status:received, status:success in batch consumption
+- Feature: Allow pre processing of messages prior to bulk consumption
+- Feature: Add global configuration for custom `bulk_import_id_generator` proc for all consumers
+- Feature: Add individual configuration for custom `bulk_import_id_generator` proc per consumer
+- Feature: Add global `replace_assocations` value for  for all consumers
+- Feature: Add individual `replace_assocations` value for for individual consumers
+- Feature: `should_consume?` method accepts BatchRecord associations
+- Feature: Reintroduce `filter_records` for bulk filtering of records prior to insertion
+- Feature: Return valid and invalid records saved during consumption for further processing in `batch_consumption.valid_records` and `batch_consumption.invalid_records` ActiveSupport Notifications
+
 # 1.22.5 - 2023-07-18
 - Fix: Fixed buffer overflow crash with DB producer.
 

data/README.md

@@ -189,6 +189,12 @@ produced by Phobos and RubyKafka):
    * exception_object
    * messages - the batch of messages (in the form of `Deimos::KafkaMessage`s)
      that failed - this should have only a single message in the batch.
+* `batch_consumption.valid_records` - sent when the consumer has successfully upserted records. Limited by `max_db_batch_size`.
+  * consumer: class of the consumer that upserted these records
+  * records: Records upserted into the DB (of type `ActiveRecord::Base`)
+* `batch_consumption.invalid_records` - sent when the consumer has rejected records returned from `filtered_records`. Limited by `max_db_batch_size`.
+  * consumer: class of the consumer that rejected these records
+  * records: Rejected records (of type `Deimos::ActiveRecordConsume::BatchRecord`)
   
 Similarly: 
 ```ruby

data/docs/CONFIGURATION.md

@@ -100,6 +100,8 @@ offset_commit_threshold|0|Number of messages that can be processed before their
 offset_retention_time|nil|The time period that committed offsets will be retained, in seconds. Defaults to the broker setting.
 heartbeat_interval|10|Interval between heartbeats; must be less than the session window.
 backoff|`(1000..60_000)`|Range representing the minimum and maximum number of milliseconds to back off after a consumer error.
+replace_associations|nil| Whether to delete existing associations for records during bulk consumption for this consumer. If no value is specified the provided/default value from the `consumers` configuration will be used.
+bulk_import_id_generator|nil| Block to determine the `bulk_import_id` generated during bulk consumption. If no block is specified the provided/default block from the `consumers` configuration will be used.
 
 ## Defining Database Pollers
 
@@ -172,6 +174,8 @@ consumers.backoff|`(1000..60_000)`|Range representing the minimum and maximum nu
 consumers.reraise_errors|false|Default behavior is to swallow uncaught exceptions and log to the metrics provider. Set this to true to instead raise all errors. Note that raising an error will ensure that the message cannot be processed - if there is a bad message which will always raise that error, your consumer will not be able to proceed past it and will be stuck forever until you fix your code. See also the `fatal_error` configuration. This is automatically set to true when using the `TestHelpers` module in RSpec.
 consumers.report_lag|false|Whether to send the `consumer_lag` metric. This requires an extra thread per consumer.
 consumers.fatal_error|`proc { false }`|Block taking an exception, payload and metadata and returning true if this should be considered a fatal error and false otherwise. E.g. you can use this to always fail if the database is available. Not needed if reraise_errors is set to true.
+consumers.replace_associations|true|Whether to delete existing associations for records during bulk consumption prior to inserting new associated records
+consumers.bulk_import_id_generator|`proc { SecureRandom.uuid }`| Block to determine the `bulk_import_id` generated during bulk consumption. Block will be used for all bulk consumers unless explicitly set for individual consumers
 
 ## Producer Configuration
 

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -28,18 +28,14 @@ module Deimos
           zip(metadata[:keys]).
           map { |p, k| Deimos::Message.new(p, nil, key: k) }
 
-        tags = %W(topic:#{metadata[:topic]})
-
-        Deimos.instrument('ar_consumer.consume_batch', tags) do
-          # The entire batch should be treated as one transaction so that if
-          # any message fails, the whole thing is rolled back or retried
-          # if there is deadlock
-          Deimos::Utils::DeadlockRetry.wrap(tags) do
-            if @compacted || self.class.config[:no_keys]
-              update_database(compact_messages(messages))
-            else
-              uncompacted_update(messages)
-            end
+        tag = metadata[:topic]
+        Deimos.config.tracer.active_span.set_tag('topic', tag)
+
+        Deimos.instrument('ar_consumer.consume_batch', tag) do
+          if @compacted || self.class.config[:no_keys]
+            update_database(compact_messages(messages))
+          else
+            uncompacted_update(messages)
           end
         end
       end
@@ -93,8 +89,9 @@ module Deimos
       end
 
       # @param _record [ActiveRecord::Base]
+      # @param _associations [Hash]
       # @return [Boolean]
-      def should_consume?(_record)
+      def should_consume?(_record, _associations=nil)
         true
       end
 
@@ -155,8 +152,13 @@ module Deimos
       # @return [void]
       def upsert_records(messages)
         record_list = build_records(messages)
-        record_list.filter!(self.method(:should_consume?).to_proc)
-
+        invalid = filter_records(record_list)
+        if invalid.any?
+          ActiveSupport::Notifications.instrument('batch_consumption.invalid_records', {
+                                                    records: invalid,
+                                                    consumer: self.class
+                                                  })
+        end
         return if record_list.empty?
 
         key_col_proc = self.method(:key_columns).to_proc
@@ -165,13 +167,31 @@ module Deimos
         updater = MassUpdater.new(@klass,
                                   key_col_proc: key_col_proc,
                                   col_proc: col_proc,
-                                  replace_associations: self.class.config[:replace_associations])
-        updater.mass_update(record_list)
+                                  replace_associations: self.class.replace_associations,
+                                  bulk_import_id_generator: self.class.bulk_import_id_generator)
+        ActiveSupport::Notifications.instrument('batch_consumption.valid_records', {
+                                                  records: updater.mass_update(record_list),
+                                                  consumer: self.class
+                                                })
+      end
+
+      # @param record_list [BatchRecordList]
+      # @return [Array<BatchRecord>]
+      def filter_records(record_list)
+        record_list.filter!(self.method(:should_consume?).to_proc)
+      end
+
+      # Process messages prior to saving to database
+      # @param _messages [Array<Deimos::Message>]
+      # @return [Void]
+      def pre_process(_messages)
+        nil
       end
 
       # @param messages [Array<Deimos::Message>]
       # @return [BatchRecordList]
       def build_records(messages)
+        pre_process(messages)
         records = messages.map do |m|
           attrs = if self.method(:record_attributes).parameters.size == 2
                     record_attributes(m.payload, m.key)
@@ -189,7 +209,8 @@ module Deimos
 
           BatchRecord.new(klass: @klass,
                           attributes: attrs,
-                          bulk_import_column: col)
+                          bulk_import_column: col,
+                          bulk_import_id_generator: self.class.bulk_import_id_generator)
         end
         BatchRecordList.new(records.compact)
       end
@@ -199,9 +220,11 @@ module Deimos
       # deleted records.
       # @return [void]
       def remove_records(messages)
-        clause = deleted_query(messages)
+        Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
+          clause = deleted_query(messages)
 
-        clause.delete_all
+          clause.delete_all
+        end
       end
     end
   end

data/lib/deimos/active_record_consume/batch_record.rb

@@ -17,16 +17,17 @@ module Deimos
       # @return [String] The column name to use for bulk IDs - defaults to `bulk_import_id`.
       attr_accessor :bulk_import_column
 
-      delegate :valid?, to: :record
+      delegate :valid?, :errors, :send, :attributes, to: :record
 
       # @param klass [Class < ActiveRecord::Base]
       # @param attributes [Hash] the full attribute list, including associations.
       # @param bulk_import_column [String]
-      def initialize(klass:, attributes:, bulk_import_column: nil)
+      # @param bulk_import_id_generator [Proc]
+      def initialize(klass:, attributes:, bulk_import_column: nil, bulk_import_id_generator: nil)
         @klass = klass
         if bulk_import_column
           self.bulk_import_column = bulk_import_column
-          self.bulk_import_id = SecureRandom.uuid
+          self.bulk_import_id = bulk_import_id_generator&.call
           attributes[bulk_import_column] = bulk_import_id
         end
         attributes = attributes.with_indifferent_access
@@ -43,7 +44,7 @@ module Deimos
         return if @klass.column_names.include?(self.bulk_import_column.to_s)
 
         raise "Create bulk_import_id on the #{@klass.table_name} table." \
-              ' Run rails g deimos:bulk_import_id {table} to create the migration.'
+                ' Run rails g deimos:bulk_import_id {table} to create the migration.'
       end
 
       # @return [Class < ActiveRecord::Base]

data/lib/deimos/active_record_consume/batch_record_list.rb

@@ -17,10 +17,19 @@ module Deimos
         self.bulk_import_column = records.first&.bulk_import_column&.to_sym
       end
 
-      # Filter out any invalid records.
+      # Filter and return removed invalid batch records by the specified method
       # @param method [Proc]
+      # @return [Array<BatchRecord>]
       def filter!(method)
-        self.batch_records.delete_if { |record| !method.call(record.record) }
+        self.batch_records, invalid = self.batch_records.partition do |batch_record|
+          case method.parameters.size
+          when 2
+            method.call(batch_record.record, batch_record.associations)
+          else
+            method.call(batch_record.record)
+          end
+        end
+        invalid
       end
 
       # Get the original ActiveRecord objects.

data/lib/deimos/active_record_consume/mass_updater.rb

@@ -19,9 +19,11 @@ module Deimos
       # @param key_col_proc [Proc<Class < ActiveRecord::Base>]
       # @param col_proc [Proc<Class < ActiveRecord::Base>]
       # @param replace_associations [Boolean]
-      def initialize(klass, key_col_proc: nil, col_proc: nil, replace_associations: true)
+      def initialize(klass, key_col_proc: nil, col_proc: nil,
+                     replace_associations: true, bulk_import_id_generator: nil)
         @klass = klass
         @replace_associations = replace_associations
+        @bulk_import_id_generator = bulk_import_id_generator
 
         @key_cols = {}
         @key_col_proc = key_col_proc
@@ -69,7 +71,7 @@ module Deimos
       def import_associations(record_list)
         record_list.fill_primary_keys!
 
-        import_id = @replace_associations ? SecureRandom.uuid : nil
+        import_id = @replace_associations ? @bulk_import_id_generator&.call : nil
         record_list.associations.each do |assoc|
           sub_records = record_list.map { |r| r.sub_records(assoc.name, import_id) }.flatten
           next unless sub_records.any?
@@ -82,9 +84,16 @@ module Deimos
       end
 
       # @param record_list [BatchRecordList]
+      # @return [Array<ActiveRecord::Base>]
       def mass_update(record_list)
-        save_records_to_database(record_list)
-        import_associations(record_list) if record_list.associations.any?
+        # The entire batch should be treated as one transaction so that if
+        # any message fails, the whole thing is rolled back or retried
+        # if there is deadlock
+        Deimos::Utils::DeadlockRetry.wrap(Deimos.config.tracer.active_span.get_tag('topic')) do
+          save_records_to_database(record_list)
+          import_associations(record_list) if record_list.associations.any?
+        end
+        record_list.records
       end
 
     end

data/lib/deimos/active_record_consumer.rb

@@ -35,6 +35,16 @@ module Deimos
         config[:bulk_import_id_column]
       end
 
+      # @return [Proc]
+      def bulk_import_id_generator
+        config[:bulk_import_id_generator]
+      end
+
+      # @return [Boolean]
+      def replace_associations
+        config[:replace_associations]
+      end
+
       # @param val [Boolean] Turn pre-compaction of the batch on or off. If true,
       # only the last message for each unique key in a batch is processed.
       # @return [void]

data/lib/deimos/config/configuration.rb

@@ -79,6 +79,7 @@ module Deimos # rubocop:disable Metrics/ModuleLength
 
   # @!visibility private
   # @param kafka_config [FigTree::ConfigStruct]
+  # rubocop:disable  Metrics/PerceivedComplexity, Metrics/AbcSize
   def self.configure_producer_or_consumer(kafka_config)
     klass = kafka_config.class_name.constantize
     klass.class_eval do
@@ -90,11 +91,18 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       if kafka_config.respond_to?(:bulk_import_id_column) # consumer
         klass.config.merge!(
           bulk_import_id_column: kafka_config.bulk_import_id_column,
-          replace_associations: kafka_config.replace_associations
+          replace_associations: if kafka_config.replace_associations.nil?
+                                  Deimos.config.consumers.replace_associations
+                                else
+                                  kafka_config.replace_associations
+                                end,
+          bulk_import_id_generator: kafka_config.bulk_import_id_generator ||
+            Deimos.config.consumers.bulk_import_id_generator
         )
       end
     end
   end
+  # rubocop:enable Metrics/PerceivedComplexity, Metrics/AbcSize
 
   define_settings do
 
@@ -242,6 +250,15 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       # Not needed if reraise_errors is set to true.
       # @return [Block]
       setting(:fatal_error, proc { false })
+
+      # The default function to generate a bulk ID for bulk consumers
+      # @return [Block]
+      setting(:bulk_import_id_generator, proc { SecureRandom.uuid })
+
+      # If true, multi-table consumers will blow away associations rather than appending to them.
+      # Applies to all consumers unless specified otherwise
+      # @return [Boolean]
+      setting :replace_associations, true
     end
 
     setting :producers do
@@ -445,7 +462,13 @@ module Deimos # rubocop:disable Metrics/ModuleLength
       setting :bulk_import_id_column, :bulk_import_id
       # If true, multi-table consumers will blow away associations rather than appending to them.
       # @return [Boolean]
-      setting :replace_associations, true
+      setting :replace_associations, nil
+
+      # The default function to generate a bulk ID for this consumer
+      # Uses the consumers proc defined in the consumers config by default unless
+      # specified for individual consumers
+      # @return [Block]
+      setting :bulk_import_id_generator, nil
 
       # These are the phobos "listener" configs. See CONFIGURATION.md for more
       # info.

data/lib/deimos/consume/batch_consumption.rb

@@ -64,7 +64,7 @@ module Deimos
           ))
         Deimos.config.metrics&.increment(
           'handler',
-          by: metadata['batch_size'],
+          by: metadata[:batch_size],
           tags: %W(
             status:received
             topic:#{metadata[:topic]}
@@ -115,7 +115,7 @@ module Deimos
           ))
         Deimos.config.metrics&.increment(
           'handler',
-          by: metadata['batch_size'],
+          by: metadata[:batch_size],
           tags: %W(
             status:success
             topic:#{metadata[:topic]}

data/lib/deimos/test_helpers.rb

@@ -28,6 +28,7 @@ module Deimos
           deimos_config.kafka.seed_brokers ||= ['test_broker']
           deimos_config.schema.backend = Deimos.schema_backend_class.mock_backend
           deimos_config.producers.backend = :test
+          deimos_config.tracer = Deimos::Tracing::Mock.new
         end
       end
 

data/lib/deimos/tracing/datadog.rb

@@ -15,11 +15,7 @@ module Deimos
 
       # :nodoc:
       def start(span_name, options={})
-        span = if ::Datadog.respond_to?(:tracer)
-                 ::Datadog.tracer.trace(span_name)
-               else
-                 ::Datadog::Tracing.trace(span_name)
-               end
+        span = tracer.trace(span_name)
         span.service = @service
         span.resource = options[:resource]
         span
@@ -30,9 +26,14 @@ module Deimos
         span.finish
       end
 
+      # :nodoc:
+      def tracer
+        @tracer ||= ::Datadog.respond_to?(:tracer) ? ::Datadog.tracer : ::Datadog::Tracing
+      end
+
       # :nodoc:
       def active_span
-        ::Datadog.tracer.active_span
+        tracer.active_span
       end
 
       # :nodoc:
@@ -45,6 +46,11 @@ module Deimos
         (span || active_span).set_tag(tag, value)
       end
 
+      # :nodoc:
+      def get_tag(tag)
+        active_span.get_tag(tag)
+      end
+
     end
   end
 end

data/lib/deimos/tracing/mock.rb

@@ -10,6 +10,7 @@ module Deimos
       def initialize(logger=nil)
         @logger = logger || Logger.new(STDOUT)
         @logger.info('MockTracingProvider initialized')
+        @active_span = MockSpan.new
       end
 
       # @param span_name [String]
@@ -32,12 +33,22 @@ module Deimos
 
       # :nodoc:
       def active_span
-        nil
+        @active_span ||= MockSpan.new
       end
 
       # :nodoc:
       def set_tag(tag, value, span=nil)
-        nil
+        if span
+          span.set_tag(tag, value)
+        else
+          active_span.set_tag(tag, value)
+        end
+      end
+
+      # Get a tag from a span with the specified tag.
+      # @param tag [String]
+      def get_tag(tag)
+        @span.get_tag(tag)
       end
 
       # :nodoc:
@@ -47,5 +58,23 @@ module Deimos
         @logger.info("Mock span '#{name}' set an error: #{exception}")
       end
     end
+
+    # Mock Span class
+    class MockSpan
+      # :nodoc:
+      def initialize
+        @span = {}
+      end
+
+      # :nodoc:
+      def set_tag(tag, value)
+        @span[tag] = value
+      end
+
+      # :nodoc:
+      def get_tag(tag)
+        @span[tag]
+      end
+    end
   end
 end

data/lib/deimos/tracing/provider.rb

@@ -42,6 +42,12 @@ module Deimos
         raise NotImplementedError
       end
 
+      # Get a tag from a span with the specified tag.
+      # @param tag [String]
+      def get_tag(tag)
+        raise NotImplementedError
+      end
+
     end
   end
 end

data/lib/deimos/utils/schema_class.rb

@@ -25,13 +25,21 @@ module Deimos
         def instance(payload, schema, namespace='')
           return payload if payload.is_a?(Deimos::SchemaClass::Base)
 
-          constants = modules_for(namespace) + [schema.underscore.camelize.singularize]
-          klass = constants.join('::').safe_constantize
+          klass = klass(schema, namespace)
           return payload if klass.nil? || payload.nil?
 
           klass.new(**payload.symbolize_keys)
         end
 
+        # Determine and return the SchemaClass with the provided schema and namespace
+        # @param schema [String]
+        # @param namespace [String]
+        # @return [Deimos::SchemaClass]
+        def klass(schema, namespace)
+          constants = modules_for(namespace) + [schema.underscore.camelize.singularize]
+          constants.join('::').safe_constantize
+        end
+
         # @param config [Hash] Producer or Consumer config
         # @return [Boolean]
         def use?(config)

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.22.5'
+  VERSION = '1.23.0'
 end

data/lib/deimos.rb

@@ -57,7 +57,20 @@ module Deimos
     # @param namespace [String]
     # @return [Deimos::SchemaBackends::Base]
     def schema_backend(schema:, namespace:)
-      schema_backend_class.new(schema: schema, namespace: namespace)
+      if Utils::SchemaClass.use?(config.to_h)
+        # Initialize an instance of the provided schema
+        # in the event the schema class is an override, the inherited
+        # schema and namespace will be applied
+        schema_class = Utils::SchemaClass.klass(schema, namespace)
+        if schema_class.nil?
+          schema_backend_class.new(schema: schema, namespace: namespace)
+        else
+          schema_instance = schema_class.new
+          schema_backend_class.new(schema: schema_instance.schema, namespace: schema_instance.namespace)
+        end
+      else
+        schema_backend_class.new(schema: schema, namespace: namespace)
+      end
     end
 
     # @param schema [String]

data/spec/active_record_batch_consumer_association_spec.rb

@@ -96,12 +96,17 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
         key_config plain: true
         record_class Widget
 
-        def should_consume?(record)
+        def should_consume?(record, associations)
           if self.should_consume_proc
-            return self.should_consume_proc.call(record)
+            case self.should_consume_proc.parameters.size
+            when 2
+              self.should_consume_proc.call(record, associations)
+            else
+              self.should_consume_proc.call(record)
+            end
+          else
+            true
           end
-
-          true
         end
 
         def record_attributes(payload, _key)
@@ -269,7 +274,7 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
 
     context 'with invalid models' do
       before(:each) do
-        consumer_class.should_consume_proc = proc { |val| val.some_int <= 10 }
+        consumer_class.should_consume_proc = proc { |record| record.some_int <= 10 }
       end
 
       it 'should only save valid models' do
@@ -280,5 +285,27 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
         expect(Widget.count).to eq(2)
       end
     end
+
+    context 'with invalid associations' do
+
+      before(:each) do
+        consumer_class.should_consume_proc = proc { |record, associations|
+          record.some_int <= 10 && associations['detail']['title'] != 'invalid'
+        }
+      end
+
+      it 'should only save valid associations' do
+        publish_batch([
+                        { key: 2,
+                          payload: { test_id: 'xyz', some_int: 5, title: 'valid' } },
+                        { key: 3,
+                          payload: { test_id: 'abc', some_int: 15, title: 'valid' } },
+                        { key: 4,
+                          payload: { test_id: 'abc', some_int: 9, title: 'invalid' } }
+                      ])
+        expect(Widget.count).to eq(2)
+        expect(Widget.second.some_int).to eq(5)
+      end
+    end
            end
 end