deimos-ruby 2.4.0.pre.beta18 → 2.4.0.pre.beta19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64559f1d1b0707d53a016342d32285fbce7514d8a5355fc8fc1dd6ddec2403ba
-  data.tar.gz: aac9b5fdf789ee31999debeaa611f1007576d6ade6678f96cd1f2b83e7a1a9cc
+  metadata.gz: 2cebceac8c154d892ac2b52333d39bedd44fab2031bc072599c3f87c76d4a720
+  data.tar.gz: e615c5604fdbece27c19b53026035d64e70d37ac0f6d8e83681e6cf4b223a9a6
 SHA512:
-  metadata.gz: 986940515b6370b5be6a39ed9b508e3294cb6f9f4e9dd0331bd17c0336d18670f8dae65a4f7038786302802796971cc74e9f8c76d1b42c2c73cd38664753b042
-  data.tar.gz: 3f1ff958b896c2ce78cb633ff8825ab2174ad89e84fdafe9820a3636217030206a644ce66de2c9928d59147b1db2c6c510681eddd976c24afa9ec739459abe99
+  metadata.gz: b9950799d72162f17afa70bbff778cb1a96edfe0c4279aae7381c1cde77dd64835c1dc45fd35bab356f0b219c94a9db93154e2edf639fda99f1f021856594196
+  data.tar.gz: d17195d05a9bd2fc91e6375efbc43e5c766b8b00060f7226b1d0d22253943ad08cc16210d40eb8452364249fb63791ed135b7dd08a7fe0c4141be2564a0411db

data/CHANGELOG.md

@@ -11,6 +11,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add `registry_url`, `registry_user` and `registry_password` per-topic configuration.
 - Added `have_sent_including` RSpec matcher to allow for Protobuf messages that use default values to be checked.
 
+# 2.3.3 - 2026-02-21
+
+- Fix/feature: Send `consumer.lags` and `consumer.lags_delta` metrics via MinimalDatadogListener.
+
+# 2.3.2 - 2026-02-20
+
+- Feature: Add overridable process_message? for batch consumption.
+- Feature: Add overridable post_process_batch for batch consumption.
+
+# 2.3.1 - 2026-01-22
+
+- Feature: Allow strings to be used in the `record_class` declaration.
+
 # 2.3.0 - 2026-01-13
 
 - Feature: Support broker setting per topic in producer configs.

data/README.md

@@ -403,6 +403,8 @@ class MyProducer < Deimos::ActiveRecordProducer
   # using the default functionality and don't need to override it) 
   # by setting `refetch` to false. This will avoid extra database fetches.
   record_class Widget, refetch: false
+  
+  # You can use a string here instead to avoid eager loading: record_class 'Widget'
 
   # Optionally override this if you want the message to be 
   # sent even if fields that aren't in the schema are changed.
@@ -490,6 +492,7 @@ A sample consumer would look as follows:
 ```ruby
 class MyConsumer < Deimos::ActiveRecordConsumer
   record_class Widget
+  # or use a string: record_class 'Widget'
 
   # Optional override of the way to fetch records based on payload and
   # key. Default is to use the key to search the primary key of the table.
@@ -567,6 +570,7 @@ A sample batch consumer would look as follows:
 ```ruby
 class MyConsumer < Deimos::ActiveRecordConsumer
   record_class Widget
+  # or use a string: record_class 'Widget'
 
   # Controls whether the batch is compacted before consuming.
   # If true, only the last message for each unique key in a batch will be

data/deimos-ruby.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('guard-rubocop', '~> 1')
   spec.add_development_dependency('karafka-testing', '~> 2.0')
   spec.add_development_dependency('pg', '~> 1.1')
+  spec.add_development_dependency('pry', '~> 0.14.1')
   spec.add_development_dependency('rails', '~> 8.0')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -25,7 +25,15 @@ module Deimos
       # they are split
       # @return [void]
       def consume_batch
-        deimos_messages = messages.map { |p| Deimos::Message.new(p.payload, key: p.key) }
+        filtered = messages.select { |p| process_message?(p.payload) }
+        skipped_count = messages.size - filtered.size
+        if skipped_count.positive?
+          Deimos::Logging.log_debug(
+            message: 'Skipping processing of messages in batch',
+            skipped_count: skipped_count
+          )
+        end
+        deimos_messages = filtered.map { |p| Deimos::Message.new(p.payload, key: p.key) }
 
         tag = topic.name
         Deimos.config.tracer.active_span.set_tag('topic', tag)
@@ -37,6 +45,8 @@ module Deimos
             uncompacted_update(deimos_messages)
           end
         end
+
+        post_process_batch(deimos_messages)
       end
 
     protected
@@ -94,6 +104,14 @@ module Deimos
         true
       end
 
+      # Perform any post-processing after a batch has been consumed.
+      # Called once per batch with the list of Deimos::Message that were processed (after filtering and compaction).
+      # @param messages [Array<Deimos::Message>] The batch of messages that were processed
+      # @return [void]
+      def post_process_batch(_messages)
+        nil
+      end
+
     private
 
       # Compact a batch of messages, taking only the last message for each

data/lib/deimos/active_record_consumer.rb

@@ -72,6 +72,7 @@ module Deimos
     # Setup
     def initialize
       @klass = self.class.config[:record_class]
+      @klass = @klass.constantize if @klass.is_a?(String)
       @compacted = self.class.config[:compacted] != false
     end
 

data/lib/deimos/active_record_producer.rb

@@ -13,18 +13,24 @@ module Deimos
   class ActiveRecordProducer < Producer
     class << self
       # Indicate the class this producer is working on.
-      # @param klass [Class]
+      # @param klass [Class,String]
       # @param refetch [Boolean] if true, and we are given a hash instead of
       # a record object, refetch the record to pass into the `generate_payload`
       # method.
       # @return [void]
       def record_class(klass=nil, refetch: true)
-        return @record_class if klass.nil?
+        return record_klass if klass.nil?
 
         @record_class = klass
         @refetch_record = refetch
       end
 
+      # @return [Class,nil]
+      def record_klass
+        @record_class = @record_class.constantize if @record_class.is_a?(String)
+        @record_class
+      end
+
       # @param record [ActiveRecord::Base]
       # @param force_send [Boolean]
       # @return [void]
@@ -38,14 +44,14 @@ module Deimos
       def send_events(records, force_send: false)
         return if Deimos.producers_disabled?(self)
 
-        primary_key = @record_class&.primary_key
+        primary_key = record_klass&.primary_key
         messages = records.map do |record|
           if record.respond_to?(:attributes)
             attrs = record.attributes.with_indifferent_access
           else
             attrs = record.with_indifferent_access
             if @refetch_record && attrs[primary_key]
-              record = @record_class.find(attrs[primary_key])
+              record = record_klass.find(attrs[primary_key])
             end
           end
           generate_payload(attrs, record).with_indifferent_access
@@ -96,7 +102,7 @@ module Deimos
       # than this value).
       # @return [ActiveRecord::Relation]
       def poll_query(time_from:, time_to:, min_id:, column_name: :updated_at)
-        klass = @record_class
+        klass = record_klass
         table = ActiveRecord::Base.connection.quote_table_name(klass.table_name)
         column = ActiveRecord::Base.connection.quote_column_name(column_name)
         primary = ActiveRecord::Base.connection.quote_column_name(klass.primary_key)

data/lib/deimos/metrics/minimal_datadog.rb

@@ -6,8 +6,12 @@ require 'deimos/metrics/minimal_datadog_listener'
 
 module Deimos
   module Metrics
-    # A Metrics wrapper class for Datadog, with only minimal metrics being sent. This will not
-    # send any rdkafka metrics, and only the following:
+    # A Metrics wrapper class for Datadog, with only minimal metrics being sent. This will only
+    # send the following rdkafka metrics:
+    # * consumer.lags
+    # * consumer.lags_delta
+    #
+    # and only the following other metrics:
     # * consumer_group
     # * error_occurred
     # * consumer.messages
@@ -28,7 +32,10 @@ module Deimos
           if config[:karafka_distribution_mode]
             karafka_config.distribution_mode = config[:karafka_distribution_mode]
           end
-          karafka_config.rd_kafka_metrics = []
+          karafka_config.rd_kafka_metrics = [
+            RdKafkaMetric.new(:gauge, :topics, 'consumer.lags', 'consumer_lag_stored'),
+            RdKafkaMetric.new(:gauge, :topics, 'consumer.lags_delta', 'consumer_lag_stored_d')
+          ]
         end
         Karafka.monitor.subscribe(karafka_listener)
       end

data/lib/deimos/schema_backends/avro_schema_registry.rb

@@ -27,7 +27,6 @@ module Deimos
           password: @registry_info&.password || Deimos.config.schema.password,
           schema_type: SchemaRegistry::Schema::Avro.new(schema_store: @schema_store)
         )
-        @schema_registry
       end
     end
   end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.4.0-beta18'
+  VERSION = '2.4.0-beta19'
 end

data/spec/active_record_batch_consumer_association_spec.rb

@@ -70,7 +70,7 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
     end
 
     before(:each) do
-      ActiveRecord::Base.connection.truncate_tables(%i(widgets details locales))
+      ActiveRecord::Base.connection.truncate_tables(:widgets, :details, :locales)
       Widget.create!(test_id: 'bad_id', some_int: 100) # should not show up
     end
 

data/spec/active_record_batch_consumer_spec.rb

@@ -57,6 +57,12 @@ module ActiveRecordBatchConsumerTest
       Class.new(described_class) do
         record_class Widget
         compacted false
+
+        def process_message?(payload)
+          return true if payload.nil?
+
+          payload['test_id'] != 'skipme'
+        end
       end
     end
 
@@ -103,6 +109,27 @@ module ActiveRecordBatchConsumerTest
               )
           end
 
+          it 'should not create records for messages that return false from process_message?' do
+            publish_batch(
+              [
+                { key: 1,
+                  payload: { test_id: 'abc', some_int: 3 } },
+                { key: 2,
+                  payload: { test_id: 'skipme', some_int: 4 } },
+                { key: 3,
+                  payload: { test_id: 'ghi', some_int: 5 } }
+              ]
+            )
+
+            expect(all_widgets).
+              to contain_exactly(
+                have_attributes(id: 1, test_id: 'abc', some_int: 3, updated_at: start,
+                                created_at: start),
+                have_attributes(id: 3, test_id: 'ghi', some_int: 5, updated_at: start,
+                                created_at: start)
+              )
+          end
+
           it 'should handle deleting a record that doesn\'t exist' do
             publish_batch(
               [
@@ -805,6 +832,64 @@ module ActiveRecordBatchConsumerTest
 
     end
 
+    describe 'post_process_batch' do
+      before(:each) do
+        register_consumer(consumer_class,
+                          'MySchema',
+                          key_config: { plain: true })
+        MyBatchConsumer.last_post_process_batch = nil
+      end
+
+      let(:consumer_class) do
+        Class.new(described_class) do
+          class << self
+            attr_accessor :last_post_process_batch
+          end
+
+          record_class Widget
+          compacted false
+
+          def post_process_batch(messages)
+            self.class.last_post_process_batch = messages
+          end
+
+          def process_message?(payload)
+            payload['test_id'] != 'skipme'
+          end
+        end
+      end
+
+      it 'is called after the batch is processed with the processed messages' do
+        publish_batch(
+          [
+            { key: 1,
+              payload: { test_id: 'abc', some_int: 3 } },
+            { key: 2,
+              payload: { test_id: 'def', some_int: 4 } }
+          ]
+        )
+        expect(MyBatchConsumer.last_post_process_batch.map(&:payload).map do |p|
+ p[:test_id]
+        end).to contain_exactly('abc', 'def')
+      end
+
+      it 'is called with filtered messages when process_message? excludes some' do
+        publish_batch(
+          [
+            { key: 1,
+              payload: { test_id: 'abc', some_int: 3 } },
+            { key: 2,
+              payload: { test_id: 'skipme', some_int: 4 } },
+            { key: 3,
+              payload: { test_id: 'ghi', some_int: 5 } }
+          ]
+        )
+
+        expect(MyBatchConsumer.last_post_process_batch.size).to eq(2)
+        expect(MyBatchConsumer.last_post_process_batch.map(&:key).map(&:to_i)).to contain_exactly(1, 3)
+      end
+    end
+
   end
 end
 # rubocop:enable Lint/ConstantDefinitionInBlock

data/spec/deimos_spec.rb

@@ -164,6 +164,34 @@ describe Deimos do
     end
   end
 
+  describe 'configure' do
+    it 'should reset cached backends when configure is called again' do
+      Karafka::App.routes.redraw do
+        topic 'configure-test-topic' do
+          active false
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config none: true
+        end
+      end
+
+      topic_config = described_class.karafka_configs.find { |c| c.name == 'configure-test-topic' }
+      payload_transcoder = topic_config.deserializers[:payload]
+
+      # Access the backend to cache it
+      first_backend = payload_transcoder.backend
+
+      # Call configure again (simulating a second configure call after topics are defined)
+      described_class.configure do |config|
+        config.schema.backend = :avro_validation
+      end
+
+      # The cached backend should have been reset, so a new one is created
+      second_backend = payload_transcoder.backend
+      expect(second_backend).not_to equal(first_backend)
+    end
+  end
+
   describe '#schema_backend_class with mock_backends' do
     after(:each) do
       described_class.mock_backends = false

data/spec/schema_backends/base_spec.rb

@@ -26,6 +26,18 @@ describe Deimos::SchemaBackends::Base do
     backend.decode(payload, schema: 'schema2')
   end
 
+  it 'should use explicit schema as subject when topic is nil' do
+    expect(backend).to receive(:validate).with(payload, schema: 'schema2')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', subject: 'schema2-value')
+    backend.encode(payload, schema: 'schema2')
+  end
+
+  it 'should use explicit schema as subject for key when topic is nil' do
+    expect(backend).to receive(:validate).with(payload, schema: 'schema2')
+    expect(backend).to receive(:encode_payload).with(payload, schema: 'schema2', subject: 'schema2-key')
+    backend.encode(payload, schema: 'schema2', is_key: true)
+  end
+
   it 'should return nil if passed nil' do
     expect(backend.decode(nil, schema: 'schema2')).to be_nil
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 2.4.0.pre.beta18
+  version: 2.4.0.pre.beta19
 platform: ruby
 authors:
 - Daniel Orner
@@ -225,6 +225,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement