deimos-ruby 2.3.0 → 2.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9811f05eec0d9b6777e2ed047a2e81b0a23bd770b52866bf8c8a79f84c8ec586
-  data.tar.gz: c84cba2ecf750f4349ba86e47ddab2d01a4dd8e5f8fd9ab008b3fcfeb14b1425
+  metadata.gz: 11656e5d925198470fb1c6918808e2dda7b769d98397e9b0f973316a21fe1616
+  data.tar.gz: 4f34eae61a408809aaa6c0d70d0fe535c27ea300e479612c8273bd63cf553b62
 SHA512:
-  metadata.gz: b8319ada2f2b715e750e7de5caf6c6117d6154da6277988e12851f420c69a2f3adc0ab30594a23bca86ec8ffa57ef2cc3e5ab9758421e26fbe91cf682ad28e04
-  data.tar.gz: 2174b8d776b547689422d6af381f423680d61df368be813e53f213eb8d084971b338a3382d8775e07ef324b9443fc1f5427fed834c54629d52c88728d016fd49
+  metadata.gz: 60c5d0fe69d3787c972c647d907de9159c4f33a63c8f1e3913c9816bd18e056f24721549377bf3ebdc25578c800c62ad7ab2f5a056e0562cacd1026858ea5d2f
+  data.tar.gz: 252a59f9db351029fd2cd67c679bddd50a854e4797855a8c7b5e7248e6b0ab71aad041d7b7e3dd900eb5c90c6cc5f82301461f58a749fe997f99d8f65d8b3f3e

data/CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 2.3.2 - 2026-02-20
+
+- Feature: Add overridable process_message? for batch consumption.
+- Feature: Add overridable post_process_batch for batch consumption.
+
+# 2.3.1 - 2026-01-22
+
+- Feature: Allow strings to be used in the `record_class` declaration.
+
 # 2.3.0 - 2026-01-13
 
 - Feature: Support broker setting per topic in producer configs.

data/README.md

@@ -398,6 +398,8 @@ class MyProducer < Deimos::ActiveRecordProducer
   # using the default functionality and don't need to override it) 
   # by setting `refetch` to false. This will avoid extra database fetches.
   record_class Widget, refetch: false
+  
+  # You can use a string here instead to avoid eager loading: record_class 'Widget'
 
   # Optionally override this if you want the message to be 
   # sent even if fields that aren't in the schema are changed.
@@ -485,6 +487,7 @@ A sample consumer would look as follows:
 ```ruby
 class MyConsumer < Deimos::ActiveRecordConsumer
   record_class Widget
+  # or use a string: record_class 'Widget'
 
   # Optional override of the way to fetch records based on payload and
   # key. Default is to use the key to search the primary key of the table.
@@ -562,6 +565,7 @@ A sample batch consumer would look as follows:
 ```ruby
 class MyConsumer < Deimos::ActiveRecordConsumer
   record_class Widget
+  # or use a string: record_class 'Widget'
 
   # Controls whether the batch is compacted before consuming.
   # If true, only the last message for each unique key in a batch will be

data/deimos-ruby.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('karafka-testing', '~> 2.0')
   spec.add_development_dependency('pg', '~> 1.1')
   spec.add_development_dependency('proto_turf')
+  spec.add_development_dependency('pry', '~> 0.14.1')
   spec.add_development_dependency('rails', '~> 8.0')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -25,7 +25,15 @@ module Deimos
       # they are split
       # @return [void]
       def consume_batch
-        deimos_messages = messages.map { |p| Deimos::Message.new(p.payload, key: p.key) }
+        filtered = messages.select { |p| process_message?(p.payload) }
+        skipped_count = messages.size - filtered.size
+        if skipped_count.positive?
+          Deimos::Logging.log_debug(
+            message: 'Skipping processing of messages in batch',
+            skipped_count: skipped_count
+          )
+        end
+        deimos_messages = filtered.map { |p| Deimos::Message.new(p.payload, key: p.key) }
 
         tag = topic.name
         Deimos.config.tracer.active_span.set_tag('topic', tag)
@@ -37,6 +45,8 @@ module Deimos
             uncompacted_update(deimos_messages)
           end
         end
+
+        post_process_batch(deimos_messages)
       end
 
     protected
@@ -94,6 +104,14 @@ module Deimos
         true
       end
 
+      # Perform any post-processing after a batch has been consumed.
+      # Called once per batch with the list of Deimos::Message that were processed (after filtering and compaction).
+      # @param messages [Array<Deimos::Message>] The batch of messages that were processed
+      # @return [void]
+      def post_process_batch(_messages)
+        nil
+      end
+
     private
 
       # Compact a batch of messages, taking only the last message for each

data/lib/deimos/active_record_consumer.rb

@@ -72,6 +72,7 @@ module Deimos
     # Setup
     def initialize
       @klass = self.class.config[:record_class]
+      @klass = @klass.constantize if @klass.is_a?(String)
       @compacted = self.class.config[:compacted] != false
     end
 

data/lib/deimos/active_record_producer.rb

@@ -13,18 +13,24 @@ module Deimos
   class ActiveRecordProducer < Producer
     class << self
       # Indicate the class this producer is working on.
-      # @param klass [Class]
+      # @param klass [Class,String]
       # @param refetch [Boolean] if true, and we are given a hash instead of
       # a record object, refetch the record to pass into the `generate_payload`
       # method.
       # @return [void]
       def record_class(klass=nil, refetch: true)
-        return @record_class if klass.nil?
+        return record_klass if klass.nil?
 
         @record_class = klass
         @refetch_record = refetch
       end
 
+      # @return [Class,nil]
+      def record_klass
+        @record_class = @record_class.constantize if @record_class.is_a?(String)
+        @record_class
+      end
+
       # @param record [ActiveRecord::Base]
       # @param force_send [Boolean]
       # @return [void]
@@ -38,14 +44,14 @@ module Deimos
       def send_events(records, force_send: false)
         return if Deimos.producers_disabled?(self)
 
-        primary_key = @record_class&.primary_key
+        primary_key = record_klass&.primary_key
         messages = records.map do |record|
           if record.respond_to?(:attributes)
             attrs = record.attributes.with_indifferent_access
           else
             attrs = record.with_indifferent_access
             if @refetch_record && attrs[primary_key]
-              record = @record_class.find(attrs[primary_key])
+              record = record_klass.find(attrs[primary_key])
             end
           end
           generate_payload(attrs, record).with_indifferent_access
@@ -96,7 +102,7 @@ module Deimos
       # than this value).
       # @return [ActiveRecord::Relation]
       def poll_query(time_from:, time_to:, min_id:, column_name: :updated_at)
-        klass = @record_class
+        klass = record_klass
         table = ActiveRecord::Base.connection.quote_table_name(klass.table_name)
         column = ActiveRecord::Base.connection.quote_column_name(column_name)
         primary = ActiveRecord::Base.connection.quote_column_name(klass.primary_key)

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.3.0'
+  VERSION = '2.3.2'
 end

data/spec/active_record_batch_consumer_association_spec.rb

@@ -70,7 +70,7 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
     end
 
     before(:each) do
-      ActiveRecord::Base.connection.truncate_tables(%i(widgets details locales))
+      ActiveRecord::Base.connection.truncate_tables(:widgets, :details, :locales)
       Widget.create!(test_id: 'bad_id', some_int: 100) # should not show up
     end
 

data/spec/active_record_batch_consumer_spec.rb

@@ -57,6 +57,12 @@ module ActiveRecordBatchConsumerTest
       Class.new(described_class) do
         record_class Widget
         compacted false
+
+        def process_message?(payload)
+          return true if payload.nil?
+
+          payload['test_id'] != 'skipme'
+        end
       end
     end
 
@@ -103,6 +109,27 @@ module ActiveRecordBatchConsumerTest
               )
           end
 
+          it 'should not create records for messages that return false from process_message?' do
+            publish_batch(
+              [
+                { key: 1,
+                  payload: { test_id: 'abc', some_int: 3 } },
+                { key: 2,
+                  payload: { test_id: 'skipme', some_int: 4 } },
+                { key: 3,
+                  payload: { test_id: 'ghi', some_int: 5 } }
+              ]
+            )
+
+            expect(all_widgets).
+              to contain_exactly(
+                have_attributes(id: 1, test_id: 'abc', some_int: 3, updated_at: start,
+                                created_at: start),
+                have_attributes(id: 3, test_id: 'ghi', some_int: 5, updated_at: start,
+                                created_at: start)
+              )
+          end
+
           it 'should handle deleting a record that doesn\'t exist' do
             publish_batch(
               [
@@ -805,6 +832,64 @@ module ActiveRecordBatchConsumerTest
 
     end
 
+    describe 'post_process_batch' do
+      before(:each) do
+        register_consumer(consumer_class,
+                          'MySchema',
+                          key_config: { plain: true })
+        MyBatchConsumer.last_post_process_batch = nil
+      end
+
+      let(:consumer_class) do
+        Class.new(described_class) do
+          class << self
+            attr_accessor :last_post_process_batch
+          end
+
+          record_class Widget
+          compacted false
+
+          def post_process_batch(messages)
+            self.class.last_post_process_batch = messages
+          end
+
+          def process_message?(payload)
+            payload['test_id'] != 'skipme'
+          end
+        end
+      end
+
+      it 'is called after the batch is processed with the processed messages' do
+        publish_batch(
+          [
+            { key: 1,
+              payload: { test_id: 'abc', some_int: 3 } },
+            { key: 2,
+              payload: { test_id: 'def', some_int: 4 } }
+          ]
+        )
+        expect(MyBatchConsumer.last_post_process_batch.map(&:payload).map do |p|
+ p[:test_id]
+        end).to contain_exactly('abc', 'def')
+      end
+
+      it 'is called with filtered messages when process_message? excludes some' do
+        publish_batch(
+          [
+            { key: 1,
+              payload: { test_id: 'abc', some_int: 3 } },
+            { key: 2,
+              payload: { test_id: 'skipme', some_int: 4 } },
+            { key: 3,
+              payload: { test_id: 'ghi', some_int: 5 } }
+          ]
+        )
+
+        expect(MyBatchConsumer.last_post_process_batch.size).to eq(2)
+        expect(MyBatchConsumer.last_post_process_batch.map(&:key).map(&:to_i)).to contain_exactly(1, 3)
+      end
+    end
+
   end
 end
 # rubocop:enable Lint/ConstantDefinitionInBlock

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.3.2
 platform: ruby
 authors:
 - Daniel Orner
@@ -245,6 +245,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement