deimos-ruby 2.3.1 → 2.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce1117d7a1d29304fcb0cafca935db90c4378ceaf9aa089dfcf9fa56fe5ea25d
-  data.tar.gz: 964f517ef411af2dcb3dbbc9f6e507d34fefcac6ef9321b8937d686bd3e307b7
+  metadata.gz: 11656e5d925198470fb1c6918808e2dda7b769d98397e9b0f973316a21fe1616
+  data.tar.gz: 4f34eae61a408809aaa6c0d70d0fe535c27ea300e479612c8273bd63cf553b62
 SHA512:
-  metadata.gz: 5684bbf8cc1e2a0831369b47f28ffb84795a3d21a47b1b11d159b2909e9af36fd83b1756a47a9902d04c792a0a389cb8d675e0451f50e75c7d8c3ccf50a389ac
-  data.tar.gz: 9946b5a3ceeae405778731c82e1bfcc24dfae89a19bb0b2553620446815b712abc81a2bb9303e31ecb1d8d7aa68c3856df6cb2c46c1264f3305e6dbe0e3b530b
+  metadata.gz: 60c5d0fe69d3787c972c647d907de9159c4f33a63c8f1e3913c9816bd18e056f24721549377bf3ebdc25578c800c62ad7ab2f5a056e0562cacd1026858ea5d2f
+  data.tar.gz: 252a59f9db351029fd2cd67c679bddd50a854e4797855a8c7b5e7248e6b0ab71aad041d7b7e3dd900eb5c90c6cc5f82301461f58a749fe997f99d8f65d8b3f3e

data/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## UNRELEASED
 
+# 2.3.2 - 2026-02-20
+
+- Feature: Add overridable process_message? for batch consumption.
+- Feature: Add overridable post_process_batch for batch consumption.
+
 # 2.3.1 - 2026-01-22
 
 - Feature: Allow strings to be used in the `record_class` declaration.

data/deimos-ruby.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency('karafka-testing', '~> 2.0')
   spec.add_development_dependency('pg', '~> 1.1')
   spec.add_development_dependency('proto_turf')
+  spec.add_development_dependency('pry', '~> 0.14.1')
   spec.add_development_dependency('rails', '~> 8.0')
   spec.add_development_dependency('rake', '~> 13')
   spec.add_development_dependency('rspec', '~> 3')

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -25,7 +25,15 @@ module Deimos
       # they are split
       # @return [void]
       def consume_batch
-        deimos_messages = messages.map { |p| Deimos::Message.new(p.payload, key: p.key) }
+        filtered = messages.select { |p| process_message?(p.payload) }
+        skipped_count = messages.size - filtered.size
+        if skipped_count.positive?
+          Deimos::Logging.log_debug(
+            message: 'Skipping processing of messages in batch',
+            skipped_count: skipped_count
+          )
+        end
+        deimos_messages = filtered.map { |p| Deimos::Message.new(p.payload, key: p.key) }
 
         tag = topic.name
         Deimos.config.tracer.active_span.set_tag('topic', tag)
@@ -37,6 +45,8 @@ module Deimos
             uncompacted_update(deimos_messages)
           end
         end
+
+        post_process_batch(deimos_messages)
       end
 
     protected
@@ -94,6 +104,14 @@ module Deimos
         true
       end
 
+      # Perform any post-processing after a batch has been consumed.
+      # Called once per batch with the list of Deimos::Message that were processed (after filtering and compaction).
+      # @param messages [Array<Deimos::Message>] The batch of messages that were processed
+      # @return [void]
+      def post_process_batch(_messages)
+        nil
+      end
+
     private
 
       # Compact a batch of messages, taking only the last message for each

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.3.1'
+  VERSION = '2.3.2'
 end

data/spec/active_record_batch_consumer_spec.rb

@@ -57,6 +57,12 @@ module ActiveRecordBatchConsumerTest
       Class.new(described_class) do
         record_class Widget
         compacted false
+
+        def process_message?(payload)
+          return true if payload.nil?
+
+          payload['test_id'] != 'skipme'
+        end
       end
     end
 
@@ -103,6 +109,27 @@ module ActiveRecordBatchConsumerTest
               )
           end
 
+          it 'should not create records for messages that return false from process_message?' do
+            publish_batch(
+              [
+                { key: 1,
+                  payload: { test_id: 'abc', some_int: 3 } },
+                { key: 2,
+                  payload: { test_id: 'skipme', some_int: 4 } },
+                { key: 3,
+                  payload: { test_id: 'ghi', some_int: 5 } }
+              ]
+            )
+
+            expect(all_widgets).
+              to contain_exactly(
+                have_attributes(id: 1, test_id: 'abc', some_int: 3, updated_at: start,
+                                created_at: start),
+                have_attributes(id: 3, test_id: 'ghi', some_int: 5, updated_at: start,
+                                created_at: start)
+              )
+          end
+
           it 'should handle deleting a record that doesn\'t exist' do
             publish_batch(
               [
@@ -805,6 +832,64 @@ module ActiveRecordBatchConsumerTest
 
     end
 
+    describe 'post_process_batch' do
+      before(:each) do
+        register_consumer(consumer_class,
+                          'MySchema',
+                          key_config: { plain: true })
+        MyBatchConsumer.last_post_process_batch = nil
+      end
+
+      let(:consumer_class) do
+        Class.new(described_class) do
+          class << self
+            attr_accessor :last_post_process_batch
+          end
+
+          record_class Widget
+          compacted false
+
+          def post_process_batch(messages)
+            self.class.last_post_process_batch = messages
+          end
+
+          def process_message?(payload)
+            payload['test_id'] != 'skipme'
+          end
+        end
+      end
+
+      it 'is called after the batch is processed with the processed messages' do
+        publish_batch(
+          [
+            { key: 1,
+              payload: { test_id: 'abc', some_int: 3 } },
+            { key: 2,
+              payload: { test_id: 'def', some_int: 4 } }
+          ]
+        )
+        expect(MyBatchConsumer.last_post_process_batch.map(&:payload).map do |p|
+ p[:test_id]
+        end).to contain_exactly('abc', 'def')
+      end
+
+      it 'is called with filtered messages when process_message? excludes some' do
+        publish_batch(
+          [
+            { key: 1,
+              payload: { test_id: 'abc', some_int: 3 } },
+            { key: 2,
+              payload: { test_id: 'skipme', some_int: 4 } },
+            { key: 3,
+              payload: { test_id: 'ghi', some_int: 5 } }
+          ]
+        )
+
+        expect(MyBatchConsumer.last_post_process_batch.size).to eq(2)
+        expect(MyBatchConsumer.last_post_process_batch.map(&:key).map(&:to_i)).to contain_exactly(1, 3)
+      end
+    end
+
   end
 end
 # rubocop:enable Lint/ConstantDefinitionInBlock

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.3.2
 platform: ruby
 authors:
 - Daniel Orner
@@ -245,6 +245,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.14.1
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement