deimos-ruby 2.4.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8ba2f3149ce2a7c10a5af09c4d7c07f25e33c8bac9f9783953e10bbbca5e74d
-  data.tar.gz: 3a5a112a7886b3acdf0b1aa0f18f984fbe93aa9e83f6912a667aa1eb113bbc5d
+  metadata.gz: 7a0b8bc495e0ef1c93a1f3a7d4b721ecd0ec149df866c0281e572c032ba25de5
+  data.tar.gz: 9426813c28bb569ee16e1f453b9f636baf309197ed59a25a337146852aec029a
 SHA512:
-  metadata.gz: e5796d8f41a4a6f86be7659d7dde670bc30651a32124fd2434ff90263fa418bb56478ef25e620e0b59a5e55d88c21d01e6ce77444b8b34aba523d924be452807
-  data.tar.gz: 107c63551a2606108e02dc0414d6ade8466b53dc89f6a54af12e02a1eff76386c8f864aca7746c27c2ff1a8bfad279b40d8efeff01938f5f5b4ee55b737f30d3
+  metadata.gz: bc7752b7345504f327c2c415dc90c58b31a3d6a92175eef71c31dcb66ecc358d7580bc79d7cd6b78c2df71bc8a45fab554d89018d6d85c2d23f91d652569f55c
+  data.tar.gz: b21aead2768acf0045a91175405a38352763a6ed85b7adc78fc70bdf1ea6dbf4cd993a94ac038c1275e7be6821bd623ab2e5b665f4509d7ce299dfba465e964d

data/.rubocop.yml

@@ -340,3 +340,8 @@ RSpec/SpecFilePathFormat:
 Security/YAMLLoad:
   Exclude:
     - 'lib/deimos.rb'
+
+Style/OneClassPerFile:
+  Exclude:
+    - karafka.rb
+    - spec/spec_helper.rb

data/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+## 2.5.0 - 2026-03-04
+
+- Breaking: `process_message?` in batch consumption now receives the full Kafka message object instead of just the payload, allowing tombstones to be identified via the message key and metadata.
+- Breaking: `test_consume_batch` and `test_consume_message` will now raise an error if the topic is configured incorrectly (i.e. the `each_message` setting doesn't match the method used).
+
 ## 2.4.0 - 2026-03-02
 
 - Major change: Switch from using `avro_turf` and `proto_turf` to use `schema_registry_client`, which handles both Avro and Protobuf.

data/lib/deimos/active_record_consume/batch_consumption.rb

@@ -25,7 +25,7 @@ module Deimos
       # they are split
       # @return [void]
       def consume_batch
-        filtered = messages.select { |p| process_message?(p.payload) }
+        filtered = messages.select { |message| process_message?(message) }
         skipped_count = messages.size - filtered.size
         if skipped_count.positive?
           Deimos::Logging.log_debug(

data/lib/deimos/active_record_consumer.rb

@@ -105,11 +105,14 @@ module Deimos
       message.payload.nil?
     end
 
-    # Override this message to conditionally save records
-    # @param _payload [Hash,Deimos::SchemaClass::Record] The kafka message
+    # Override this method to conditionally save records.
+    #
+    # This receives the full Kafka message so tombstones can still be
+    # identified and acted on using message metadata/key.
+    # @param _message [Karafka::Messages::Message] The Kafka message
     # @return [Boolean] if true, record is created/update.
     #   If false, record processing is skipped but message offset is still committed.
-    def process_message?(_payload)
+    def process_message?(_message)
       true
     end
   end

data/lib/deimos/test_helpers.rb

@@ -232,6 +232,9 @@ module Deimos
                    else
         Deimos.topic_for_consumer(handler_class_or_topic)
       end
+
+      _validate_consumer_config(topic_name, single)
+
       consumer = karafka.consumer_for(topic_name)
 
       Deimos.karafka_config_for(topic: topic_name).each_message(single)
@@ -259,5 +262,25 @@ module Deimos
 
       consumer.consume
     end
+
+  private
+
+    # Validate that the consumer config matches the test method being called.
+    # @param topic_name [String]
+    # @param single [Boolean] true if testing single message consumption
+    # @return [void]
+    def _validate_consumer_config(topic_name, single)
+      config = Deimos.karafka_config_for(topic: topic_name)
+      return if config.nil?
+
+      each_message = config.each_message
+      if single && !each_message
+        raise "Topic #{topic_name} is not configured with `each_message true` but " \
+              'you are trying to test with `test_consume_message`. Use `test_consume_batch` instead.'
+      elsif !single && each_message
+        raise "Topic #{topic_name} is configured with `each_message true` but " \
+              'you are trying to test with `test_consume_batch`. Use `test_consume_message` instead.'
+      end
+    end
   end
 end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '2.4.0'
+  VERSION = '2.5.0'
 end

data/spec/active_record_batch_consumer_spec.rb

@@ -58,10 +58,10 @@ module ActiveRecordBatchConsumerTest
         record_class Widget
         compacted false
 
-        def process_message?(payload)
-          return true if payload.nil?
+        def process_message?(message)
+          return true if message.payload.nil?
 
-          payload['test_id'] != 'skipme'
+          message.payload['test_id'] != 'skipme'
         end
       end
     end
@@ -853,8 +853,8 @@ module ActiveRecordBatchConsumerTest
             self.class.last_post_process_batch = messages
           end
 
-          def process_message?(payload)
-            payload['test_id'] != 'skipme'
+          def process_message?(message)
+            message.payload['test_id'] != 'skipme'
           end
         end
       end

data/spec/active_record_consumer_spec.rb

@@ -135,18 +135,21 @@ module ActiveRecordConsumerTest
           schema 'MySchemaWithDateTimes'
           namespace 'com.my-namespace'
           key_config plain: true
+          each_message true
         end
         topic 'my-topic2' do
           consumer MyConsumerWithKey
           schema 'MySchemaWithDateTimes'
           namespace 'com.my-namespace'
           key_config schema: 'MySchemaId_key'
+          each_message true
         end
         topic 'my-topic3' do
           consumer MyCustomFetchConsumer
           schema 'MySchema'
           namespace 'com.my-namespace'
           key_config none: true
+          each_message true
         end
       end
     end
@@ -266,6 +269,7 @@ module ActiveRecordConsumerTest
                 schema 'MySchema'
                 namespace 'com.my-namespace'
                 key_config plain: true
+                each_message true
               end
             end
 

data/spec/consumer_spec.rb

@@ -33,6 +33,7 @@ module ConsumerTest
           namespace 'com.my-namespace'
           key_config field: 'test_id'
           consumer consumer_class
+          each_message true
           reraise_errors true
           use_schema_classes route_usc
           reraise_errors route_rre
@@ -91,6 +92,7 @@ module ConsumerTest
                 namespace 'com.my-namespace'
                 key_config plain: true
                 consumer MyConsumer
+                each_message true
                 reraise_errors true
                 use_schema_classes use_schema_classes
                 reraise_errors true
@@ -177,6 +179,7 @@ module ConsumerTest
               namespace 'com.my-namespace'
               key_config field: 'test_id'
               consumer consumer_class
+              each_message true
               reraise_errors true
             end
           end
@@ -217,6 +220,7 @@ module ConsumerTest
               namespace 'com.my-namespace'
               key_config field: 'test_id'
               consumer ConsumerTest::MyConsumer
+              each_message true
               reraise_errors true
               use_schema_classes true
               reraise_errors true

data/spec/karafka_config/karafka_spec.rb

@@ -138,6 +138,7 @@ RSpec.describe Karafka::Routing::Topic do
         schema 'MySchema'
         namespace 'com.my-namespace'
         key_config({ field: :test_id })
+        each_message true
       end
     end
 

data/spec/test_helpers_spec.rb

@@ -72,4 +72,56 @@ RSpec.describe Deimos::TestHelpers do
     end
   end
 
+  describe 'consumer config validation' do
+    let(:consumer_class) do
+      Class.new(Deimos::Consumer) do
+        def consume_message(message)
+          message.payload
+        end
+      end
+    end
+
+    context 'when calling test_consume_message on a batch topic' do
+      before(:each) do
+        stub_const('MyBatchConsumer', consumer_class)
+        Karafka::App.routes.redraw do
+          topic 'my-batch-topic' do
+            consumer MyBatchConsumer
+            schema 'MySchema'
+            namespace 'com.my-namespace'
+            key_config field: 'test_id'
+            each_message false
+          end
+        end
+      end
+
+      it 'should raise an error' do
+        expect {
+          test_consume_message('my-batch-topic', { 'test_id' => 'foo', 'some_int' => 1 })
+        }.to raise_error(RuntimeError, /not configured with `each_message true`.*test_consume_batch/)
+      end
+    end
+
+    context 'when calling test_consume_batch on a single-message topic' do
+      before(:each) do
+        stub_const('MySingleConsumer', consumer_class)
+        Karafka::App.routes.redraw do
+          topic 'my-single-topic' do
+            consumer MySingleConsumer
+            schema 'MySchema'
+            namespace 'com.my-namespace'
+            key_config field: 'test_id'
+            each_message true
+          end
+        end
+      end
+
+      it 'should raise an error' do
+        expect {
+          test_consume_batch('my-single-topic', [{ 'test_id' => 'foo', 'some_int' => 1 }])
+        }.to raise_error(RuntimeError, /configured with `each_message true`.*test_consume_message/)
+      end
+    end
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.0
 platform: ruby
 authors:
 - Daniel Orner