deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

data/lib/generators/deimos/v2_generator.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+# THINGS TO REMEMBER
+# logger
+# fatal_error
+# bulk_import_id_generator
+
+require 'rails/generators'
+require 'rails/version'
+
+# Generates a new consumer.
+module Deimos
+  module Generators
+    # Generator for ActiveRecord model and migration.
+    class V2Generator < Rails::Generators::Base
+
+    class ProcString < String
+      def inspect
+        self.to_s
+      end
+    end
+
+      source_root File.expand_path('v2/templates', __dir__)
+
+      no_commands do
+        def deimos_config
+          Deimos.config
+        end
+
+        def deimos_configs
+          configs = {
+            producers: %i(topic_prefix disabled backend),
+            schema: %i(backend registry_url user password path generated_class_path use_schema_classes
+                       nest_child_schemas use_full_namespace schema_namespace_map),
+            db_producer: %i(log_topics compact_topics),
+          }
+
+          response = {}
+          configs.each do |group, settings|
+            group_setting = deimos_config.send(group)
+            next if settings.all? { |s| group_setting.default_value?(s)}
+
+            response[group] = {}
+            settings.each do |field|
+              unless group_setting.default_value?(field.to_sym)
+                response[group][field.to_s] = group_setting.send(field.to_sym)
+              end
+            end
+          end
+          response
+        end
+
+        def setup_configs
+          configs = {}
+          configs[:client_id] = if deimos_config.kafka.client_id && deimos_config.kafka.client_id != 'phobos'
+                                  deimos_config.kafka.client_id
+                                else
+                                  Rails::Application.subclasses.first&.name&.gsub('::Application', '')&.underscore
+                                end
+          if deimos_config.consumer_objects.any? { |c| c.max_concurrency.present? }
+            configs[:concurrency] = deimos_config.consumer_objects.map(&:max_concurrency).compact.max
+          end
+          if deimos_config.consumer_objects.any? { |c| c.max_wait_time.present? }
+            configs[:max_wait_time] = deimos_config.consumer_objects.map(&:max_wait_time).compact.max
+          end
+          configs.compact
+        end
+
+        def default_kafka_configs
+          configs = {}
+          configs["bootstrap.servers"] = deimos_config.kafka.seed_brokers.join(',')
+          configs["socket.connection.setup.timeout.ms"] = deimos_config.kafka.connect_timeout * 1000
+          configs["socket.timeout.ms"] = deimos_config.kafka.socket_timeout * 1000
+          configs["security.protocol"] = if deimos_config.kafka.ssl.enabled
+                                           "ssl"
+                                         elsif deimos_config.kafka.sasl.enabled
+                                           if deimos_config.kafka.sasl.enforce_ssl
+                                             "sasl_ssl"
+                                           else
+                                             "sasl_plain"
+                                           end
+                                         end
+          configs["ssl.ca.pem"] = deimos_config.kafka.ssl.ca_cert
+          configs["ssl.certificate.pem"] = deimos_config.kafka.ssl.client_cert
+          configs["ssl.key.pem"] = deimos_config.kafka.ssl.client_cert_key
+          configs["ssl.endpoint.identification.algorithm"] = "https" if deimos_config.kafka.ssl.verify_hostname
+          configs["sasl.kerberos.principal"] = deimos_config.kafka.sasl.gssapi_principal
+          configs["sasl.kerberos.keytab"] = deimos_config.kafka.sasl.gssapi_keytab
+          configs["sasl.username"] = deimos_config.kafka.sasl.plain_username || deimos_config.kafka.sasl.scram_username
+          configs["sasl.password"] = deimos_config.kafka.sasl.plain_password || deimos_config.kafka.sasl.scram_password
+          configs["sasl.mechanisms"] = deimos_config.kafka.sasl.scram_mechanism
+          configs["request.required.acks"] = deimos_config.producers.required_acks
+          configs["message.send.max.retries"] = deimos_config.producers.max_retries
+          configs["retry.backoff.ms"] = deimos_config.producers.retry_backoff * 1000 if deimos_config.producers.retry_backoff
+          configs["compression.codec"] = deimos_config.producers.compression_codec
+          configs.compact
+        end
+
+        def default_configs
+          {
+            payload_log: deimos_config.payload_log,
+            reraise_errors: deimos_config.consumers.reraise_errors,
+            replace_associations: deimos_config.consumers.replace_associations,
+            namespace: deimos_config.producers.schema_namespace,
+            use_schema_classes: deimos_config.schema.use_schema_classes
+          }.compact
+        end
+
+        def consumer_configs
+          deimos_config.consumer_objects.group_by(&:group_id).map do |group_id, consumers|
+            [group_id, consumers.map do |consumer|
+              kafka_configs = {}
+              kafka_configs["auto.offset.reset"] = consumer.start_from_beginning ? 'earliest' : 'latest'
+              kafka_configs["session.timeout.ms"] = consumer.session_timeout * 1000 unless consumer.default_value?(:session_timeout)
+              kafka_configs["auto.commit.interval.ms"] = consumer.offset_commit_interval * 1000 unless consumer.default_value?(:offset_commit_interval)
+              kafka_configs["heartbeat.interval.ms"] = consumer.heartbeat_interval * 1000 unless consumer.default_value?(:heartbeat_interval)
+              configs = {
+                kafka: kafka_configs.compact,
+                topic: consumer.topic,
+                consumer: ProcString.new(consumer.class_name),
+                schema: consumer.schema,
+                namespace: consumer.namespace,
+                key_config: consumer.key_config,
+              }
+              configs[:use_schema_classes] = consumer.use_schema_classes unless consumer.default_value?(:use_schema_classes)
+              configs[:max_db_batch_size] = consumer.max_db_batch_size unless consumer.default_value?(:max_db_batch_size)
+              configs[:bulk_import_id_column] = consumer.bulk_import_id_column unless consumer.default_value?(:bulk_import_id_column)
+              configs[:replace_associations] = consumer.replace_associations unless consumer.default_value?(:replace_associations)
+              configs[:active] = false if consumer.disabled
+              configs[:each_message] = true unless consumer.delivery.to_s == 'inline_batch'
+              configs
+            end]
+          end.to_h
+        end
+
+        def producer_configs
+          deimos_config.producer_objects.map do |producer|
+            {
+              topic: producer.topic,
+              producer_class: ProcString.new(producer.class_name),
+              schema: producer.schema,
+              namespace: producer.namespace || deimos_config.producers.schema_namespace,
+              key_config: producer.key_config,
+              use_schema_classes: producer.use_schema_classes
+            }.compact
+          end
+        end
+
+        def rename_consumer_methods
+          deimos_config.consumer_objects.each do |consumer|
+            consumer.class_name.constantize
+            file = Object.const_source_location(consumer.class_name)[0]
+            if file.to_s.include?(Rails.root.to_s)
+              gsub_file(file, /([\t ]+)def consume\((\w+)(, *(\w+)?)\)/,
+                        "\\1def consume_message(message)\n\\1  \\2 = message.payload\n\\1  \\4 = message.metadata")
+              gsub_file(file, /([\t ]+)def consume_batch\((\w+)(, *(\w+)?)\)/,
+                        "\\1def consume_batch\n\\1  \\2 = messages.payloads\n\\1  \\4 = messages.metadata")
+              gsub_file(file, /def record_attributes\((\w+)\)/,
+                        "def record_attributes(\\1, key)")
+            end
+          end
+        end
+
+        def fix_specs
+          Dir["*/**/*_spec.rb"].each do |file|
+            gsub_file(file, /,\s*call_original: true/, "")
+            gsub_file(file, 'Deimos::Backends::Test.sent_messages', "Deimos::TestHelpers.sent_messages")
+          end
+        end
+
+        def process_all_files
+          template('karafka.rb.tt', "karafka.rb", force: true)
+          rename_consumer_methods
+          fix_specs
+        end
+
+      end
+
+      desc 'Generate and update app files for version 2.0'
+      # @return [void]
+      def generate
+        process_all_files
+        say "Generation complete! You are safe to remove the existing initializer that configures Deimos.", :green
+        say "Note: The following settings cannot be determined by the generator:", :yellow
+        say "*  logger / phobos_logger (dynamic object, cannot be printed out)", :yellow
+        say "*  kafka.sasl.oauth_token_provider", :yellow
+        say "*  producers.max_buffer_size", :yellow
+        say "*  consumer.backoff (only handles minimum, not maximum)", :yellow
+        say "For more information, see https://github.com/flipp-oss/deimos/blob/master/docs/UPGRADING.md", :yellow
+      end
+    end
+  end
+end

data/lib/tasks/deimos.rake

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'phobos'
-require 'phobos/cli'
 require 'generators/deimos/schema_class_generator'
 require 'optparse'
 
@@ -15,17 +13,17 @@ namespace :deimos do
     ENV['DEIMOS_TASK_NAME'] = 'consumer'
     STDOUT.sync = true
     Rails.logger.info('Running deimos:start rake task.')
-    Phobos::CLI::Commands.start(%w(start --skip_config))
+    Karafka::Server.run
   end
 
   desc 'Starts the Deimos database producer'
-  task db_producer: :environment do
+  task outbox: :environment do
     ENV['DEIMOS_RAKE_TASK'] = 'true'
-    ENV['DEIMOS_TASK_NAME'] = 'db_producer'
+    ENV['DEIMOS_TASK_NAME'] = 'outbox'
     STDOUT.sync = true
-    Rails.logger.info('Running deimos:db_producer rake task.')
+    Rails.logger.info('Running deimos:outbox rake task.')
     thread_count = ENV['THREAD_COUNT'].to_i.zero? ? 1 : ENV['THREAD_COUNT'].to_i
-    Deimos.start_db_backend!(thread_count: thread_count)
+    Deimos.start_outbox_backend!(thread_count: thread_count)
   end
 
   task db_poller: :environment do

data/spec/active_record_batch_consumer_association_spec.rb

@@ -74,9 +74,21 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
     end
 
     prepend_before(:each) do
-      consumer_class.config[:bulk_import_id_column] = :bulk_import_id
       stub_const('MyBatchConsumer', consumer_class)
       stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+      klass = consumer_class
+      col = bulk_import_id_column
+      rep = replace_associations
+      Karafka::App.routes.redraw do
+        topic 'my-topic' do
+          consumer klass
+          schema 'MySchemaWithTitle'
+          namespace 'com.my-namespace'
+          key_config plain: true
+          bulk_import_id_column col
+          replace_associations rep
+        end
+      end
     end
 
     # Helper to publish a list of messages and call the consumer
@@ -84,16 +96,16 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
       keys = messages.map { |m| m[:key] }
       payloads = messages.map { |m| m[:payload] }
 
-      test_consume_batch(MyBatchConsumer, payloads, keys: keys, call_original: true)
+      test_consume_batch(MyBatchConsumer, payloads, keys: keys)
     end
 
+    let(:bulk_import_id_column) { :bulk_import_id }
+    let(:replace_associations) { true }
+
     let(:consumer_class) do
       klass = Class.new(described_class) do
         cattr_accessor :record_attributes_proc
         cattr_accessor :should_consume_proc
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        key_config plain: true
         record_class Widget
 
         def should_consume?(record, associations)
@@ -154,7 +166,6 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
 
     context 'when association configured in consumer without model changes' do
       before(:each) do
-        consumer_class.config[:bulk_import_id_column] = :bulk_import_id
         ActiveRecord::Base.connection.remove_column(:widgets, :bulk_import_id)
         Widget.reset_column_information
       end
@@ -164,6 +175,7 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
       end
 
       it 'should raise error when bulk_import_id is not found' do
+        set_karafka_config(:reraise_errors, true)
         expect {
           publish_batch([{ key: 2,
                            payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
@@ -173,10 +185,8 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
     end
 
     context 'with one-to-one relation in association and custom bulk_import_id' do
-      before(:each) do
-        consumer_class.config[:bulk_import_id_column] = :custom_id
-        consumer_class.config[:replace_associations] = false
-      end
+      let(:bulk_import_id_column) { :custom_id }
+      let(:replace_associations) { false }
 
       before(:all) do
         ActiveRecord::Base.connection.add_column(:widgets, :custom_id, :string, if_not_exists: true)
@@ -193,8 +203,8 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
     end
 
     context 'with one-to-many relationship in association and default bulk_import_id' do
+      let(:replace_associations) { false }
       before(:each) do
-        consumer_class.config[:replace_associations] = false
         consumer_class.record_attributes_proc = proc do |payload|
           {
             test_id: payload['test_id'],
@@ -214,7 +224,6 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
       end
 
       it 'should save item to widget and associated details' do
-        consumer_class.config[:replace_associations] = false
         publish_batch([{ key: 2,
                          payload: { test_id: 'xyz', some_int: 5, title: 'Widget Title' } }])
         expect(Widget.count).to eq(2)
@@ -233,8 +242,8 @@ module ActiveRecordBatchConsumerTest # rubocop:disable Metrics/ModuleLength
     end
 
     context 'with replace_associations on' do
+      let(:replace_associations) { true }
       before(:each) do
-        consumer_class.config[:replace_associations] = true
         consumer_class.record_attributes_proc = proc do |payload|
           {
             test_id: payload['test_id'],

data/spec/active_record_batch_consumer_spec.rb

@@ -34,7 +34,11 @@ module ActiveRecordBatchConsumerTest
     prepend_before(:each) do
       stub_const('MyBatchConsumer', consumer_class)
       stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
-      consumer_class.config[:bulk_import_id_column] = :bulk_import_id # default
+      register_consumer(MyBatchConsumer,
+                        'MySchema',
+                        key_config: {plain: true},
+                        configs: {bulk_import_id_column: :bulk_import_id})
+      Widget.delete_all
     end
 
     around(:each) do |ex|
@@ -50,9 +54,6 @@ module ActiveRecordBatchConsumerTest
     # Basic uncompacted consumer
     let(:consumer_class) do
       Class.new(described_class) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        key_config plain: true
         record_class Widget
         compacted false
       end
@@ -68,7 +69,7 @@ module ActiveRecordBatchConsumerTest
       keys = messages.map { |m| m[:key] }
       payloads = messages.map { |m| m[:payload] }
 
-      test_consume_batch(MyBatchConsumer, payloads, keys: keys, call_original: true)
+      test_consume_batch(MyBatchConsumer, payloads, keys: keys)
     end
 
     describe 'consume_batch' do
@@ -82,10 +83,6 @@ module ActiveRecordBatchConsumerTest
             end
           end
 
-          it 'should handle an empty batch' do
-            expect { publish_batch([]) }.not_to raise_error
-          end
-
           it 'should create records from a batch' do
             publish_batch(
               [
@@ -257,12 +254,16 @@ module ActiveRecordBatchConsumerTest
     end
 
     describe 'compacted mode' do
+      before(:each) do
+      register_consumer(consumer_class,
+                        'MySchema',
+                        key_config: {plain: true})
+
+      end
+
       # Create a compacted consumer
       let(:consumer_class) do
         Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config plain: true
           record_class Widget
 
           # :no-doc:
@@ -302,11 +303,14 @@ module ActiveRecordBatchConsumerTest
     end
 
     describe 'compound keys' do
+      before(:each) do
+        register_consumer(consumer_class,
+                          'MySchema',
+                          key_config: {schema: 'MySchemaCompound_key'})
+      end
+
       let(:consumer_class) do
         Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config schema: 'MySchemaCompound_key'
           record_class Widget
           compacted false
 
@@ -353,13 +357,10 @@ module ActiveRecordBatchConsumerTest
     end
 
     describe 'no keys' do
-      let(:consumer_class) do
-        Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config none: true
-          record_class Widget
-        end
+      before(:each) do
+        register_consumer(consumer_class,
+                          'MySchema',
+                          key_config: {none: true})
       end
 
       it 'should handle unkeyed topics' do
@@ -385,11 +386,13 @@ module ActiveRecordBatchConsumerTest
     end
 
     describe 'soft deletion' do
+      before(:each) do
+        register_consumer(consumer_class,
+                          'MySchema',
+                          key_config: {plain: true})
+      end
       let(:consumer_class) do
         Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config plain: true
           record_class Widget
           compacted false
 
@@ -451,11 +454,13 @@ module ActiveRecordBatchConsumerTest
     end
 
     describe 'skipping records' do
+      before(:each) do
+        register_consumer(consumer_class,
+                          'MySchema',
+                          key_config: {plain: true})
+      end
       let(:consumer_class) do
         Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config plain: true
           record_class Widget
 
           # Sample customization: Skipping records
@@ -471,9 +476,9 @@ module ActiveRecordBatchConsumerTest
         publish_batch(
           [
             { key: 1, # Record that consumer can decide to skip
-              payload: { test_id: 'skipme' } },
+              payload: { test_id: 'skipme', some_int: 3 } },
             { key: 2,
-              payload: { test_id: 'abc123' } }
+              payload: { test_id: 'abc123', some_int: 3 } }
           ]
         )
 
@@ -484,11 +489,13 @@ module ActiveRecordBatchConsumerTest
 
     describe 'pre processing' do
       context 'with uncompacted messages' do
+        before(:each) do
+          register_consumer(consumer_class,
+                            'MySchema',
+                            key_config: {plain: true})
+        end
         let(:consumer_class) do
           Class.new(described_class) do
-            schema 'MySchema'
-            namespace 'com.my-namespace'
-            key_config plain: true
             record_class Widget
             compacted false
 
@@ -527,9 +534,11 @@ module ActiveRecordBatchConsumerTest
       context 'with a global bulk_import_id_generator' do
 
         before(:each) do
-          Deimos.configure do
-            consumers.bulk_import_id_generator(proc { 'global' })
-          end
+          register_consumer(consumer_class,
+                            'MySchema',
+                            key_config: {plain: true},
+                            configs: {bulk_import_id_generator: proc { 'global' }}
+                            )
         end
 
         it 'should call the default bulk_import_id_generator proc' do
@@ -559,10 +568,17 @@ module ActiveRecordBatchConsumerTest
       context 'with a class defined bulk_import_id_generator' do
 
         before(:each) do
-          Deimos.configure do
-            consumers.bulk_import_id_generator(proc { 'global' })
+          Karafka::App.routes.clear
+          Karafka::App.routes.draw do
+            defaults do
+              bulk_import_id_generator(proc { 'global'})
+            end
           end
-          consumer_class.config[:bulk_import_id_generator] = proc { 'custom' }
+          register_consumer(consumer_class,
+                            'MySchema',
+                            key_config: {plain: true},
+                            configs: {bulk_import_id_generator: proc { 'custom' }}
+                            )
         end
 
         it 'should call the default bulk_import_id_generator proc' do
@@ -593,11 +609,13 @@ module ActiveRecordBatchConsumerTest
 
     describe 'should_consume?' do
 
+      before(:each) do
+        register_consumer(consumer_class,
+                          'MySchema',
+                          key_config: {plain: true})
+      end
       let(:consumer_class) do
         Class.new(described_class) do
-          schema 'MySchema'
-          namespace 'com.my-namespace'
-          key_config plain: true
           record_class Widget
           compacted false
 
@@ -609,9 +627,8 @@ module ActiveRecordBatchConsumerTest
             nil
           end
 
-          ActiveSupport::Notifications.subscribe('batch_consumption.invalid_records') do |*args|
-            payload = ActiveSupport::Notifications::Event.new(*args).payload
-            payload[:consumer].process_invalid_records(payload[:records])
+          Karafka.monitor.subscribe('deimos.batch_consumption.invalid_records') do |event|
+            event[:consumer].process_invalid_records(event[:records])
           end
 
         end
@@ -649,11 +666,13 @@ module ActiveRecordBatchConsumerTest
     describe 'post processing' do
 
       context 'with uncompacted messages' do
+        before(:each) do
+          register_consumer(consumer_class,
+                            'MySchema',
+                            key_config: {plain: true})
+        end
         let(:consumer_class) do
           Class.new(described_class) do
-            schema 'MySchema'
-            namespace 'com.my-namespace'
-            key_config plain: true
             record_class Widget
             compacted false
 
@@ -673,13 +692,11 @@ module ActiveRecordBatchConsumerTest
               Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: attrs['some_int'])
             end
 
-            ActiveSupport::Notifications.subscribe('batch_consumption.invalid_records') do |*args|
-              payload = ActiveSupport::Notifications::Event.new(*args).payload
+            Karafka.monitor.subscribe('deimos.batch_consumption.invalid_records') do |payload|
               payload[:consumer].process_invalid_records(payload[:records])
             end
 
-            ActiveSupport::Notifications.subscribe('batch_consumption.valid_records') do |*args|
-              payload = ActiveSupport::Notifications::Event.new(*args).payload
+            Karafka.monitor.subscribe('deimos.batch_consumption.valid_records') do |payload|
               payload[:consumer].process_valid_records(payload[:records])
             end
 
@@ -707,11 +724,13 @@ module ActiveRecordBatchConsumerTest
       end
 
       context 'with compacted messages' do
+        before(:each) do
+          register_consumer(consumer_class,
+                            'MySchema',
+                            key_config: {plain: true})
+        end
         let(:consumer_class) do
           Class.new(described_class) do
-            schema 'MySchema'
-            namespace 'com.my-namespace'
-            key_config plain: true
             record_class Widget
             compacted true
 
@@ -731,13 +750,11 @@ module ActiveRecordBatchConsumerTest
               Widget.find_by(id: attrs['id'], test_id: attrs['test_id']).update!(some_int: attrs['some_int'])
             end
 
-            ActiveSupport::Notifications.subscribe('batch_consumption.invalid_records') do |*args|
-              payload = ActiveSupport::Notifications::Event.new(*args).payload
+            Karafka.monitor.subscribe('deimos.batch_consumption.invalid_records') do |payload|
               payload[:consumer].process_invalid_records(payload[:records])
             end
 
-            ActiveSupport::Notifications.subscribe('batch_consumption.valid_records') do |*args|
-              payload = ActiveSupport::Notifications::Event.new(*args).payload
+            Karafka.monitor.subscribe('deimos.batch_consumption.valid_records') do |payload|
               payload[:consumer].process_valid_records(payload[:records])
             end
 
@@ -765,11 +782,13 @@ module ActiveRecordBatchConsumerTest
       end
 
       context 'with post processing errors' do
+        before(:each) do
+          register_consumer(consumer_class,
+                            'MySchema',
+                            key_config: {plain: true})
+        end
         let(:consumer_class) do
           Class.new(described_class) do
-            schema 'MySchema'
-            namespace 'com.my-namespace'
-            key_config plain: true
             record_class Widget
             compacted false
 
@@ -777,15 +796,15 @@ module ActiveRecordBatchConsumerTest
               raise StandardError, 'Something went wrong'
             end
 
-            ActiveSupport::Notifications.subscribe('batch_consumption.valid_records') do |*args|
-              payload = ActiveSupport::Notifications::Event.new(*args).payload
-              payload[:consumer].process_valid_records(payload[:records])
+            Karafka.monitor.subscribe('deimos.batch_consumption.valid_records') do |event|
+              event[:consumer].process_valid_records(event[:records])
             end
 
           end
         end
 
         it 'should save records if an exception occurs in post processing' do
+          set_karafka_config(:reraise_errors, true)
           Widget.create!(id: 1, test_id: 'abc', some_int: 1)
           Widget.create!(id: 2, test_id: 'def', some_int: 2)
 

data/spec/active_record_consume/batch_consumption_spec.rb

@@ -9,11 +9,11 @@ RSpec.describe Deimos::ActiveRecordConsume::BatchConsumption do
     describe 'upsert_records' do
       let(:records) do
         [
-          Deimos::Message.new({ v: 1 }, nil, key: 1),
-          Deimos::Message.new({ v: 2 }, nil, key: 2),
-          Deimos::Message.new({ v: 3 }, nil, key: 3),
-          Deimos::Message.new({ v: 4 }, nil, key: 4),
-          Deimos::Message.new({ v: 5 }, nil, key: 5)
+          Deimos::Message.new({ v: 1 }, key: 1),
+          Deimos::Message.new({ v: 2 }, key: 2),
+          Deimos::Message.new({ v: 3 }, key: 3),
+          Deimos::Message.new({ v: 4 }, key: 4),
+          Deimos::Message.new({ v: 5 }, key: 5)
         ]
       end
 
@@ -56,11 +56,11 @@ RSpec.describe Deimos::ActiveRecordConsume::BatchConsumption do
     describe 'remove_records' do
       let(:records) do
         [
-          Deimos::Message.new(nil, nil, key: 1),
-          Deimos::Message.new(nil, nil, key: 2),
-          Deimos::Message.new(nil, nil, key: 3),
-          Deimos::Message.new(nil, nil, key: 4),
-          Deimos::Message.new(nil, nil, key: 5)
+          Deimos::Message.new(nil, key: 1),
+          Deimos::Message.new(nil, key: 2),
+          Deimos::Message.new(nil, key: 3),
+          Deimos::Message.new(nil, key: 4),
+          Deimos::Message.new(nil, key: 5)
         ]
       end