deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

data/spec/spec_helper.rb

@@ -14,6 +14,7 @@ require 'handlers/my_batch_consumer'
 require 'handlers/my_consumer'
 require 'rspec/rails'
 require 'rspec/snapshot'
+require 'karafka/testing/rspec/helpers'
 require "trilogy_adapter/connection"
 ActiveRecord::Base.public_send :extend, TrilogyAdapter::Connection
 Dir['./spec/schemas/**/*.rb'].sort.each { |f| require f }
@@ -23,9 +24,33 @@ SCHEMA_CLASS_SETTINGS = { off: false, on: true }.freeze
 
 class DeimosApp < Rails::Application
 end
+DeimosApp.initializer("setup_root_dir", before: "karafka.require_karafka_boot_file") do
+  ENV['KARAFKA_ROOT_DIR'] = "#{Rails.root}/spec/karafka"
+end
 DeimosApp.initialize!
 
-# Helpers for Executor/DbProducer
+module Helpers
+
+  def set_karafka_config(method, val)
+    Deimos.karafka_configs.each { |c| c.send(method.to_sym, val) }
+  end
+
+  def register_consumer(klass, schema, namespace='com.my-namespace', key_config:{none: true}, configs: {})
+    Karafka::App.routes.redraw do
+      topic 'my-topic' do
+        consumer klass
+        schema schema
+        namespace namespace
+        key_config key_config
+        configs.each do |k, v|
+          public_send(k, v)
+        end
+      end
+    end
+  end
+end
+
+# Helpers for Executor/OutboxProducer
 module TestRunners
   # Execute a block until it stops failing. This is helpful for testing threads
   # where we need to wait for them to continue but don't want to rely on
@@ -73,11 +98,7 @@ module DbConfigs
   # @param topic [String]
   # @param key [String]
   def build_message(payload, topic, key)
-    message = Deimos::Message.new(payload, Deimos::Producer,
-                                  topic: topic, key: key)
-    message.encoded_payload = message.payload
-    message.encoded_key = message.key
-    message
+    { payload: payload, topic: topic, key: key}
   end
 
   DB_OPTIONS = [
@@ -85,12 +106,12 @@ module DbConfigs
       adapter: 'postgresql',
       port: 5432,
       username: 'postgres',
-      password: 'root',
+      password: 'password',
       database: 'postgres',
       host: ENV['PG_HOST'] || 'localhost'
     },
     {
-      adapter: 'mysql2',
+      adapter: 'trilogy',
       port: 3306,
       username: 'root',
       database: 'test',
@@ -123,14 +144,14 @@ module DbConfigs
   end
 
   # :nodoc:
-  def run_db_backend_migration
-    migration_class_name = 'DbBackendMigration'
+  def run_outbox_backend_migration
+    migration_class_name = 'OutboxBackendMigration'
     migration_version = '[5.2]'
     migration = ERB.new(
-      File.read('lib/generators/deimos/db_backend/templates/migration')
+      File.read('lib/generators/deimos/outbox_backend/templates/migration')
     ).result(binding)
     eval(migration) # rubocop:disable Security/Eval
-    ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
+    ActiveRecord::Migration.new.run(OutboxBackendMigration, direction: :up)
   end
 
   # :nodoc:
@@ -147,7 +168,7 @@ module DbConfigs
   # Set up the given database.
   def setup_db(options)
     ActiveRecord::Base.establish_connection(options)
-    run_db_backend_migration
+    run_outbox_backend_migration
     run_db_poller_migration
 
     ActiveRecord::Base.descendants.each do |klass|
@@ -163,7 +184,10 @@ end
 RSpec.configure do |config|
   config.extend(DbConfigs)
   include DbConfigs
+  config.include Karafka::Testing::RSpec::Helpers
+
   config.include TestRunners
+  config.include Helpers
   config.full_backtrace = true
 
   config.snapshot_dir = "spec/snapshots"
@@ -199,13 +223,11 @@ RSpec.configure do |config|
   config.before(:each) do
     Deimos.config.reset!
     Deimos.configure do |deimos_config|
-      deimos_config.producers.backend = :test
+      deimos_config.producers.backend = :kafka
       deimos_config.schema.nest_child_schemas = true
-      deimos_config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
       deimos_config.schema.path = File.join(File.expand_path(__dir__), 'schemas')
       deimos_config.consumers.reraise_errors = true
       deimos_config.schema.registry_url = ENV['SCHEMA_REGISTRY'] || 'http://localhost:8081'
-      deimos_config.kafka.seed_brokers = ENV['KAFKA_SEED_BROKER'] || 'localhost:9092'
       deimos_config.logger = Logger.new('/dev/null')
       deimos_config.logger.level = Logger::INFO
       deimos_config.schema.backend = :avro_validation
@@ -213,6 +235,10 @@ RSpec.configure do |config|
     end
   end
 
+  config.after(:each) do
+    Deimos::EVENT_TYPES.each { |type| Karafka.monitor.notifications_bus.clear(type) }
+  end
+
   config.around(:each) do |example|
     use_cleaner = !example.metadata[:integration]
 
@@ -262,26 +288,32 @@ end
 
 RSpec.shared_context('with publish_backend') do
   before(:each) do
-    producer_class = Class.new(Deimos::Producer) do
-      schema 'MySchema'
-      namespace 'com.my-namespace'
-      topic 'my-topic'
-      key_config field: 'test_id'
-    end
+    producer_class = Class.new(Deimos::Producer)
     stub_const('MyProducer', producer_class)
 
-    producer_class = Class.new(Deimos::Producer) do
-      schema 'MySchema'
-      namespace 'com.my-namespace'
-      topic 'my-topic'
-      key_config none: true
-    end
+    producer_class_no_key = Class.new(Deimos::Producer)
     stub_const('MyNoKeyProducer', producer_class)
+
+    Karafka::App.routes.redraw do
+      topic 'my-topic-no-key' do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config none: true
+        producer_class producer_class_no_key
+      end
+      topic 'my-topic' do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config field: 'test_id'
+        producer_class producer_class
+      end
+    end
+
   end
 
   let(:messages) do
     (1..3).map do |i|
-      build_message({ foo: i }, 'my-topic', "foo#{i}")
+      build_message({ test_id: "foo#{i}", some_int: i }, 'my-topic', "foo#{i}")
     end
   end
 end

data/spec/utils/db_poller_spec.rb

@@ -20,21 +20,27 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
   describe '#start!' do
 
     before(:each) do
-      producer_class = Class.new(Deimos::Producer) do
-        schema 'MySchema'
-        namespace 'com.my-namespace'
-        topic 'my-topic'
-        key_config field: 'test_id'
-      end
+      producer_class = Class.new(Deimos::Producer)
       stub_const('MyProducer', producer_class)
 
-      producer_class = Class.new(Deimos::Producer) do
-        schema 'MySchemaWithId'
-        namespace 'com.my-namespace'
-        topic 'my-topic'
-        key_config plain: true
-      end
+      producer_class = Class.new(Deimos::Producer)
       stub_const('MyProducerWithID', producer_class)
+
+      Karafka::App.routes.redraw do
+        topic 'my-topic' do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config field: 'test_id'
+          producer_class MyProducer
+        end
+        topic 'my-topic-with-id' do
+          schema 'MySchemaWithId'
+          namespace 'com.my-namespace'
+          key_config plain: true
+          producer_class MyProducerWithID
+        end
+      end
+
     end
 
     it 'should raise an error if no pollers configured' do
@@ -76,12 +82,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
     let(:config) { Deimos.config.db_poller_objects.first.dup }
 
     before(:each) do
-      Widget.delete_all
       producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchemaWithId'
-        namespace 'com.my-namespace'
-        topic 'my-topic-with-id'
-        key_config none: true
         record_class Widget
 
         # :nodoc:
@@ -91,6 +92,16 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
       end
       stub_const('MyProducer', producer_class)
 
+      Widget.delete_all
+      Karafka::App.routes.redraw do
+        topic 'my-topic-with-id' do
+          schema 'MySchemaWithId'
+          namespace 'com.my-namespace'
+          key_config none: true
+          producer_class MyProducer
+        end
+      end
+
       Deimos.configure do
         db_poller do
           producer_class 'MyProducer'
@@ -195,7 +206,9 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
         before(:each) { config.skip_too_large_messages = true }
 
         it 'should skip and move on' do
-          error = Kafka::MessageSizeTooLarge.new('OH NOES')
+          rdkafka_error = instance_double(Rdkafka::RdkafkaError, code: :msg_size_too_large)
+          error = WaterDrop::Errors::ProduceManyError.new(nil, nil)
+          allow(error).to receive(:cause).and_return(rdkafka_error)
           allow(poller).to receive(:sleep)
           allow(poller).to receive(:process_batch) do
             raise error
@@ -331,7 +344,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
       end
 
       it 'should send events across multiple batches' do
-        allow(Deimos.config.logger).to receive(:info)
+        allow(Deimos::Logging).to receive(:log_info)
         allow(MyProducer).to receive(:poll_query).and_call_original
         expect(poller).to receive(:process_and_touch_info).ordered.
           with([widgets[0], widgets[1], widgets[2]], anything).and_call_original
@@ -376,7 +389,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
                time_to: time_value(secs: 120), # yes this is weird but it's because of travel_to
                column_name: :updated_at,
                min_id: last_widget.id)
-        expect(Deimos.config.logger).to have_received(:info).
+        expect(Deimos::Logging).to have_received(:log_info).
           with('Poll MyProducer: ["my-topic-with-id"] complete at 2015-05-05 00:59:58 -0400 (3 batches, 0 errored batches, 7 processed messages)')
       end
 
@@ -398,7 +411,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
       describe 'errors' do
         before(:each) do
           poller.config.retries = 0
-          allow(Deimos.config.logger).to receive(:info)
+          allow(Deimos::Logging).to receive(:log_info)
         end
 
         after(:each) do
@@ -428,7 +441,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
           info = Deimos::PollInfo.last
           expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
           expect(info.last_sent_id).to eq(widgets[6].id)
-          expect(Deimos.config.logger).to have_received(:info).
+          expect(Deimos::Logging).to have_received(:log_info).
             with('Poll MyProducer: ["my-topic-with-id"] complete at 2015-05-05 00:59:58 -0400 (2 batches, 1 errored batches, 7 processed messages)')
         end
       end
@@ -449,10 +462,6 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
     before(:each) do
       Widget.delete_all
       producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchemaWithId'
-        namespace 'com.my-namespace'
-        topic 'my-topic-with-id'
-        key_config none: true
         record_class Widget
 
         # :nodoc:
@@ -461,20 +470,22 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
         end
       end
       stub_const('ProducerOne', producer_class)
+      stub_const('ProducerTwo', producer_class)
 
-      producer_class = Class.new(Deimos::ActiveRecordProducer) do
-        schema 'MySchemaWithId'
-        namespace 'com.my-namespace'
-        topic 'my-topic-with-id'
-        key_config none: true
-        record_class Widget
-
-        # :nodoc:
-        def self.generate_payload(attrs, widget)
-          super.merge(message_id: widget.generated_id)
+      Karafka::App.routes.redraw do
+        topic 'my-topic-with-id' do
+          schema 'MySchemaWithId'
+          namespace 'com.my-namespace'
+          key_config none: true
+          producer_class ProducerOne
+        end
+        topic 'my-topic-with-id2' do
+          schema 'MySchemaWithId'
+          namespace 'com.my-namespace'
+          key_config none: true
+          producer_class ProducerTwo
         end
       end
-      stub_const('ProducerTwo', producer_class)
 
       poller_class = Class.new(Deimos::Utils::DbPoller::StateBased) do
         def self.producers
@@ -513,8 +524,7 @@ each_db_config(Deimos::Utils::DbPoller::Base) do
       expect(Deimos::Utils::DbPoller::MultiProducerPoller).to receive(:poll_query).at_least(:once)
       poller.process_updates
 
-      expect(ProducerOne).to have_received(:send_events).with(widgets)
-      expect(ProducerTwo).to have_received(:send_events).with(widgets)
+      expect(ProducerOne).to have_received(:send_events).twice.with(widgets)
       expect(widgets.map(&:reload).map(&:publish_status)).to eq(%w(PUBLISHED PUBLISHED PUBLISHED))
     end
 

data/spec/utils/{db_producer_spec.rb → outbox_producer_spec.rb}

@@ -1,23 +1,17 @@
 # frozen_string_literal: true
 
-each_db_config(Deimos::Utils::DbProducer) do
+each_db_config(Deimos::Utils::OutboxProducer) do
   let(:producer) do
     producer = described_class.new(logger)
     allow(producer).to receive(:sleep)
-    allow(producer).to receive(:producer).and_return(phobos_producer)
     producer
   end
 
-  let(:logger) { nil }
-  let(:phobos_producer) do
-    pp = instance_double(Phobos::Producer::PublicAPI)
-    allow(pp).to receive(:publish_list)
-    pp
-  end
+  let(:logger) { instance_double(Logger, error: nil, info: nil, debug: nil )}
 
   before(:each) do
-    stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
-    stub_const('Deimos::Utils::DbProducer::DELETE_BATCH_SIZE', 1)
+    stub_const('Deimos::Utils::OutboxProducer::BATCH_SIZE', 2)
+    stub_const('Deimos::Utils::OutboxProducer::DELETE_BATCH_SIZE', 1)
   end
 
   specify '#process_next_messages' do
@@ -47,7 +41,7 @@ each_db_config(Deimos::Utils::DbProducer) do
                                    message: 'blah',
                                    key: "key#{i}")
     end
-    stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
+    stub_const('Deimos::Utils::OutboxProducer::BATCH_SIZE', 5)
     producer.current_topic = 'topic1'
     messages = producer.retrieve_messages
     expect(messages.size).to eq(3)
@@ -58,71 +52,8 @@ each_db_config(Deimos::Utils::DbProducer) do
 
     it 'should produce normally' do
       batch = ['A'] * 1000
-      expect(phobos_producer).to receive(:publish_list).with(batch).once
-      expect(Deimos.config.metrics).to receive(:increment).with('publish',
-                                                                tags: %w(status:success topic:),
-                                                                by: 1000).once
-      producer.produce_messages(batch)
-    end
-
-    it 'should split the batch size on buffer overflow' do
-      class_producer = double(Phobos::Producer::ClassMethods::PublicAPI, # rubocop:disable RSpec/VerifiedDoubles
-                              sync_producer_shutdown: nil)
-      allow(producer.class).to receive(:producer).and_return(class_producer)
-      expect(class_producer).to receive(:sync_producer_shutdown).twice
-      count = 0
-      allow(phobos_producer).to receive(:publish_list) do
-        count += 1
-        raise Kafka::BufferOverflow if count < 3
-      end
-      allow(Deimos.config.metrics).to receive(:increment)
-      batch = ['A'] * 1000
+      expect(Karafka.producer).to receive(:produce_many_sync).with(batch).once
       producer.produce_messages(batch)
-      expect(phobos_producer).to have_received(:publish_list).with(batch)
-      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100)
-      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(100).times
-      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
-                                                                      tags: %w(status:success topic:),
-                                                                      by: 10).exactly(100).times
-    end
-
-    it "should raise an error if it can't split any more" do
-      allow(phobos_producer).to receive(:publish_list) do
-        raise Kafka::BufferOverflow
-      end
-      expect(Deimos.config.metrics).not_to receive(:increment)
-      batch = ['A'] * 1000
-      expect { producer.produce_messages(batch) }.to raise_error(Kafka::BufferOverflow)
-      expect(phobos_producer).to have_received(:publish_list).with(batch)
-      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).once
-      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).once
-      expect(phobos_producer).to have_received(:publish_list).with(['A']).once
-    end
-
-    it 'should not resend batches of sent messages' do
-      allow(phobos_producer).to receive(:publish_list) do |group|
-        raise Kafka::BufferOverflow if group.any?('A') && group.size >= 1000
-        raise Kafka::BufferOverflow if group.any?('BIG') && group.size >= 10
-      end
-      allow(Deimos.config.metrics).to receive(:increment)
-      batch = ['A'] * 450 + ['BIG'] * 550
-      producer.produce_messages(batch)
-
-      expect(phobos_producer).to have_received(:publish_list).with(batch)
-      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).exactly(4).times
-      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 50 + ['BIG'] * 50)
-      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(5).times
-      expect(phobos_producer).to have_received(:publish_list).with(['BIG'] * 1).exactly(550).times
-
-      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
-                                                                      tags: %w(status:success topic:),
-                                                                      by: 100).exactly(4).times
-      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
-                                                                      tags: %w(status:success topic:),
-                                                                      by: 10).exactly(5).times
-      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
-                                                                      tags: %w(status:success topic:),
-                                                                      by: 1).exactly(550).times
     end
 
     describe '#compact_messages' do
@@ -149,17 +80,17 @@ each_db_config(Deimos::Utils::DbProducer) do
       let(:deduped_batch) { batch[1..2] }
 
       it 'should dedupe messages when :all is set' do
-        Deimos.configure { |c| c.db_producer.compact_topics = :all }
+        Deimos.configure { |c| c.outbox.compact_topics = :all }
         expect(producer.compact_messages(batch)).to eq(deduped_batch)
       end
 
       it 'should dedupe messages when topic is included' do
-        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic my-topic2) }
+        Deimos.configure { |c| c.outbox.compact_topics = %w(my-topic my-topic2) }
         expect(producer.compact_messages(batch)).to eq(deduped_batch)
       end
 
       it 'should not dedupe messages when topic is not included' do
-        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic3 my-topic2) }
+        Deimos.configure { |c| c.outbox.compact_topics = %w(my-topic3 my-topic2) }
         expect(producer.compact_messages(batch)).to eq(batch)
       end
 
@@ -176,13 +107,13 @@ each_db_config(Deimos::Utils::DbProducer) do
             message: 'BBB'
           }
         ].map { |h| Deimos::KafkaMessage.create!(h) }
-        Deimos.configure { |c| c.db_producer.compact_topics = :all }
+        Deimos.configure { |c| c.outbox.compact_topics = :all }
         expect(producer.compact_messages(unkeyed_batch)).to eq(unkeyed_batch)
-        Deimos.configure { |c| c.db_producer.compact_topics = [] }
+        Deimos.configure { |c| c.outbox.compact_topics = [] }
       end
 
       it 'should compact messages when all messages are unique' do
-        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic my-topic2) }
+        Deimos.configure { |c| c.outbox.compact_topics = %w(my-topic my-topic2) }
         expect(producer.compact_messages(deduped_batch)).to eq(deduped_batch)
       end
     end
@@ -228,7 +159,7 @@ each_db_config(Deimos::Utils::DbProducer) do
                                                                     }
                                                                   ])
       expect(Deimos.config.metrics).to receive(:increment).ordered.with(
-        'db_producer.process',
+        'outbox.process',
         tags: %w(topic:my-topic),
         by: 2
       )
@@ -249,7 +180,7 @@ each_db_config(Deimos::Utils::DbProducer) do
                                                                     }
                                                                   ])
       expect(Deimos.config.metrics).to receive(:increment).ordered.with(
-        'db_producer.process',
+        'outbox.process',
         tags: %w(topic:my-topic),
         by: 2
       )
@@ -263,13 +194,11 @@ each_db_config(Deimos::Utils::DbProducer) do
     end
 
     it 'should register an error if it gets an error' do
-      allow(producer).to receive(:shutdown_producer)
       expect(producer).to receive(:retrieve_messages).and_raise('OH NOES')
       expect(Deimos::KafkaTopicInfo).to receive(:register_error).
         with('my-topic', 'abc')
       expect(producer).not_to receive(:produce_messages)
       producer.process_topic('my-topic')
-      expect(producer).to have_received(:shutdown_producer)
     end
 
     it 'should move on if it gets a partial batch' do
@@ -299,14 +228,14 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(Deimos::KafkaTopicInfo).to receive(:register_error)
 
       expect(Deimos::KafkaMessage.count).to eq(4)
-      subscriber = Deimos.subscribe('db_producer.produce') do |event|
+      Karafka.monitor.subscribe('deimos.outbox.produce') do |event|
         expect(event.payload[:exception_object].message).to eq('OH NOES')
         expect(event.payload[:messages]).to eq(messages)
       end
       producer.process_topic('my-topic')
       # don't delete for regular errors
       expect(Deimos::KafkaMessage.count).to eq(4)
-      Deimos.unsubscribe(subscriber)
+      Karafka.monitor.notifications_bus.clear('deimos.outbox.produce')
     end
 
     it 'should retry deletes and not re-publish' do
@@ -340,7 +269,7 @@ each_db_config(Deimos::Utils::DbProducer) do
         with('my-topic', 'abc').and_return(true)
       expect(producer).to receive(:retrieve_messages).ordered.and_return(messages)
       expect(producer).to receive(:retrieve_messages).ordered.and_return([])
-      expect(phobos_producer).to receive(:publish_list).once.with(messages.map(&:phobos_message))
+      expect(Karafka.producer).to receive(:produce_many_sync).once.with(messages.map(&:karafka_message))
 
       expect(Deimos::KafkaMessage.count).to eq(8)
       producer.process_topic('my-topic')
@@ -360,102 +289,6 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
     end
 
-    context 'with buffer overflow exception' do
-      let(:messages) do
-        (1..4).map do |i|
-          Deimos::KafkaMessage.create!(
-            id: i,
-            key: i,
-            topic: 'my-topic',
-            message: { message: "mess#{i}" },
-            partition_key: "key#{i}"
-          )
-        end
-      end
-      let(:logger) do
-        logger = instance_double(Logger)
-        allow(logger).to receive(:error)
-        logger
-      end
-      let(:message_producer) do
-        Deimos.config.schema.backend = :mock
-        Deimos::ActiveRecordProducer.topic('my-topic')
-        Deimos::ActiveRecordProducer.key_config
-        Deimos::ActiveRecordProducer
-      end
-
-      around(:each) do |example|
-        config = Deimos::ActiveRecordProducer.config.clone
-        backend = Deimos.config.schema.backend
-
-        example.run
-      ensure
-        Deimos::ActiveRecordProducer.instance_variable_set(:@config, config)
-        Deimos.config.schema.backend = backend
-      end
-
-      before(:each) do
-        message_producer
-        (5..8).each do |i|
-          Deimos::KafkaMessage.create!(
-            id: i,
-            topic: 'my-topic2',
-            message: "mess#{i}",
-            partition_key: "key#{i}"
-          )
-        end
-        allow(Deimos::KafkaTopicInfo).to receive(:lock).
-          with('my-topic', 'abc').and_return(true)
-        allow(producer).to receive(:produce_messages).and_raise(Kafka::BufferOverflow)
-        allow(producer).to receive(:retrieve_messages).and_return(messages)
-        allow(Deimos::KafkaTopicInfo).to receive(:register_error)
-      end
-
-      it 'should delete messages on buffer overflow' do
-        expect(Deimos::KafkaMessage.count).to eq(8)
-        producer.process_topic('my-topic')
-        expect(Deimos::KafkaMessage.count).to eq(4)
-      end
-
-      it 'should notify on buffer overflow' do
-        subscriber = Deimos.subscribe('db_producer.produce') do |event|
-          expect(event.payload[:exception_object].message).to eq('Kafka::BufferOverflow')
-          expect(event.payload[:messages]).to eq(messages)
-        end
-        producer.process_topic('my-topic')
-        Deimos.unsubscribe(subscriber)
-        expect(logger).to have_received(:error).with('Message batch too large, deleting...')
-        expect(logger).to have_received(:error).with(
-          [
-            { key: '1', payload: 'payload-decoded' },
-            { key: '2', payload: 'payload-decoded' },
-            { key: '3', payload: 'payload-decoded' },
-            { key: '4', payload: 'payload-decoded' }
-          ]
-        )
-      end
-
-      context 'with exception on error logging attempt' do
-        let(:message_producer) do
-          Deimos::ActiveRecordProducer.topic('my-topic')
-          Deimos::ActiveRecordProducer
-        end
-
-        it 'should notify on buffer overflow disregarding decoding exception' do
-          subscriber = Deimos.subscribe('db_producer.produce') do |event|
-            expect(event.payload[:exception_object].message).to eq('Kafka::BufferOverflow')
-            expect(event.payload[:messages]).to eq(messages)
-          end
-          producer.process_topic('my-topic')
-          Deimos.unsubscribe(subscriber)
-          expect(logger).to have_received(:error).with('Message batch too large, deleting...')
-          expect(logger).to have_received(:error).with(
-            'Large message details logging failure: '\
-            'No key config given - if you are not decoding keys, please use `key_config plain: true`'
-          )
-        end
-      end
-    end
   end
 
   describe '#send_pending_metrics' do