deimos-temp-fork 0.0.1

data/spec/utils/db_poller_spec.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+# @param seconds [Integer]
+# @return [Time]
+def time_value(secs: 0, mins: 0)
+  Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
+end
+
+each_db_config(Deimos::Utils::DbPoller) do
+
+  before(:each) do
+    Deimos::PollInfo.delete_all
+  end
+
+  describe '#start!' do
+
+    before(:each) do
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config field: 'test_id'
+      end
+      stub_const('MyProducer', producer_class)
+
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config plain: true
+      end
+      stub_const('MyProducerWithID', producer_class)
+    end
+
+    it 'should raise an error if no pollers configured' do
+      Deimos.configure {}
+      expect { described_class.start! }.to raise_error('No pollers configured!')
+    end
+
+    it 'should start pollers as configured' do
+      Deimos.configure do
+        db_poller do
+          producer_class 'MyProducer'
+        end
+        db_poller do
+          producer_class 'MyProducerWithID'
+        end
+      end
+
+      allow(Deimos::Utils::DbPoller).to receive(:new)
+      signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
+      allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
+      described_class.start!
+      expect(Deimos::Utils::DbPoller).to have_received(:new).twice
+      expect(Deimos::Utils::DbPoller).to have_received(:new).
+        with(Deimos.config.db_poller_objects[0])
+      expect(Deimos::Utils::DbPoller).to have_received(:new).
+        with(Deimos.config.db_poller_objects[1])
+    end
+  end
+
+  describe 'pollers' do
+    include_context 'with widgets'
+
+    let(:poller) do
+      poller = described_class.new(config)
+      allow(poller).to receive(:sleep)
+      poller
+    end
+
+    let(:config) { Deimos.config.db_poller_objects.first.dup }
+
+    before(:each) do
+      Widget.delete_all
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-id'
+        key_config none: true
+        record_class Widget
+
+        # :nodoc:
+        def self.generate_payload(attrs, widget)
+          super.merge(message_id: widget.generated_id)
+        end
+      end
+      stub_const('MyProducer', producer_class)
+
+      Deimos.configure do
+        db_poller do
+          producer_class 'MyProducer'
+          run_every 1.minute
+        end
+      end
+    end
+
+    after(:each) do
+      travel_back
+    end
+
+    it 'should crash if initialized with an invalid producer' do
+      config.producer_class = 'NoProducer'
+      expect { described_class.new(config) }.to raise_error('Class NoProducer not found!')
+    end
+
+    describe '#retrieve_poll_info' do
+
+      it 'should start from beginning when configured' do
+        poller.retrieve_poll_info
+        expect(Deimos::PollInfo.count).to eq(1)
+        info = Deimos::PollInfo.last
+        expect(info.producer).to eq('MyProducer')
+        expect(info.last_sent).to eq(Time.new(0))
+        expect(info.last_sent_id).to eq(0)
+      end
+
+      it 'should start from now when configured' do
+        travel_to time_value
+        config.start_from_beginning = false
+        poller.retrieve_poll_info
+        expect(Deimos::PollInfo.count).to eq(1)
+        info = Deimos::PollInfo.last
+        expect(info.producer).to eq('MyProducer')
+        expect(info.last_sent).to eq(time_value)
+        expect(info.last_sent_id).to eq(0)
+      end
+
+    end
+
+    specify '#start' do
+      i = 0
+      expect(poller).to receive(:process_updates).twice do
+        i += 1
+        poller.stop if i == 2
+      end
+      poller.start
+    end
+
+    specify '#should_run?' do
+      Deimos::PollInfo.create!(producer: 'MyProducer',
+                               last_sent: time_value)
+      poller.retrieve_poll_info
+
+      # run_every is set to 1 minute
+      travel_to time_value(secs: 62)
+      expect(poller.should_run?).to eq(true)
+
+      travel_to time_value(secs: 30)
+      expect(poller.should_run?).to eq(false)
+
+      travel_to time_value(mins: -1) # this shouldn't be possible but meh
+      expect(poller.should_run?).to eq(false)
+
+      # take the 2 seconds of delay_time into account
+      travel_to time_value(secs: 60)
+      expect(poller.should_run?).to eq(false)
+    end
+
+    specify '#process_batch' do
+      travel_to time_value
+      widgets = (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) }
+      widgets.last.update_attribute(:updated_at, time_value(mins: -30))
+      expect(MyProducer).to receive(:send_events).with(widgets)
+      poller.retrieve_poll_info
+      poller.process_batch(widgets)
+      info = Deimos::PollInfo.last
+      expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
+      expect(info.last_sent_id).to eq(widgets.last.id)
+    end
+
+    describe '#process_updates' do
+      before(:each) do
+        Deimos::PollInfo.create!(producer: 'MyProducer',
+                                 last_sent: time_value(mins: -61),
+                                 last_sent_id: 0)
+        poller.retrieve_poll_info
+        travel_to time_value
+        stub_const('Deimos::Utils::DbPoller::BATCH_SIZE', 3)
+      end
+
+      let!(:old_widget) do
+        # old widget, earlier than window
+        Widget.create!(test_id: 'some_id', some_int: 40,
+                       updated_at: time_value(mins: -200))
+      end
+
+      let!(:last_widget) do
+        # new widget, before delay
+        Widget.create!(test_id: 'some_id', some_int: 10,
+                       updated_at: time_value(secs: -1))
+      end
+
+      let!(:widgets) do
+        (1..7).map do |i|
+          Widget.create!(test_id: 'some_id', some_int: i,
+                         updated_at: time_value(mins: -61, secs: 30 + i))
+        end
+      end
+
+      it 'should update the full table' do
+        info = Deimos::PollInfo.last
+        config.full_table = true
+        expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
+            m.call(*args)
+            expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
+            expect(info.last_sent_id).to eq(widgets[1].id)
+          end
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[2], widgets[3], widgets[4]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[5], widgets[6]]).and_call_original
+        poller.process_updates
+
+        # this is the updated_at of widgets[6]
+        expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
+        expect(info.last_sent_id).to eq(widgets[6].id)
+
+        last_widget.update_attribute(:updated_at, time_value(mins: -250))
+
+        travel 61.seconds
+        # should reprocess the table
+        expect(poller).to receive(:process_batch).ordered.
+          with([last_widget, old_widget, widgets[0]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[1], widgets[2], widgets[3]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[4], widgets[5], widgets[6]]).and_call_original
+        poller.process_updates
+
+        expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
+        expect(info.last_sent_id).to eq(widgets[6].id)
+      end
+
+      it 'should send events across multiple batches' do
+        allow(MyProducer).to receive(:poll_query).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[0], widgets[1], widgets[2]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[3], widgets[4], widgets[5]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[6]]).and_call_original
+        poller.process_updates
+
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(mins: -61),
+               time_to: time_value(secs: -2),
+               column_name: :updated_at,
+               min_id: 0)
+
+        travel 61.seconds
+        # process the last widget which came in during the delay
+        expect(poller).to receive(:process_batch).with([last_widget]).
+          and_call_original
+        poller.process_updates
+
+        # widgets[6] updated_at value
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(mins: -61, secs: 37),
+               time_to: time_value(secs: 59), # plus 61 seconds minus 2 seconds for delay
+               column_name: :updated_at,
+               min_id: widgets[6].id)
+
+        travel 61.seconds
+        # nothing else to process
+        expect(poller).not_to receive(:process_batch)
+        poller.process_updates
+        poller.process_updates
+
+        expect(MyProducer).to have_received(:poll_query).twice.
+          with(time_from: time_value(secs: -1),
+               time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
+               column_name: :updated_at,
+               min_id: last_widget.id)
+      end
+
+      it 'should recover correctly with errors and save the right ID' do
+        widgets.each do |w|
+          w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
+        end
+        allow(MyProducer).to receive(:poll_query).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[0], widgets[1], widgets[2]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
+
+        expect { poller.process_updates }.to raise_exception('OH NOES')
+
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(mins: -61),
+               time_to: time_value(secs: -2),
+               column_name: :updated_at,
+               min_id: 0)
+
+        info = Deimos::PollInfo.last
+        expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
+        expect(info.last_sent_id).to eq(widgets[2].id)
+
+        travel 61.seconds
+        # process the last widget which came in during the delay
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[3], widgets[4], widgets[5]]).and_call_original
+        expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
+          and_call_original
+        poller.process_updates
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(mins: -61, secs: 30),
+               time_to: time_value(secs: 59),
+               column_name: :updated_at,
+               min_id: widgets[2].id)
+
+        expect(info.reload.last_sent.in_time_zone).to eq(time_value(secs: -1))
+        expect(info.last_sent_id).to eq(last_widget.id)
+      end
+
+    end
+
+  end
+end

data/spec/utils/db_producer_spec.rb

@@ -0,0 +1,514 @@
+# frozen_string_literal: true
+
+each_db_config(Deimos::Utils::DbProducer) do
+  let(:producer) do
+    producer = described_class.new
+    allow(producer).to receive(:sleep)
+    allow(producer).to receive(:producer).and_return(phobos_producer)
+    producer
+  end
+
+  let(:phobos_producer) do
+    pp = instance_double(Phobos::Producer::PublicAPI)
+    allow(pp).to receive(:publish_list)
+    pp
+  end
+
+  before(:each) do
+    stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
+    stub_const('Deimos::Utils::DbProducer::DELETE_BATCH_SIZE', 1)
+  end
+
+  specify '#process_next_messages' do
+    expect(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
+    expect(producer).to receive(:process_topic).twice
+    expect(Deimos::KafkaTopicInfo).to receive(:ping_empty_topics).with(%w(topic1 topic2))
+    expect(producer).to receive(:sleep).with(0.5)
+    producer.process_next_messages
+  end
+
+  specify '#retrieve_topics' do
+    (1..3).each do |i|
+      Deimos::KafkaMessage.create!(topic: "topic#{i}",
+                                   key: 'blergkey',
+                                   message: 'blerg')
+    end
+    expect(producer.retrieve_topics).
+      to contain_exactly('topic1', 'topic2', 'topic3')
+  end
+
+  specify '#retrieve_messages' do
+    (1..3).each do |i|
+      Deimos::KafkaMessage.create!(topic: 'topic1',
+                                   message: 'blah',
+                                   key: "key#{i}")
+      Deimos::KafkaMessage.create!(topic: 'topic2',
+                                   message: 'blah',
+                                   key: "key#{i}")
+    end
+    stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
+    producer.current_topic = 'topic1'
+    messages = producer.retrieve_messages
+    expect(messages.size).to eq(3)
+    expect(messages).to all(be_a_kind_of(Deimos::KafkaMessage))
+  end
+
+  describe '#produce_messages' do
+
+    it 'should produce normally' do
+      batch = ['A'] * 1000
+      expect(phobos_producer).to receive(:publish_list).with(batch).once
+      expect(Deimos.config.metrics).to receive(:increment).with('publish',
+                                                                tags: %w(status:success topic:),
+                                                                by: 1000).once
+      producer.produce_messages(batch)
+    end
+
+    it 'should split the batch size on buffer overflow' do
+      class_producer = double(Phobos::Producer::ClassMethods::PublicAPI, # rubocop:disable RSpec/VerifiedDoubles
+                              sync_producer_shutdown: nil)
+      allow(producer.class).to receive(:producer).and_return(class_producer)
+      expect(class_producer).to receive(:sync_producer_shutdown).twice
+      count = 0
+      allow(phobos_producer).to receive(:publish_list) do
+        count += 1
+        raise Kafka::BufferOverflow if count < 3
+      end
+      allow(Deimos.config.metrics).to receive(:increment)
+      batch = ['A'] * 1000
+      producer.produce_messages(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(100).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 10).exactly(100).times
+    end
+
+    it "should raise an error if it can't split any more" do
+      allow(phobos_producer).to receive(:publish_list) do
+        raise Kafka::BufferOverflow
+      end
+      expect(Deimos.config.metrics).not_to receive(:increment)
+      batch = ['A'] * 1000
+      expect { producer.produce_messages(batch) }.to raise_error(Kafka::BufferOverflow)
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).once
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).once
+      expect(phobos_producer).to have_received(:publish_list).with(['A']).once
+    end
+
+    it 'should not resend batches of sent messages' do
+      allow(phobos_producer).to receive(:publish_list) do |group|
+        raise Kafka::BufferOverflow if group.any?('A') && group.size >= 1000
+        raise Kafka::BufferOverflow if group.any?('BIG') && group.size >= 10
+      end
+      allow(Deimos.config.metrics).to receive(:increment)
+      batch = ['A'] * 450 + ['BIG'] * 550
+      producer.produce_messages(batch)
+
+      expect(phobos_producer).to have_received(:publish_list).with(batch)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 100).exactly(4).times
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 50 + ['BIG'] * 50)
+      expect(phobos_producer).to have_received(:publish_list).with(['A'] * 10).exactly(5).times
+      expect(phobos_producer).to have_received(:publish_list).with(['BIG'] * 1).exactly(550).times
+
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 100).exactly(4).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 10).exactly(5).times
+      expect(Deimos.config.metrics).to have_received(:increment).with('publish',
+                                                                      tags: %w(status:success topic:),
+                                                                      by: 1).exactly(550).times
+    end
+
+    describe '#compact_messages' do
+      let(:batch) do
+        [
+          {
+            key: 1,
+            topic: 'my-topic',
+            message: 'AAA'
+          },
+          {
+            key: 2,
+            topic: 'my-topic',
+            message: 'BBB'
+          },
+          {
+            key: 1,
+            topic: 'my-topic',
+            message: 'CCC'
+          }
+        ].map { |h| Deimos::KafkaMessage.create!(h) }
+      end
+
+      let(:deduped_batch) { batch[1..2] }
+
+      it 'should dedupe messages when :all is set' do
+        Deimos.configure { |c| c.db_producer.compact_topics = :all }
+        expect(producer.compact_messages(batch)).to eq(deduped_batch)
+      end
+
+      it 'should dedupe messages when topic is included' do
+        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic my-topic2) }
+        expect(producer.compact_messages(batch)).to eq(deduped_batch)
+      end
+
+      it 'should not dedupe messages when topic is not included' do
+        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic3 my-topic2) }
+        expect(producer.compact_messages(batch)).to eq(batch)
+      end
+
+      it 'should not dedupe messages without keys' do
+        unkeyed_batch = [
+          {
+            key: nil,
+            topic: 'my-topic',
+            message: 'AAA'
+          },
+          {
+            key: nil,
+            topic: 'my-topic',
+            message: 'BBB'
+          }
+        ].map { |h| Deimos::KafkaMessage.create!(h) }
+        Deimos.configure { |c| c.db_producer.compact_topics = :all }
+        expect(producer.compact_messages(unkeyed_batch)).to eq(unkeyed_batch)
+        Deimos.configure { |c| c.db_producer.compact_topics = [] }
+      end
+
+      it 'should compact messages when all messages are unique' do
+        Deimos.configure { |c| c.db_producer.compact_topics = %w(my-topic my-topic2) }
+        expect(producer.compact_messages(deduped_batch)).to eq(deduped_batch)
+      end
+    end
+  end
+
+  describe '#process_topic' do
+    before(:each) do
+      producer.id = 'abc'
+    end
+
+    it 'should do nothing if lock fails' do
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(false)
+      expect(producer).not_to receive(:retrieve_messages)
+      producer.process_topic('my-topic')
+    end
+
+    it 'should complete successfully' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.new(
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:retrieve_messages).ordered.
+        and_return(messages[0..1])
+      expect(producer).to receive(:send_pending_metrics).twice
+      expect(producer).to receive(:produce_messages).ordered.with([
+                                                                    {
+                                                                      payload: 'mess1',
+                                                                      key: nil,
+                                                                      partition_key: 'key1',
+                                                                      topic: 'my-topic'
+                                                                    },
+                                                                    {
+                                                                      payload: 'mess2',
+                                                                      key: nil,
+                                                                      partition_key: 'key2',
+                                                                      topic: 'my-topic'
+                                                                    }
+                                                                  ])
+      expect(Deimos.config.metrics).to receive(:increment).ordered.with(
+        'db_producer.process',
+        tags: %w(topic:my-topic),
+        by: 2
+      )
+      expect(producer).to receive(:retrieve_messages).ordered.
+        and_return(messages[2..3])
+      expect(producer).to receive(:produce_messages).ordered.with([
+                                                                    {
+                                                                      payload: 'mess3',
+                                                                      partition_key: 'key3',
+                                                                      key: nil,
+                                                                      topic: 'my-topic'
+                                                                    },
+                                                                    {
+                                                                      payload: 'mess4',
+                                                                      partition_key: 'key4',
+                                                                      key: nil,
+                                                                      topic: 'my-topic'
+                                                                    }
+                                                                  ])
+      expect(Deimos.config.metrics).to receive(:increment).ordered.with(
+        'db_producer.process',
+        tags: %w(topic:my-topic),
+        by: 2
+      )
+      expect(producer).to receive(:retrieve_messages).ordered.
+        and_return([])
+      expect(Deimos::KafkaTopicInfo).to receive(:heartbeat).
+        with('my-topic', 'abc').twice
+      expect(Deimos::KafkaTopicInfo).to receive(:clear_lock).
+        with('my-topic', 'abc').once
+      producer.process_topic('my-topic')
+    end
+
+    it 'should register an error if it gets an error' do
+      allow(producer).to receive(:shutdown_producer)
+      expect(producer).to receive(:retrieve_messages).and_raise('OH NOES')
+      expect(Deimos::KafkaTopicInfo).to receive(:register_error).
+        with('my-topic', 'abc')
+      expect(producer).not_to receive(:produce_messages)
+      producer.process_topic('my-topic')
+      expect(producer).to have_received(:shutdown_producer)
+    end
+
+    it 'should move on if it gets a partial batch' do
+      expect(producer).to receive(:retrieve_messages).ordered.
+        and_return([Deimos::KafkaMessage.new(
+          topic: 'my-topic',
+          message: 'mess1'
+        )])
+      expect(producer).to receive(:produce_messages).once
+      producer.process_topic('my-topic')
+    end
+
+    it 'should notify on error' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:produce_messages).and_raise('OH NOES')
+      expect(producer).to receive(:retrieve_messages).and_return(messages)
+      expect(Deimos::KafkaTopicInfo).to receive(:register_error)
+
+      expect(Deimos::KafkaMessage.count).to eq(4)
+      subscriber = Deimos.subscribe('db_producer.produce') do |event|
+        expect(event.payload[:exception_object].message).to eq('OH NOES')
+        expect(event.payload[:messages]).to eq(messages)
+      end
+      producer.process_topic('my-topic')
+      # don't delete for regular errors
+      expect(Deimos::KafkaMessage.count).to eq(4)
+      Deimos.unsubscribe(subscriber)
+    end
+
+    it 'should delete messages on buffer overflow' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      (5..8).each do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:produce_messages).and_raise(Kafka::BufferOverflow)
+      expect(producer).to receive(:retrieve_messages).and_return(messages)
+      expect(Deimos::KafkaTopicInfo).to receive(:register_error)
+
+      expect(Deimos::KafkaMessage.count).to eq(8)
+      producer.process_topic('my-topic')
+      expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should retry deletes and not re-publish' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      (5..8).each do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+
+      raise_error = true
+      expect(Deimos::KafkaMessage).to receive(:where).exactly(5).times.and_wrap_original do |m, *args|
+        if raise_error
+          raise_error = false
+          raise 'Lock wait timeout'
+        end
+        m.call(*args)
+      end
+
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return(messages)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return([])
+      expect(phobos_producer).to receive(:publish_list).once.with(messages.map(&:phobos_message))
+
+      expect(Deimos::KafkaMessage.count).to eq(8)
+      producer.process_topic('my-topic')
+      expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should re-raise misc errors on delete' do
+      messages = (1..3).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      expect(Deimos::KafkaMessage).to receive(:where).once.and_raise('OH NOES')
+      expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
+    end
+
+  end
+
+  describe '#send_pending_metrics' do
+    it 'should use the first created_at for each topic' do |example|
+      # sqlite date-time strings don't work correctly
+      next if example.metadata[:db_config][:adapter] == 'sqlite3'
+
+      freeze_time do
+        (1..2).each do |i|
+          Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
+                                       created_at: (3 + i).minutes.ago)
+          Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
+                                       created_at: (2 + i).minutes.ago)
+          Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
+                                       created_at: (1 + i).minute.ago)
+        end
+        Deimos::KafkaTopicInfo.create!(topic: 'topic1',
+                                       last_processed_at: 6.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic2',
+                                       last_processed_at: 3.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic3',
+                                       last_processed_at: 5.minutes.ago)
+        allow(Deimos.config.metrics).to receive(:gauge)
+        producer.send_pending_metrics
+        expect(Deimos.config.metrics).to have_received(:gauge).exactly(6).times
+        # topic1 has the earliest message 4 minutes ago and last processed 6
+        # minutes ago, so the most amount of time we've seen nothing is 4 minutes
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_max_wait', 4.minutes.to_i, tags: ['topic:topic1'])
+        # topic2 has earliest message 5 minutes ago and last processed 3 minutes
+        # ago, so we should give it 3 minutes
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_max_wait', 3.minutes.to_i, tags: ['topic:topic2'])
+        # topic3 has no messages, so should get 0
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_max_wait', 0, tags: ['topic:topic3'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic1'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic2'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 0, tags: ['topic:topic3'])
+      end
+    end
+  end
+
+  example 'Full integration test' do
+    (1..4).each do |i|
+      (1..2).each do |j|
+        Deimos::KafkaMessage.create!(topic: "topic#{j}",
+                                     message: "mess#{i}",
+                                     partition_key: "key#{i}")
+        Deimos::KafkaMessage.create!(topic: "topic#{j + 2}",
+                                     key: "key#{i}",
+                                     partition_key: "key#{i}",
+                                     message: "mess#{i}")
+      end
+    end
+    allow(producer).to receive(:produce_messages)
+
+    producer.process_next_messages
+    expect(Deimos::KafkaTopicInfo.count).to eq(4)
+    topics = Deimos::KafkaTopicInfo.select('distinct topic').map(&:topic)
+    expect(topics).to contain_exactly('topic1', 'topic2', 'topic3', 'topic4')
+    expect(Deimos::KafkaMessage.count).to eq(0)
+
+    expect(producer).to have_received(:produce_messages).with([
+                                                                {
+                                                                  payload: 'mess1',
+                                                                  partition_key: 'key1',
+                                                                  key: nil,
+                                                                  topic: 'topic1'
+                                                                },
+                                                                {
+                                                                  payload: 'mess2',
+                                                                  key: nil,
+                                                                  partition_key: 'key2',
+                                                                  topic: 'topic1'
+                                                                }
+                                                              ])
+    expect(producer).to have_received(:produce_messages).with([
+                                                                {
+                                                                  payload: 'mess3',
+                                                                  key: nil,
+                                                                  partition_key: 'key3',
+                                                                  topic: 'topic1'
+                                                                },
+                                                                {
+                                                                  payload: 'mess4',
+                                                                  key: nil,
+                                                                  partition_key: 'key4',
+                                                                  topic: 'topic1'
+                                                                }
+                                                              ])
+    expect(producer).to have_received(:produce_messages).with([
+                                                                {
+                                                                  payload: 'mess1',
+                                                                  key: 'key1',
+                                                                  partition_key: 'key1',
+                                                                  topic: 'topic3'
+                                                                },
+                                                                {
+                                                                  payload: 'mess2',
+                                                                  partition_key: 'key2',
+                                                                  key: 'key2',
+                                                                  topic: 'topic3'
+                                                                }
+                                                              ])
+    expect(producer).to have_received(:produce_messages).with([
+                                                                {
+                                                                  payload: 'mess3',
+                                                                  key: 'key3',
+                                                                  partition_key: 'key3',
+                                                                  topic: 'topic3'
+                                                                },
+                                                                {
+                                                                  payload: 'mess4',
+                                                                  partition_key: 'key4',
+                                                                  key: 'key4',
+                                                                  topic: 'topic3'
+                                                                }
+                                                              ])
+  end
+end