deimos-ruby 1.6.3 → 1.8.1.pre.beta1

data/spec/spec_helper.rb

@@ -2,6 +2,7 @@
 
 $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'active_record'
+require 'database_cleaner'
 require 'deimos'
 require 'deimos/metrics/mock'
 require 'deimos/tracing/mock'
@@ -31,6 +32,7 @@ module TestRunners
   # Test runner
   class TestRunner
     attr_accessor :id, :started, :stopped, :should_error
+
     # :nodoc:
     def initialize(id=nil)
       @id = id
@@ -100,9 +102,8 @@ module DbConfigs
     end
   end
 
-  # Set up the given database.
-  def setup_db(options)
-    ActiveRecord::Base.establish_connection(options)
+  # :nodoc:
+  def run_db_backend_migration
     migration_class_name = 'DbBackendMigration'
     migration_version = '[5.2]'
     migration = ERB.new(
@@ -110,6 +111,24 @@ module DbConfigs
     ).result(binding)
     eval(migration) # rubocop:disable Security/Eval
     ActiveRecord::Migration.new.run(DbBackendMigration, direction: :up)
+  end
+
+  # :nodoc:
+  def run_db_poller_migration
+    migration_class_name = 'DbPollerMigration'
+    migration_version = '[5.2]'
+    migration = ERB.new(
+      File.read('lib/generators/deimos/db_poller/templates/migration')
+    ).result(binding)
+    eval(migration) # rubocop:disable Security/Eval
+    ActiveRecord::Migration.new.run(DbPollerMigration, direction: :up)
+  end
+
+  # Set up the given database.
+  def setup_db(options)
+    ActiveRecord::Base.establish_connection(options)
+    run_db_backend_migration
+    run_db_poller_migration
 
     ActiveRecord::Base.descendants.each do |klass|
       klass.reset_sequence_name if klass.respond_to?(:reset_sequence_name)
@@ -130,8 +149,11 @@ RSpec.configure do |config|
   # true by default for RSpec 4.0
   config.shared_context_metadata_behavior = :apply_to_host_groups
 
+  config.filter_run(focus: true)
+  config.run_all_when_everything_filtered = true
+
   config.before(:all) do
-    Time.zone = 'EST'
+    Time.zone = 'Eastern Time (US & Canada)'
     ActiveRecord::Base.logger = Logger.new('/dev/null')
     ActiveRecord::Base.establish_connection(
       'adapter' => 'sqlite3',
@@ -141,9 +163,10 @@ RSpec.configure do |config|
   config.include Deimos::TestHelpers
   config.include ActiveSupport::Testing::TimeHelpers
   config.before(:suite) do
-    Time.zone = 'EST'
-    ActiveRecord::Base.logger = Logger.new('/dev/null')
     setup_db(DbConfigs::DB_OPTIONS.last)
+
+    DatabaseCleaner.strategy = :transaction
+    DatabaseCleaner.clean_with(:truncation)
   end
 
   config.mock_with(:rspec) do |mocks|
@@ -164,6 +187,39 @@ RSpec.configure do |config|
       deimos_config.schema.backend = :avro_validation
     end
   end
+
+  config.around(:each) do |example|
+    use_cleaner = !example.metadata[:integration]
+
+    DatabaseCleaner.start if use_cleaner
+
+    example.run
+
+    DatabaseCleaner.clean if use_cleaner
+  end
+end
+
+RSpec.shared_context('with widgets') do
+  before(:all) do
+    ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+      t.string(:test_id)
+      t.integer(:some_int)
+      t.boolean(:some_bool)
+      t.timestamps
+    end
+
+    # :nodoc:
+    class Widget < ActiveRecord::Base
+      # @return [String]
+      def generated_id
+        'generated_id'
+      end
+    end
+  end
+
+  after(:all) do
+    ActiveRecord::Base.connection.drop_table(:widgets)
+  end
 end
 
 RSpec.shared_context('with DB') do

data/spec/utils/db_poller_spec.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+# @param seconds [Integer]
+# @return [Time]
+def time_value(secs: 0, mins: 0)
+  Time.local(2015, 5, 5, 1, 0, 0) + (secs + (mins * 60))
+end
+
+each_db_config(Deimos::Utils::DbPoller) do
+
+  before(:each) do
+    Deimos::PollInfo.delete_all
+  end
+
+  describe '#start!' do
+
+    before(:each) do
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config field: 'test_id'
+      end
+      stub_const('MyProducer', producer_class)
+
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config plain: true
+      end
+      stub_const('MyProducerWithID', producer_class)
+    end
+
+    it 'should raise an error if no pollers configured' do
+      Deimos.configure {}
+      expect { described_class.start! }.to raise_error('No pollers configured!')
+    end
+
+    it 'should start pollers as configured' do
+      Deimos.configure do
+        db_poller do
+          producer_class 'MyProducer'
+        end
+        db_poller do
+          producer_class 'MyProducerWithID'
+        end
+      end
+
+      allow(Deimos::Utils::DbPoller).to receive(:new)
+      signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
+      allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
+      described_class.start!
+      expect(Deimos::Utils::DbPoller).to have_received(:new).twice
+      expect(Deimos::Utils::DbPoller).to have_received(:new).
+        with(Deimos.config.db_poller_objects[0])
+      expect(Deimos::Utils::DbPoller).to have_received(:new).
+        with(Deimos.config.db_poller_objects[1])
+    end
+  end
+
+  describe 'pollers' do
+    include_context 'with widgets'
+
+    let(:poller) do
+      poller = described_class.new(config)
+      allow(poller).to receive(:sleep)
+      poller
+    end
+
+    let(:config) { Deimos.config.db_poller_objects.first.dup }
+
+    before(:each) do
+      Widget.delete_all
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-id'
+        key_config none: true
+        record_class Widget
+
+        # :nodoc:
+        def self.generate_payload(attrs, widget)
+          super.merge(message_id: widget.generated_id)
+        end
+      end
+      stub_const('MyProducer', producer_class)
+
+      Deimos.configure do
+        db_poller do
+          producer_class 'MyProducer'
+          run_every 1.minute
+        end
+      end
+    end
+
+    after(:each) do
+      travel_back
+    end
+
+    it 'should crash if initialized with an invalid producer' do
+      config.producer_class = 'NoProducer'
+      expect { described_class.new(config) }.to raise_error('Class NoProducer not found!')
+    end
+
+    describe '#retrieve_poll_info' do
+
+      it 'should start from beginning when configured' do
+        poller.retrieve_poll_info
+        expect(Deimos::PollInfo.count).to eq(1)
+        info = Deimos::PollInfo.last
+        expect(info.producer).to eq('MyProducer')
+        expect(info.last_sent).to eq(Time.new(0))
+        expect(info.last_sent_id).to eq(0)
+      end
+
+      it 'should start from now when configured' do
+        travel_to time_value
+        config.start_from_beginning = false
+        poller.retrieve_poll_info
+        expect(Deimos::PollInfo.count).to eq(1)
+        info = Deimos::PollInfo.last
+        expect(info.producer).to eq('MyProducer')
+        expect(info.last_sent).to eq(time_value)
+        expect(info.last_sent_id).to eq(0)
+      end
+
+    end
+
+    specify '#start' do
+      i = 0
+      expect(poller).to receive(:process_updates).twice do
+        i += 1
+        poller.stop if i == 2
+      end
+      poller.start
+    end
+
+    specify '#should_run?' do
+      Deimos::PollInfo.create!(producer: 'MyProducer',
+                               last_sent: time_value)
+      poller.retrieve_poll_info
+
+      # run_every is set to 1 minute
+      travel_to time_value(secs: 62)
+      expect(poller.should_run?).to eq(true)
+
+      travel_to time_value(secs: 30)
+      expect(poller.should_run?).to eq(false)
+
+      travel_to time_value(mins: -1) # this shouldn't be possible but meh
+      expect(poller.should_run?).to eq(false)
+
+      # take the 2 seconds of delay_time into account
+      travel_to time_value(secs: 60)
+      expect(poller.should_run?).to eq(false)
+    end
+
+    specify '#process_batch' do
+      travel_to time_value
+      widgets = (1..3).map { Widget.create!(test_id: 'some_id', some_int: 4) }
+      widgets.last.update_attribute(:updated_at, time_value(mins: -30))
+      expect(MyProducer).to receive(:send_events).with(widgets)
+      poller.retrieve_poll_info
+      poller.process_batch(widgets)
+      info = Deimos::PollInfo.last
+      expect(info.last_sent.in_time_zone).to eq(time_value(mins: -30))
+      expect(info.last_sent_id).to eq(widgets.last.id)
+    end
+
+    describe '#process_updates' do
+      before(:each) do
+        Deimos::PollInfo.create!(producer: 'MyProducer',
+                                 last_sent: time_value(mins: -61),
+                                 last_sent_id: 0)
+        poller.retrieve_poll_info
+        travel_to time_value
+        stub_const('Deimos::Utils::DbPoller::BATCH_SIZE', 3)
+      end
+
+      let!(:old_widget) do
+        # old widget, earlier than window
+        Widget.create!(test_id: 'some_id', some_int: 40,
+                       updated_at: time_value(mins: -200))
+      end
+
+      let!(:last_widget) do
+        # new widget, before delay
+        Widget.create!(test_id: 'some_id', some_int: 10,
+                       updated_at: time_value(secs: -1))
+      end
+
+      let!(:widgets) do
+        (1..7).map do |i|
+          Widget.create!(test_id: 'some_id', some_int: i,
+                         updated_at: time_value(mins: -61, secs: 30 + i))
+        end
+      end
+
+      it 'should update the full table' do
+        info = Deimos::PollInfo.last
+        config.full_table = true
+        expect(MyProducer).to receive(:poll_query).at_least(:once).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([old_widget, widgets[0], widgets[1]]).and_wrap_original do |m, *args|
+            m.call(*args)
+            expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 32))
+            expect(info.last_sent_id).to eq(widgets[1].id)
+          end
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[2], widgets[3], widgets[4]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[5], widgets[6]]).and_call_original
+        poller.process_updates
+
+        # this is the updated_at of widgets[6]
+        expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
+        expect(info.last_sent_id).to eq(widgets[6].id)
+
+        last_widget.update_attribute(:updated_at, time_value(mins: -250))
+
+        travel 61.seconds
+        # should reprocess the table
+        expect(poller).to receive(:process_batch).ordered.
+          with([last_widget, old_widget, widgets[0]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[1], widgets[2], widgets[3]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[4], widgets[5], widgets[6]]).and_call_original
+        poller.process_updates
+
+        expect(info.reload.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 37))
+        expect(info.last_sent_id).to eq(widgets[6].id)
+      end
+
+      it 'should send events across multiple batches' do
+        allow(MyProducer).to receive(:poll_query).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[0], widgets[1], widgets[2]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[3], widgets[4], widgets[5]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[6]]).and_call_original
+        poller.process_updates
+
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(mins: -61),
+               time_to: time_value(secs: -2),
+               column_name: :updated_at,
+               min_id: 0)
+
+        travel 61.seconds
+        # process the last widget which came in during the delay
+        expect(poller).to receive(:process_batch).with([last_widget]).
+          and_call_original
+        poller.process_updates
+
+        # widgets[6] updated_at value
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(mins: -61, secs: 37),
+               time_to: time_value(secs: 59), # plus 61 seconds minus 2 seconds for delay
+               column_name: :updated_at,
+               min_id: widgets[6].id)
+
+        travel 61.seconds
+        # nothing else to process
+        expect(poller).not_to receive(:process_batch)
+        poller.process_updates
+        poller.process_updates
+
+        expect(MyProducer).to have_received(:poll_query).twice.
+          with(time_from: time_value(secs: -1),
+               time_to: time_value(secs: 120), # plus 122 seconds minus 2 seconds
+               column_name: :updated_at,
+               min_id: last_widget.id)
+      end
+
+      it 'should recover correctly with errors and save the right ID' do
+        widgets.each do |w|
+          w.update_attribute(:updated_at, time_value(mins: -61, secs: 30))
+        end
+        allow(MyProducer).to receive(:poll_query).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[0], widgets[1], widgets[2]]).and_call_original
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[3], widgets[4], widgets[5]]).and_raise('OH NOES')
+
+        expect { poller.process_updates }.to raise_exception('OH NOES')
+
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(mins: -61),
+               time_to: time_value(secs: -2),
+               column_name: :updated_at,
+               min_id: 0)
+
+        info = Deimos::PollInfo.last
+        expect(info.last_sent.in_time_zone).to eq(time_value(mins: -61, secs: 30))
+        expect(info.last_sent_id).to eq(widgets[2].id)
+
+        travel 61.seconds
+        # process the last widget which came in during the delay
+        expect(poller).to receive(:process_batch).ordered.
+          with([widgets[3], widgets[4], widgets[5]]).and_call_original
+        expect(poller).to receive(:process_batch).with([widgets[6], last_widget]).
+          and_call_original
+        poller.process_updates
+        expect(MyProducer).to have_received(:poll_query).
+          with(time_from: time_value(mins: -61, secs: 30),
+               time_to: time_value(secs: 59),
+               column_name: :updated_at,
+               min_id: widgets[2].id)
+
+        expect(info.reload.last_sent.in_time_zone).to eq(time_value(secs: -1))
+        expect(info.last_sent_id).to eq(last_widget.id)
+      end
+
+    end
+
+  end
+end

data/spec/utils/db_producer_spec.rb

@@ -16,11 +16,13 @@ each_db_config(Deimos::Utils::DbProducer) do
 
   before(:each) do
     stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
+    stub_const('Deimos::Utils::DbProducer::DELETE_BATCH_SIZE', 1)
   end
 
   specify '#process_next_messages' do
     expect(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
     expect(producer).to receive(:process_topic).twice
+    expect(Deimos::KafkaTopicInfo).to receive(:ping_empty_topics).with(%w(topic1 topic2))
     expect(producer).to receive(:sleep).with(0.5)
     producer.process_next_messages
   end
@@ -40,6 +42,9 @@ each_db_config(Deimos::Utils::DbProducer) do
       Deimos::KafkaMessage.create!(topic: 'topic1',
                                    message: 'blah',
                                    key: "key#{i}")
+      Deimos::KafkaMessage.create!(topic: 'topic2',
+                                   message: 'blah',
+                                   key: "key#{i}")
     end
     stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
     producer.current_topic = 'topic1'
@@ -280,6 +285,12 @@ each_db_config(Deimos::Utils::DbProducer) do
           message: "mess#{i}",
           partition_key: "key#{i}"
         )
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
       end
 
       expect(Deimos::KafkaTopicInfo).to receive(:lock).
@@ -288,9 +299,60 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(producer).to receive(:retrieve_messages).and_return(messages)
       expect(Deimos::KafkaTopicInfo).to receive(:register_error)
 
+      expect(Deimos::KafkaMessage.count).to eq(8)
+      producer.process_topic('my-topic')
       expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should retry deletes and not re-publish' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      (5..8).each do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+
+      raise_error = true
+      expect(Deimos::KafkaMessage).to receive(:where).exactly(5).times.and_wrap_original do |m, *args|
+        if raise_error
+          raise_error = false
+          raise 'Lock wait timeout'
+        end
+        m.call(*args)
+      end
+
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return(messages)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return([])
+      expect(phobos_producer).to receive(:publish_list).once.with(messages.map(&:phobos_message))
+
+      expect(Deimos::KafkaMessage.count).to eq(8)
       producer.process_topic('my-topic')
-      expect(Deimos::KafkaMessage.count).to eq(0)
+      expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should re-raise misc errors on delete' do
+      messages = (1..3).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      expect(Deimos::KafkaMessage).to receive(:where).once.and_raise('OH NOES')
+      expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
     end
 
   end
@@ -309,21 +371,34 @@ each_db_config(Deimos::Utils::DbProducer) do
           Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
                                        created_at: (1 + i).minute.ago)
         end
+        Deimos::KafkaTopicInfo.create!(topic: 'topic1',
+                                       last_processed_at: 6.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic2',
+                                       last_processed_at: 3.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic3',
+                                       last_processed_at: 5.minutes.ago)
         allow(Deimos.config.metrics).to receive(:gauge)
         producer.send_pending_metrics
-        expect(Deimos.config.metrics).to have_received(:gauge).twice
+        expect(Deimos.config.metrics).to have_received(:gauge).exactly(6).times
+        # topic1 has the earliest message 4 minutes ago and last processed 6
+        # minutes ago, so the most amount of time we've seen nothing is 4 minutes
         expect(Deimos.config.metrics).to have_received(:gauge).
           with('pending_db_messages_max_wait', 4.minutes.to_i, tags: ['topic:topic1'])
+        # topic2 has earliest message 5 minutes ago and last processed 3 minutes
+        # ago, so we should give it 3 minutes
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_max_wait', 3.minutes.to_i, tags: ['topic:topic2'])
+        # topic3 has no messages, so should get 0
         expect(Deimos.config.metrics).to have_received(:gauge).
-          with('pending_db_messages_max_wait', 5.minutes.to_i, tags: ['topic:topic2'])
+          with('pending_db_messages_max_wait', 0, tags: ['topic:topic3'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic1'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic2'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 0, tags: ['topic:topic3'])
       end
     end
-
-    it 'should send 0 if no messages' do
-      expect(Deimos.config.metrics).to receive(:gauge).
-        with('pending_db_messages_max_wait', 0)
-      producer.send_pending_metrics
-    end
   end
 
   example 'Full integration test' do
@@ -403,5 +478,4 @@ each_db_config(Deimos::Utils::DbProducer) do
                                                                 }
                                                               ])
   end
-
 end