deimos-ruby 1.7.0.pre.beta1 → 1.8.1.pre.beta3

data/spec/schemas/com/my-namespace/request/Index.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.request",
+  "name": "Index",
+  "type": "record",
+  "fields": [
+    {
+      "name": "request_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/request/UpdateRequest.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.request",
+  "name": "UpdateRequest",
+  "type": "record",
+  "fields": [
+    {
+      "name": "update_request_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/response/Index.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.response",
+  "name": "Index",
+  "type": "record",
+  "fields": [
+    {
+      "name": "response_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/schemas/com/my-namespace/response/UpdateResponse.avsc

@@ -0,0 +1,11 @@
+{
+  "namespace": "com.my-namespace.response",
+  "name": "UpdateResponse",
+  "type": "record",
+  "fields": [
+    {
+      "name": "update_response_id",
+      "type": "string"
+    }
+  ]
+}

data/spec/spec_helper.rb

@@ -2,6 +2,8 @@
 
 $LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 require 'active_record'
+require 'action_controller/railtie'
+require 'database_cleaner'
 require 'deimos'
 require 'deimos/metrics/mock'
 require 'deimos/tracing/mock'
@@ -10,6 +12,11 @@ require 'active_support/testing/time_helpers'
 require 'activerecord-import'
 require 'handlers/my_batch_consumer'
 require 'handlers/my_consumer'
+require 'rspec/rails'
+
+class DeimosApp < Rails::Application
+end
+DeimosApp.initialize!
 
 # Helpers for Executor/DbProducer
 module TestRunners
@@ -31,6 +38,7 @@ module TestRunners
   # Test runner
   class TestRunner
     attr_accessor :id, :started, :stopped, :should_error
+
     # :nodoc:
     def initialize(id=nil)
       @id = id
@@ -147,6 +155,9 @@ RSpec.configure do |config|
   # true by default for RSpec 4.0
   config.shared_context_metadata_behavior = :apply_to_host_groups
 
+  config.filter_run(focus: true)
+  config.run_all_when_everything_filtered = true
+
   config.before(:all) do
     Time.zone = 'Eastern Time (US & Canada)'
     ActiveRecord::Base.logger = Logger.new('/dev/null')
@@ -159,6 +170,9 @@ RSpec.configure do |config|
   config.include ActiveSupport::Testing::TimeHelpers
   config.before(:suite) do
     setup_db(DbConfigs::DB_OPTIONS.last)
+
+    DatabaseCleaner.strategy = :transaction
+    DatabaseCleaner.clean_with(:truncation)
   end
 
   config.mock_with(:rspec) do |mocks|
@@ -179,6 +193,16 @@ RSpec.configure do |config|
       deimos_config.schema.backend = :avro_validation
     end
   end
+
+  config.around(:each) do |example|
+    use_cleaner = !example.metadata[:integration]
+
+    DatabaseCleaner.start if use_cleaner
+
+    example.run
+
+    DatabaseCleaner.clean if use_cleaner
+  end
 end
 
 RSpec.shared_context('with widgets') do

data/spec/utils/db_poller_spec.rb

@@ -48,8 +48,8 @@ each_db_config(Deimos::Utils::DbPoller) do
       end
 
       allow(Deimos::Utils::DbPoller).to receive(:new)
-      signal_double = instance_double(Deimos::Utils::SignalHandler, run!: nil)
-      allow(Deimos::Utils::SignalHandler).to receive(:new).and_return(signal_double)
+      signal_double = instance_double(Sigurd::SignalHandler, run!: nil)
+      allow(Sigurd::SignalHandler).to receive(:new).and_return(signal_double)
       described_class.start!
       expect(Deimos::Utils::DbPoller).to have_received(:new).twice
       expect(Deimos::Utils::DbPoller).to have_received(:new).

data/spec/utils/db_producer_spec.rb

@@ -16,11 +16,13 @@ each_db_config(Deimos::Utils::DbProducer) do
 
   before(:each) do
     stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 2)
+    stub_const('Deimos::Utils::DbProducer::DELETE_BATCH_SIZE', 1)
   end
 
   specify '#process_next_messages' do
     expect(producer).to receive(:retrieve_topics).and_return(%w(topic1 topic2))
     expect(producer).to receive(:process_topic).twice
+    expect(Deimos::KafkaTopicInfo).to receive(:ping_empty_topics).with(%w(topic1 topic2))
     expect(producer).to receive(:sleep).with(0.5)
     producer.process_next_messages
   end
@@ -40,6 +42,9 @@ each_db_config(Deimos::Utils::DbProducer) do
       Deimos::KafkaMessage.create!(topic: 'topic1',
                                    message: 'blah',
                                    key: "key#{i}")
+      Deimos::KafkaMessage.create!(topic: 'topic2',
+                                   message: 'blah',
+                                   key: "key#{i}")
     end
     stub_const('Deimos::Utils::DbProducer::BATCH_SIZE', 5)
     producer.current_topic = 'topic1'
@@ -280,6 +285,12 @@ each_db_config(Deimos::Utils::DbProducer) do
           message: "mess#{i}",
           partition_key: "key#{i}"
         )
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
       end
 
       expect(Deimos::KafkaTopicInfo).to receive(:lock).
@@ -288,9 +299,60 @@ each_db_config(Deimos::Utils::DbProducer) do
       expect(producer).to receive(:retrieve_messages).and_return(messages)
       expect(Deimos::KafkaTopicInfo).to receive(:register_error)
 
+      expect(Deimos::KafkaMessage.count).to eq(8)
+      producer.process_topic('my-topic')
       expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should retry deletes and not re-publish' do
+      messages = (1..4).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      (5..8).each do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic2',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+
+      raise_error = true
+      expect(Deimos::KafkaMessage).to receive(:where).exactly(5).times.and_wrap_original do |m, *args|
+        if raise_error
+          raise_error = false
+          raise 'Lock wait timeout'
+        end
+        m.call(*args)
+      end
+
+      expect(Deimos::KafkaTopicInfo).to receive(:lock).
+        with('my-topic', 'abc').and_return(true)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return(messages)
+      expect(producer).to receive(:retrieve_messages).ordered.and_return([])
+      expect(phobos_producer).to receive(:publish_list).once.with(messages.map(&:phobos_message))
+
+      expect(Deimos::KafkaMessage.count).to eq(8)
       producer.process_topic('my-topic')
-      expect(Deimos::KafkaMessage.count).to eq(0)
+      expect(Deimos::KafkaMessage.count).to eq(4)
+    end
+
+    it 'should re-raise misc errors on delete' do
+      messages = (1..3).map do |i|
+        Deimos::KafkaMessage.create!(
+          id: i,
+          topic: 'my-topic',
+          message: "mess#{i}",
+          partition_key: "key#{i}"
+        )
+      end
+      expect(Deimos::KafkaMessage).to receive(:where).once.and_raise('OH NOES')
+      expect { producer.delete_messages(messages) }.to raise_exception('OH NOES')
     end
 
   end
@@ -309,21 +371,34 @@ each_db_config(Deimos::Utils::DbProducer) do
           Deimos::KafkaMessage.create!(topic: "topic#{i}", message: nil,
                                        created_at: (1 + i).minute.ago)
         end
+        Deimos::KafkaTopicInfo.create!(topic: 'topic1',
+                                       last_processed_at: 6.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic2',
+                                       last_processed_at: 3.minutes.ago)
+        Deimos::KafkaTopicInfo.create!(topic: 'topic3',
+                                       last_processed_at: 5.minutes.ago)
         allow(Deimos.config.metrics).to receive(:gauge)
         producer.send_pending_metrics
-        expect(Deimos.config.metrics).to have_received(:gauge).twice
+        expect(Deimos.config.metrics).to have_received(:gauge).exactly(6).times
+        # topic1 has the earliest message 4 minutes ago and last processed 6
+        # minutes ago, so the most amount of time we've seen nothing is 4 minutes
         expect(Deimos.config.metrics).to have_received(:gauge).
           with('pending_db_messages_max_wait', 4.minutes.to_i, tags: ['topic:topic1'])
+        # topic2 has earliest message 5 minutes ago and last processed 3 minutes
+        # ago, so we should give it 3 minutes
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_max_wait', 3.minutes.to_i, tags: ['topic:topic2'])
+        # topic3 has no messages, so should get 0
         expect(Deimos.config.metrics).to have_received(:gauge).
-          with('pending_db_messages_max_wait', 5.minutes.to_i, tags: ['topic:topic2'])
+          with('pending_db_messages_max_wait', 0, tags: ['topic:topic3'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic1'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 3, tags: ['topic:topic2'])
+        expect(Deimos.config.metrics).to have_received(:gauge).
+          with('pending_db_messages_count', 0, tags: ['topic:topic3'])
       end
     end
-
-    it 'should send 0 if no messages' do
-      expect(Deimos.config.metrics).to receive(:gauge).
-        with('pending_db_messages_max_wait', 0)
-      producer.send_pending_metrics
-    end
   end
 
   example 'Full integration test' do
@@ -403,5 +478,4 @@ each_db_config(Deimos::Utils::DbProducer) do
                                                                 }
                                                               ])
   end
-
 end

data/spec/utils/deadlock_retry_spec.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Utils::DeadlockRetry do
+  include_context 'with widgets'
+
+  before(:each) do
+    allow(described_class).to receive(:sleep)
+  end
+
+  describe 'deadlock handling' do
+    let(:batch) { [{ key: 1, payload: { test_id: 'abc', some_int: 3 } }] }
+
+    it 'should retry deadlocks 3 times' do
+      # Should receive original attempt + 2 retries
+      expect(Widget).
+        to receive(:create).
+        and_raise(ActiveRecord::Deadlocked.new('Lock wait timeout exceeded')).
+        exactly(3).times
+
+      # After 3 tries, should let it bubble up
+      expect {
+        described_class.wrap do
+          Widget.create(test_id: 'abc')
+        end
+      }.to raise_error(ActiveRecord::Deadlocked)
+    end
+
+    it 'should stop retrying deadlocks after success' do
+      allow(Widget).
+        to receive(:create).
+        with(hash_including(test_id: 'first')).
+        and_call_original
+
+      # Fail on first attempt, succeed on second
+      expect(Widget).
+        to receive(:create).
+        with(hash_including(test_id: 'second')).
+        and_raise(ActiveRecord::Deadlocked.new('Deadlock found when trying to get lock')).
+        once.
+        ordered
+
+      expect(Widget).
+        to receive(:create).
+        with(hash_including(test_id: 'second')).
+        once.
+        ordered.
+        and_call_original
+
+      # Should not raise anything
+      described_class.wrap do
+        Widget.create(test_id: 'first')
+        Widget.create(test_id: 'second')
+      end
+
+      expect(Widget.all).to match_array([
+                                          have_attributes(test_id: 'first'),
+                                          have_attributes(test_id: 'second')
+                                        ])
+    end
+
+    it 'should not retry non-deadlock exceptions' do
+      expect(Widget).
+        to receive(:create).
+        and_raise(ActiveRecord::StatementInvalid.new('Oops!!')).
+        once
+
+      expect {
+        described_class.wrap do
+          Widget.create(test_id: 'abc')
+        end
+      }.to raise_error(ActiveRecord::StatementInvalid, 'Oops!!')
+    end
+  end
+end

data/spec/utils/lag_reporter_spec.rb

@@ -2,8 +2,11 @@
 
 describe Deimos::Utils::LagReporter do
 
+  let(:kafka_client) { instance_double(Kafka::Client) }
+  let(:partition1_tags) { %w(consumer_group:group1 partition:1 topic:my-topic) }
+  let(:partition2_tags) { %w(consumer_group:group1 partition:2 topic:my-topic) }
+
   before(:each) do
-    kafka_client = instance_double(Kafka::Client)
     allow(kafka_client).to receive(:last_offset_for).and_return(100)
     allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
     Deimos.configure { |c| c.consumers.report_lag = true }
@@ -20,38 +23,22 @@ describe Deimos::Utils::LagReporter do
       'heartbeat.consumer.kafka',
       group_id: 'group1', topic_partitions: { 'my-topic': [1] }
     )
-
   end
 
   it 'should report lag' do
     expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
-      with('consumer_lag', 95,
-           tags: %w(
-             consumer_group:group1
-             partition:1
-             topic:my-topic
-           ))
+      with('consumer_lag', 95, tags: partition1_tags)
     expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 80,
-           tags: %w(
-             consumer_group:group1
-             partition:2
-             topic:my-topic
-           ))
+      with('consumer_lag', 80, tags: partition2_tags)
     expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 0,
-           tags: %w(
-             consumer_group:group1
-             partition:2
-             topic:my-topic
-           ))
+      with('consumer_lag', 0, tags: partition2_tags)
     ActiveSupport::Notifications.instrument(
       'seek.consumer.kafka',
       offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
     )
     ActiveSupport::Notifications.instrument(
       'start_process_message.consumer.kafka',
-      offset_lag: 80, topic: 'my-topic', group_id: 'group1', partition: 2
+      offset: 20, topic: 'my-topic', group_id: 'group1', partition: 2
     )
     ActiveSupport::Notifications.instrument(
       'heartbeat.consumer.kafka',
@@ -59,8 +46,28 @@ describe Deimos::Utils::LagReporter do
     )
     ActiveSupport::Notifications.instrument(
       'start_process_batch.consumer.kafka',
-      offset_lag: 0, topic: 'my-topic', group_id: 'group1', partition: 2
+      last_offset: 100, topic: 'my-topic', group_id: 'group1', partition: 2
+    )
+    ActiveSupport::Notifications.instrument(
+      'heartbeat.consumer.kafka',
+      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
+    )
+  end
+
+  it 'should update lag after heartbeat' do
+    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
+      with('consumer_lag', 94, tags: partition2_tags)
+    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
+      with('consumer_lag', 95, tags: partition2_tags)
+    ActiveSupport::Notifications.instrument(
+      'seek.consumer.kafka',
+      offset: 6, topic: 'my-topic', group_id: 'group1', partition: 2
+    )
+    ActiveSupport::Notifications.instrument(
+      'heartbeat.consumer.kafka',
+      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
     )
+    allow(kafka_client).to receive(:last_offset_for).and_return(101)
     ActiveSupport::Notifications.instrument(
       'heartbeat.consumer.kafka',
       group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }

data/spec/utils/schema_controller_mixin_spec.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'deimos/utils/schema_controller_mixin'
+require 'deimos/schema_backends/avro_local'
+
+RSpec.describe Deimos::Utils::SchemaControllerMixin, type: :controller do
+
+  before(:each) do
+    Deimos.configure do
+      schema.backend(:avro_local)
+    end
+  end
+
+  controller(ActionController::Base) do
+    include Deimos::Utils::SchemaControllerMixin # rubocop:disable RSpec/DescribedClass
+
+    request_namespace 'com.my-namespace.request'
+    response_namespace 'com.my-namespace.response'
+    schemas :index, :show
+    schemas :update, request: 'UpdateRequest', response: 'UpdateResponse'
+
+    # :nodoc:
+    def index
+      render_schema({ 'response_id' => payload[:request_id] + ' mom' })
+    end
+
+    # :nodoc:
+    def show
+      render_schema({ 'response_id' => payload[:request_id] + ' dad' })
+    end
+
+    # :nodoc:
+    def update
+      render_schema({ 'update_response_id' => payload[:update_request_id] + ' sis' })
+    end
+  end
+
+  it 'should render the correct response for index' do
+    request_backend = Deimos.schema_backend(schema: 'Index',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'Index',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    get :index, body: request_backend.encode({ 'request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi mom' })
+  end
+
+  it 'should render the correct response for show' do
+    request_backend = Deimos.schema_backend(schema: 'Index',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'Index',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    get :show, params: { id: 1 }, body: request_backend.encode({ 'request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'response_id' => 'hi dad' })
+  end
+
+  it 'should render the correct response for update' do
+    request_backend = Deimos.schema_backend(schema: 'UpdateRequest',
+                                            namespace: 'com.my-namespace.request')
+    response_backend = Deimos.schema_backend(schema: 'UpdateResponse',
+                                             namespace: 'com.my-namespace.response')
+    request.content_type = 'avro/binary'
+    post :update, params: { id: 1 }, body: request_backend.encode({ 'update_request_id' => 'hi' })
+    expect(response_backend.decode(response.body)).to eq({ 'update_response_id' => 'hi sis' })
+  end
+
+end