deimos-kafka 1.0.0.pre.beta15

data/spec/kafka_source_spec.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require 'activerecord-import'
+
+# Wrap in a module so our classes don't leak out afterwards
+module KafkaSourceSpec
+  RSpec.describe Deimos::KafkaSource do
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:widgets, force: true) do |t|
+        t.integer(:widget_id)
+        t.string(:description)
+        t.string(:model_id, default: '')
+        t.string(:name)
+        t.timestamps
+      end
+      ActiveRecord::Base.connection.add_index(:widgets, :widget_id)
+
+      # Dummy producer which mimicks the behavior of a real producer
+      class WidgetProducer < Deimos::ActiveRecordProducer
+        topic 'my-topic'
+        namespace 'com.my-namespace'
+        schema 'Widget'
+        key_config field: :id
+      end
+
+      # Dummy producer which mimicks the behavior of a real producer
+      class WidgetProducerTheSecond < Deimos::ActiveRecordProducer
+        topic 'my-topic-the-second'
+        namespace 'com.my-namespace'
+        schema 'WidgetTheSecond'
+        key_config field: :id
+      end
+
+      # Dummy class we can include the mixin in. Has a backing table created
+      # earlier.
+      class Widget < ActiveRecord::Base
+        include Deimos::KafkaSource
+
+        # :nodoc:
+        def self.kafka_producers
+          [WidgetProducer, WidgetProducerTheSecond]
+        end
+      end
+      Widget.reset_column_information
+
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:widgets)
+    end
+
+    before(:each) do
+      Widget.delete_all
+    end
+
+    it 'should send events on creation, update, and deletion' do
+      widget = Widget.create!(widget_id: 1, name: 'widget')
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'widget',
+                                        id: widget.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, 1)
+      expect('my-topic-the-second').to have_sent({
+                                                   widget_id: 1,
+                                                   model_id: '',
+                                                   id: widget.id,
+                                                   created_at: anything,
+                                                   updated_at: anything
+                                                 }, 1)
+      widget.update_attribute(:name, 'widget 2')
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'widget 2',
+                                        id: widget.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, 1)
+      expect('my-topic-the-second').to have_sent({
+                                                   widget_id: 1,
+                                                   model_id: '',
+                                                   id: widget.id,
+                                                   created_at: anything,
+                                                   updated_at: anything
+                                                 }, 1)
+      widget.destroy
+      expect('my-topic').to have_sent(nil, 1)
+      expect('my-topic-the-second').to have_sent(nil, 1)
+    end
+
+    it 'should send events on import' do
+      widgets = (1..3).map do |i|
+        Widget.new(widget_id: i, name: "Widget #{i}")
+      end
+      Widget.import(widgets)
+      widgets = Widget.all
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: 'Widget 1',
+                                        id: widgets[0].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[0].id)
+      expect('my-topic').to have_sent({
+                                        widget_id: 2,
+                                        name: 'Widget 2',
+                                        id: widgets[1].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[1].id)
+      expect('my-topic').to have_sent({
+                                        widget_id: 3,
+                                        name: 'Widget 3',
+                                        id: widgets[2].id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widgets[2].id)
+    end
+
+    it 'should send events even if the save fails' do
+      widget = Widget.create!(widget_id: 1, name: 'widget')
+      expect('my-topic').to have_sent({
+                                        widget_id: 1,
+                                        name: widget.name,
+                                        id: widget.id,
+                                        created_at: anything,
+                                        updated_at: anything
+                                      }, widget.id)
+      clear_kafka_messages!
+      Widget.transaction do
+        widget.update_attribute(:name, 'widget 3')
+        raise ActiveRecord::Rollback
+      end
+      expect('my-topic').to have_sent(anything)
+    end
+
+    it 'should not send events if an unrelated field changes' do
+      widget = Widget.create!(widget_id: 1, name: 'widget')
+      clear_kafka_messages!
+      widget.update_attribute(:description, 'some description')
+      expect('my-topic').not_to have_sent(anything)
+    end
+
+    context 'with DB backend' do
+      before(:each) do
+        Deimos.configure do |config|
+          config.publish_backend = :db
+        end
+        setup_db(DB_OPTIONS.last) # sqlite
+        allow(Deimos::Producer).to receive(:produce_batch).and_call_original
+      end
+
+      it 'should save to the DB' do
+        Widget.create!(widget_id: 1, name: 'widget')
+        expect(Deimos::KafkaMessage.count).to eq(2) # 2 producers
+      end
+
+      it 'should not save with a rollback' do
+        Widget.transaction do
+          Widget.create!(widget_id: 1, name: 'widget')
+          raise ActiveRecord::Rollback
+        end
+        expect(Deimos::KafkaMessage.count).to eq(0)
+      end
+    end
+  end
+end

data/spec/kafka_topic_info_spec.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+each_db_config(Deimos::KafkaTopicInfo) do
+
+  it 'should lock the topic' do
+    expect(described_class.lock('my-topic', 'abc')).to be_truthy
+    expect(described_class.lock('my-topic', 'def')).to be_falsey
+    expect(described_class.lock('my-topic2', 'def')).to be_truthy
+    expect(described_class.count).to eq(2)
+    expect(described_class.first.locked_by).to eq('abc')
+    expect(described_class.last.locked_by).to eq('def')
+  end
+
+  it "should lock the topic if it's old" do
+    described_class.create!(topic: 'my-topic', locked_by: 'abc', error: true,
+                            locked_at: 2.minutes.ago)
+    expect(described_class.lock('my-topic', 'abc')).to be_truthy
+    expect(described_class.count).to eq(1)
+    expect(described_class.first.locked_by).to eq('abc')
+
+  end
+
+  it "should lock the topic if it's not currently locked" do
+    described_class.create!(topic: 'my-topic', locked_by: nil,
+                            locked_at: nil)
+    expect(described_class.lock('my-topic', 'abc')).to be_truthy
+    expect(described_class.count).to eq(1)
+    expect(described_class.first.locked_by).to eq('abc')
+  end
+
+  it "should not lock the topic if it's errored" do
+    described_class.create!(topic: 'my-topic', locked_by: nil,
+                            locked_at: nil, error: true)
+    expect(described_class.lock('my-topic', 'abc')).to be_falsey
+    expect(described_class.count).to eq(1)
+    expect(described_class.first.locked_by).to eq(nil)
+  end
+
+  specify '#clear_lock' do
+    described_class.create!(topic: 'my-topic', locked_by: 'abc',
+                            locked_at: 10.seconds.ago, error: true, retries: 1)
+    described_class.create!(topic: 'my-topic2', locked_by: 'def',
+                            locked_at: 10.seconds.ago, error: true, retries: 1)
+    described_class.clear_lock('my-topic', 'abc')
+    expect(described_class.count).to eq(2)
+    record = described_class.first
+    expect(record.locked_by).to eq(nil)
+    expect(record.locked_at).to eq(nil)
+    expect(record.error).to eq(false)
+    expect(record.retries).to eq(0)
+    record = described_class.last
+    expect(record.locked_by).not_to eq(nil)
+    expect(record.locked_at).not_to eq(nil)
+    expect(record.error).not_to eq(false)
+    expect(record.retries).not_to eq(0)
+  end
+
+  specify '#register_error' do
+    freeze_time do
+      described_class.create!(topic: 'my-topic', locked_by: 'abc',
+                              locked_at: 10.seconds.ago)
+      described_class.create!(topic: 'my-topic2', locked_by: 'def',
+                              locked_at: 10.seconds.ago, error: true, retries: 1)
+      described_class.register_error('my-topic', 'abc')
+      record = described_class.first
+      expect(record.locked_by).to be_nil
+      expect(record.locked_at).to eq(Time.zone.now)
+      expect(record.error).to be_truthy
+      expect(record.retries).to eq(1)
+
+      described_class.register_error('my-topic2', 'def')
+      record = described_class.last
+      expect(record.error).to be_truthy
+      expect(record.retries).to eq(2)
+      expect(record.locked_at).to eq(Time.zone.now)
+    end
+  end
+
+  specify '#heartbeat' do
+    freeze_time do
+      described_class.create!(topic: 'my-topic', locked_by: 'abc',
+                              locked_at: 10.seconds.ago)
+      described_class.heartbeat('my-topic', 'abc')
+      expect(described_class.last.locked_at).to eq(Time.zone.now)
+    end
+  end
+
+end

data/spec/phobos.bad_db.yml

@@ -0,0 +1,73 @@
+logger:
+  # Optional log file, set to false or remove to disable it
+  file: log/phobos.log
+  # Optional output format for stdout, default is false (human readable).
+  # Set to true to enable json output.
+  stdout_json: false
+  level: debug
+  # Comment the block to disable ruby-kafka logs
+  ruby_kafka:
+    level: debug
+
+kafka:
+  # identifier for this application
+  client_id: phobos
+  # timeout setting for connecting to brokers
+  connect_timeout: 15
+  # timeout setting for socket connections
+  socket_timeout: 15
+
+producer:
+  # number of seconds a broker can wait for replicas to acknowledge
+  # a write before responding with a timeout
+  ack_timeout: 5
+  # number of replicas that must acknowledge a write, or `:all`
+  # if all in-sync replicas must acknowledge
+  required_acks: 1
+  # number of retries that should be attempted before giving up sending
+  # messages to the cluster. Does not include the original attempt
+  max_retries: 2
+  # number of seconds to wait between retries
+  retry_backoff: 1
+  # number of messages allowed in the buffer before new writes will
+  # raise {BufferOverflow} exceptions
+  max_buffer_size: 10000
+  # maximum size of the buffer in bytes. Attempting to produce messages
+  # when the buffer reaches this size will result in {BufferOverflow} being raised
+  max_buffer_bytesize: 10000000
+  # name of the compression codec to use, or nil if no compression should be performed.
+  # Valid codecs: `:snappy` and `:gzip`
+  compression_codec:
+  # number of messages that needs to be in a message set before it should be compressed.
+  # Note that message sets are per-partition rather than per-topic or per-producer
+  compression_threshold: 1
+  # maximum number of messages allowed in the queue. Only used for async_producer
+  max_queue_size: 10000
+  # if greater than zero, the number of buffered messages that will automatically
+  # trigger a delivery. Only used for async_producer
+  delivery_threshold: 0
+  # if greater than zero, the number of seconds between automatic message
+  # deliveries. Only used for async_producer
+  delivery_interval: 0
+
+consumer:
+  # number of seconds after which, if a client hasn't contacted the Kafka cluster,
+  # it will be kicked out of the group
+  session_timeout: 300
+  # interval between offset commits, in seconds
+  offset_commit_interval: 10
+  # number of messages that can be processed before their offsets are committed.
+  # If zero, offset commits are not triggered by message processing
+  offset_commit_threshold: 0
+  # interval between heartbeats; must be less than the session window
+  heartbeat_interval: 10
+
+backoff:
+  min_ms: 1000
+  max_ms: 60000
+
+listeners:
+  - handler: ConsumerTest::MyConsumer
+    topic: my_consume_topic
+    group_id: my_group_id
+    max_bytes_per_partition: 524288 # 512 KB

data/spec/phobos.yml

@@ -0,0 +1,73 @@
+logger:
+  # Optional log file, set to false or remove to disable it
+  file: log/phobos.log
+  # Optional output format for stdout, default is false (human readable).
+  # Set to true to enable json output.
+  stdout_json: false
+  level: debug
+  # Comment the block to disable ruby-kafka logs
+  ruby_kafka:
+    level: debug
+
+kafka:
+  # identifier for this application
+  client_id: phobos
+  # timeout setting for connecting to brokers
+  connect_timeout: 15
+  # timeout setting for socket connections
+  socket_timeout: 15
+
+producer:
+  # number of seconds a broker can wait for replicas to acknowledge
+  # a write before responding with a timeout
+  ack_timeout: 5
+  # number of replicas that must acknowledge a write, or `:all`
+  # if all in-sync replicas must acknowledge
+  required_acks: :all
+  # number of retries that should be attempted before giving up sending
+  # messages to the cluster. Does not include the original attempt
+  max_retries: 2
+  # number of seconds to wait between retries
+  retry_backoff: 1
+  # number of messages allowed in the buffer before new writes will
+  # raise {BufferOverflow} exceptions
+  max_buffer_size: 10000
+  # maximum size of the buffer in bytes. Attempting to produce messages
+  # when the buffer reaches this size will result in {BufferOverflow} being raised
+  max_buffer_bytesize: 10000000
+  # name of the compression codec to use, or nil if no compression should be performed.
+  # Valid codecs: `:snappy` and `:gzip`
+  compression_codec:
+  # number of messages that needs to be in a message set before it should be compressed.
+  # Note that message sets are per-partition rather than per-topic or per-producer
+  compression_threshold: 1
+  # maximum number of messages allowed in the queue. Only used for async_producer
+  max_queue_size: 10000
+  # if greater than zero, the number of buffered messages that will automatically
+  # trigger a delivery. Only used for async_producer
+  delivery_threshold: 0
+  # if greater than zero, the number of seconds between automatic message
+  # deliveries. Only used for async_producer
+  delivery_interval: 0
+
+consumer:
+  # number of seconds after which, if a client hasn't contacted the Kafka cluster,
+  # it will be kicked out of the group
+  session_timeout: 300
+  # interval between offset commits, in seconds
+  offset_commit_interval: 10
+  # number of messages that can be processed before their offsets are committed.
+  # If zero, offset commits are not triggered by message processing
+  offset_commit_threshold: 0
+  # interval between heartbeats; must be less than the session window
+  heartbeat_interval: 10
+
+backoff:
+  min_ms: 1000
+  max_ms: 60000
+
+listeners:
+  - handler: ConsumerTest::MyConsumer
+    topic: my_consume_topic
+    group_id: my_group_id
+    max_bytes_per_partition: 524288 # 512 KB

data/spec/producer_spec.rb

@@ -0,0 +1,397 @@
+# frozen_string_literal: true
+
+# :nodoc:
+module ProducerTest
+  describe Deimos::Producer do
+
+    prepend_before(:each) do
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config field: 'test_id'
+      end
+      stub_const('MyProducer', producer_class)
+
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config plain: true
+      end
+      stub_const('MyProducerWithID', producer_class)
+
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config plain: true
+        # :nodoc:
+        def self.partition_key(payload)
+          payload[:payload_key] ? payload[:payload_key] + '1' : nil
+        end
+      end
+      stub_const('MyNonEncodedProducer', producer_class)
+
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic2'
+        key_config none: true
+      end
+      stub_const('MyNoKeyProducer', producer_class)
+
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic2'
+        key_config schema: 'MySchema-key'
+      end
+      stub_const('MySchemaProducer', producer_class)
+
+      producer_class = Class.new(Deimos::Producer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+      end
+      stub_const('MyErrorProducer', producer_class)
+
+    end
+
+    it 'should fail on invalid message with error handler' do
+      subscriber = Deimos.subscribe('produce') do |event|
+        expect(event.payload[:payloads]).to eq([{ 'invalid' => 'key' }])
+      end
+      expect { MyProducer.publish('invalid' => 'key', :payload_key => 'key') }.
+        to raise_error(Avro::SchemaValidator::ValidationError)
+      Deimos.unsubscribe(subscriber)
+    end
+
+    it 'should produce a message' do
+      expect(described_class).to receive(:produce_batch).once.with(
+        Deimos::Backends::KafkaAsync,
+        [
+          Deimos::Message.new({ 'test_id' => 'foo', 'some_int' => 123 },
+                              MyProducer,
+                              topic: 'my-topic',
+                              partition_key: 'foo',
+                              key: 'foo'),
+          Deimos::Message.new({ 'test_id' => 'bar', 'some_int' => 124 },
+                              MyProducer,
+                              topic: 'my-topic',
+                              partition_key: 'bar',
+                              key: 'bar')
+        ]
+      )
+
+      MyProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123 },
+         { 'test_id' => 'bar', 'some_int' => 124 }]
+      )
+      expect(
+        was_message_sent?({ 'test_id' => 'foo', 'some_int' => 123 }, 'my-topic')
+      ).to be(true)
+      expect(
+        was_message_sent?({ 'test_id' => 'foo', 'some_int' => 123 }, 'your-topic')
+      ).to be(false)
+      expect(
+        was_message_sent?({ 'test_id' => 'foo2', 'some_int' => 123 }, 'my-topic')
+      ).to be(false)
+    end
+
+    it 'should add a message ID' do
+      payload = { 'test_id' => 'foo',
+                  'some_int' => 123,
+                  'message_id' => a_kind_of(String),
+                  'timestamp' => a_kind_of(String) }
+      expect(described_class).to receive(:produce_batch).once do |_, messages|
+        expect(messages.size).to eq(1)
+        expect(messages[0].to_h).
+          to match(
+            payload: payload,
+            topic: 'my-topic',
+            partition_key: 'key',
+            metadata: {
+              producer_name: 'MyProducerWithID',
+              decoded_payload: payload
+            },
+            key: 'key'
+          )
+      end
+      MyProducerWithID.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'key' }]
+      )
+    end
+
+    it 'should not publish if publish disabled' do
+      expect(described_class).not_to receive(:produce_batch)
+      Deimos.configure { |c| c.disable_producers = true }
+      MyProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123 },
+         { 'test_id' => 'bar', 'some_int' => 124 }]
+      )
+      expect(MyProducer.topic).not_to have_sent(anything)
+    end
+
+    it 'should not send messages if inside a disable_producers block' do
+      Deimos.disable_producers do
+        MyProducer.publish_list(
+          [{ 'test_id' => 'foo', 'some_int' => 123 },
+           { 'test_id' => 'bar', 'some_int' => 124 }]
+        )
+      end
+      expect(MyProducer.topic).not_to have_sent(anything)
+      MyProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123 },
+         { 'test_id' => 'bar', 'some_int' => 124 }]
+      )
+      expect(MyProducer.topic).to have_sent(anything)
+    end
+
+    it 'should produce to a prefixed topic' do
+      Deimos.configure { |c| c.producer_topic_prefix = 'prefix.' }
+      payload = { 'test_id' => 'foo', 'some_int' => 123 }
+      expect(described_class).to receive(:produce_batch).once do |_, messages|
+        expect(messages.size).to eq(1)
+        expect(messages[0].to_h).
+          to eq(
+            payload: payload,
+            topic: 'prefix.my-topic',
+            partition_key: 'foo',
+            metadata: {
+              producer_name: 'MyProducer',
+              decoded_payload: payload
+            },
+            key: 'foo'
+          )
+      end
+
+      MyProducer.publish_list([payload])
+      Deimos.configure { |c| c.producer_topic_prefix = nil }
+      expect(described_class).to receive(:produce_batch).once do |_, messages|
+        expect(messages.size).to eq(1)
+        expect(messages[0].to_h).
+          to eq(
+            payload: payload,
+            topic: 'my-topic',
+            partition_key: 'foo',
+            metadata: {
+              producer_name: 'MyProducer',
+              decoded_payload: payload
+            },
+            key: 'foo'
+          )
+      end
+
+      MyProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123 }]
+      )
+    end
+
+    it 'should encode the key' do
+      encoder = instance_double(Deimos::AvroDataEncoder)
+      allow(Deimos::Message).to receive(:new).and_wrap_original do |m, hash, producer|
+        message = m.call(hash, producer)
+        allow(message).to receive(:add_fields)
+        allow(message).to receive(:coerce_fields)
+        message
+      end
+      allow(MyProducer).to receive(:encoder).and_return(encoder).at_least(:once)
+      allow(encoder).to receive(:avro_schema)
+      expect(encoder).to receive(:encode_key).with('test_id', 'foo', 'my-topic-key')
+      expect(encoder).to receive(:encode_key).with('test_id', 'bar', 'my-topic-key')
+      expect(encoder).to receive(:encode).with({
+                                                 'test_id' => 'foo',
+                                                 'some_int' => 123
+                                               }, { topic: 'my-topic-value' })
+      expect(encoder).to receive(:encode).with({
+                                                 'test_id' => 'bar',
+                                                 'some_int' => 124
+                                               }, { topic: 'my-topic-value' })
+
+      MyProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123 },
+         { 'test_id' => 'bar', 'some_int' => 124 }]
+      )
+    end
+
+    it 'should not encode with plaintext key' do
+      key_encoder = Deimos::AvroDataEncoder.new(
+        schema: 'MySchema',
+        namespace: 'com.my-namespace'
+      )
+      allow(key_encoder).to receive(:encode)
+      allow(MyNonEncodedProducer).to receive(:encoder).and_return(key_encoder)
+      expect(key_encoder).not_to receive(:encode_key)
+
+      MyNonEncodedProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => 'foo_key' },
+         { 'test_id' => 'bar', 'some_int' => 124, :payload_key => 'bar_key' }]
+      )
+    end
+
+    it 'should encode with a schema' do
+
+      encoder = instance_double(Deimos::AvroDataEncoder)
+      expect(MySchemaProducer).to receive(:key_encoder).and_return(encoder).
+        at_least(:once)
+      expect(encoder).to receive(:encode).with({ 'test_id' => 'foo_key' },
+                                               { topic: 'my-topic2-key' })
+      expect(encoder).to receive(:encode).with({ 'test_id' => 'bar_key' },
+                                               { topic: 'my-topic2-key' })
+
+      MySchemaProducer.publish_list(
+        [{ 'test_id' => 'foo', 'some_int' => 123,
+           :payload_key => { 'test_id' => 'foo_key' } },
+         { 'test_id' => 'bar', 'some_int' => 124,
+           :payload_key => { 'test_id' => 'bar_key' } }]
+      )
+    end
+
+    it 'should error with nothing set' do
+      expect {
+        MyErrorProducer.publish_list(
+          [{ 'test_id' => 'foo', 'some_int' => 123, :payload_key => '123' }]
+        )
+      }.to raise_error('No key config given - if you are not encoding keys, please use `key_config plain: true`')
+    end
+
+    it 'should error if no key given and none is not the config' do
+      expect {
+        MyNonEncodedProducer.publish_list(
+          [{ 'test_id' => 'foo', 'some_int' => 123 }]
+        )
+      }      .to raise_error('No key given but a key is required! Use `key_config none: true` to avoid using keys.')
+    end
+
+    it 'should allow nil keys if none: true is configured' do
+      expect {
+        MyNoKeyProducer.publish_list(
+          [{ 'test_id' => 'foo', 'some_int' => 123 }]
+        )
+      }      .not_to raise_error
+    end
+
+    it 'should use a partition key' do
+      MyNonEncodedProducer.publish_list([{
+                                          'test_id' => 'foo',
+                                          'some_int' => 123,
+                                          :payload_key => '123'
+                                        },
+                                         {
+                                           'test_id' => 'bar',
+                                           'some_int' => 456,
+                                           :payload_key => '456'
+                                         }])
+      expect(MyNonEncodedProducer.topic).to have_sent({
+                                                        'test_id' => 'foo',
+                                                        'some_int' => 123
+                                                      }, '123', '1231')
+      expect(MyNonEncodedProducer.topic).to have_sent({
+                                                        'test_id' => 'bar',
+                                                        'some_int' => 456
+                                                      }, '456', '4561')
+    end
+
+    describe 'disabling' do
+      it 'should disable globally' do
+        Deimos.disable_producers do
+          Deimos.disable_producers do # test nested
+            MyProducer.publish(
+              'test_id' => 'foo',
+              'some_int' => 123,
+              :payload_key => '123'
+            )
+            MyProducerWithID.publish(
+              'test_id' => 'foo', 'some_int' => 123
+            )
+            expect('my-topic').not_to have_sent(anything)
+            expect(Deimos).to be_producers_disabled
+            expect(Deimos).to be_producers_disabled([MyProducer])
+          end
+        end
+
+        MyProducerWithID.publish(
+          'test_id' => 'foo', 'some_int' => 123, :payload_key => 123
+        )
+        expect('my-topic').
+          to have_sent('test_id' => 'foo', 'some_int' => 123,
+                       'message_id' => anything, 'timestamp' => anything)
+        expect(Deimos).not_to be_producers_disabled
+        expect(Deimos).not_to be_producers_disabled([MyProducer])
+      end
+
+      it 'should disable a single producer' do
+        Deimos.disable_producers(MyProducer) do # test nested
+          Deimos.disable_producers(MyProducer) do
+            MySchemaProducer.publish(
+              'test_id' => 'foo', 'some_int' => 123,
+              :payload_key => { 'test_id' => 'foo_key' }
+            )
+            MyProducer.publish(
+              'test_id' => 'foo',
+              'some_int' => 123,
+              :payload_key => '123'
+            )
+            expect('my-topic').not_to have_sent(anything)
+            expect('my-topic2').to have_sent('test_id' => 'foo', 'some_int' => 123)
+            expect(Deimos).not_to be_producers_disabled
+            expect(Deimos).to be_producers_disabled(MyProducer)
+            expect(Deimos).not_to be_producers_disabled(MySchemaProducer)
+          end
+        end
+        expect(Deimos).not_to be_producers_disabled
+        expect(Deimos).not_to be_producers_disabled(MyProducer)
+        expect(Deimos).not_to be_producers_disabled(MySchemaProducer)
+        MyProducer.publish(
+          'test_id' => 'foo',
+          'some_int' => 123,
+          :payload_key => '123'
+        )
+        expect('my-topic').
+          to have_sent('test_id' => 'foo', 'some_int' => 123)
+      end
+
+    end
+
+    describe '#determine_backend_class' do
+      it 'should return kafka_async if sync is false' do
+        expect(described_class.determine_backend_class(false, false)).
+          to eq(Deimos::Backends::KafkaAsync)
+        expect(described_class.determine_backend_class(nil, false)).
+          to eq(Deimos::Backends::KafkaAsync)
+      end
+
+      it 'should return kafka if sync is true' do
+        expect(described_class.determine_backend_class(true, false)).
+          to eq(Deimos::Backends::Kafka)
+      end
+
+      it 'should return db if db is set' do
+        allow(Deimos.config).to receive(:publish_backend).and_return(:db)
+        expect(described_class.determine_backend_class(true, false)).
+          to eq(Deimos::Backends::Db)
+        expect(described_class.determine_backend_class(false, false)).
+          to eq(Deimos::Backends::Db)
+      end
+
+      it 'should return kafka if force_send is true' do
+        allow(Deimos.config).to receive(:publish_backend).and_return(:db)
+        expect(described_class.determine_backend_class(true, true)).
+          to eq(Deimos::Backends::Kafka)
+        expect(described_class.determine_backend_class(false, true)).
+          to eq(Deimos::Backends::KafkaAsync)
+      end
+
+      it 'should use the default sync if set' do
+        expect(described_class.determine_backend_class(true, true)).
+          to eq(Deimos::Backends::Kafka)
+        expect(described_class.determine_backend_class(false, true)).
+          to eq(Deimos::Backends::KafkaAsync)
+        expect(described_class.determine_backend_class(nil, true)).
+          to eq(Deimos::Backends::Kafka)
+      end
+    end
+
+  end
+end