deimos-kafka 1.0.0.pre.beta15

data/spec/active_record_producer_spec.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+# :nodoc:
+module ActiveRecordProducerTest
+  describe Deimos::ActiveRecordProducer do
+
+    before(:all) do
+      ActiveRecord::Base.connection.create_table(:widgets) do |t|
+        t.string(:test_id)
+        t.integer(:some_int)
+        t.boolean(:some_bool)
+        t.timestamps
+      end
+
+      # :nodoc:
+      class Widget < ActiveRecord::Base
+        # @return [String]
+        def generated_id
+          'generated_id'
+        end
+      end
+    end
+
+    after(:all) do
+      ActiveRecord::Base.connection.drop_table(:widgets)
+    end
+
+    prepend_before(:each) do
+
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        topic 'my-topic'
+        key_config none: true
+      end
+      stub_const('MyProducer', producer_class)
+
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithBooleans'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-boolean'
+        key_config none: true
+      end
+      stub_const('MyBooleanProducer', producer_class)
+
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithId'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-id'
+        key_config none: true
+        record_class Widget
+
+        # :nodoc:
+        def self.generate_payload(attrs, widget)
+          super.merge(message_id: widget.generated_id)
+        end
+
+      end
+      stub_const('MyProducerWithID', producer_class)
+
+      producer_class = Class.new(Deimos::ActiveRecordProducer) do
+        schema 'MySchemaWithUniqueId'
+        namespace 'com.my-namespace'
+        topic 'my-topic-with-unique-id'
+        key_config field: :id
+        record_class Widget
+      end
+      stub_const('MyProducerWithUniqueID', producer_class)
+    end
+
+    it 'should send events correctly' do
+      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 3))
+      expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+    end
+
+    it 'should coerce values' do
+      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: '3'))
+      MyProducer.send_event(Widget.new(test_id: 'abc', some_int: 4.5))
+      expect('my-topic').to have_sent(test_id: 'abc', some_int: 3)
+      expect('my-topic').to have_sent(test_id: 'abc', some_int: 4)
+      expect {
+        MyProducer.send_event(Widget.new(test_id: 'abc', some_int: nil))
+      }.to raise_error(Avro::SchemaValidator::ValidationError)
+
+      MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: nil))
+      MyBooleanProducer.send_event(Widget.new(test_id: 'abc', some_bool: true))
+      expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: false)
+      expect('my-topic-with-boolean').to have_sent(test_id: 'abc', some_bool: true)
+    end
+
+    it 'should be able to call the record' do
+      widget = Widget.create!(test_id: 'abc2', some_int: 3)
+      MyProducerWithID.send_event(id: widget.id, test_id: 'abc2', some_int: 3)
+      expect('my-topic-with-id').to have_sent(
+        test_id: 'abc2',
+        some_int: 3,
+        message_id: 'generated_id',
+        timestamp: anything
+      )
+    end
+
+    specify '#watched_attributes' do
+      expect(MyProducer.watched_attributes).to eq(%w(test_id some_int))
+    end
+
+  end
+end

data/spec/avro_data_decoder_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+describe Deimos::AvroDataDecoder do
+
+  let(:decoder) do
+    decoder = described_class.new(schema: 'MySchema',
+                                  namespace: 'com.my-namespace')
+    allow(decoder).to(receive(:decode)) { |payload| payload }
+    decoder
+  end
+
+  it 'should decode a key' do
+    # reset stub from TestHelpers
+    allow(described_class).to receive(:new).and_call_original
+    expect(decoder.decode_key({ 'test_id' => '123' }, 'test_id')).to eq('123')
+  end
+
+end

data/spec/avro_data_encoder_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'avro_turf/messaging'
+
+describe Deimos::AvroDataEncoder do
+
+  let(:encoder) do
+    encoder = described_class.new(schema: 'MySchema',
+                                  namespace: 'com.my-namespace')
+    allow(encoder).to(receive(:encode)) { |payload| payload }
+    encoder
+  end
+
+  specify 'generate_key_schema' do
+    expect_any_instance_of(AvroTurf::SchemaStore).
+      to receive(:add_schema).with(
+        'type' => 'record',
+        'name' => 'MySchema_key',
+        'namespace' => 'com.my-namespace',
+        'doc' => 'Key for com.my-namespace.MySchema',
+        'fields' => [
+          {
+            'name' => 'test_id',
+            'type' => 'string'
+          }
+        ]
+      )
+    encoder.send(:_generate_key_schema, 'test_id')
+  end
+
+  it 'should encode a key' do
+    # reset stub from TestHelpers
+    allow(described_class).to receive(:new).and_call_original
+    expect(encoder.encode_key('test_id', '123')).to eq('test_id' => '123')
+  end
+
+end

data/spec/backends/db_spec.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+each_db_config(Deimos::Backends::Db) do
+  include_context 'with publish_backend'
+
+  it 'should save to the database' do
+    described_class.publish(producer_class: MyProducer, messages: messages)
+    records = Deimos::KafkaMessage.all
+    expect(records.size).to eq(3)
+    expect(records[0].attributes.to_h).to include(
+      'message' => '{"foo"=>1}',
+      'topic' => 'my-topic',
+      'key' => 'foo1'
+    )
+    expect(records[1].attributes.to_h).to include(
+      'message' => '{"foo"=>2}',
+      'topic' => 'my-topic',
+      'key' => 'foo2'
+    )
+    expect(records[2].attributes.to_h).to include(
+      'message' => '{"foo"=>3}',
+      'topic' => 'my-topic',
+      'key' => 'foo3'
+    )
+  end
+  it 'should add to non-keyed messages' do
+    described_class.publish(producer_class: MyNoKeyProducer,
+                            messages: messages)
+    expect(Deimos::KafkaMessage.count).to eq(3)
+    described_class.publish(producer_class: MyNoKeyProducer,
+                            messages: [messages.first])
+    expect(Deimos::KafkaMessage.count).to eq(4)
+
+  end
+end

data/spec/backends/kafka_async_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Backends::KafkaAsync do
+  include_context 'with publish_backend'
+  it 'should publish to Kafka asynchronously' do
+    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
+    expect(producer).to receive(:async_publish_list).with(messages.map(&:encoded_hash))
+    expect(described_class).to receive(:producer).and_return(producer)
+    described_class.publish(producer_class: MyProducer, messages: messages)
+  end
+end

data/spec/backends/kafka_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Backends::Kafka do
+  include_context 'with publish_backend'
+  it 'should publish to Kafka synchronously' do
+    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
+    expect(producer).to receive(:publish_list).with(messages.map(&:encoded_hash))
+    expect(described_class).to receive(:producer).and_return(producer)
+    described_class.publish(producer_class: MyProducer, messages: messages)
+  end
+end

data/spec/consumer_spec.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+# :nodoc:
+module ConsumerTest
+  describe Deimos::Consumer do
+
+    prepend_before(:each) do
+      # :nodoc:
+      consumer_class = Class.new(Deimos::Consumer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config field: 'test_id'
+
+        # :nodoc:
+        def consume(_payload, _metadata)
+          raise 'This should not be called unless call_original is set'
+        end
+      end
+      stub_const('ConsumerTest::MyConsumer', consumer_class)
+    end
+
+    it 'should consume a message' do
+      test_consume_message(MyConsumer,
+                           'test_id' => 'foo',
+                           'some_int' => 123) do |payload, _metadata|
+                             expect(payload['test_id']).to eq('foo')
+                           end
+    end
+
+    it 'should consume a message on a topic' do
+      test_consume_message('my_consume_topic',
+                           'test_id' => 'foo',
+                           'some_int' => 123) do |payload, _metadata|
+                             expect(payload['test_id']).to eq('foo')
+                           end
+    end
+
+    it 'should fail on invalid message' do
+      test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
+    end
+
+    it 'should fail on message with extra fields' do
+      test_consume_invalid_message(MyConsumer,
+                                   'test_id' => 'foo',
+                                   'some_int' => 123,
+                                   'extra_field' => 'field name')
+    end
+
+    it 'should not fail when before_consume fails without reraising errors' do
+      Deimos.configure { |config| config.reraise_consumer_errors = false }
+      expect {
+        test_consume_message(
+          MyConsumer,
+          { 'test_id' => 'foo',
+            'some_int' => 123 },
+          { skip_expectation: true }
+        ) { raise 'OH NOES' }
+      } .not_to raise_error
+    end
+
+    it 'should not fail when consume fails without reraising errors' do
+      Deimos.configure { |config| config.reraise_consumer_errors = false }
+      expect {
+        test_consume_message(
+          MyConsumer,
+          { 'invalid' => 'key' },
+          { skip_expectation: true }
+        )
+      } .not_to raise_error
+    end
+
+    it 'should call original' do
+      expect {
+        test_consume_message(MyConsumer,
+                             { 'test_id' => 'foo', 'some_int' => 123 },
+                             { call_original: true })
+      }.to raise_error('This should not be called unless call_original is set')
+    end
+
+    describe 'decode_key' do
+
+      it 'should use the key field in the value if set' do
+        # actual decoding is disabled in test
+        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
+        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
+      end
+
+      it 'should use the key schema if set' do
+        consumer_class = Class.new(Deimos::Consumer) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config schema: 'MySchema_key'
+        end
+        stub_const('ConsumerTest::MySchemaConsumer', consumer_class)
+        expect(MyConsumer.new.decode_key('test_id' => '123')).to eq('123')
+        expect { MyConsumer.new.decode_key(123) }.to raise_error(NoMethodError)
+      end
+
+      it 'should not encode if plain is set' do
+        consumer_class = Class.new(Deimos::Consumer) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+        end
+        stub_const('ConsumerTest::MyNonEncodedConsumer', consumer_class)
+        expect(MyNonEncodedConsumer.new.decode_key('123')).to eq('123')
+      end
+
+      it 'should error with nothing set' do
+        consumer_class = Class.new(Deimos::Consumer) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+        end
+        stub_const('ConsumerTest::MyErrorConsumer', consumer_class)
+        expect { MyErrorConsumer.new.decode_key('123') }.
+          to raise_error('No key config given - if you are not decoding keys, please use `key_config plain: true`')
+      end
+
+    end
+
+    describe 'timestamps' do
+      before(:each) do
+        # :nodoc:
+        consumer_class = Class.new(Deimos::Consumer) do
+          schema 'MySchemaWithDateTimes'
+          namespace 'com.my-namespace'
+          key_config plain: true
+
+          # :nodoc:
+          def consume(_payload, _metadata)
+            raise 'This should not be called unless call_original is set'
+          end
+        end
+        stub_const('ConsumerTest::MyConsumer', consumer_class)
+        stub_consumer(consumer_class)
+      end
+
+      it 'should consume a message' do
+        expect(Deimos.config.metrics).to receive(:histogram).twice
+        test_consume_message('my_consume_topic',
+                             'test_id' => 'foo',
+                             'some_int' => 123,
+                             'updated_at' => Time.now.to_i,
+                             'timestamp' => 2.minutes.ago.to_s) do |payload, _metadata|
+                               expect(payload['test_id']).to eq('foo')
+                             end
+      end
+
+      it 'should fail nicely when timestamp wrong format' do
+        expect(Deimos.config.metrics).to receive(:histogram).twice
+        test_consume_message('my_consume_topic',
+                             'test_id' => 'foo',
+                             'some_int' => 123,
+                             'updated_at' => Time.now.to_i,
+                             'timestamp' => 'dffdf') do |payload, _metadata|
+                               expect(payload['test_id']).to eq('foo')
+                             end
+        test_consume_message('my_consume_topic',
+                             'test_id' => 'foo',
+                             'some_int' => 123,
+                             'updated_at' => Time.now.to_i,
+                             'timestamp' => '') do |payload, _metadata|
+                               expect(payload['test_id']).to eq('foo')
+                             end
+      end
+
+    end
+  end
+end

data/spec/deimos_spec.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+describe Deimos do
+
+  it 'should have a version number' do
+    expect(Deimos::VERSION).not_to be_nil
+  end
+
+  specify 'configure' do
+    phobos_configuration = { 'logger' =>
+  { 'file' => 'log/phobos.log',
+    'stdout_json' => false,
+    'level' => 'debug',
+    'ruby_kafka' =>
+     { 'level' => 'debug' } },
+                             'kafka' =>
+  { 'client_id' => 'phobos',
+    'connect_timeout' => 15,
+    'socket_timeout' => 15,
+    'seed_brokers' => 'my_seed_broker.com',
+    'ssl_ca_cert' => 'my_ssl_ca_cert',
+    'ssl_client_cert' => 'my_ssl_client_cert',
+    'ssl_client_cert_key' => 'my_ssl_client_cert_key' },
+                             'producer' =>
+  { 'ack_timeout' => 5,
+    'required_acks' => :all,
+    'max_retries' => 2,
+    'retry_backoff' => 1,
+    'max_buffer_size' => 10_000,
+    'max_buffer_bytesize' => 10_000_000,
+    'compression_codec' => nil,
+    'compression_threshold' => 1,
+    'max_queue_size' => 10_000,
+    'delivery_threshold' => 0,
+    'delivery_interval' => 0 },
+                             'consumer' =>
+  { 'session_timeout' => 300,
+    'offset_commit_interval' => 10,
+    'offset_commit_threshold' => 0,
+    'heartbeat_interval' => 10 },
+                             'backoff' =>
+                              { 'min_ms' => 1000,
+                                'max_ms' => 60_000 },
+                             'listeners' => [
+                               { 'handler' => 'ConsumerTest::MyConsumer',
+                                 'topic' => 'my_consume_topic',
+                                 'group_id' => 'my_group_id',
+                                 'max_bytes_per_partition' => 524_288 }
+                             ],
+                             'custom_logger' => nil,
+                             'custom_kafka_logger' => nil }
+
+    expect(Phobos).to receive(:configure).with(phobos_configuration)
+    allow(described_class).to receive(:ssl_var_contents) { |key| key }
+    described_class.configure do |config|
+      config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
+      config.seed_broker = 'my_seed_broker.com'
+      config.ssl_enabled = true
+      config.ssl_ca_cert = 'my_ssl_ca_cert'
+      config.ssl_client_cert = 'my_ssl_client_cert'
+      config.ssl_client_cert_key = 'my_ssl_client_cert_key'
+    end
+  end
+
+  it 'should error if required_acks is not all' do
+    expect {
+      described_class.configure do |config|
+        config.publish_backend = :db
+        config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.bad_db.yml')
+      end
+    }.to raise_error('Cannot set publish_backend to :db unless required_acks is set to ":all" in phobos.yml!')
+  end
+
+  describe '#start_db_backend!' do
+    before(:each) do
+      allow(described_class).to receive(:run_db_backend_in_thread)
+    end
+
+    it 'should start if backend is db and num_producer_threads is > 0' do
+      signal_handler = instance_double(Deimos::Utils::SignalHandler)
+      allow(signal_handler).to receive(:run!)
+      expect(Deimos::Utils::SignalHandler).to receive(:new) do |executor|
+        expect(executor.runners.size).to eq(2)
+        signal_handler
+      end
+      described_class.configure do |config|
+        config.publish_backend = :db
+      end
+      described_class.start_db_backend!(thread_count: 2)
+    end
+
+    it 'should not start if backend is not db' do
+      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      described_class.configure do |config|
+        config.publish_backend = :kafka
+      end
+      described_class.start_db_backend!(thread_count: 2)
+    end
+
+    it 'should not start if num_producer_threads is nil' do
+      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      described_class.configure do |config|
+        config.publish_backend = :db
+      end
+      described_class.start_db_backend!(thread_count: nil)
+    end
+
+    it 'should not start if num_producer_threads is 0' do
+      expect(Deimos::Utils::SignalHandler).not_to receive(:new)
+      described_class.configure do |config|
+        config.publish_backend = :db
+      end
+      described_class.start_db_backend!(thread_count: 0)
+    end
+
+  end
+end