deimos-temp-fork 0.0.1

data/spec/backends/kafka_async_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Backends::KafkaAsync do
+  include_context 'with publish_backend'
+  it 'should publish to Kafka asynchronously' do
+    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
+    expect(producer).to receive(:async_publish_list).with(messages.map(&:encoded_hash))
+    expect(described_class).to receive(:producer).and_return(producer)
+    described_class.publish(producer_class: MyProducer, messages: messages)
+  end
+end

data/spec/backends/kafka_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+RSpec.describe Deimos::Backends::Kafka do
+  include_context 'with publish_backend'
+  it 'should publish to Kafka synchronously' do
+    producer = instance_double(Phobos::Producer::ClassMethods::PublicAPI)
+    expect(producer).to receive(:publish_list).with(messages.map(&:encoded_hash))
+    expect(described_class).to receive(:producer).and_return(producer)
+    described_class.publish(producer_class: MyProducer, messages: messages)
+  end
+end

data/spec/batch_consumer_spec.rb

@@ -0,0 +1,256 @@
+# frozen_string_literal: true
+
+# :nodoc:
+module ConsumerTest
+  describe Deimos::Consumer, 'Batch Consumer' do
+
+    prepend_before(:each) do
+      # :nodoc:
+      consumer_class = Class.new(described_class) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config field: 'test_id'
+
+        # :nodoc:
+        def consume_batch(_payloads, _metadata)
+          raise 'This should not be called unless call_original is set'
+        end
+      end
+      stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+    end
+
+    let(:batch) do
+      [
+        { 'test_id' => 'foo', 'some_int' => 123 },
+        { 'test_id' => 'bar', 'some_int' => 456 }
+      ]
+    end
+
+    let(:invalid_payloads) do
+      batch.concat([{ 'invalid' => 'key' }])
+    end
+
+    it 'should provide backwards compatibility for BatchConsumer class' do
+      consumer_class = Class.new(Deimos::BatchConsumer) do
+        schema 'MySchema'
+        namespace 'com.my-namespace'
+        key_config field: 'test_id'
+
+        # :nodoc:
+        def consume_batch(_payloads, _metadata)
+          raise 'This should not be called unless call_original is set'
+        end
+      end
+      stub_const('ConsumerTest::MyOldBatchConsumer', consumer_class)
+
+      test_consume_batch(MyOldBatchConsumer, batch) do |received, _metadata|
+        expect(received).to eq(batch)
+      end
+    end
+
+    it 'should consume a batch of messages' do
+      test_consume_batch(MyBatchConsumer, batch) do |received, _metadata|
+        expect(received).to eq(batch)
+      end
+    end
+
+    it 'should consume a message on a topic' do
+      test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
+        expect(received).to eq(batch)
+      end
+    end
+
+    it 'should fail on an invalid message in the batch' do
+      test_consume_batch_invalid_message(MyBatchConsumer, batch.concat(invalid_payloads))
+    end
+
+    describe 'when reraising errors is disabled' do
+      before(:each) do
+        Deimos.configure { |config| config.consumers.reraise_errors = false }
+      end
+
+      it 'should not fail when before_consume_batch fails' do
+        expect {
+          test_consume_batch(
+            MyBatchConsumer,
+            batch,
+            skip_expectation: true
+          ) { raise 'OH NOES' }
+        }.not_to raise_error
+      end
+
+      it 'should not fail when consume_batch fails' do
+        expect {
+          test_consume_batch(
+            MyBatchConsumer,
+            invalid_payloads,
+            skip_expectation: true
+          )
+        }.not_to raise_error
+      end
+    end
+
+    describe 'decoding' do
+      let(:keys) do
+        batch.map { |v| v.slice('test_id') }
+      end
+
+      it 'should decode payloads for all messages in the batch' do
+        test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
+          # Mock decoder simply returns the payload
+          expect(received).to eq(batch)
+        end
+      end
+
+      it 'should decode keys for all messages in the batch' do
+        expect_any_instance_of(ConsumerTest::MyBatchConsumer).
+          to receive(:decode_key).with(keys[0]).and_call_original
+        expect_any_instance_of(ConsumerTest::MyBatchConsumer).
+          to receive(:decode_key).with(keys[1]).and_call_original
+
+        test_consume_batch('my_batch_consume_topic', batch, keys: keys) do |_received, metadata|
+          # Mock decode_key extracts the value of the first field as the key
+          expect(metadata[:keys]).to eq(%w(foo bar))
+          expect(metadata[:first_offset]).to eq(1)
+        end
+      end
+
+      it 'should decode plain keys for all messages in the batch' do
+        consumer_class = Class.new(described_class) do
+          schema 'MySchema'
+          namespace 'com.my-namespace'
+          key_config plain: true
+        end
+        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+
+        test_consume_batch('my_batch_consume_topic', batch, keys: [1, 2]) do |_received, metadata|
+          expect(metadata[:keys]).to eq([1, 2])
+        end
+      end
+    end
+
+    describe 'timestamps' do
+      before(:each) do
+        # :nodoc:
+        consumer_class = Class.new(described_class) do
+          schema 'MySchemaWithDateTimes'
+          namespace 'com.my-namespace'
+          key_config plain: true
+
+          # :nodoc:
+          def consume_batch(_payloads, _metadata)
+            raise 'This should not be called unless call_original is set'
+          end
+        end
+        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+        allow(Deimos.config.metrics).to receive(:histogram)
+      end
+
+      let(:batch_with_time) do
+        [
+          {
+            'test_id' => 'foo',
+            'some_int' => 123,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => 2.minutes.ago.to_s
+          },
+          {
+            'test_id' => 'bar',
+            'some_int' => 456,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => 3.minutes.ago.to_s
+          }
+        ]
+      end
+
+      let(:invalid_times) do
+        [
+          {
+            'test_id' => 'baz',
+            'some_int' => 123,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => 'yesterday morning'
+          },
+          {
+            'test_id' => 'ok',
+            'some_int' => 456,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => ''
+          },
+          {
+            'test_id' => 'hello',
+            'some_int' => 456,
+            'updated_at' => Time.now.to_i,
+            'timestamp' => '1234567890'
+          }
+        ]
+      end
+
+      it 'should consume a batch' do
+        expect(Deimos.config.metrics).
+          to receive(:histogram).with('handler',
+                                      a_kind_of(Numeric),
+                                      tags: %w(time:time_delayed topic:my-topic)).twice
+
+        test_consume_batch('my_batch_consume_topic', batch_with_time) do |received, _metadata|
+          expect(received).to eq(batch_with_time)
+        end
+      end
+
+      it 'should fail nicely and ignore timestamps with the wrong format' do
+        batch = invalid_times.concat(batch_with_time)
+
+        expect(Deimos.config.metrics).
+          to receive(:histogram).with('handler',
+                                      a_kind_of(Numeric),
+                                      tags: %w(time:time_delayed topic:my-topic)).twice
+
+        test_consume_batch('my_batch_consume_topic', batch) do |received, _metadata|
+          expect(received).to eq(batch)
+        end
+      end
+    end
+
+    describe 'logging' do
+      before(:each) do
+        # :nodoc:
+        consumer_class = Class.new(described_class) do
+          schema 'MySchemaWithUniqueId'
+          namespace 'com.my-namespace'
+          key_config plain: true
+
+          # :nodoc:
+          def consume_batch(_payloads, _metadata)
+            raise 'This should not be called unless call_original is set'
+          end
+        end
+        stub_const('ConsumerTest::MyBatchConsumer', consumer_class)
+        allow(Deimos.config.metrics).to receive(:histogram)
+      end
+
+      it 'should log message identifiers' do
+        batch_with_message_id = [
+          { 'id' => 1, 'test_id' => 'foo', 'some_int' => 5,
+            'timestamp' => 3.minutes.ago.to_s, 'message_id' => 'one' },
+          { 'id' => 2, 'test_id' => 'bar', 'some_int' => 6,
+            'timestamp' => 2.minutes.ago.to_s, 'message_id' => 'two' }
+        ]
+
+        allow(Deimos.config.logger).
+          to receive(:info)
+
+        expect(Deimos.config.logger).
+          to receive(:info).
+          with(hash_including(
+                 message_ids: [
+                   { key: 1, message_id: 'one' },
+                   { key: 2, message_id: 'two' }
+                 ]
+               )).
+          twice
+
+        test_consume_batch('my_batch_consume_topic', batch_with_message_id, keys: [1, 2])
+      end
+    end
+  end
+end

data/spec/config/configuration_spec.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+# Mock consumer
+class MyConfigConsumer < Deimos::Consumer
+  # :no-doc:
+  def consume
+  end
+end
+
+# Mock consumer 2
+class MyConfigConsumer2 < Deimos::Consumer
+  # :no-doc:
+  def consume
+  end
+end
+
+describe Deimos, 'configuration' do
+  it 'should configure with deprecated fields' do
+    logger = Logger.new(nil)
+    described_class.configure do
+      kafka_logger logger
+      reraise_consumer_errors true
+      schema_registry_url 'http://schema.registry'
+      seed_broker 'whatever'
+      schema_path 'some_path'
+      producer_schema_namespace 'namespace'
+      producer_topic_prefix 'prefix'
+      disable_producers true
+      ssl_enabled true
+      ssl_ca_cert 'cert'
+      ssl_client_cert 'cert'
+      ssl_client_cert_key 'key'
+      publish_backend 'db'
+      report_lag true
+    end
+
+    expect(described_class.config.kafka.logger).to eq(logger)
+    expect(described_class.config.consumers.reraise_errors).to eq(true)
+    expect(described_class.config.schema.registry_url).to eq('http://schema.registry')
+    expect(described_class.config.kafka.seed_brokers).to eq('whatever')
+    expect(described_class.config.producers.schema_namespace).to eq('namespace')
+    expect(described_class.config.producers.topic_prefix).to eq('prefix')
+    expect(described_class.config.producers.disabled).to eq(true)
+    expect(described_class.config.kafka.ssl.enabled).to eq(true)
+    expect(described_class.config.kafka.ssl.ca_cert).to eq('cert')
+    expect(described_class.config.kafka.ssl.client_cert).to eq('cert')
+    expect(described_class.config.kafka.ssl.client_cert_key).to eq('key')
+    expect(described_class.config.producers.backend).to eq('db')
+    expect(described_class.config.consumers.report_lag).to eq(true)
+  end
+
+  it 'reads existing Phobos config YML files' do
+    described_class.config.reset!
+    described_class.configure { |c| c.phobos_config_file = File.join(File.dirname(__FILE__), '..', 'phobos.yml') }
+    expect(described_class.config.phobos_config).to match(
+      logger: an_instance_of(Logger),
+      backoff: { min_ms: 1000, max_ms: 60_000 },
+      consumer: {
+        session_timeout: 300,
+        offset_commit_interval: 10,
+        offset_commit_threshold: 0,
+        heartbeat_interval: 10
+      },
+      custom_kafka_logger: an_instance_of(Logger),
+      custom_logger: an_instance_of(Logger),
+      kafka: {
+        client_id: 'phobos',
+        connect_timeout: 15,
+        socket_timeout: 15,
+        ssl_verify_hostname: true,
+        seed_brokers: ['localhost:9092']
+      },
+      listeners: [
+        {
+          topic: 'my_consume_topic',
+          group_id: 'my_group_id',
+          max_concurrency: 1,
+          start_from_beginning: true,
+          max_bytes_per_partition: 524_288,
+          min_bytes: 1,
+          max_wait_time: 5,
+          force_encoding: nil,
+          delivery: 'batch',
+          session_timeout: 300,
+          offset_commit_interval: 10,
+          offset_commit_threshold: 0,
+          offset_retention_time: nil,
+          heartbeat_interval: 10,
+          handler: 'ConsumerTest::MyConsumer'
+        }, {
+          topic: 'my_batch_consume_topic',
+          group_id: 'my_batch_group_id',
+          max_concurrency: 1,
+          start_from_beginning: true,
+          max_bytes_per_partition: 500.kilobytes,
+          min_bytes: 1,
+          max_wait_time: 5,
+          force_encoding: nil,
+          delivery: 'inline_batch',
+          session_timeout: 300,
+          offset_commit_interval: 10,
+          offset_commit_threshold: 0,
+          offset_retention_time: nil,
+          heartbeat_interval: 10,
+          handler: 'ConsumerTest::MyBatchConsumer'
+        }
+      ],
+      producer: {
+        ack_timeout: 5,
+        required_acks: :all,
+        max_retries: 2,
+        retry_backoff: 1,
+        max_buffer_size: 10_000,
+        max_buffer_bytesize: 10_000_000,
+        compression_codec: nil,
+        compression_threshold: 1,
+        max_queue_size: 10_000,
+        delivery_threshold: 0,
+        delivery_interval: 0
+      }
+    )
+  end
+
+  specify '#phobos_config' do
+    logger1 = Logger.new(nil)
+    logger2 = Logger.new(nil)
+    described_class.config.reset!
+    described_class.configure do
+      phobos_logger logger1
+      kafka do
+        logger logger2
+        seed_brokers 'my-seed-brokers'
+        client_id 'phobos2'
+        connect_timeout 30
+        socket_timeout 30
+        ssl.enabled(true)
+        ssl.ca_cert('cert')
+        ssl.client_cert('cert')
+        ssl.client_cert_key('key')
+        ssl.verify_hostname(false)
+      end
+      consumers do
+        session_timeout 30
+        offset_commit_interval 5
+        offset_commit_threshold 0
+        heartbeat_interval 5
+        backoff 5..10
+      end
+      producers do
+        ack_timeout 3
+        required_acks 1
+        max_retries 1
+        retry_backoff 2
+        max_buffer_size 5
+        max_buffer_bytesize 5
+        compression_codec :snappy
+        compression_threshold 2
+        max_queue_size 10
+        delivery_threshold 1
+        delivery_interval 1
+        persistent_connections true
+      end
+      consumer do
+        class_name 'MyConfigConsumer'
+        schema 'blah'
+        topic 'blah'
+        group_id 'myconsumerid'
+        max_concurrency 1
+        start_from_beginning true
+        max_bytes_per_partition 10
+        min_bytes 5
+        max_wait_time 5
+        force_encoding true
+        delivery :message
+        backoff 100..200
+        session_timeout 10
+        offset_commit_interval 13
+        offset_commit_threshold 13
+        offset_retention_time 13
+        heartbeat_interval 13
+      end
+      consumer do
+        disabled true
+        class_name 'MyConfigConsumer2'
+        schema 'blah2'
+        topic 'blah2'
+        group_id 'myconsumerid2'
+      end
+    end
+
+    expect(described_class.config.phobos_config).
+      to match(
+        logger: an_instance_of(Logger),
+        backoff: { min_ms: 5, max_ms: 10 },
+        consumer: {
+          session_timeout: 30,
+          offset_commit_interval: 5,
+          offset_commit_threshold: 0,
+          heartbeat_interval: 5
+        },
+        custom_kafka_logger: logger2,
+        custom_logger: logger1,
+        kafka: {
+          client_id: 'phobos2',
+          connect_timeout: 30,
+          socket_timeout: 30,
+          ssl_ca_cert: 'cert',
+          ssl_client_cert: 'cert',
+          ssl_client_cert_key: 'key',
+          ssl_verify_hostname: false,
+          seed_brokers: ['my-seed-brokers']
+        },
+        listeners: [
+          {
+            topic: 'blah',
+            group_id: 'myconsumerid',
+            max_concurrency: 1,
+            start_from_beginning: true,
+            max_bytes_per_partition: 10,
+            min_bytes: 5,
+            max_wait_time: 5,
+            force_encoding: true,
+            delivery: 'message',
+            backoff: { min_ms: 100, max_ms: 200 },
+            session_timeout: 10,
+            offset_commit_interval: 13,
+            offset_commit_threshold: 13,
+            offset_retention_time: 13,
+            heartbeat_interval: 13,
+            handler: 'MyConfigConsumer'
+          }
+        ],
+        producer: {
+          ack_timeout: 3,
+          required_acks: 1,
+          max_retries: 1,
+          retry_backoff: 2,
+          max_buffer_size: 5,
+          max_buffer_bytesize: 5,
+          compression_codec: :snappy,
+          compression_threshold: 2,
+          max_queue_size: 10,
+          delivery_threshold: 1,
+          delivery_interval: 1
+        }
+      )
+  end
+end