deimos-ruby 1.24.3 → 2.0.0.pre.alpha1

data/spec/config/configuration_spec.rb

@@ -1,329 +0,0 @@
-# frozen_string_literal: true
-
-# Mock consumer
-class MyConfigConsumer < Deimos::Consumer
-  # :no-doc:
-  def consume
-  end
-end
-
-# Mock consumer 2
-class MyConfigConsumer2 < Deimos::Consumer
-  # :no-doc:
-  def consume
-  end
-end
-
-describe Deimos, 'configuration' do
-  it 'should configure with deprecated fields' do
-    logger = Logger.new(nil)
-    described_class.configure do
-      kafka_logger logger
-      reraise_consumer_errors true
-      schema_registry_url 'http://schema.registry'
-      schema.use_schema_classes false
-      seed_broker 'whatever'
-      schema_path 'some_path'
-      producer_schema_namespace 'namespace'
-      producer_topic_prefix 'prefix'
-      disable_producers true
-      ssl_enabled true
-      ssl_ca_cert 'cert'
-      ssl_client_cert 'cert'
-      ssl_client_cert_key 'key'
-      publish_backend 'db'
-      report_lag true
-    end
-
-    expect(described_class.config.kafka.logger).to eq(logger)
-    expect(described_class.config.consumers.reraise_errors).to eq(true)
-    expect(described_class.config.schema.registry_url).to eq('http://schema.registry')
-    expect(described_class.config.schema.use_schema_classes).to eq(false)
-    expect(described_class.config.kafka.seed_brokers).to eq('whatever')
-    expect(described_class.config.producers.schema_namespace).to eq('namespace')
-    expect(described_class.config.producers.topic_prefix).to eq('prefix')
-    expect(described_class.config.producers.disabled).to eq(true)
-    expect(described_class.config.kafka.ssl.enabled).to eq(true)
-    expect(described_class.config.kafka.ssl.ca_cert).to eq('cert')
-    expect(described_class.config.kafka.ssl.client_cert).to eq('cert')
-    expect(described_class.config.kafka.ssl.client_cert_key).to eq('key')
-    expect(described_class.config.producers.backend).to eq('db')
-    expect(described_class.config.consumers.report_lag).to eq(true)
-  end
-
-  it 'reads existing Phobos config YML files' do
-    described_class.config.reset!
-    described_class.configure { |c| c.phobos_config_file = File.join(File.dirname(__FILE__), '..', 'phobos.yml') }
-    expect(described_class.config.phobos_config).to match(
-      logger: an_instance_of(Logger),
-      backoff: { min_ms: 1000, max_ms: 60_000 },
-      consumer: {
-        session_timeout: 300,
-        offset_commit_interval: 10,
-        offset_commit_threshold: 0,
-        heartbeat_interval: 10
-      },
-      custom_kafka_logger: an_instance_of(Logger),
-      custom_logger: an_instance_of(Logger),
-      kafka: {
-        client_id: 'phobos',
-        connect_timeout: 15,
-        socket_timeout: 15,
-        ssl_verify_hostname: true,
-        ssl_ca_certs_from_system: false,
-        seed_brokers: ['localhost:9092']
-      },
-      listeners: [
-        {
-          topic: 'my_consume_topic',
-          group_id: 'my_group_id',
-          max_concurrency: 1,
-          start_from_beginning: true,
-          max_bytes_per_partition: 524_288,
-          min_bytes: 1,
-          max_wait_time: 5,
-          force_encoding: nil,
-          delivery: 'batch',
-          session_timeout: 300,
-          offset_commit_interval: 10,
-          offset_commit_threshold: 0,
-          offset_retention_time: nil,
-          heartbeat_interval: 10,
-          handler: 'ConsumerTest::MyConsumer',
-          use_schema_classes: nil,
-          max_db_batch_size: nil,
-          bulk_import_id_generator: nil,
-          save_associations_first: false
-        }, {
-          topic: 'my_batch_consume_topic',
-          group_id: 'my_batch_group_id',
-          max_concurrency: 1,
-          start_from_beginning: true,
-          max_bytes_per_partition: 500.kilobytes,
-          min_bytes: 1,
-          max_wait_time: 5,
-          force_encoding: nil,
-          delivery: 'inline_batch',
-          session_timeout: 300,
-          offset_commit_interval: 10,
-          offset_commit_threshold: 0,
-          offset_retention_time: nil,
-          heartbeat_interval: 10,
-          handler: 'ConsumerTest::MyBatchConsumer',
-          use_schema_classes: nil,
-          max_db_batch_size: nil,
-          bulk_import_id_generator: nil,
-          save_associations_first: false
-        }
-      ],
-      producer: {
-        ack_timeout: 5,
-        required_acks: :all,
-        max_retries: 2,
-        retry_backoff: 1,
-        max_buffer_size: 10_000,
-        max_buffer_bytesize: 10_000_000,
-        compression_codec: nil,
-        compression_threshold: 1,
-        max_queue_size: 10_000,
-        delivery_threshold: 0,
-        delivery_interval: 0,
-        persistent_connections: false
-      }
-    )
-  end
-
-  specify '#phobos_config' do
-    logger1 = Logger.new(nil)
-    logger2 = Logger.new(nil)
-    described_class.config.reset!
-    described_class.configure do
-      phobos_logger logger1
-      kafka do
-        logger logger2
-        seed_brokers 'my-seed-brokers'
-        client_id 'phobos2'
-        connect_timeout 30
-        socket_timeout 30
-        ssl.enabled(true)
-        ssl.ca_certs_from_system(true)
-        ssl.ca_cert('cert')
-        ssl.client_cert('cert')
-        ssl.client_cert_key('key')
-        ssl.verify_hostname(false)
-        sasl.enabled true
-        sasl.gssapi_principal 'gssapi_principal'
-        sasl.gssapi_keytab 'gssapi_keytab'
-        sasl.plain_authzid 'plain_authzid'
-        sasl.plain_username 'plain_username'
-        sasl.plain_password 'plain_password'
-        sasl.scram_username 'scram_username'
-        sasl.scram_password 'scram_password'
-        sasl.scram_mechanism 'scram_mechanism'
-        sasl.enforce_ssl true
-        sasl.oauth_token_provider 'oauth_token_provider'
-      end
-      consumers do
-        session_timeout 30
-        offset_commit_interval 5
-        offset_commit_threshold 0
-        heartbeat_interval 5
-        backoff 5..10
-      end
-      producers do
-        ack_timeout 3
-        required_acks 1
-        max_retries 1
-        retry_backoff 2
-        max_buffer_size 5
-        max_buffer_bytesize 5
-        compression_codec :snappy
-        compression_threshold 2
-        max_queue_size 10
-        delivery_threshold 1
-        delivery_interval 1
-        persistent_connections true
-      end
-      consumer do
-        class_name 'MyConfigConsumer'
-        schema 'blah'
-        topic 'blah'
-        group_id 'myconsumerid'
-        max_concurrency 1
-        start_from_beginning true
-        max_bytes_per_partition 10
-        min_bytes 5
-        max_wait_time 5
-        force_encoding true
-        delivery :message
-        backoff 100..200
-        session_timeout 10
-        offset_commit_interval 13
-        offset_commit_threshold 13
-        offset_retention_time 13
-        heartbeat_interval 13
-        use_schema_classes false
-      end
-      consumer do
-        disabled true
-        class_name 'MyConfigConsumer2'
-        schema 'blah2'
-        topic 'blah2'
-        group_id 'myconsumerid2'
-        use_schema_classes false
-      end
-    end
-
-    expect(described_class.config.phobos_config).
-      to match(
-        logger: an_instance_of(Logger),
-        backoff: { min_ms: 5, max_ms: 10 },
-        consumer: {
-          session_timeout: 30,
-          offset_commit_interval: 5,
-          offset_commit_threshold: 0,
-          heartbeat_interval: 5
-        },
-        custom_kafka_logger: logger2,
-        custom_logger: logger1,
-        kafka: {
-          client_id: 'phobos2',
-          connect_timeout: 30,
-          socket_timeout: 30,
-          ssl_ca_certs_from_system: true,
-          ssl_ca_cert: 'cert',
-          ssl_client_cert: 'cert',
-          ssl_client_cert_key: 'key',
-          ssl_verify_hostname: false,
-          seed_brokers: ['my-seed-brokers'],
-          sasl_gssapi_principal: 'gssapi_principal',
-          sasl_gssapi_keytab: 'gssapi_keytab',
-          sasl_plain_authzid: 'plain_authzid',
-          sasl_plain_username: 'plain_username',
-          sasl_plain_password: 'plain_password',
-          sasl_scram_username: 'scram_username',
-          sasl_scram_password: 'scram_password',
-          sasl_scram_mechanism: 'scram_mechanism',
-          sasl_over_ssl: true,
-          sasl_oauth_token_provider: 'oauth_token_provider',
-        },
-        listeners: [
-          {
-            topic: 'blah',
-            group_id: 'myconsumerid',
-            max_concurrency: 1,
-            start_from_beginning: true,
-            max_bytes_per_partition: 10,
-            min_bytes: 5,
-            max_wait_time: 5,
-            force_encoding: true,
-            delivery: 'message',
-            backoff: { min_ms: 100, max_ms: 200 },
-            session_timeout: 10,
-            offset_commit_interval: 13,
-            offset_commit_threshold: 13,
-            offset_retention_time: 13,
-            heartbeat_interval: 13,
-            handler: 'MyConfigConsumer',
-            use_schema_classes: false,
-            max_db_batch_size: nil,
-            bulk_import_id_generator: nil,
-            save_associations_first: false
-          }
-        ],
-        producer: {
-          ack_timeout: 3,
-          required_acks: 1,
-          max_retries: 1,
-          retry_backoff: 2,
-          max_buffer_size: 5,
-          max_buffer_bytesize: 5,
-          compression_codec: :snappy,
-          compression_threshold: 2,
-          max_queue_size: 10,
-          delivery_threshold: 1,
-          delivery_interval: 1,
-          persistent_connections: true
-        }
-      )
-  end
-
-  it 'should override global configurations' do
-    described_class.configure do
-      consumers.bulk_import_id_generator(-> { 'global' })
-      consumers.replace_associations true
-
-      consumer do
-        class_name 'MyConfigConsumer'
-        schema 'blah'
-        topic 'blah'
-        group_id 'myconsumerid'
-        bulk_import_id_generator(-> { 'consumer' })
-        replace_associations false
-        save_associations_first true
-      end
-
-      consumer do
-        class_name 'MyConfigConsumer2'
-        schema 'blah'
-        topic 'blah'
-        group_id 'myconsumerid'
-      end
-    end
-
-    consumers = described_class.config.consumers
-    expect(consumers.replace_associations).to eq(true)
-    expect(consumers.bulk_import_id_generator.call).to eq('global')
-
-    custom = MyConfigConsumer.config
-    expect(custom[:replace_associations]).to eq(false)
-    expect(custom[:bulk_import_id_generator].call).to eq('consumer')
-    expect(custom[:save_associations_first]).to eq(true)
-
-    default = MyConfigConsumer2.config
-    expect(default[:replace_associations]).to eq(true)
-    expect(default[:bulk_import_id_generator].call).to eq('global')
-    expect(default[:save_associations_first]).to eq(false)
-
-  end
-end

data/spec/kafka_listener_spec.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-describe Deimos::KafkaListener do
-  include_context 'with widgets'
-
-  prepend_before(:each) do
-    producer_class = Class.new(Deimos::Producer) do
-      schema 'MySchema'
-      namespace 'com.my-namespace'
-      topic 'my-topic'
-      key_config none: true
-    end
-    stub_const('MyProducer', producer_class)
-  end
-
-  before(:each) do
-    Deimos.configure do |c|
-      c.producers.backend = :kafka
-      c.schema.backend = :avro_local
-    end
-    allow_any_instance_of(Kafka::Cluster).to receive(:add_target_topics)
-    allow_any_instance_of(Kafka::Cluster).to receive(:partitions_for).
-      and_raise(Kafka::Error)
-  end
-
-  describe '.send_produce_error' do
-    let(:payloads) do
-      [{ 'test_id' => 'foo', 'some_int' => 123 },
-       { 'test_id' => 'bar', 'some_int' => 124 }]
-    end
-
-    it 'should listen to publishing errors and republish as Deimos events' do
-      allow(Deimos::Producer).to receive(:descendants).and_return([MyProducer])
-      Deimos.subscribe('produce_error') do |event|
-        expect(event.payload).to include(
-          producer: MyProducer,
-          topic: 'my-topic',
-          payloads: payloads
-        )
-      end
-      expect(Deimos.config.metrics).to receive(:increment).
-        with('publish_error', tags: %w(topic:my-topic), by: 2)
-      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
-    end
-
-    it 'should not send any notifications when producer is not found' do
-      Deimos.subscribe('produce_error') do |_|
-        raise 'OH NOES'
-      end
-      allow(Deimos::Producer).to receive(:descendants).and_return([])
-      expect(Deimos.config.metrics).not_to receive(:increment).with('publish_error', anything)
-      expect { MyProducer.publish_list(payloads) }.to raise_error(Kafka::DeliveryFailed)
-    end
-  end
-end

data/spec/phobos.bad_db.yml

@@ -1,73 +0,0 @@
-logger:
-  # Optional log file, set to false or remove to disable it
-  file: log/phobos.log
-  # Optional output format for stdout, default is false (human readable).
-  # Set to true to enable json output.
-  stdout_json: false
-  level: debug
-  # Comment the block to disable ruby-kafka logs
-  ruby_kafka:
-    level: debug
-
-kafka:
-  # identifier for this application
-  client_id: phobos
-  # timeout setting for connecting to brokers
-  connect_timeout: 15
-  # timeout setting for socket connections
-  socket_timeout: 15
-
-producer:
-  # number of seconds a broker can wait for replicas to acknowledge
-  # a write before responding with a timeout
-  ack_timeout: 5
-  # number of replicas that must acknowledge a write, or `:all`
-  # if all in-sync replicas must acknowledge
-  required_acks: 1
-  # number of retries that should be attempted before giving up sending
-  # messages to the cluster. Does not include the original attempt
-  max_retries: 2
-  # number of seconds to wait between retries
-  retry_backoff: 1
-  # number of messages allowed in the buffer before new writes will
-  # raise {BufferOverflow} exceptions
-  max_buffer_size: 10000
-  # maximum size of the buffer in bytes. Attempting to produce messages
-  # when the buffer reaches this size will result in {BufferOverflow} being raised
-  max_buffer_bytesize: 10000000
-  # name of the compression codec to use, or nil if no compression should be performed.
-  # Valid codecs: `:snappy` and `:gzip`
-  compression_codec:
-  # number of messages that needs to be in a message set before it should be compressed.
-  # Note that message sets are per-partition rather than per-topic or per-producer
-  compression_threshold: 1
-  # maximum number of messages allowed in the queue. Only used for async_producer
-  max_queue_size: 10000
-  # if greater than zero, the number of buffered messages that will automatically
-  # trigger a delivery. Only used for async_producer
-  delivery_threshold: 0
-  # if greater than zero, the number of seconds between automatic message
-  # deliveries. Only used for async_producer
-  delivery_interval: 0
-
-consumer:
-  # number of seconds after which, if a client hasn't contacted the Kafka cluster,
-  # it will be kicked out of the group
-  session_timeout: 300
-  # interval between offset commits, in seconds
-  offset_commit_interval: 10
-  # number of messages that can be processed before their offsets are committed.
-  # If zero, offset commits are not triggered by message processing
-  offset_commit_threshold: 0
-  # interval between heartbeats; must be less than the session window
-  heartbeat_interval: 10
-
-backoff:
-  min_ms: 1000
-  max_ms: 60000
-
-listeners:
-  - handler: ConsumerTest::MyConsumer
-    topic: my_consume_topic
-    group_id: my_group_id
-    max_bytes_per_partition: 524288 # 512 KB

data/spec/phobos.yml

@@ -1,77 +0,0 @@
-logger:
-  # Optional log file, set to false or remove to disable it
-  file: log/phobos.log
-  # Optional output format for stdout, default is false (human readable).
-  # Set to true to enable json output.
-  stdout_json: false
-  level: debug
-  # Comment the block to disable ruby-kafka logs
-  ruby_kafka:
-    level: debug
-
-kafka:
-  # identifier for this application
-  client_id: phobos
-  # timeout setting for connecting to brokers
-  connect_timeout: 15
-  # timeout setting for socket connections
-  socket_timeout: 15
-
-producer:
-  # number of seconds a broker can wait for replicas to acknowledge
-  # a write before responding with a timeout
-  ack_timeout: 5
-  # number of replicas that must acknowledge a write, or `:all`
-  # if all in-sync replicas must acknowledge
-  required_acks: :all
-  # number of retries that should be attempted before giving up sending
-  # messages to the cluster. Does not include the original attempt
-  max_retries: 2
-  # number of seconds to wait between retries
-  retry_backoff: 1
-  # number of messages allowed in the buffer before new writes will
-  # raise {BufferOverflow} exceptions
-  max_buffer_size: 10000
-  # maximum size of the buffer in bytes. Attempting to produce messages
-  # when the buffer reaches this size will result in {BufferOverflow} being raised
-  max_buffer_bytesize: 10000000
-  # name of the compression codec to use, or nil if no compression should be performed.
-  # Valid codecs: `:snappy` and `:gzip`
-  compression_codec:
-  # number of messages that needs to be in a message set before it should be compressed.
-  # Note that message sets are per-partition rather than per-topic or per-producer
-  compression_threshold: 1
-  # maximum number of messages allowed in the queue. Only used for async_producer
-  max_queue_size: 10000
-  # if greater than zero, the number of buffered messages that will automatically
-  # trigger a delivery. Only used for async_producer
-  delivery_threshold: 0
-  # if greater than zero, the number of seconds between automatic message
-  # deliveries. Only used for async_producer
-  delivery_interval: 0
-
-consumer:
-  # number of seconds after which, if a client hasn't contacted the Kafka cluster,
-  # it will be kicked out of the group
-  session_timeout: 300
-  # interval between offset commits, in seconds
-  offset_commit_interval: 10
-  # number of messages that can be processed before their offsets are committed.
-  # If zero, offset commits are not triggered by message processing
-  offset_commit_threshold: 0
-  # interval between heartbeats; must be less than the session window
-  heartbeat_interval: 10
-
-backoff:
-  min_ms: 1000
-  max_ms: 60000
-
-listeners:
-  - handler: ConsumerTest::MyConsumer
-    topic: my_consume_topic
-    group_id: my_group_id
-    max_bytes_per_partition: 524288 # 512 KB
-  - handler: ConsumerTest::MyBatchConsumer
-    topic: my_batch_consume_topic
-    group_id: my_batch_group_id
-    delivery: inline_batch

data/spec/utils/inline_consumer_spec.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-describe Deimos::Utils::SeekListener do
-
-  describe '#start_listener' do
-    let(:consumer) { instance_double(Kafka::Consumer) }
-    let(:handler) { class_double(Deimos::Utils::MessageBankHandler) }
-
-    before(:each) do
-      allow(handler).to receive(:start)
-      allow(consumer).to receive(:subscribe)
-      allow_any_instance_of(Phobos::Listener).to receive(:create_kafka_consumer).and_return(consumer)
-      allow_any_instance_of(Kafka::Client).to receive(:last_offset_for).and_return(100)
-      stub_const('Deimos::Utils::SeekListener::MAX_SEEK_RETRIES', 2)
-    end
-
-    it 'should seek offset' do
-      allow(consumer).to receive(:seek)
-      expect(consumer).to receive(:seek).once
-      seek_listener = described_class.new(handler: handler, group_id: 999, topic: 'test_topic')
-      seek_listener.start_listener
-    end
-
-    it 'should retry on errors when seeking offset' do
-      allow(consumer).to receive(:seek).and_raise(StandardError)
-      expect(consumer).to receive(:seek).twice
-      seek_listener = described_class.new(handler: handler, group_id: 999, topic: 'test_topic')
-      seek_listener.start_listener
-    end
-  end
-end

data/spec/utils/lag_reporter_spec.rb

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-describe Deimos::Utils::LagReporter do
-
-  let(:kafka_client) { instance_double(Kafka::Client) }
-  let(:partition1_tags) { %w(consumer_group:group1 partition:1 topic:my-topic) }
-  let(:partition2_tags) { %w(consumer_group:group1 partition:2 topic:my-topic) }
-
-  before(:each) do
-    allow(kafka_client).to receive(:last_offset_for).and_return(100)
-    allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
-    Deimos.configure { |c| c.consumers.report_lag = true }
-  end
-
-  after(:each) do
-    described_class.reset
-    Deimos.configure { |c| c.consumers.report_lag = false }
-  end
-
-  it 'should not report lag before ready' do
-    expect(Deimos.config.metrics).not_to receive(:gauge)
-    ActiveSupport::Notifications.instrument(
-      'heartbeat.consumer.kafka',
-      group_id: 'group1', topic_partitions: { 'my-topic': [1] }
-    )
-  end
-
-  it 'should report lag' do
-    expect(Deimos.config.metrics).to receive(:gauge).ordered.twice.
-      with('consumer_lag', 95, tags: partition1_tags)
-    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 80, tags: partition2_tags)
-    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 0, tags: partition2_tags)
-    ActiveSupport::Notifications.instrument(
-      'seek.consumer.kafka',
-      offset: 5, topic: 'my-topic', group_id: 'group1', partition: 1
-    )
-    ActiveSupport::Notifications.instrument(
-      'start_process_message.consumer.kafka',
-      offset: 20, topic: 'my-topic', group_id: 'group1', partition: 2
-    )
-    ActiveSupport::Notifications.instrument(
-      'heartbeat.consumer.kafka',
-      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
-    )
-    ActiveSupport::Notifications.instrument(
-      'start_process_batch.consumer.kafka',
-      last_offset: 100, topic: 'my-topic', group_id: 'group1', partition: 2
-    )
-    ActiveSupport::Notifications.instrument(
-      'heartbeat.consumer.kafka',
-      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
-    )
-  end
-
-  it 'should update lag after heartbeat' do
-    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 94, tags: partition2_tags)
-    expect(Deimos.config.metrics).to receive(:gauge).ordered.once.
-      with('consumer_lag', 95, tags: partition2_tags)
-    ActiveSupport::Notifications.instrument(
-      'seek.consumer.kafka',
-      offset: 6, topic: 'my-topic', group_id: 'group1', partition: 2
-    )
-    ActiveSupport::Notifications.instrument(
-      'heartbeat.consumer.kafka',
-      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
-    )
-    allow(kafka_client).to receive(:last_offset_for).and_return(101)
-    ActiveSupport::Notifications.instrument(
-      'heartbeat.consumer.kafka',
-      group_id: 'group1', topic_partitions: { 'my-topic': [1, 2] }
-    )
-  end
-end