deimos-ruby 1.3.0.pre.beta5 → 1.4.0.pre.beta1

data/spec/config/configuration_spec.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+class MyConfigConsumer < Deimos::Consumer
+end
+describe Deimos, 'configuration' do
+  it 'should configure with deprecated fields' do
+    logger = Logger.new(nil)
+    Deimos.configure do
+      kafka_logger logger
+      reraise_consumer_errors true
+      schema_registry_url 'http://schema.registry'
+      seed_broker 'whatever'
+      schema_path 'some_path'
+      producer_schema_namespace 'namespace'
+      producer_topic_prefix 'prefix'
+      disable_producers true
+      ssl_enabled true
+      ssl_ca_cert 'cert'
+      ssl_client_cert 'cert'
+      ssl_client_cert_key 'key'
+      publish_backend 'db'
+      report_lag true
+    end
+
+    expect(Deimos.config.kafka.logger).to eq(logger)
+    expect(Deimos.config.consumers.reraise_errors).to eq(true)
+    expect(Deimos.config.schema.registry_url).to eq('http://schema.registry')
+    expect(Deimos.config.kafka.seed_brokers).to eq('whatever')
+    expect(Deimos.config.producers.schema_namespace).to eq('namespace')
+    expect(Deimos.config.producers.topic_prefix).to eq('prefix')
+    expect(Deimos.config.producers.disabled).to eq(true)
+    expect(Deimos.config.kafka.ssl.enabled).to eq(true)
+    expect(Deimos.config.kafka.ssl.ca_cert).to eq('cert')
+    expect(Deimos.config.kafka.ssl.client_cert).to eq('cert')
+    expect(Deimos.config.kafka.ssl.client_cert_key).to eq('key')
+    expect(Deimos.config.producers.backend).to eq('db')
+    expect(Deimos.config.consumers.report_lag).to eq(true)
+  end
+
+  it 'reads existing Phobos config YML files' do
+    Deimos.config.reset!
+    Deimos.configure { |c| c.phobos_config_file = File.join(File.dirname(__FILE__), '..', 'phobos.yml') }
+    expect(Deimos.config.phobos_config).to match(
+      logger: an_instance_of(Logger),
+      backoff: { min_ms: 1000, max_ms: 60_000 },
+      consumer: {
+        session_timeout: 300,
+        offset_commit_interval: 10,
+        offset_commit_threshold: 0,
+        heartbeat_interval: 10
+      },
+      custom_kafka_logger: an_instance_of(Logger),
+      custom_logger: an_instance_of(Logger),
+      kafka: {
+        client_id: 'phobos',
+        connect_timeout: 15,
+        socket_timeout: 15,
+        ssl_verify_hostname: true,
+        seed_brokers: ['localhost:9092']
+      },
+      listeners: [
+        {
+          topic: 'my_consume_topic',
+          group_id: 'my_group_id',
+          max_concurrency: nil,
+          start_from_beginning: nil,
+          max_bytes_per_partition: 524_288,
+          min_bytes: nil,
+          max_wait_time: nil,
+          force_encoding: nil,
+          delivery: nil,
+          session_timeout: nil,
+          offset_commit_interval: nil,
+          offset_commit_threshold: nil,
+          offset_retention_time: nil,
+          heartbeat_interval: nil,
+          handler: 'ConsumerTest::MyConsumer'
+        }, {
+          topic: 'my_batch_consume_topic',
+          group_id: 'my_batch_group_id',
+          max_concurrency: nil,
+          start_from_beginning: nil,
+          max_bytes_per_partition: nil,
+          min_bytes: nil,
+          max_wait_time: nil,
+          force_encoding: nil,
+          delivery: 'inline_batch',
+          session_timeout: nil,
+          offset_commit_interval: nil,
+          offset_commit_threshold: nil,
+          offset_retention_time: nil,
+          heartbeat_interval: nil,
+          handler: 'ConsumerTest::MyBatchConsumer'
+        }
+      ],
+      producer: {
+        ack_timeout: 5,
+        required_acks: :all,
+        max_retries: 2,
+        retry_backoff: 1,
+        max_buffer_size: 10_000,
+        max_buffer_bytesize: 10_000_000,
+        compression_codec: nil,
+        compression_threshold: 1,
+        max_queue_size: 10_000,
+        delivery_threshold: 0,
+        delivery_interval: 0
+      }
+    )
+  end
+
+  specify '#phobos_config' do
+    logger1 = Logger.new(nil)
+    logger2 = Logger.new(nil)
+    Deimos.config.reset!
+    Deimos.configure do
+      phobos_logger logger1
+      kafka do
+        logger logger2
+        seed_brokers 'my-seed-brokers'
+        client_id 'phobos2'
+        connect_timeout 30
+        socket_timeout 30
+        ssl.enabled(true)
+        ssl.ca_cert('cert')
+        ssl.client_cert('cert')
+        ssl.client_cert_key('key')
+        ssl.verify_hostname(false)
+      end
+      consumers do
+        session_timeout 30
+        offset_commit_interval 5
+        offset_commit_threshold 0
+        heartbeat_interval 5
+        backoff 5..10
+      end
+      producers do
+        ack_timeout 3
+        required_acks 1
+        max_retries 1
+        retry_backoff 2
+        max_buffer_size 5
+        max_buffer_bytesize 5
+        compression_codec :snappy
+        compression_threshold 2
+        max_queue_size 10
+        delivery_threshold 1
+        delivery_interval 1
+        persistent_connections true
+      end
+      consumer do
+        class_name 'MyConfigConsumer'
+        schema 'blah'
+        topic 'blah'
+        group_id 'myconsumerid'
+        max_concurrency 1
+        start_from_beginning true
+        max_bytes_per_partition 10
+        min_bytes 5
+        max_wait_time 5
+        force_encoding true
+        delivery :message
+        backoff 100..200
+        session_timeout 10
+        offset_commit_interval 13
+        offset_commit_threshold 13
+        offset_retention_time 13
+        heartbeat_interval 13
+      end
+    end
+
+    expect(Deimos.config.phobos_config).
+      to match(
+        logger: an_instance_of(Logger),
+        backoff: { min_ms: 5, max_ms: 10 },
+        consumer: {
+          session_timeout: 30,
+          offset_commit_interval: 5,
+          offset_commit_threshold: 0,
+          heartbeat_interval: 5
+        },
+        custom_kafka_logger: logger2,
+        custom_logger: logger1,
+        kafka: {
+          client_id: 'phobos2',
+          connect_timeout: 30,
+          socket_timeout: 30,
+          ssl_ca_cert: 'cert',
+          ssl_client_cert: 'cert',
+          ssl_client_cert_key: 'key',
+          ssl_verify_hostname: false,
+          seed_brokers: ['my-seed-brokers']
+        },
+        listeners: [
+          {
+            topic: 'blah',
+            group_id: 'myconsumerid',
+            max_concurrency: 1,
+            start_from_beginning: true,
+            max_bytes_per_partition: 10,
+            min_bytes: 5,
+            max_wait_time: 5,
+            force_encoding: true,
+            delivery: 'message',
+            backoff: { min_ms: 100, max_ms: 200 },
+            session_timeout: 10,
+            offset_commit_interval: 13,
+            offset_commit_threshold: 13,
+            offset_retention_time: 13,
+            heartbeat_interval: 13,
+            handler: 'MyConfigConsumer'
+          }
+        ],
+        producer: {
+          ack_timeout: 3,
+          required_acks: 1,
+          max_retries: 1,
+          retry_backoff: 2,
+          max_buffer_size: 5,
+          max_buffer_bytesize: 5,
+          compression_codec: :snappy,
+          compression_threshold: 2,
+          max_queue_size: 10,
+          delivery_threshold: 1,
+          delivery_interval: 1
+        }
+      )
+  end
+end

data/spec/consumer_spec.rb

@@ -45,14 +45,14 @@ module ConsumerTest
     end
 
     it 'should fail if reraise is false but fatal_error is true' do
-      Deimos.configure { |config| config.reraise_consumer_errors = false }
+      Deimos.configure { |config| config.consumers.reraise_errors = false }
       test_consume_invalid_message(MyConsumer, 'fatal')
     end
 
     it 'should fail if fatal_error is true globally' do
       Deimos.configure do |config|
-        config.fatal_error { true }
-        config.reraise_consumer_errors = false
+        config.consumers.fatal_error = proc { true }
+        config.consumers.reraise_errors = false
       end
       test_consume_invalid_message(MyConsumer, 'invalid' => 'key')
     end
@@ -65,7 +65,7 @@ module ConsumerTest
     end
 
     it 'should not fail when before_consume fails without reraising errors' do
-      Deimos.configure { |config| config.reraise_consumer_errors = false }
+      Deimos.configure { |config| config.consumers.reraise_errors = false }
       expect {
         test_consume_message(
           MyConsumer,
@@ -77,7 +77,7 @@ module ConsumerTest
     end
 
     it 'should not fail when consume fails without reraising errors' do
-      Deimos.configure { |config| config.reraise_consumer_errors = false }
+      Deimos.configure { |config| config.consumers.reraise_errors = false }
       expect {
         test_consume_message(
           MyConsumer,

data/spec/deimos_spec.rb

@@ -57,26 +57,13 @@ describe Deimos do
     expect(Deimos::VERSION).not_to be_nil
   end
 
-  specify 'configure' do
-    expect(Phobos).to receive(:configure).with(phobos_configuration)
-    allow(described_class).to receive(:ssl_var_contents) { |key| key }
-    described_class.configure do |config|
-      config.phobos_config_file = config_path
-      config.seed_broker = 'my_seed_broker.com'
-      config.ssl_enabled = true
-      config.ssl_ca_cert = 'my_ssl_ca_cert'
-      config.ssl_client_cert = 'my_ssl_client_cert'
-      config.ssl_client_cert_key = 'my_ssl_client_cert_key'
-    end
-  end
-
   it 'should error if required_acks is not all' do
     expect {
       described_class.configure do |config|
-        config.publish_backend = :db
+        config.producers.backend = :db
         config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.bad_db.yml')
       end
-    }.to raise_error('Cannot set publish_backend to :db unless required_acks is set to ":all" in phobos.yml!')
+    }.to raise_error('Cannot set producers.backend to :db unless producers.required_acks is set to ":all"!')
   end
 
   describe '#start_db_backend!' do
@@ -94,7 +81,7 @@ describe Deimos do
         signal_handler
       end
       described_class.configure do |config|
-        config.publish_backend = :db
+        config.producers.backend = :db
       end
       described_class.start_db_backend!(thread_count: 2)
     end
@@ -102,7 +89,7 @@ describe Deimos do
     it 'should not start if backend is not db' do
       expect(Deimos::Utils::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
-        config.publish_backend = :kafka
+        config.producers.backend = :kafka
       end
       expect { described_class.start_db_backend!(thread_count: 2) }.
         to raise_error('Publish backend is not set to :db, exiting')
@@ -111,7 +98,7 @@ describe Deimos do
     it 'should not start if thread_count is nil' do
       expect(Deimos::Utils::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
-        config.publish_backend = :db
+        config.producers.backend = :db
       end
       expect { described_class.start_db_backend!(thread_count: nil) }.
         to raise_error('Thread count is not given or set to zero, exiting')
@@ -120,7 +107,7 @@ describe Deimos do
     it 'should not start if thread_count is 0' do
       expect(Deimos::Utils::SignalHandler).not_to receive(:new)
       described_class.configure do |config|
-        config.publish_backend = :db
+        config.producers.backend = :db
       end
       expect { described_class.start_db_backend!(thread_count: 0) }.
         to raise_error('Thread count is not given or set to zero, exiting')
@@ -128,11 +115,11 @@ describe Deimos do
 
     describe 'delivery configuration' do
       before(:each) do
-        described_class.config = nil
         allow(YAML).to receive(:load).and_return(phobos_configuration)
       end
 
       it 'should not raise an error with properly configured handlers' do
+        path = config_path # for scope issues in the block below
         # Add explicit consumers
         phobos_configuration['listeners'] << { 'handler' => 'ConsumerTest::MyConsumer',
                                                'delivery' => 'message' }
@@ -140,47 +127,39 @@ describe Deimos do
                                                'delivery' => 'batch' }
 
         expect {
-          described_class.configure { |c| c.phobos_config_file = config_path }
+          described_class.configure { |c| c.phobos_config_file = path }
         }.not_to raise_error
       end
 
       it 'should raise an error if BatchConsumers do not have inline_batch delivery' do
+        path = config_path # for scope issues in the block below
         phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyBatchConsumer',
                                                'delivery' => 'message' }]
 
         expect {
-          described_class.configure { |c| c.phobos_config_file = config_path }
+          described_class.configure { |c| c.phobos_config_file = path }
         }.to raise_error('BatchConsumer ConsumerTest::MyBatchConsumer must have delivery set to `inline_batch`')
       end
 
       it 'should raise an error if Consumers do not have message or batch delivery' do
+        path = config_path # for scope issues in the block below
         phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer',
                                                'delivery' => 'inline_batch' }]
 
         expect {
-          described_class.configure { |c| c.phobos_config_file = config_path }
+          described_class.configure { |c| c.phobos_config_file = path }
         }.to raise_error('Non-batch Consumer ConsumerTest::MyConsumer must have delivery set to `message` or `batch`')
       end
 
       it 'should treat nil as `batch`' do
+        path = config_path # for scope issues in the block below
         phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyConsumer' }]
 
         expect {
-          described_class.configure { |c| c.phobos_config_file = config_path }
+          described_class.configure { |c| c.phobos_config_file = path }
         }.not_to raise_error
       end
 
-      it 'should ignore non-Deimos listeners' do
-        consumer_class = Class.new { include Phobos::Handler }
-        stub_const('ConsumerTest::MyOtherConsumer', consumer_class)
-        phobos_configuration['listeners'] = [{ 'handler' => 'ConsumerTest::MyOtherConsumer',
-                                               'topic' => 'my_consume_topic',
-                                               'group_id' => 'my_group_id' }]
-
-        expect {
-          described_class.configure { |c| c.phobos_config_file = config_path }
-        }.not_to raise_error
-      end
     end
   end
 end

data/spec/kafka_source_spec.rb

@@ -145,7 +145,7 @@ module KafkaSourceSpec
     context 'with DB backend' do
       before(:each) do
         Deimos.configure do |config|
-          config.publish_backend = :db
+          config.producers.backend = :db
         end
         setup_db(DB_OPTIONS.last) # sqlite
         allow(Deimos::Producer).to receive(:produce_batch).and_call_original

data/spec/producer_spec.rb

@@ -125,7 +125,7 @@ module ProducerTest
 
     it 'should not publish if publish disabled' do
       expect(described_class).not_to receive(:produce_batch)
-      Deimos.configure { |c| c.disable_producers = true }
+      Deimos.configure { |c| c.producers.disabled = true }
       MyProducer.publish_list(
         [{ 'test_id' => 'foo', 'some_int' => 123 },
          { 'test_id' => 'bar', 'some_int' => 124 }]
@@ -158,7 +158,7 @@ module ProducerTest
     end
 
     it 'should produce to a prefixed topic' do
-      Deimos.configure { |c| c.producer_topic_prefix = 'prefix.' }
+      Deimos.configure { |c| c.producers.topic_prefix = 'prefix.' }
       payload = { 'test_id' => 'foo', 'some_int' => 123 }
       expect(described_class).to receive(:produce_batch).once do |_, messages|
         expect(messages.size).to eq(1)
@@ -176,7 +176,7 @@ module ProducerTest
       end
 
       MyProducer.publish_list([payload])
-      Deimos.configure { |c| c.producer_topic_prefix = nil }
+      Deimos.configure { |c| c.producers.topic_prefix = nil }
       expect(described_class).to receive(:produce_batch).once do |_, messages|
         expect(messages.size).to eq(1)
         expect(messages[0].to_h).
@@ -377,7 +377,7 @@ module ProducerTest
       end
 
       it 'should return db if db is set' do
-        allow(Deimos.config).to receive(:publish_backend).and_return(:db)
+        Deimos.configure { producers.backend = :db }
         expect(described_class.determine_backend_class(true, false)).
           to eq(Deimos::Backends::Db)
         expect(described_class.determine_backend_class(false, false)).
@@ -385,7 +385,7 @@ module ProducerTest
       end
 
       it 'should return kafka if force_send is true' do
-        allow(Deimos.config).to receive(:publish_backend).and_return(:db)
+        Deimos.configure { producers.backend = :db }
         expect(described_class.determine_backend_class(true, true)).
           to eq(Deimos::Backends::Kafka)
         expect(described_class.determine_backend_class(false, true)).

data/spec/spec_helper.rb

@@ -144,33 +144,21 @@ RSpec.configure do |config|
     Time.zone = 'EST'
     ActiveRecord::Base.logger = Logger.new('/dev/null')
     setup_db(DbConfigs::DB_OPTIONS.last)
+  end
+
+  config.before(:each) do |ex|
+    Deimos.config.reset!
     Deimos.configure do |deimos_config|
       deimos_config.phobos_config_file = File.join(File.dirname(__FILE__), 'phobos.yml')
-      deimos_config.schema_path = File.join(File.expand_path(__dir__), 'schemas')
-      deimos_config.reraise_consumer_errors = true
-      deimos_config.schema_registry_url = ENV['SCHEMA_REGISTRY'] || 'http://localhost:8081'
-      deimos_config.seed_broker = ENV['KAFKA_SEED_BROKER'] || 'localhost:9092'
+      deimos_config.schema.path = File.join(File.expand_path(__dir__), 'schemas')
+      deimos_config.consumers.reraise_errors = true
+      deimos_config.schema.registry_url = ENV['SCHEMA_REGISTRY'] || 'http://localhost:8081'
+      deimos_config.kafka.seed_brokers = ENV['KAFKA_SEED_BROKER'] || 'localhost:9092'
       deimos_config.logger = Logger.new('/dev/null')
       deimos_config.logger.level = Logger::INFO
-
-      # Use Mock Metrics and Tracing for rspecs
-      deimos_config.metrics = Deimos::Metrics::Mock.new
-      deimos_config.tracer = Deimos::Tracing::Mock.new
     end
-  end
-
-  config.before(:each) do |ex|
     stub_producers_and_consumers! unless ex.metadata[:integration]
-
-    @previous_config = Deimos.config.dup
-    @previous_phobos_config = Phobos.config.dup
   end
-
-  config.after(:each) do
-    Deimos.config = @previous_config
-    Phobos.instance_variable_set(:@config, @previous_phobos_config)
-  end
-
 end
 
 RSpec.shared_context('with DB') do

data/spec/updateable_schema_store_spec.rb

@@ -3,7 +3,7 @@
 describe AvroTurf::SchemaStore do
 
   it 'should add an in-memory schema' do
-    schema_store = described_class.new(path: Deimos.config.schema_path)
+    schema_store = described_class.new(path: Deimos.config.schema.path)
     schema_store.load_schemas!
     found_schema = schema_store.find('MySchema', 'com.my-namespace').as_json
     expect(found_schema['name']).to eq('MySchema')

data/spec/utils/db_producer_spec.rb

@@ -192,11 +192,6 @@ each_db_config(Deimos::Utils::DbProducer) do
                                                                       topic: 'my-topic'
                                                                     }
                                                                   ])
-      expect(Deimos.config.metrics).to receive(:increment).ordered.with(
-        'db_producer.process',
-        tags: %w(topic:my-topic),
-        by: 2
-      )
       expect(producer).to receive(:retrieve_messages).ordered.
         and_return(messages[2..3])
       expect(producer).to receive(:produce_messages).ordered.with([
@@ -213,11 +208,6 @@ each_db_config(Deimos::Utils::DbProducer) do
                                                                       topic: 'my-topic'
                                                                     }
                                                                   ])
-      expect(Deimos.config.metrics).to receive(:increment).ordered.with(
-        'db_producer.process',
-        tags: %w(topic:my-topic),
-        by: 2
-      )
       expect(producer).to receive(:retrieve_messages).ordered.
         and_return([])
       expect(Deimos::KafkaTopicInfo).to receive(:heartbeat).

data/spec/utils/lag_reporter_spec.rb

@@ -6,12 +6,12 @@ describe Deimos::Utils::LagReporter do
     kafka_client = instance_double(Kafka::Client)
     allow(kafka_client).to receive(:last_offset_for).and_return(100)
     allow(Phobos).to receive(:create_kafka_client).and_return(kafka_client)
-    Deimos.configure { |c| c.report_lag = true }
+    Deimos.configure { |c| c.consumers.report_lag = true }
   end
 
   after(:each) do
     described_class.reset
-    Deimos.configure { |c| c.report_lag = false }
+    Deimos.configure { |c| c.consumers.report_lag = false }
   end
 
   it 'should not report lag before ready' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.3.0.pre.beta5
+  version: 1.4.0.pre.beta1
 platform: ruby
 authors:
 - Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-14 00:00:00.000000000 Z
+date: 2019-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro-patches
@@ -206,20 +206,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
-- !ruby/object:Gem::Dependency
-  name: rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.2'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -331,6 +317,7 @@ files:
 - bin/deimos
 - deimos-ruby.gemspec
 - docker-compose.yml
+- docs/CONFIGURATION.md
 - docs/DATABASE_BACKEND.md
 - docs/PULL_REQUEST_TEMPLATE.md
 - lib/deimos.rb
@@ -344,7 +331,9 @@ files:
 - lib/deimos/backends/kafka_async.rb
 - lib/deimos/base_consumer.rb
 - lib/deimos/batch_consumer.rb
-- lib/deimos/configuration.rb
+- lib/deimos/config/configurable.rb
+- lib/deimos/config/configuration.rb
+- lib/deimos/config/phobos_config.rb
 - lib/deimos/consumer.rb
 - lib/deimos/instrumentation.rb
 - lib/deimos/kafka_message.rb
@@ -386,6 +375,8 @@ files:
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
 - spec/batch_consumer_spec.rb
+- spec/config/configurable_spec.rb
+- spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
 - spec/handlers/my_batch_consumer.rb
@@ -396,7 +387,6 @@ files:
 - spec/phobos.yml
 - spec/producer_spec.rb
 - spec/publish_backend_spec.rb
-- spec/rake_spec.rb
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc
@@ -448,6 +438,8 @@ test_files:
 - spec/backends/kafka_async_spec.rb
 - spec/backends/kafka_spec.rb
 - spec/batch_consumer_spec.rb
+- spec/config/configurable_spec.rb
+- spec/config/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/deimos_spec.rb
 - spec/handlers/my_batch_consumer.rb
@@ -458,7 +450,6 @@ test_files:
 - spec/phobos.yml
 - spec/producer_spec.rb
 - spec/publish_backend_spec.rb
-- spec/rake_spec.rb
 - spec/schemas/com/my-namespace/MySchema-key.avsc
 - spec/schemas/com/my-namespace/MySchema.avsc
 - spec/schemas/com/my-namespace/MySchemaWithBooleans.avsc