deimos-ruby 1.3.0.pre.beta5 → 1.4.0.pre.beta1

data/lib/deimos/config/phobos_config.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/array'
+
+module Deimos
+  # Module to handle phobos.yml as well as outputting the configuration to save
+  # to Phobos itself.
+  module PhobosConfig
+    extend ActiveSupport::Concern
+
+    # @return [Hash]
+    def to_h
+      (FIELDS + [:handler]).map { |f|
+        val = self.send(f)
+        if f == :backoff && val
+          [:backoff, _backoff(val)]
+        elsif val.present?
+          [f, val]
+        end
+      }.to_h
+    end
+
+    # :nodoc:
+    def reset!
+      super
+      Phobos.configure(self.phobos_config)
+    end
+
+    # Create a hash representing the config that Phobos expects.
+    # @return [Hash]
+    def phobos_config
+      p_config = {
+        logger: Logger.new(STDOUT),
+        custom_logger: self.phobos_logger,
+        custom_kafka_logger: self.kafka.logger,
+        kafka: {
+          client_id: self.kafka.client_id,
+          connect_timeout: self.kafka.connect_timeout,
+          socket_timeout: self.kafka.socket_timeout,
+          ssl_verify_hostname: self.kafka.ssl.verify_hostname,
+          seed_brokers: Array.wrap(self.kafka.seed_brokers)
+        },
+        producer: {
+          ack_timeout: self.producers.ack_timeout,
+          required_acks: self.producers.required_acks,
+          max_retries: self.producers.max_retries,
+          retry_backoff: self.producers.retry_backoff,
+          max_buffer_size: self.producers.max_buffer_size,
+          max_buffer_bytesize: self.producers.max_buffer_bytesize,
+          compression_codec: self.producers.compression_codec,
+          compression_threshold: self.producers.compression_threshold,
+          max_queue_size: self.producers.max_queue_size,
+          delivery_threshold: self.producers.delivery_threshold,
+          delivery_interval: self.producers.delivery_interval
+        },
+        consumer: {
+          session_timeout: self.consumers.session_timeout,
+          offset_commit_interval: self.consumers.offset_commit_interval,
+          offset_commit_threshold: self.consumers.offset_commit_threshold,
+          heartbeat_interval: self.consumers.heartbeat_interval
+        },
+        backoff: _backoff(self.consumers.backoff.to_a)
+      }
+
+      p_config[:listeners] = self.consumer_objects.map do |consumer|
+        hash = consumer.to_h.reject do |k, _|
+          %i(class_name schema namespace key_config backoff).include?(k)
+        end
+        hash = hash.map { |k, v| [k, v.is_a?(Symbol) ? v.to_s : v] }.to_h
+        hash[:handler] = consumer.class_name
+        if consumer.backoff
+          hash[:backoff] = _backoff(consumer.backoff.to_a)
+        end
+        hash
+      end
+
+      if self.kafka.ssl.enabled
+        %w(ca_cert client_cert client_cert_key).each do |key|
+          next if self.kafka.ssl.send(key).blank?
+
+          p_config[:kafka]["ssl_#{key}".to_sym] = ssl_var_contents(self.kafka.ssl.send(key))
+        end
+      end
+      p_config
+    end
+
+    # @param key [String]
+    # @return [String]
+    def ssl_var_contents(key)
+      File.exist?(key) ? File.read(key) : key
+    end
+
+    # Legacy method to parse Phobos config file
+    def phobos_config_file=(file)
+      pconfig = YAML.load(ERB.new(File.read(File.expand_path(file))).result). # rubocop:disable Security/YAMLLoad
+        with_indifferent_access
+      self.logger&.warn('phobos.yml is deprecated - use direct configuration instead.')
+      pconfig[:kafka].each do |k, v|
+        if k.starts_with?('ssl')
+          k = k.sub('ssl_', '')
+          self.kafka.ssl.send("#{k}=", v)
+        else
+          self.kafka.send("#{k}=", v)
+        end
+      end
+      pconfig[:producer].each do |k, v|
+        self.producers.send("#{k}=", v)
+      end
+      pconfig[:consumer].each do |k, v|
+        self.consumers.send("#{k}=", v)
+      end
+      self.consumers.backoff = pconfig[:backoff][:min_ms]..pconfig[:backoff][:max_ms]
+      pconfig[:listeners].each do |listener_hash|
+        self.consumer do
+          listener_hash.each do |k, v|
+            k = 'class_name' if k == 'handler'
+            send(k, v)
+          end
+        end
+      end
+    end
+
+  private
+
+    # @param values [Array<Integer>]
+    # @return [Hash<Integer>]
+    def _backoff(values)
+      {
+        min_ms: values[0],
+        max_ms: values[-1]
+      }
+    end
+  end
+end

data/lib/deimos/kafka_message.rb

@@ -14,6 +14,12 @@ module Deimos
       write_attribute(:message, mess ? mess.to_s : nil)
     end
 
+    # Decoded payload for this message.
+    # @return [Hash]
+    def decoded_message
+      self.class.decoded([self]).first
+    end
+
     # Get a decoder to decode a set of messages on the given topic.
     # @param topic [String]
     # @return [Deimos::Consumer]

data/lib/deimos/producer.rb

@@ -63,7 +63,7 @@ module Deimos
       def config
         @config ||= {
           encode_key: true,
-          namespace: Deimos.config.producer_schema_namespace
+          namespace: Deimos.config.producers.schema_namespace
         }
       end
 
@@ -76,7 +76,7 @@ module Deimos
           return
         end
         # accessor
-        "#{Deimos.config.producer_topic_prefix}#{config[:topic]}"
+        "#{Deimos.config.producers.topic_prefix}#{config[:topic]}"
       end
 
       # Override the default partition key (which is the payload key).
@@ -100,8 +100,8 @@ module Deimos
       # @param force_send [Boolean] if true, ignore the configured backend
       # and send immediately to Kafka.
       def publish_list(payloads, sync: nil, force_send: false)
-        return if Deimos.config.seed_broker.blank? ||
-                  Deimos.config.disable_producers ||
+        return if Deimos.config.kafka.seed_brokers.blank? ||
+                  Deimos.config.producers.disabled ||
                   Deimos.producers_disabled?(self)
 
         backend_class = determine_backend_class(sync, force_send)
@@ -126,7 +126,7 @@ module Deimos
         backend = if force_send
                     :kafka
                   else
-                    Deimos.config.publish_backend
+                    Deimos.config.producers.backend
                   end
         if backend == :kafka_async && sync
           backend = :kafka

data/lib/deimos/test_helpers.rb

@@ -61,7 +61,6 @@ module Deimos
             encoder = Deimos::AvroDataEncoder.new(schema: schema,
                                                   namespace: decoder.namespace)
             encoder.schema_store = decoder.schema_store
-            payload = payload.respond_to?(:stringify_keys) ? payload.stringify_keys : payload
             encoder.encode_local(payload)
           end
           payload
@@ -77,9 +76,8 @@ module Deimos
         config.before(:suite) do
           Deimos.configure do |fr_config|
             fr_config.logger = Logger.new(STDOUT)
-            fr_config.seed_broker ||= 'test_broker'
-            fr_config.tracer = Deimos::Tracing::Mock.new
-            fr_config.metrics = Deimos::Metrics::Mock.new
+            fr_config.consumers.reraise_errors = true
+            fr_config.kafka.seed_brokers ||= ['test_broker']
           end
         end
 
@@ -469,12 +467,6 @@ module Deimos
       allow(klass).to receive(:decoder) do
         create_decoder(klass.config[:schema], klass.config[:namespace])
       end
-
-      if klass.config[:key_schema] # rubocop:disable Style/GuardClause
-        allow(klass).to receive(:key_decoder) do
-          create_decoder(klass.config[:key_schema], klass.config[:namespace])
-        end
-      end
     end
   end
 end

data/lib/deimos/utils/db_producer.rb

@@ -93,11 +93,6 @@ module Deimos
           end
         end
         Deimos::KafkaMessage.where(id: messages.map(&:id)).delete_all
-        Deimos.config.metrics&.increment(
-          'db_producer.process',
-          tags: %W(topic:#{@current_topic}),
-          by: messages.size
-        )
         return false if batch_size < BATCH_SIZE
 
         KafkaTopicInfo.heartbeat(@current_topic, @id) # keep alive

data/lib/deimos/utils/lag_reporter.rb

@@ -153,28 +153,28 @@ module Deimos
   end
 
   ActiveSupport::Notifications.subscribe('start_process_message.consumer.kafka') do |*args|
-    next unless Deimos.config.report_lag
+    next unless Deimos.config.consumers.report_lag
 
     event = ActiveSupport::Notifications::Event.new(*args)
     Deimos::Utils::LagReporter.message_processed(event.payload)
   end
 
   ActiveSupport::Notifications.subscribe('start_process_batch.consumer.kafka') do |*args|
-    next unless Deimos.config.report_lag
+    next unless Deimos.config.consumers.report_lag
 
     event = ActiveSupport::Notifications::Event.new(*args)
     Deimos::Utils::LagReporter.message_processed(event.payload)
   end
 
   ActiveSupport::Notifications.subscribe('seek.consumer.kafka') do |*args|
-    next unless Deimos.config.report_lag
+    next unless Deimos.config.consumers.report_lag
 
     event = ActiveSupport::Notifications::Event.new(*args)
     Deimos::Utils::LagReporter.offset_seek(event.payload)
   end
 
   ActiveSupport::Notifications.subscribe('heartbeat.consumer.kafka') do |*args|
-    next unless Deimos.config.report_lag
+    next unless Deimos.config.consumers.report_lag
 
     event = ActiveSupport::Notifications::Event.new(*args)
     Deimos::Utils::LagReporter.heartbeat(event.payload)

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.3.0-beta5'
+  VERSION = '1.4.0-beta1'
 end

data/lib/deimos.rb

@@ -4,6 +4,7 @@ require 'avro-patches'
 require 'avro_turf'
 require 'phobos'
 require 'deimos/version'
+require 'deimos/config/configuration'
 require 'deimos/avro_data_encoder'
 require 'deimos/avro_data_decoder'
 require 'deimos/producer'
@@ -11,7 +12,6 @@ require 'deimos/active_record_producer'
 require 'deimos/active_record_consumer'
 require 'deimos/consumer'
 require 'deimos/batch_consumer'
-require 'deimos/configuration'
 require 'deimos/instrumentation'
 require 'deimos/utils/lag_reporter'
 
@@ -40,48 +40,10 @@ require 'erb'
 # Parent module.
 module Deimos
   class << self
-    attr_accessor :config
-
-    # Configure Deimos.
-    def configure
-      first_time_config = self.config.nil?
-      self.config ||= Configuration.new
-      old_config = self.config.dup
-      yield(config)
-
-      # Don't re-configure Phobos every time
-      if first_time_config || config.phobos_config_changed?(old_config)
-
-        file = config.phobos_config_file
-        phobos_config = YAML.load(ERB.new(File.read(File.expand_path(file))).result)
-
-        configure_kafka_for_phobos(phobos_config)
-        configure_loggers(phobos_config)
-
-        Phobos.configure(phobos_config)
-
-        validate_consumers
-      end
-
-      validate_db_backend if self.config.publish_backend == :db
-    end
-
-    # Ensure everything is set up correctly for the DB backend.
-    def validate_db_backend
-      begin
-        require 'activerecord-import'
-      rescue LoadError
-        raise 'Cannot set publish_backend to :db without activerecord-import! Please add it to your Gemfile.'
-      end
-      if Phobos.config.producer_hash[:required_acks] != :all
-        raise 'Cannot set publish_backend to :db unless required_acks is set to ":all" in phobos.yml!'
-      end
-    end
-
     # Start the DB producers to send Kafka messages.
     # @param thread_count [Integer] the number of threads to start.
     def start_db_backend!(thread_count: 1)
-      if self.config.publish_backend != :db
+      if self.config.producers.backend != :db
         raise('Publish backend is not set to :db, exiting')
       end
 
@@ -99,52 +61,6 @@ module Deimos
       signal_handler = Deimos::Utils::SignalHandler.new(executor)
       signal_handler.run!
     end
-
-    # @param phobos_config [Hash]
-    def configure_kafka_for_phobos(phobos_config)
-      if config.ssl_enabled
-        %w(ssl_ca_cert ssl_client_cert ssl_client_cert_key).each do |key|
-          next if config.send(key).blank?
-
-          phobos_config['kafka'][key] = ssl_var_contents(config.send(key))
-        end
-      end
-      phobos_config['kafka']['seed_brokers'] = config.seed_broker if config.seed_broker
-    end
-
-    # @param phobos_config [Hash]
-    def configure_loggers(phobos_config)
-      phobos_config['custom_logger'] = config.phobos_logger
-      phobos_config['custom_kafka_logger'] = config.kafka_logger
-    end
-
-    # @param filename [String] a file to read, or the contents of the SSL var
-    # @return [String] the contents of the file
-    def ssl_var_contents(filename)
-      File.exist?(filename) ? File.read(filename) : filename
-    end
-
-    # Validate that consumers are configured correctly, including their
-    # delivery mode.
-    def validate_consumers
-      Phobos.config.listeners.each do |listener|
-        handler_class = listener.handler.constantize
-        delivery = listener.delivery
-
-        # Validate that Deimos consumers use proper delivery configs
-        if handler_class < Deimos::BatchConsumer
-          unless delivery == 'inline_batch'
-            raise "BatchConsumer #{listener.handler} must have delivery set to"\
-              ' `inline_batch`'
-          end
-        elsif handler_class < Deimos::Consumer
-          if delivery.present? && !%w(message batch).include?(delivery)
-            raise "Non-batch Consumer #{listener.handler} must have delivery"\
-              ' set to `message` or `batch`'
-          end
-        end
-      end
-    end
   end
 end
 

data/lib/tasks/deimos.rake

@@ -7,7 +7,7 @@ namespace :deimos do
   desc 'Starts Deimos in the rails environment'
   task start: :environment do
     Deimos.configure do |config|
-      config.publish_backend = :kafka if config.publish_backend == :kafka_async
+      config.producers.backend = :kafka_sync if config.producers.backend == :kafka_async
     end
     ENV['DEIMOS_RAKE_TASK'] = 'true'
     STDOUT.sync = true

data/spec/backends/db_spec.rb

@@ -4,11 +4,6 @@ each_db_config(Deimos::Backends::Db) do
   include_context 'with publish_backend'
 
   it 'should save to the database' do
-    expect(Deimos.config.metrics).to receive(:increment).with(
-      'db_producer.insert',
-      tags: %w(topic:my-topic),
-      by: 3
-    )
     described_class.publish(producer_class: MyProducer, messages: messages)
     records = Deimos::KafkaMessage.all
     expect(records.size).to eq(3)

data/spec/batch_consumer_spec.rb

@@ -50,7 +50,7 @@ module ConsumerTest
 
     describe 'when reraising errors is disabled' do
       before(:each) do
-        Deimos.configure { |config| config.reraise_consumer_errors = false }
+        Deimos.configure { |config| config.consumers.reraise_errors = false }
       end
 
       it 'should not fail when before_consume_batch fails' do

data/spec/config/configurable_spec.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+# :nodoc:
+class MyConfig
+  include Deimos::Configurable
+
+  configure do
+    setting :set1
+    setting :set2, 'hi mom'
+    setting :group do
+      setting :set3, default_proc: proc { false }
+      setting :set5, (proc { 5 })
+    end
+
+    setting_object :listy do
+      setting :list1, 10
+      setting :list2, 5
+    end
+  end
+end
+
+describe Deimos::Configurable do
+  it 'should configure correctly with default values' do
+    expect(MyConfig.config.set1).to be_nil
+    expect(MyConfig.config.set2).to eq('hi mom')
+    expect(MyConfig.config.group.set3).to eq(false)
+    expect(MyConfig.config.listy_objects).to be_empty
+    expect { MyConfig.config.blah }.to raise_error(NameError)
+    expect { MyConfig.config.group.set4 }.to raise_error(NameError)
+  end
+
+  it "should raise error when setting configs that don't exist" do
+    expect { MyConfig.configure { set15 'some_value' } }.to raise_error(NameError)
+  end
+
+  it 'should add values' do
+    MyConfig.configure do |config|
+      config.set1 = 5 # config.x syntax
+      set2 nil # method_missing syntax
+      config.group.set3 = true
+    end
+
+    # second configure should not blow anything away
+    MyConfig.configure do
+      listy do
+        list1 0
+        list2 1
+      end
+      listy do
+        list1 100
+        list2 200
+      end
+    end
+
+    expect(MyConfig.config.set1).to eq(5)
+    expect(MyConfig.config.set2).to be_nil
+    expect(MyConfig.config.group.set3).to eq(true)
+    expect(MyConfig.config.listy_objects.map(&:to_h)).
+      to eq([
+              { list1: 0, list2: 1 },
+              { list1: 100, list2: 200 }
+            ])
+
+    # test reset!
+    MyConfig.config.reset!
+    expect(MyConfig.config.set1).to be_nil
+    expect(MyConfig.config.set2).to eq('hi mom')
+    expect(MyConfig.config.group.set3).to eq(false)
+    expect(MyConfig.config.listy_objects).to be_empty
+  end
+
+  it 'should add with block syntax' do
+    MyConfig.configure do
+      group do
+        set5(proc { 10 })
+      end
+    end
+    expect(MyConfig.config.group.set5.call).to eq(10)
+  end
+
+  it 'should add or redefine settings' do
+    MyConfig.configure do
+      setting :group do
+        setting :set6, 15
+        setting :set5, (proc { 15 })
+      end
+      setting_object :notey do
+        setting :note_title, 'some-title'
+      end
+    end
+
+    expect(MyConfig.config.group.set6).to eq(15)
+    expect(MyConfig.config.group.set5.call).to eq(15)
+    expect(MyConfig.config.listy_objects).to be_empty
+    expect(MyConfig.config.notey_objects).to be_empty
+
+    MyConfig.configure do
+      notey do
+        note_title 'hi mom'
+      end
+      listy do
+        list1 0
+      end
+    end
+    expect(MyConfig.config.notey_objects.size).to eq(1)
+    expect(MyConfig.config.notey_objects.first.note_title).to eq('hi mom')
+    expect(MyConfig.config.listy_objects.size).to eq(1)
+    expect(MyConfig.config.listy_objects.first.list1).to eq(0)
+
+    # This should not remove any keys
+    MyConfig.configure do
+      setting :group do
+        setting :set6, 20
+      end
+    end
+    expect(MyConfig.config.group.set6).to eq(20)
+    expect(MyConfig.config.group.set5.call).to eq(15)
+
+  end
+
+end