deimos-ruby 1.24.2 → 2.0.0.pre.alpha1

data/lib/deimos/utils/db_poller/base.rb

@@ -11,6 +11,7 @@ module Deimos
       # Base poller class for retrieving and publishing messages.
       class Base
 
+        FATAL_CODES = %i(invalid_msg_size msg_size_too_large)
         # @return [Integer]
         BATCH_SIZE = 1000
 
@@ -58,14 +59,14 @@ module Deimos
           if Deimos.config.producers.backend == :kafka_async
             Deimos.config.producers.backend = :kafka
           end
-          Deimos.config.logger.info('Starting...')
+          Deimos::Logging.log_info('Starting...')
           @signal_to_stop = false
           ActiveRecord::Base.connection.reconnect! unless ActiveRecord::Base.connection.open_transactions.positive?
 
           retrieve_poll_info
           loop do
             if @signal_to_stop
-              Deimos.config.logger.info('Shutting down')
+              Deimos::Logging.log_info('Shutting down')
               break
             end
             process_updates if should_run?
@@ -95,7 +96,7 @@ module Deimos
         # Stop the poll.
         # @return [void]
         def stop
-          Deimos.config.logger.info('Received signal to stop')
+          Deimos::Logging.log_info('Received signal to stop')
           @signal_to_stop = true
         end
 
@@ -111,9 +112,9 @@ module Deimos
         # @param span [Object]
         # @return [Boolean]
         def handle_message_too_large(exception, batch, status, span)
-          Deimos.config.logger.error("Error publishing through DB Poller: #{exception.message}")
+          Deimos::Logging.log_error("Error publishing through DB Poller: #{exception.message}")
           if @config.skip_too_large_messages
-            Deimos.config.logger.error("Skipping messages #{batch.map(&:id).join(', ')} since they are too large")
+            Deimos::Logging.log_error("Skipping messages #{batch.map(&:id).join(', ')} since they are too large")
             Deimos.config.tracer&.set_error(span, exception)
             status.batches_errored += 1
             true
@@ -137,21 +138,22 @@ module Deimos
             process_batch(batch)
             Deimos.config.tracer&.finish(span)
             status.batches_processed += 1
-          rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,
-                 Kafka::RecordListTooLarge => e
-            retry unless handle_message_too_large(e, batch, status, span)
-          rescue Kafka::Error => e # keep trying till it fixes itself
-            Deimos.config.logger.error("Error publishing through DB Poller: #{e.message}")
-            sleep(0.5)
-            retry
+          rescue WaterDrop::Errors::ProduceManyError => e
+            if FATAL_CODES.include?(e.cause.try(:code))
+              retry unless handle_message_too_large(e, batch, status, span)
+            else
+              Deimos::Logging.log_error("Error publishing through DB Poller: #{e.message}")
+              sleep(0.5)
+              retry
+            end
           rescue StandardError => e
-            Deimos.config.logger.error("Error publishing through DB poller: #{e.message}}")
+            Deimos::Logging.log_error("Error publishing through DB poller: #{e.message}}")
             if @config.retries.nil? || retries < @config.retries
               retries += 1
               sleep(0.5)
               retry
             else
-              Deimos.config.logger.error('Retries exceeded, moving on to next batch')
+              Deimos::Logging.log_error('Retries exceeded, moving on to next batch')
               Deimos.config.tracer&.set_error(span, e)
               status.batches_errored += 1
               return false

data/lib/deimos/utils/db_poller/state_based.rb

@@ -10,14 +10,14 @@ module Deimos
         # Send messages for updated data.
         # @return [void]
         def process_updates
-          Deimos.config.logger.info("Polling #{log_identifier}")
+          Deimos::Logging.log_info("Polling #{log_identifier}")
           status = PollStatus.new(0, 0, 0)
           first_batch = true
 
           # poll_query gets all the relevant data from the database, as defined
           # by the producer itself.
           loop do
-            Deimos.config.logger.debug("Polling #{log_identifier}, batch #{status.current_batch}")
+            Deimos::Logging.log_debug("Polling #{log_identifier}, batch #{status.current_batch}")
             batch = fetch_results.to_a
             break if batch.empty?
 
@@ -29,7 +29,7 @@ module Deimos
           # If there were no results at all, we update last_sent so that we still get a wait
           # before the next poll.
           @info.touch(:last_sent) if first_batch
-          Deimos.config.logger.info("Poll #{log_identifier} complete (#{status.report}")
+          Deimos::Logging.log_info("Poll #{log_identifier} complete (#{status.report}")
         end
 
         # @return [ActiveRecord::Relation]

data/lib/deimos/utils/db_poller/time_based.rb

@@ -28,14 +28,14 @@ module Deimos
         def process_updates
           time_from = @config.full_table ? Time.new(0) : @info.last_sent.in_time_zone
           time_to = Time.zone.now - @config.delay_time
-          Deimos.config.logger.info("Polling #{log_identifier} from #{time_from} to #{time_to}")
+          Deimos::Logging.log_info("Polling #{log_identifier} from #{time_from} to #{time_to}")
           status = PollStatus.new(0, 0, 0)
           first_batch = true
 
           # poll_query gets all the relevant data from the database, as defined
           # by the producer itself.
           loop do
-            Deimos.config.logger.debug("Polling #{log_identifier}, batch #{status.current_batch}")
+            Deimos::Logging.log_debug("Polling #{log_identifier}, batch #{status.current_batch}")
             batch = fetch_results(time_from, time_to).to_a
             break if batch.empty?
 
@@ -47,14 +47,14 @@ module Deimos
           # If there were no results at all, we update last_sent so that we still get a wait
           # before the next poll.
           @info.touch(:last_sent) if first_batch
-          Deimos.config.logger.info("Poll #{log_identifier} complete at #{time_to} (#{status.report})")
+          Deimos::Logging.log_info("Poll #{log_identifier} complete at #{time_to} (#{status.report})")
         end
 
         # @param time_from [ActiveSupport::TimeWithZone]
         # @param time_to [ActiveSupport::TimeWithZone]
         # @return [ActiveRecord::Relation]
         def fetch_results(time_from, time_to)
-          id = self.producer_classes.first.config[:record_class].primary_key
+          id = self.producer_classes.first.record_class.primary_key
           quoted_timestamp = ActiveRecord::Base.connection.quote_column_name(@config.timestamp_column)
           quoted_id = ActiveRecord::Base.connection.quote_column_name(id)
           @resource_class.poll_query(time_from: time_from,

data/lib/deimos/utils/db_poller.rb

@@ -16,7 +16,7 @@ module Deimos
         end
         executor = Sigurd::Executor.new(pollers,
                                         sleep_seconds: 5,
-                                        logger: Deimos.config.logger)
+                                        logger: Karafka.logger)
         signal_handler = Sigurd::SignalHandler.new(executor)
         signal_handler.run!
       end

data/lib/deimos/utils/deadlock_retry.rb

@@ -45,7 +45,7 @@ module Deimos
             # Reraise if all retries exhausted
             raise if count <= 0
 
-            Deimos.config.logger.warn(
+            Deimos::Logging.log_warn(
               message: 'Deadlock encountered when trying to execute query. '\
                 "Retrying. #{count} attempt(s) remaining",
               tags: tags

data/lib/deimos/utils/{db_producer.rb → outbox_producer.rb}

@@ -4,8 +4,7 @@ module Deimos
   module Utils
     # Class which continually polls the kafka_messages table
     # in the database and sends Kafka messages.
-    class DbProducer
-      include Phobos::Producer
+    class OutboxProducer
       attr_accessor :id, :current_topic
 
       # @return [Integer]
@@ -14,17 +13,19 @@ module Deimos
       DELETE_BATCH_SIZE = 10
       # @return [Integer]
       MAX_DELETE_ATTEMPTS = 3
+      # @return [Array<Symbol>]
+      FATAL_CODES = %i(invalid_msg_size msg_size_too_large)
 
       # @param logger [Logger]
       def initialize(logger=Logger.new(STDOUT))
         @id = SecureRandom.uuid
         @logger = logger
-        @logger.push_tags("DbProducer #{@id}") if @logger.respond_to?(:push_tags)
+        @logger.push_tags("OutboxProducer #{@id}") if @logger.respond_to?(:push_tags)
       end
 
       # @return [FigTree]
       def config
-        Deimos.config.db_producer
+        Deimos.config.outbox
       end
 
       # Start the poll.
@@ -82,7 +83,6 @@ module Deimos
       rescue StandardError => e
         @logger.error("Error processing messages for topic #{@current_topic}: #{e.class.name}: #{e.message} #{e.backtrace.join("\n")}")
         KafkaTopicInfo.register_error(@current_topic, @id)
-        shutdown_producer
       end
 
       # Process a single batch in a topic.
@@ -94,24 +94,23 @@ module Deimos
         batch_size = messages.size
         compacted_messages = compact_messages(messages)
         log_messages(compacted_messages)
-        Deimos.instrument('db_producer.produce', topic: @current_topic, messages: compacted_messages) do
+        Karafka.monitor.instrument('deimos.outbox.produce', topic: @current_topic, messages: compacted_messages) do
           begin
-            produce_messages(compacted_messages.map(&:phobos_message))
-          rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge, Kafka::RecordListTooLarge => e
-            delete_messages(messages)
-            @logger.error('Message batch too large, deleting...')
-            begin
-              @logger.error(Deimos::KafkaMessage.decoded(messages))
-            rescue StandardError => logging_exception # rubocop:disable Naming/RescuedExceptionsVariableName
-              @logger.error("Large message details logging failure: #{logging_exception.message}")
-            ensure
+            produce_messages(compacted_messages.map(&:karafka_message))
+          rescue WaterDrop::Errors::ProduceManyError => e
+            if FATAL_CODES.include?(e.cause.try(:code))
+              @logger.error('Message batch too large, deleting...')
+              delete_messages(messages)
               raise e
+            else
+              Deimos.log_error("Got error #{e.cause.class.name} when publishing #{batch_size} messages, retrying...")
+              retry
             end
           end
         end
         delete_messages(messages)
         Deimos.config.metrics&.increment(
-          'db_producer.process',
+          'outbox.process',
           tags: %W(topic:#{@current_topic}),
           by: messages.size
         )
@@ -197,16 +196,6 @@ module Deimos
         end
       end
 
-      # Shut down the sync producer if we have to. Phobos will automatically
-      # create a new one. We should call this if the producer can be in a bad
-      # state and e.g. we need to clear the buffer.
-      # @return [void]
-      def shutdown_producer
-        if self.class.producer.respond_to?(:sync_producer_shutdown) # Phobos 1.8.3
-          self.class.producer.sync_producer_shutdown
-        end
-      end
-
       # Produce messages in batches, reducing the size 1/10 if the batch is too
       # large. Does not retry batches of messages that have already been sent.
       # @param batch [Array<Hash>]
@@ -217,30 +206,10 @@ module Deimos
         begin
           batch[current_index..-1].in_groups_of(batch_size, false).each do |group|
             @logger.debug("Publishing #{group.size} messages to #{@current_topic}")
-            producer.publish_list(group)
-            Deimos.config.metrics&.increment(
-              'publish',
-              tags: %W(status:success topic:#{@current_topic}),
-              by: group.size
-            )
+            Karafka.producer.produce_many_sync(group)
             current_index += group.size
             @logger.info("Sent #{group.size} messages to #{@current_topic}")
           end
-        rescue Kafka::BufferOverflow, Kafka::MessageSizeTooLarge,
-               Kafka::RecordListTooLarge => e
-          if batch_size == 1
-            shutdown_producer
-            raise
-          end
-
-          @logger.error("Got error #{e.class.name} when publishing #{batch.size} in groups of #{batch_size}, retrying...")
-          batch_size = if batch_size < 10
-                         1
-                       else
-                         (batch_size / 10).to_i
-                       end
-          shutdown_producer
-          retry
         end
       end
 

data/lib/deimos/utils/schema_class.rb

@@ -55,13 +55,6 @@ module Deimos
           constants.join('::').safe_constantize
         end
 
-        # @param config [Hash] Producer or Consumer config
-        # @return [Boolean]
-        def use?(config)
-          use_schema_classes = config[:use_schema_classes]
-          use_schema_classes.present? ? use_schema_classes : Deimos.config.schema.use_schema_classes
-        end
-
       end
     end
   end

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.24.2'
+  VERSION = '2.0.0-alpha1'
 end

data/lib/deimos.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
 require 'active_support'
+require 'deimos/ext/redraw'
+require 'karafka'
 
-require 'phobos'
 require 'deimos/version'
+require 'deimos/logging'
 require 'deimos/config/configuration'
 require 'deimos/producer'
 require 'deimos/active_record_producer'
 require 'deimos/active_record_consumer'
 require 'deimos/consumer'
-require 'deimos/batch_consumer'
-require 'deimos/instrumentation'
-require 'deimos/utils/lag_reporter'
 
 require 'deimos/backends/base'
 require 'deimos/backends/kafka'
@@ -23,27 +22,38 @@ require 'deimos/utils/schema_class'
 require 'deimos/schema_class/enum'
 require 'deimos/schema_class/record'
 
-require 'deimos/monkey_patches/phobos_cli'
+require 'deimos/ext/schema_route'
+require 'deimos/ext/consumer_route'
+require 'deimos/ext/producer_route'
+require 'deimos/ext/producer_middleware'
+require 'deimos/ext/routing_defaults'
 
 require 'deimos/railtie' if defined?(Rails)
-require 'deimos/utils/schema_controller_mixin' if defined?(ActionController)
 
 if defined?(ActiveRecord)
   require 'deimos/kafka_source'
   require 'deimos/kafka_topic_info'
-  require 'deimos/backends/db'
+  require 'deimos/backends/outbox'
   require 'sigurd'
-  require 'deimos/utils/db_producer'
+  require 'deimos/utils/outbox_producer'
   require 'deimos/utils/db_poller'
 end
 
-require 'deimos/utils/inline_consumer'
 require 'yaml'
 require 'erb'
 
 # Parent module.
 module Deimos
+  EVENT_TYPES = %w(
+    deimos.ar_consumer.consume_batch
+    deimos.encode_message
+    deimos.batch_consumption.invalid_records
+    deimos.batch_consumption.valid_records
+    deimos.outbox.produce
+  )
+
   class << self
+
     # @return [Class<Deimos::SchemaBackends::Base>]
     def schema_backend_class
       backend = Deimos.config.schema.backend.to_s
@@ -57,7 +67,7 @@ module Deimos
     # @param namespace [String]
     # @return [Deimos::SchemaBackends::Base]
     def schema_backend(schema:, namespace:)
-      if Utils::SchemaClass.use?(config.to_h)
+      if config.schema.use_schema_classes
         # Initialize an instance of the provided schema
         # in the event the schema class is an override, the inherited
         # schema and namespace will be applied
@@ -91,13 +101,26 @@ module Deimos
       self.schema_backend(schema: schema, namespace: namespace).decode(payload)
     end
 
+    # @param message [Hash] a Karafka message with keys :payload, :key and :topic
+    def decode_message(message)
+      topic = message[:topic]
+      if Deimos.config.producers.topic_prefix
+        topic = topic.sub(Deimos.config.producers.topic_prefix, '')
+      end
+      config = karafka_config_for(topic: topic)
+      message[:payload] = config.deserializers[:payload].decode_message_hash(message[:payload])
+      if message[:key] && config.deserializers[:key].respond_to?(:decode_message_hash)
+        message[:key] = config.deserializers[:key].decode_message_hash(message[:key])
+      end
+    end
+
     # Start the DB producers to send Kafka messages.
     # @param thread_count [Integer] the number of threads to start.
     # @return [void]
-    def start_db_backend!(thread_count: 1)
+    def start_outbox_backend!(thread_count: 1)
       Sigurd.exit_on_signal = true
-      if self.config.producers.backend != :db
-        raise('Publish backend is not set to :db, exiting')
+      if self.config.producers.backend != :outbox
+        raise('Publish backend is not set to :outbox, exiting')
       end
 
       if thread_count.nil? || thread_count.zero?
@@ -105,25 +128,55 @@ module Deimos
       end
 
       producers = (1..thread_count).map do
-        Deimos::Utils::DbProducer.
-          new(self.config.db_producer.logger || self.config.logger)
+        Deimos::Utils::OutboxProducer.
+          new(self.config.outbox.logger || Karafka.logger)
       end
       executor = Sigurd::Executor.new(producers,
                                       sleep_seconds: 5,
-                                      logger: self.config.logger)
+                                      logger: Karafka.logger)
       signal_handler = Sigurd::SignalHandler.new(executor)
       signal_handler.run!
     end
-  end
-end
 
-at_exit do
-  begin
-    Deimos::Backends::KafkaAsync.shutdown_producer
-    Deimos::Backends::Kafka.shutdown_producer
-  rescue StandardError => e
-    Deimos.config.logger.error(
-      "Error closing producer on shutdown: #{e.message} #{e.backtrace.join("\n")}"
-    )
+    def setup_karafka
+      Karafka.producer.middleware.append(Deimos::ProducerMiddleware)
+      EVENT_TYPES.each { |type| Karafka.monitor.notifications_bus.register_event(type) }
+
+      Karafka.producer.monitor.subscribe('error.occurred') do |event|
+        if event.payload.key?(:messages)
+          topic = event[:messages].first[:topic]
+          config = Deimos.karafka_config_for(topic: topic)
+          message = Deimos::Logging.messages_log_text(config&.payload_log, event[:messages])
+          Karafka.logger.error("Error producing messages: #{event[:error].message} #{message.to_json}")
+        end
+      end
+    end
+
+    # @return [Array<Karafka::Routing::Topic]
+    def karafka_configs
+      Karafka::App.routes.flat_map(&:topics).flat_map(&:to_a)
+    end
+
+    # @param topic [String]
+    # @return [Karafka::Routing::Topic,nil]
+    def karafka_config_for(topic: nil, producer: nil)
+      if topic
+        karafka_configs.find { |t| t.name == topic}
+      elsif producer
+        karafka_configs.find { |t| t.producer_class == producer}
+      end
+    end
+
+    # @param handler_class [Class]
+    # @return [String,nil]
+    def topic_for_consumer(handler_class)
+      Deimos.karafka_configs.each do |topic|
+        if topic.consumer == handler_class
+          return topic.name
+        end
+      end
+      nil
+    end
+
   end
 end

data/lib/generators/deimos/{db_backend_generator.rb → outbox_backend_generator.rb}

@@ -6,14 +6,14 @@ require 'rails/generators/active_record/migration'
 module Deimos
   module Generators
     # Generate the database backend migration.
-    class DbBackendGenerator < Rails::Generators::Base
+    class OutboxBackendGenerator < Rails::Generators::Base
       include Rails::Generators::Migration
       if Rails.version < '4'
         extend(ActiveRecord::Generators::Migration)
       else
         include ActiveRecord::Generators::Migration
       end
-      source_root File.expand_path('db_backend/templates', __dir__)
+      source_root File.expand_path('outbox_backend/templates', __dir__)
       desc 'Add migrations for the database backend'
 
       # @return [String]
@@ -38,10 +38,10 @@ module Deimos
       def generate
         if Rails.version < '4'
           migration_template('rails3_migration',
-                             "#{db_migrate_path}/create_db_backend.rb")
+                             "#{db_migrate_path}/create_outbox_backend.rb")
         else
           migration_template('migration',
-                             "#{db_migrate_path}/create_db_backend.rb")
+                             "#{db_migrate_path}/create_outbox_backend.rb")
         end
       end
     end

data/lib/generators/deimos/schema_class_generator.rb

@@ -3,7 +3,6 @@
 require 'rails/generators'
 require 'deimos'
 require 'deimos/schema_backends/avro_base'
-require 'deimos/config/configuration'
 
 # Generates new schema classes.
 module Deimos

data/lib/generators/deimos/v2/templates/karafka.rb.tt

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require 'deimos'
+require 'karafka'
+
+Deimos.configure do
+<%- deimos_configs.each do |group, settings| -%>
+  <%= group -%> do
+  <%- settings.each do |k, v| -%>
+    <%= k.to_s %> <%= v.inspect %>
+  <%- end -%>
+  end
+  <%- end -%>
+  <%- deimos_config.db_poller_objects.each do |poller| -%>
+  db_poller do
+    <%- poller.non_default_settings! -%>
+    <%- poller.to_h.each do |k, v| -%>
+    <%= k.to_s %> <%= v.inspect %>
+    <%- end -%>
+  <%- end -%>
+  end
+end
+
+class KarafkaApp < Karafka::App
+  setup do |config|
+    <%- setup_configs.each do |k, v| -%>
+    config.<%= k %> = <%= v.inspect %>
+    <%- end -%>
+    config.kafka = {
+      <%- default_kafka_configs.compact.each do |k, v| -%>
+      "<%= k.to_s %>": <%= v.inspect %>,
+      <%- end -%>
+    }
+    # Recreate consumers with each batch. This will allow Rails code reload to work in the
+    # development mode. Otherwise Karafka process would not be aware of code changes
+    config.consumer_persistence = !Rails.env.development?
+  end
+
+  # Comment out this part if you are not using instrumentation and/or you are not
+  # interested in logging events for certain environments. Since instrumentation
+  # notifications add extra boilerplate, if you want to achieve max performance,
+  # listen to only what you really need for given environment.
+  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
+  # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
+
+  # This logger prints the producer development info using the Karafka logger.
+  # It is similar to the consumer logger listener but producer oriented.
+  Karafka.producer.monitor.subscribe(
+    WaterDrop::Instrumentation::LoggerListener.new(
+      # Log producer operations using the Karafka logger
+      Karafka.logger,
+      # If you set this to true, logs will contain each message details
+      # Please note, that this can be extensive
+      log_messages: false
+    )
+  )
+
+  # You can subscribe to all consumer related errors and record/track then that way
+  #
+  # Karafka.monitor.subscribe 'error.occurred' do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
+  # You can subscribe to all producer related errors and record/track then that way
+  # Please note, that producer and consumer have their own notifications pipeline so you need to
+  # setup error tracking independently for each of them
+  #
+  # Karafka.producer.monitor.subscribe('error.occurred') do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
+  routes.draw do
+    defaults do
+    <%- default_configs.each do |k, v| -%>
+      <%= k.to_s %> <%= v.inspect %>
+    <%- end -%>
+    end
+
+    <%- producer_configs.each do |producer| -%>
+    topic "<%= producer[:topic] %>" do
+      <%- producer.except(:topic).each do |k, v| -%>
+      <%- if k.to_sym == :key_config -%>
+      <%= k.to_s %>(<%= v.inspect %>)
+      <%- else -%>
+      <%= k.to_s %> <%= v.inspect %>
+      <%- end -%>
+      <%- end -%>
+    end
+    <%- end -%>
+
+    <%- consumer_configs.each do |group_id, topics| -%>
+    <%- if consumer_configs.length > 1 -%>
+    consumer_group :<%= group_id %> do<%- end -%>
+    <%- topics.each do |consumer| %>
+    topic "<%= consumer[:topic] -%>" do
+      <%- if consumer[:kafka].present? -%>
+      kafka(
+        <%- consumer[:kafka].each do |k, v| -%>
+        "<%= k.to_s %>": <%= v.inspect %>,
+        <%- end -%>
+      )
+      <%- end -%>
+      <%- consumer.except(:topic, :kafka).each do |k, v| -%>
+      <%- if k.to_sym == :key_config -%>
+      <%= k.to_s %>(<%= v.inspect %>)
+      <%- else -%>
+      <%= k.to_s %> <%= v.inspect %>
+      <%- end -%>
+      <%- end -%>
+    end
+    <%- end -%>
+    <%- if consumer_configs.length > 1 -%>
+    end<%- end %>
+    <%- end -%>
+
+    # Uncomment this if you use Karafka with ActiveJob
+    # You need to define the topic per each queue name you use
+    # active_job_topic :default
+    #    topic :example do
+      # Uncomment this if you want Karafka to manage your topics configuration
+      # Managing topics configuration via routing will allow you to ensure config consistency
+      # across multiple environments
+      #
+      # config(partitions: 2, 'cleanup.policy': 'compact')
+    # consumer ExampleConsumer
+    # end
+  end
+end
+
+Deimos.setup_karafka
+
+# Karafka now features a Web UI!
+# Visit the setup documentation to get started and enhance your experience.
+#
+# https://karafka.io/docs/Web-UI-Getting-Started
+
+# Karafka::Web.setup do |config|
+#   # You may want to set it per ENV. This value was randomly generated.
+#   config.ui.sessions.secret = '<%= SecureRandom.hex %>'
+# end
+
+# Karafka::Web.enable!
+