karafka 2.0.33 → 2.0.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 511cc39ce92e82706fcd6ac9c54b3416c7908cf5e9773ee15b972219046778ca
-  data.tar.gz: 737059dc4f9de33577f4883590f607da1ee4ee330ff9f5c18bd42dfe261c567d
+  metadata.gz: e2d2197fb0fe7eae8db43cc45ffbdcdc6da83d60731ae63b82dc4a24d852736e
+  data.tar.gz: 76b4b3f8ffb915b96a16f1eb83503283e5da46039fb952af383322773c4be551
 SHA512:
-  metadata.gz: d91e5de5bddd7b9ff2e423c986eda7f8e795528dc5ffb7a3694c82a2a096f8b3ee31679ac221bb0da4cd6efc7efa55ab195f081fbd2cf393df7d21549d950cd9
-  data.tar.gz: dbea8026eb3385b6ee14a749b3bc0187f4e91e9e66b60f2e8ac74fcdd409ad238b8e6f7dd5e5d5fbb5f0cc36299795ed63b794f1dd4a2a2b53e26776a7523876
+  metadata.gz: 35d69a8fa9a26462f2167174d8700648909bbcf38363c5adef2f7ee42995b524c0d244bffe0bd76698f65df5475db78b38914f38d7eda239376c4031555ee885
+  data.tar.gz: 49a2256930ce83bc46b6471cf702b1ea2fe6f897a805f92e4dee7c3d9034a9625a44ecff0222b8396ed73858af5571cca4eaabca8d188a81312312c2225daa97

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Karafka framework changelog
 
+## 2.0.35 (2023-03-13)
+- **[Feature]** Allow for defining topics config via the DSL and its automatic creation via CLI command.
+- **[Feature]** Allow for full topics reset and topics repartitioning via the CLI. 
+
+## 2.0.34 (2023-03-04)
+- [Improvement] Attach an `embedded` tag to Karafka processes started using the embedded API.
+- [Change] Renamed `Datadog::Listener` to `Datadog::MetricsListener` for consistency. (#1124)
+
+### Upgrade notes
+
+1. Replace `Datadog::Listener` references to `Datadog::MetricsListener`.
+
 ## 2.0.33 (2023-02-24)
 - **[Feature]** Support `perform_all_later` in ActiveJob adapter for Rails `7.1+`
 - **[Feature]** Introduce ability to assign and re-assign tags in consumer instances. This can be used for extra instrumentation that is context aware.
@@ -99,7 +111,7 @@ class KarafkaApp < Karafka::App
 - [Improvement] Expand `LoggerListener` with `client.resume` notification.
 - [Improvement] Replace random anonymous subscription groups ids with stable once.
 - [Improvement] Add `consumer.consume`, `consumer.revoke` and `consumer.shutting_down` notification events and move the revocation logic calling to strategies.
-- [Change] Rename job queue statistics `processing` key to `busy`. No changes needed because naming in the DataDog listener stays the same. 
+- [Change] Rename job queue statistics `processing` key to `busy`. No changes needed because naming in the DataDog listener stays the same.
 - [Fix] Fix proctitle listener state changes reporting on new states.
 - [Fix] Make sure all files descriptors are closed in the integration specs.
 - [Fix] Fix a case where empty subscription groups could leak into the execution flow.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.33)
+    karafka (2.0.35)
       karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
@@ -19,7 +19,7 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     byebug (11.1.3)
-    concurrent-ruby (1.2.0)
+    concurrent-ruby (1.2.2)
     diff-lcs (1.5.0)
     docile (1.4.0)
     factory_bot (6.2.1)
@@ -37,7 +37,7 @@ GEM
       mini_portile2 (~> 2.6)
       rake (> 12)
     mini_portile2 (2.8.1)
-    minitest (5.17.0)
+    minitest (5.18.0)
     rake (13.0.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -61,12 +61,13 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.10)
-      karafka-core (>= 2.0.9, < 3.0.0)
+    waterdrop (2.5.0)
+      karafka-core (>= 2.0.12, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.7)
 
 PLATFORMS
+  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -78,4 +79,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.4.6
+   2.4.7

data/config/locales/errors.yml

@@ -45,6 +45,10 @@ en:
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
       active_format: needs to be either true or false
+      structurable.partitions_format: needs to be more or equal to 1
+      structurable.active_format: needs to be true
+      structurable.replication_factor_format: needs to be more or equal to 1
+      structurable.details_format: needs to be a hash with only symbol keys
       inconsistent_namespacing: |
         needs to be consistent namespacing style
         disable this validation by setting config.strict_topics_namespacing to false

data/lib/karafka/cli/topics.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli < Thor
+    # CLI actions related to Kafka cluster topics management
+    class Topics < Base
+      include Helpers::Colorize
+
+      desc 'Allows for the topics management (create, delete, reset, repartition)'
+      # @param action [String] action we want to take
+      def call(action = 'missing')
+        case action
+        when 'create'
+          create
+        when 'delete'
+          delete
+        when 'reset'
+          reset
+        when 'repartition'
+          repartition
+        when 'migrate'
+          migrate
+        else
+          raise ::ArgumentError, "Invalid topics action: #{action}"
+        end
+      end
+
+      private
+
+      # Creates topics based on the routing setup and configuration
+      def create
+        structurable_routing_topics.each do |topic|
+          name = topic.name
+
+          if existing_topics_names.include?(name)
+            puts "#{yellow('Skipping')} because topic #{name} already exists."
+          else
+            puts "Creating topic #{name}..."
+            Admin.create_topic(
+              name,
+              topic.structurable.partitions,
+              topic.structurable.replication_factor,
+              topic.structurable.details
+            )
+            puts "#{green('Created')} topic #{name}."
+          end
+        end
+      end
+
+      # Deletes routing based topics
+      def delete
+        structurable_routing_topics.each do |topic|
+          name = topic.name
+
+          if existing_topics_names.include?(name)
+            puts "Deleting topic #{name}..."
+            Admin.delete_topic(name)
+            puts "#{green('Deleted')} topic #{name}."
+          else
+            puts "#{yellow('Skipping')} because topic #{name} does not exist."
+          end
+        end
+      end
+
+      # Deletes routing based topics and re-creates them
+      def reset
+        delete
+
+        # We need to invalidate the metadata cache, otherwise we will think, that the topic
+        # already exists
+        @existing_topics = nil
+
+        create
+      end
+
+      # Creates missing topics and aligns the partitions count
+      def migrate
+        create
+
+        @existing_topics = nil
+
+        repartition
+      end
+
+      # Increases number of partitions on topics that have less partitions than defined
+      # Will **not** create topics if missing.
+      def repartition
+        existing_partitions = existing_topics.map do |topic|
+          [topic.fetch(:topic_name), topic.fetch(:partition_count)]
+        end.to_h
+
+        structurable_routing_topics.each do |topic|
+          name = topic.name
+
+          desired_count = topic.config.partitions
+          existing_count = existing_partitions.fetch(name, false)
+
+          if existing_count && existing_count < desired_count
+            puts "Increasing number of partitions to #{desired_count} on topic #{name}..."
+            Admin.create_partitions(name, desired_count)
+            change = desired_count - existing_count
+            puts "#{green('Created')} #{change} additional partitions on topic #{name}."
+          elsif existing_count
+            puts "#{yellow('Skipping')} because topic #{name} has #{existing_count} partitions."
+          else
+            puts "#{yellow('Skipping')} because topic #{name} does not exist."
+          end
+        end
+      end
+
+      # @return [Array<Karafka::Routing::Topic>] all available topics that can be managed
+      # @note If topic is defined in multiple consumer groups, first config will be used. This
+      #   means, that this CLI will not work for simultaneous management of multiple clusters from
+      #   a single CLI command execution flow.
+      def structurable_routing_topics
+        return @structurable_routing_topics if @structurable_routing_topics
+
+        collected_topics = {}
+
+        App.consumer_groups.each do |consumer_group|
+          consumer_group.topics.each do |topic|
+            # Skip topics that were explicitly disabled from management
+            next unless topic.structurable.active?
+
+            collected_topics[topic.name] ||= topic
+          end
+        end
+
+        @structurable_routing_topics = collected_topics.values
+      end
+
+      # @return [Array<Hash>] existing topics details
+      def existing_topics
+        @existing_topics ||= Admin.cluster_info.topics
+      end
+
+      # @return [Array<String>] names of already existing topics
+      def existing_topics_names
+        existing_topics.map { |topic| topic.fetch(:topic_name) }
+      end
+    end
+  end
+end

data/lib/karafka/embedded.rb

@@ -7,7 +7,10 @@ module Karafka
       # Starts Karafka without supervision and without ownership of signals in a background thread
       # so it won't interrupt other things running
       def start
-        Thread.new { Karafka::Server.start }
+        Thread.new do
+          Karafka::Process.tags.add(:execution_mode, 'embedded')
+          Karafka::Server.start
+        end
       end
 
       # Stops Karafka upon any event

data/lib/karafka/helpers/colorize.rb

@@ -15,6 +15,12 @@ module Karafka
       def red(string)
         "\033[0;31m#{string}\033[0m"
       end
+
+      # @param string [String] string we want to have in yellow
+      # @return [String] yellow string
+      def yellow(string)
+        "\033[1;33m#{string}\033[0m"
+      end
     end
   end
 end

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -1,258 +1,15 @@
 # frozen_string_literal: true
 
+require_relative 'metrics_listener'
+
 module Karafka
   module Instrumentation
     # Namespace for vendor specific instrumentation
     module Vendors
       # Datadog specific instrumentation
       module Datadog
-        # Listener that can be used to subscribe to Karafka to receive stats via StatsD
-        # and/or Datadog
-        #
-        # @note You need to setup the `dogstatsd-ruby` client and assign it
-        class Listener
-          include ::Karafka::Core::Configurable
-          extend Forwardable
-
-          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
-
-          # Value object for storing a single rdkafka metric publishing details
-          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
-
-          # Namespace under which the DD metrics should be published
-          setting :namespace, default: 'karafka'
-
-          # Datadog client that we should use to publish the metrics
-          setting :client
-
-          # Default tags we want to publish (for example hostname)
-          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
-          setting :default_tags, default: []
-
-          # All the rdkafka metrics we want to publish
-          #
-          # By default we publish quite a lot so this can be tuned
-          # Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
-          setting :rd_kafka_metrics, default: [
-            # Client metrics
-            RdKafkaMetric.new(:count, :root, 'messages.consumed', 'rxmsgs_d'),
-            RdKafkaMetric.new(:count, :root, 'messages.consumed.bytes', 'rxmsg_bytes'),
-
-            # Broker metrics
-            RdKafkaMetric.new(:count, :brokers, 'consume.attempts', 'txretries_d'),
-            RdKafkaMetric.new(:count, :brokers, 'consume.errors', 'txerrs_d'),
-            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
-            RdKafkaMetric.new(:count, :brokers, 'connection.connects', 'connects_d'),
-            RdKafkaMetric.new(:count, :brokers, 'connection.disconnects', 'disconnects_d'),
-            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
-            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
-            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99]),
-
-            # Topics metrics
-            RdKafkaMetric.new(:gauge, :topics, 'consumer.lags', 'consumer_lag_stored'),
-            RdKafkaMetric.new(:gauge, :topics, 'consumer.lags_delta', 'consumer_lag_stored_d')
-          ].freeze
-
-          configure
-
-          # @param block [Proc] configuration block
-          def initialize(&block)
-            configure
-            setup(&block) if block
-          end
-
-          # @param block [Proc] configuration block
-          # @note We define this alias to be consistent with `WaterDrop#setup`
-          def setup(&block)
-            configure(&block)
-          end
-
-          # Hooks up to WaterDrop instrumentation for emitted statistics
-          #
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_statistics_emitted(event)
-            statistics = event[:statistics]
-            consumer_group_id = event[:consumer_group_id]
-
-            base_tags = default_tags + ["consumer_group:#{consumer_group_id}"]
-
-            rd_kafka_metrics.each do |metric|
-              report_metric(metric, statistics, base_tags)
-            end
-          end
-
-          # Increases the errors count by 1
-          #
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_error_occurred(event)
-            extra_tags = ["type:#{event[:type]}"]
-
-            if event.payload[:caller].respond_to?(:messages)
-              extra_tags += consumer_tags(event.payload[:caller])
-            end
-
-            count('error_occurred', 1, tags: default_tags + extra_tags)
-          end
-
-          # Reports how many messages we've polled and how much time did we spend on it
-          #
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_connection_listener_fetch_loop_received(event)
-            time_taken = event[:time]
-            messages_count = event[:messages_buffer].size
-
-            consumer_group_id = event[:subscription_group].consumer_group_id
-
-            extra_tags = ["consumer_group:#{consumer_group_id}"]
-
-            histogram('listener.polling.time_taken', time_taken, tags: default_tags + extra_tags)
-            histogram('listener.polling.messages', messages_count, tags: default_tags + extra_tags)
-          end
-
-          # Here we report majority of things related to processing as we have access to the
-          # consumer
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_consumed(event)
-            consumer = event.payload[:caller]
-            messages = consumer.messages
-            metadata = messages.metadata
-
-            tags = default_tags + consumer_tags(consumer)
-
-            count('consumer.messages', messages.count, tags: tags)
-            count('consumer.batches', 1, tags: tags)
-            gauge('consumer.offset', metadata.last_offset, tags: tags)
-            histogram('consumer.consumed.time_taken', event[:time], tags: tags)
-            histogram('consumer.batch_size', messages.count, tags: tags)
-            histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
-            histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
-          end
-
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoked(event)
-            tags = default_tags + consumer_tags(event.payload[:caller])
-
-            count('consumer.revoked', 1, tags: tags)
-          end
-
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutdown(event)
-            tags = default_tags + consumer_tags(event.payload[:caller])
-
-            count('consumer.shutdown', 1, tags: tags)
-          end
-
-          # Worker related metrics
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_worker_process(event)
-            jq_stats = event[:jobs_queue].statistics
-
-            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
-            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
-            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
-          end
-
-          # We report this metric before and after processing for higher accuracy
-          # Without this, the utilization would not be fully reflected
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_worker_processed(event)
-            jq_stats = event[:jobs_queue].statistics
-
-            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
-          end
-
-          private
-
-          %i[
-            count
-            gauge
-            histogram
-            increment
-            decrement
-          ].each do |metric_type|
-            class_eval <<~METHODS, __FILE__, __LINE__ + 1
-              def #{metric_type}(key, *args)
-                client.#{metric_type}(
-                  namespaced_metric(key),
-                  *args
-                )
-              end
-            METHODS
-          end
-
-          # Wraps metric name in listener's namespace
-          # @param metric_name [String] RdKafkaMetric name
-          # @return [String]
-          def namespaced_metric(metric_name)
-            "#{namespace}.#{metric_name}"
-          end
-
-          # Reports a given metric statistics to Datadog
-          # @param metric [RdKafkaMetric] metric value object
-          # @param statistics [Hash] hash with all the statistics emitted
-          # @param base_tags [Array<String>] base tags we want to start with
-          def report_metric(metric, statistics, base_tags)
-            case metric.scope
-            when :root
-              public_send(
-                metric.type,
-                metric.name,
-                statistics.fetch(*metric.key_location),
-                tags: base_tags
-              )
-            when :brokers
-              statistics.fetch('brokers').each_value do |broker_statistics|
-                # Skip bootstrap nodes
-                # Bootstrap nodes have nodeid -1, other nodes have positive
-                # node ids
-                next if broker_statistics['nodeid'] == -1
-
-                public_send(
-                  metric.type,
-                  metric.name,
-                  broker_statistics.dig(*metric.key_location),
-                  tags: base_tags + ["broker:#{broker_statistics['nodename']}"]
-                )
-              end
-            when :topics
-              statistics.fetch('topics').each do |topic_name, topic_values|
-                topic_values['partitions'].each do |partition_name, partition_statistics|
-                  next if partition_name == '-1'
-                  # Skip until lag info is available
-                  next if partition_statistics['consumer_lag'] == -1
-
-                  public_send(
-                    metric.type,
-                    metric.name,
-                    partition_statistics.dig(*metric.key_location),
-                    tags: base_tags + [
-                      "topic:#{topic_name}",
-                      "partition:#{partition_name}"
-                    ]
-                  )
-                end
-              end
-            else
-              raise ArgumentError, metric.scope
-            end
-          end
-
-          # Builds basic per consumer tags for publication
-          #
-          # @param consumer [Karafka::BaseConsumer]
-          # @return [Array<String>]
-          def consumer_tags(consumer)
-            messages = consumer.messages
-            metadata = messages.metadata
-            consumer_group_id = consumer.topic.consumer_group.id
-
-            [
-              "topic:#{metadata.topic}",
-              "partition:#{metadata.partition}",
-              "consumer_group:#{consumer_group_id}"
-            ]
-          end
-        end
+        # Alias to keep backwards compatibility
+        Listener = MetricsListener
       end
     end
   end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # Listener that can be used to subscribe to Karafka to receive stats via StatsD
+        # and/or Datadog
+        #
+        # @note You need to setup the `dogstatsd-ruby` client and assign it
+        class MetricsListener
+          include ::Karafka::Core::Configurable
+          extend Forwardable
+
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          # Namespace under which the DD metrics should be published
+          setting :namespace, default: 'karafka'
+
+          # Datadog client that we should use to publish the metrics
+          setting :client
+
+          # Default tags we want to publish (for example hostname)
+          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
+          setting :default_tags, default: []
+
+          # All the rdkafka metrics we want to publish
+          #
+          # By default we publish quite a lot so this can be tuned
+          # Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
+          setting :rd_kafka_metrics, default: [
+            # Client metrics
+            RdKafkaMetric.new(:count, :root, 'messages.consumed', 'rxmsgs_d'),
+            RdKafkaMetric.new(:count, :root, 'messages.consumed.bytes', 'rxmsg_bytes'),
+
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'consume.attempts', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'consume.errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.connects', 'connects_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.disconnects', 'disconnects_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99]),
+
+            # Topics metrics
+            RdKafkaMetric.new(:gauge, :topics, 'consumer.lags', 'consumer_lag_stored'),
+            RdKafkaMetric.new(:gauge, :topics, 'consumer.lags_delta', 'consumer_lag_stored_d')
+          ].freeze
+
+          configure
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            configure
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Hooks up to WaterDrop instrumentation for emitted statistics
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+            consumer_group_id = event[:consumer_group_id]
+
+            base_tags = default_tags + ["consumer_group:#{consumer_group_id}"]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics, base_tags)
+            end
+          end
+
+          # Increases the errors count by 1
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(event)
+            extra_tags = ["type:#{event[:type]}"]
+
+            if event.payload[:caller].respond_to?(:messages)
+              extra_tags += consumer_tags(event.payload[:caller])
+            end
+
+            count('error_occurred', 1, tags: default_tags + extra_tags)
+          end
+
+          # Reports how many messages we've polled and how much time did we spend on it
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_fetch_loop_received(event)
+            time_taken = event[:time]
+            messages_count = event[:messages_buffer].size
+
+            consumer_group_id = event[:subscription_group].consumer_group_id
+
+            extra_tags = ["consumer_group:#{consumer_group_id}"]
+
+            histogram('listener.polling.time_taken', time_taken, tags: default_tags + extra_tags)
+            histogram('listener.polling.messages', messages_count, tags: default_tags + extra_tags)
+          end
+
+          # Here we report majority of things related to processing as we have access to the
+          # consumer
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consumed(event)
+            consumer = event.payload[:caller]
+            messages = consumer.messages
+            metadata = messages.metadata
+
+            tags = default_tags + consumer_tags(consumer)
+
+            count('consumer.messages', messages.count, tags: tags)
+            count('consumer.batches', 1, tags: tags)
+            gauge('consumer.offset', metadata.last_offset, tags: tags)
+            histogram('consumer.consumed.time_taken', event[:time], tags: tags)
+            histogram('consumer.batch_size', messages.count, tags: tags)
+            histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
+            histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
+          end
+
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoked(event)
+            tags = default_tags + consumer_tags(event.payload[:caller])
+
+            count('consumer.revoked', 1, tags: tags)
+          end
+
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutdown(event)
+            tags = default_tags + consumer_tags(event.payload[:caller])
+
+            count('consumer.shutdown', 1, tags: tags)
+          end
+
+          # Worker related metrics
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_worker_process(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
+            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
+            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
+          end
+
+          # We report this metric before and after processing for higher accuracy
+          # Without this, the utilization would not be fully reflected
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_worker_processed(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
+          end
+
+          private
+
+          %i[
+            count
+            gauge
+            histogram
+            increment
+            decrement
+          ].each do |metric_type|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              def #{metric_type}(key, *args)
+                client.#{metric_type}(
+                  namespaced_metric(key),
+                  *args
+                )
+              end
+            METHODS
+          end
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}.#{metric_name}"
+          end
+
+          # Reports a given metric statistics to Datadog
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          # @param base_tags [Array<String>] base tags we want to start with
+          def report_metric(metric, statistics, base_tags)
+            case metric.scope
+            when :root
+              public_send(
+                metric.type,
+                metric.name,
+                statistics.fetch(*metric.key_location),
+                tags: base_tags
+              )
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  broker_statistics.dig(*metric.key_location),
+                  tags: base_tags + ["broker:#{broker_statistics['nodename']}"]
+                )
+              end
+            when :topics
+              statistics.fetch('topics').each do |topic_name, topic_values|
+                topic_values['partitions'].each do |partition_name, partition_statistics|
+                  next if partition_name == '-1'
+                  # Skip until lag info is available
+                  next if partition_statistics['consumer_lag'] == -1
+
+                  public_send(
+                    metric.type,
+                    metric.name,
+                    partition_statistics.dig(*metric.key_location),
+                    tags: base_tags + [
+                      "topic:#{topic_name}",
+                      "partition:#{partition_name}"
+                    ]
+                  )
+                end
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+
+          # Builds basic per consumer tags for publication
+          #
+          # @param consumer [Karafka::BaseConsumer]
+          # @return [Array<String>]
+          def consumer_tags(consumer)
+            messages = consumer.messages
+            metadata = messages.metadata
+            consumer_group_id = consumer.topic.consumer_group.id
+
+            [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}",
+              "consumer_group:#{consumer_group_id}"
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/structurable/config.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Structurable < Base
+        # Config for structurable feature
+        Config = Struct.new(
+          :active,
+          :partitions,
+          :replication_factor,
+          :details,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/structurable/contract.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Structurable < Base
+        # Basic validation of the Kafka expected config details
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          nested :structurable do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
+            required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
+            required(:details) do |val|
+              val.is_a?(Hash) &&
+                val.keys.all? { |key| key.is_a?(Symbol) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/structurable/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Structurable < Base
+        # Extension for managing Kafka topic configuration
+        module Topic
+          # @param active [Boolean] is the topic structure management feature active
+          # @param partitions [Integer]
+          # @param replication_factor [Integer]
+          # @param details [Hash] extra configuration for the topic
+          # @return [Config] defined structure
+          def config(active: true, partitions: 1, replication_factor: 1, **details)
+            @structurable ||= Config.new(
+              active: active,
+              partitions: partitions,
+              replication_factor: replication_factor,
+              details: details
+            )
+          end
+
+          # @return [Config] config details
+          def structurable
+            config
+          end
+
+          # @return [true] structurable is always active
+          def structurable?
+            structurable.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus structurable
+          #   settings
+          def to_h
+            super.merge(
+              structurable: structurable.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/structurable.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # This feature allows to store per topic structure that can be later on used to bootstrap
+      # topics structure for test/development, etc. This allows to share the same set of settings
+      # for topics despite the environment. Pretty much similar to how the `structure.sql` or
+      # `schema.rb` operate for SQL dbs.
+      class Structurable < Base
+      end
+    end
+  end
+end

data/lib/karafka/templates/karafka.rb.erb

@@ -53,6 +53,11 @@ class KarafkaApp < Karafka::App
     # active_job_topic :default
 <% end -%>
     topic :example do
+      # Uncomment this if you want Karafka to manage your topics configuration
+      # Managing topics configuration via routing will allow you to ensure config consistency
+      # across multiple environments
+      #
+      # config(partitions: 2, 'cleanup.policy': 'compact')
       consumer ExampleConsumer
     end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.33'
+  VERSION = '2.0.35'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.33
+  version: 2.0.35
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-02-24 00:00:00.000000000 Z
+date: 2023-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -168,6 +168,7 @@ files:
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
+- lib/karafka/cli/topics.rb
 - lib/karafka/connection/client.rb
 - lib/karafka/connection/consumer_group_coordinator.rb
 - lib/karafka/connection/listener.rb
@@ -198,6 +199,7 @@ files:
 - lib/karafka/instrumentation/vendors/datadog/dashboard.json
 - lib/karafka/instrumentation/vendors/datadog/listener.rb
 - lib/karafka/instrumentation/vendors/datadog/logger_listener.rb
+- lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb
 - lib/karafka/licenser.rb
 - lib/karafka/messages/batch_metadata.rb
 - lib/karafka/messages/builders/batch_metadata.rb
@@ -308,6 +310,10 @@ files:
 - lib/karafka/routing/features/manual_offset_management/config.rb
 - lib/karafka/routing/features/manual_offset_management/contract.rb
 - lib/karafka/routing/features/manual_offset_management/topic.rb
+- lib/karafka/routing/features/structurable.rb
+- lib/karafka/routing/features/structurable/config.rb
+- lib/karafka/routing/features/structurable/contract.rb
+- lib/karafka/routing/features/structurable/topic.rb
 - lib/karafka/routing/proxy.rb
 - lib/karafka/routing/router.rb
 - lib/karafka/routing/subscription_group.rb