karafka 2.0.0.rc2 → 2.0.0.rc5

data/lib/karafka/instrumentation/vendors/datadog/dashboard.json

@@ -0,0 +1 @@
+{"title":"Karafka monitoring dashboard","description":"","widgets":[{"id":7444969424381053,"definition":{"title":"Stability & errors","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":8304008422587936,"definition":{"title":"Client connects and disconnects","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Connects","formula":"query1"},{"alias":"Disconnects","formula":"query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.connection.connects{*} by {host}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.connection.disconnects{*} by {host}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":3722865443336921,"definition":{"title":"Errors encountered (any)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"in-karafka errors","formula":"query1"},{"alias":"librdkafka consume errors","formula":"query2"},{"alias":"librdkafka receive errors","formula":"query3"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.error_occurred{*} by {type}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consume.errors{*}.as_count()","data_source":"metrics","name":"query2"},{"query":"sum:karafka.receive.errors{*}.as_count()","data_source":"metrics","name":"query3"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":5477381252952760,"definition":{"title":"Processing errors","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.error_occurred{type:consumer.consume.error} by {partition,topic}.as_count()","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":2357301680769076,"definition":{"title":"Processing errors rate per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"% error rate per topic","formula":"(query1 / (query1 + query2)) * 100"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.error_occurred{type:consumer.consume.error} by {topic,partition}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batches{*} by {topic,partition}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":2,"width":4,"height":2}},{"id":3902930069982135,"definition":{"title":"Batches successful vs failures","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Successfully processed batch","formula":"query1"},{"alias":"Batch processing with error","formula":"query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.batches{*} by {partition,topic}.as_count()","data_source":"metrics","name":"query1"},{"query":"avg:karafka.error_occurred{type:consumer.consume.error} by {partition,topic}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":2,"width":4,"height":2}},{"id":718749162159145,"definition":{"title":"Consumer instances revocations and shutdowns","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Consumer instances revokations","formula":"query1"},{"alias":"Consumer instances shutdowns","formula":"query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.revoked{*}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.shutdown{*}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":2,"width":4,"height":2}}]},"layout":{"x":0,"y":0,"width":12,"height":5}},{"id":5988438511387100,"definition":{"title":"Workers poll","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":8769294644934352,"definition":{"title":"Enqueued jobs","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Enqueued jobs","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.worker.enqueued_jobs.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":2714502141463873,"definition":{"title":"Workers usage","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Busy workers (p95)","formula":"query1"},{"alias":"Total workers","formula":"query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.worker.processing.95percentile{*}","data_source":"metrics","name":"query1"},{"query":"sum:karafka.worker.total_threads{*}","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":5370086629441984,"definition":{"title":"Workers % utilization","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"% workers utilization","formula":"(query1 / query2) * 100"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.worker.processing.95percentile{*}","data_source":"metrics","name":"query1"},{"query":"sum:karafka.worker.total_threads{*}","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":0,"width":4,"height":2}}]},"layout":{"x":0,"y":5,"width":12,"height":3}},{"id":8544040083223278,"definition":{"title":"Throughput ","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":3740207481939733,"definition":{"title":"Offset lag changes","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"derivative(query1)"}],"response_format":"timeseries","queries":[{"query":"max:karafka.consumer.offset{*} by {topic,partition}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":6319110548544878,"definition":{"title":"Batches processed per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.batches{*} by {partition,topic}.as_count()","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":6232784865331443,"definition":{"title":"Messages consumed per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Messages consumed","formula":"query1"},{"alias":"Average batch size","formula":"query1 / query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.messages{*} by {partition,topic}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batches{*} by {partition,topic}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":2321394598982770,"definition":{"title":"Consumption lag (in seconds)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Consumption lag in s (max)","formula":"query2 / 1000"},{"alias":"Consumption lag in s (avg)","formula":"query3 / 1000"},{"alias":"Consumption lag in s (p95)","formula":"query1 / 1000"}],"response_format":"timeseries","queries":[{"query":"max:karafka.consumer.consumption_lag.max{*}","data_source":"metrics","name":"query2"},{"query":"max:karafka.consumer.consumption_lag.avg{*}","data_source":"metrics","name":"query3"},{"query":"max:karafka.consumer.consumption_lag.95percentile{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":2,"width":4,"height":2}},{"id":1062074781483741,"definition":{"title":"Processing lag (in ms)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Processing lag in ms (p95)","formula":"query1"},{"alias":"Processing lag in ms (max)","formula":"query2"},{"alias":"Processing lag in ms (avg)","formula":"query3"}],"response_format":"timeseries","queries":[{"query":"max:karafka.consumer.processing_lag.95percentile{*}","data_source":"metrics","name":"query1"},{"query":"max:karafka.consumer.processing_lag.max{*}","data_source":"metrics","name":"query2"},{"query":"max:karafka.consumer.processing_lag.avg{*}","data_source":"metrics","name":"query3"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":2,"width":4,"height":2}},{"id":7497794728674267,"definition":{"title":"Batch processing time","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"},{"formula":"query2"},{"formula":"query3"}],"response_format":"timeseries","queries":[{"query":"max:karafka.consumer.consumed.time_taken.95percentile{*} by {topic,partition}","data_source":"metrics","name":"query1"},{"query":"max:karafka.consumer.consumed.time_taken.max{*} by {topic,partition}","data_source":"metrics","name":"query2"},{"query":"max:karafka.consumer.consumed.time_taken.avg{*} by {topic,partition}","data_source":"metrics","name":"query3"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":2,"width":4,"height":2}},{"id":4192833027984161,"definition":{"title":"Batch size per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Batch size p95","formula":"query1"},{"alias":"Batch size avg","formula":"query2"},{"alias":"Batch size max","formula":"query3"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.batch_size.95percentile{*} by {partition,topic}","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batch_size.avg{*} by {partition,topic}","data_source":"metrics","name":"query2"},{"query":"sum:karafka.consumer.batch_size.max{*} by {partition,topic}","data_source":"metrics","name":"query3"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":4,"width":4,"height":2}},{"id":4741598444771147,"definition":{"title":"Messages consumed overall","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Messages consumed","formula":"query1"},{"alias":"Average batch size","formula":"query1 / query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.messages{*}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batches{*}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":4,"width":4,"height":2}},{"id":4502534794102513,"definition":{"title":"Polling times (ms)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"p95 ms polling time","formula":"query1"},{"alias":"max ms polling time","formula":"query2"},{"alias":"average ms polling time","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:karafka.listener.polling.time_taken.95percentile{*}"},{"name":"query2","data_source":"metrics","query":"avg:karafka.listener.polling.time_taken.max{*}"},{"name":"query3","data_source":"metrics","query":"avg:karafka.listener.polling.time_taken.avg{*}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":4,"width":4,"height":2}}]},"layout":{"x":0,"y":0,"width":12,"height":7,"is_column_break":true}}],"template_variables":[],"layout_type":"ordered","is_read_only":false,"notify_list":[],"reflow_type":"fixed","id":"s3u-z47-i6u"}

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # Listener that can be used to subscribe to Karafka to receive stats via StatsD
+        # and/or Datadog
+        #
+        # @note You need to setup the `dogstatsd-ruby` client and assign it
+        class Listener
+          include ::Karafka::Core::Configurable
+          extend Forwardable
+
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          # Namespace under which the DD metrics should be published
+          setting :namespace, default: 'karafka'
+
+          # Datadog client that we should use to publish the metrics
+          setting :client
+
+          # Default tags we want to publish (for example hostname)
+          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
+          setting :default_tags, default: []
+
+          # All the rdkafka metrics we want to publish
+          #
+          # By default we publish quite a lot so this can be tuned
+          # Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
+          setting :rd_kafka_metrics, default: [
+            # Client metrics
+            RdKafkaMetric.new(:count, :root, 'messages.consumed', 'rxmsgs_d'),
+            RdKafkaMetric.new(:count, :root, 'messages.consumed.bytes', 'rxmsg_bytes'),
+
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'consume.attempts', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'consume.errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.connects', 'connects_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.disconnects', 'disconnects_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99])
+          ].freeze
+
+          configure
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            configure
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Hooks up to WaterDrop instrumentation for emitted statistics
+          #
+          # @param event [Dry::Events::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics)
+            end
+          end
+
+          # Increases the errors count by 1
+          #
+          # @param event [Dry::Events::Event]
+          def on_error_occurred(event)
+            extra_tags = ["type:#{event[:type]}"]
+
+            if event.payload[:caller].respond_to?(:messages)
+              metadata = event.payload[:caller].messages.metadata
+
+              extra_tags += [
+                "topic:#{metadata.topic}",
+                "partition:#{metadata.partition}"
+              ]
+            end
+
+            count('error_occurred', 1, tags: default_tags + extra_tags)
+          end
+
+          # Reports how many messages we've polled and how much time did we spend on it
+          #
+          # @param event [Dry::Events::Event]
+          def on_connection_listener_fetch_loop_received(event)
+            time_taken = event[:time]
+            messages_count = event[:messages_buffer].size
+
+            histogram('listener.polling.time_taken', time_taken, tags: default_tags)
+            histogram('listener.polling.messages', messages_count, tags: default_tags)
+          end
+
+          # Here we report majority of things related to processing as we have access to the
+          # consumer
+          # @param event [Dry::Events::Event]
+          def on_consumer_consumed(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.messages', messages.count, tags: tags)
+            count('consumer.batches', 1, tags: tags)
+            gauge('consumer.offset', metadata.last_offset, tags: tags)
+            histogram('consumer.consumed.time_taken', event[:time], tags: tags)
+            histogram('consumer.batch_size', messages.count, tags: tags)
+            histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
+            histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
+          end
+
+          # @param event [Dry::Events::Event]
+          def on_consumer_revoked(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.revoked', 1, tags: tags)
+          end
+
+          # @param event [Dry::Events::Event]
+          def on_consumer_shutdown(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.shutdown', 1, tags: tags)
+          end
+
+          # Worker related metrics
+          # @param event [Dry::Events::Event]
+          def on_worker_process(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
+            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
+          end
+
+          # We report this metric before and after processing for higher accuracy
+          # Without this, the utilization would not be fully reflected
+          # @param event [Dry::Events::Event]
+          def on_worker_processed(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+          end
+
+          private
+
+          %i[
+            count
+            gauge
+            histogram
+            increment
+            decrement
+          ].each do |metric_type|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              def #{metric_type}(key, *args)
+                client.#{metric_type}(
+                  namespaced_metric(key),
+                  *args
+                )
+              end
+            METHODS
+          end
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}.#{metric_name}"
+          end
+
+          # Reports a given metric statistics to Datadog
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          def report_metric(metric, statistics)
+            case metric.scope
+            when :root
+              public_send(
+                metric.type,
+                metric.name,
+                statistics.fetch(*metric.key_location),
+                tags: default_tags
+              )
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  broker_statistics.dig(*metric.key_location),
+                  tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
+                )
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/patches/rdkafka/consumer.rb

@@ -12,7 +12,7 @@ module Karafka
         # @note We need this to make sure that we allocate proper dispatched events only to
         #   callback listeners that should publish them
         def name
-          ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
+          @name ||= ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
         end
       end
     end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -23,7 +23,9 @@ module Karafka
           dispatch_method: :produce_async,
           # We don't create a dummy proc based partitioner as we would have to evaluate it with
           # each job.
-          partitioner: nil
+          partitioner: nil,
+          # Allows for usage of `:key` or `:partition_key`
+          partition_key_type: :key
         }.freeze
 
         private_constant :DEFAULTS
@@ -45,11 +47,12 @@ module Karafka
         # @return [Hash] hash with dispatch details to which we merge topic and payload
         def dispatch_details(job)
           partitioner = fetch_option(job, :partitioner, DEFAULTS)
+          key_type = fetch_option(job, :partition_key_type, DEFAULTS)
 
           return {} unless partitioner
 
           {
-            partition_key: partitioner.call(job)
+            key_type => partitioner.call(job)
           }
         end
       end

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -25,6 +25,7 @@ module Karafka
 
         optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
         optional(:partitioner) { |val| val.respond_to?(:call) }
+        optional(:partition_key_type) { |val| %i[key partition_key].include?(val) }
       end
     end
   end

data/lib/karafka/pro/base_consumer.rb

@@ -36,7 +36,7 @@ module Karafka
         end
       end
 
-      # Runs extra logic after consumption that is related to handling long running jobs
+      # Runs extra logic after consumption that is related to handling long-running jobs
       # @note This overwrites the '#on_after_consume' from the base consumer
       def on_after_consume
         coordinator.on_finished do |first_group_message, last_group_message|
@@ -59,7 +59,7 @@ module Karafka
           # Mark as consumed only if manual offset management is not on
           mark_as_consumed(last_message) unless topic.manual_offset_management? || revoked?
 
-          # If this is not a long running job there is nothing for us to do here
+          # If this is not a long-running job there is nothing for us to do here
           return unless topic.long_running_job?
 
           # Once processing is done, we move to the new offset based on commits

data/lib/karafka/pro/contracts/base.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Karafka Pro related contracts
+    module Contracts
+      # Base contract for Pro components contracts
+      class Base < ::Karafka::Contracts::Base
+      end
+    end
+  end
+end

data/lib/karafka/pro/contracts/consumer_group.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Contracts
+      # Contract for validating correct Pro components setup on a consumer group and topic levels
+      class ConsumerGroup < Base
+        virtual do |data, errors|
+          next unless errors.empty?
+          next unless data.key?(:topics)
+
+          fetched_errors = []
+
+          data.fetch(:topics).each do |topic|
+            ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
+              fetched_errors << [[topic, key].flatten, value]
+            end
+          end
+
+          fetched_errors
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/contracts/consumer_group_topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Contracts
+      # Contract for validating correct Pro components setup on a topic levels
+      class ConsumerGroupTopic < Base
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('pro_consumer_group_topic')
+        end
+
+        virtual do |data|
+          next if data[:consumer] < Karafka::Pro::BaseConsumer
+
+          [[%i[consumer], :consumer_format]]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -22,7 +22,11 @@ module Karafka
         processing/jobs_builder
         processing/coordinator
         processing/partitioner
-        routing/extensions
+        contracts/base
+        contracts/consumer_group
+        contracts/consumer_group_topic
+        routing/topic_extensions
+        routing/builder_extensions
         active_job/consumer
         active_job/dispatcher
         active_job/job_options_contract
@@ -37,6 +41,16 @@ module Karafka
         def setup(config)
           COMPONENTS.each { |component| require_relative(component) }
 
+          reconfigure(config)
+
+          load_routing_extensions
+        end
+
+        private
+
+        # Sets proper config options to use pro components
+        # @param config [WaterDrop::Configurable::Node] root config node
+        def reconfigure(config)
           icfg = config.internal
 
           icfg.processing.coordinator_class = Processing::Coordinator
@@ -48,10 +62,14 @@ module Karafka
           icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
           icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
 
-          ::Karafka::Routing::Topic.include(Routing::Extensions)
-
           config.monitor.subscribe(PerformanceTracker.instance)
         end
+
+        # Loads routing extensions
+        def load_routing_extensions
+          ::Karafka::Routing::Topic.include(Routing::TopicExtensions)
+          ::Karafka::Routing::Builder.prepend(Routing::BuilderExtensions)
+        end
       end
     end
   end

data/lib/karafka/pro/processing/partitioner.rb

@@ -27,9 +27,28 @@ module Karafka
           # process the data. With one thread it is not worth partitioning the work as the work
           # itself will be assigned to one thread (pointless work)
           if ktopic.virtual_partitioner? && @concurrency > 1
-            messages
-              .group_by { |msg| ktopic.virtual_partitioner.call(msg).hash.abs % @concurrency }
-              .each { |group_id, messages_group| yield(group_id, messages_group) }
+            # We need to reduce it to number of threads, so the group_id is not a direct effect
+            # of the end user action. Otherwise the persistence layer for consumers would cache
+            # it forever and it would cause memory leaks
+            groupings = messages
+                        .group_by { |msg| ktopic.virtual_partitioner.call(msg) }
+                        .values
+
+            # Reduce the max concurrency to a size that matches the concurrency
+            # As mentioned above we cannot use the partitioning keys directly as it could cause
+            # memory leaks
+            #
+            # The algorithm here is simple, we assume that the most costly in terms of processing,
+            # will be processing of the biggest group and we reduce the smallest once to have
+            # max of groups equal to concurrency
+            while groupings.size > @concurrency
+              groupings.sort_by! { |grouping| -grouping.size }
+
+              # Offset order needs to be maintained for virtual partitions
+              groupings << (groupings.pop + groupings.pop).sort_by!(&:offset)
+            end
+
+            groupings.each_with_index { |messages_group, index| yield(index, messages_group) }
           else
             # When no virtual partitioner, works as regular one
             yield(0, messages)

data/lib/karafka/pro/routing/builder_extensions.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro routing components
+    module Routing
+      # Routing extensions for builder to be able to validate Pro components correct usage
+      module BuilderExtensions
+        # Validate consumer groups with pro contracts
+        # @param block [Proc] routing defining block
+        def draw(&block)
+          super
+
+          each do |consumer_group|
+            ::Karafka::Pro::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/{extensions.rb → topic_extensions.rb}

@@ -14,7 +14,7 @@ module Karafka
     # Pro routing components
     module Routing
       # Routing extensions that allow to configure some extra PRO routing options
-      module Extensions
+      module TopicExtensions
         class << self
           # @param base [Class] class we extend
           def included(base)
@@ -28,7 +28,7 @@ module Karafka
           virtual_partitioner != nil
         end
 
-        # @return [Boolean] is a given job on a topic a long running one
+        # @return [Boolean] is a given job on a topic a long-running one
         def long_running_job?
           @long_running_job || false
         end

data/lib/karafka/process.rb

@@ -9,6 +9,7 @@ module Karafka
       SIGINT
       SIGQUIT
       SIGTERM
+      SIGTTIN
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|

data/lib/karafka/processing/jobs_queue.rb

@@ -119,6 +119,17 @@ module Karafka
         @semaphores[group_id].pop while wait?(group_id)
       end
 
+      # - `processing` - number of jobs that are currently being processed (active work)
+      # - `enqueued` - number of jobs in the queue that are waiting to be picked up by a worker
+      #
+      # @return [Hash] hash with basic usage statistics of this queue.
+      def statistics
+        {
+          processing: size - @queue.size,
+          enqueued: @queue.size
+        }.freeze
+      end
+
       private
 
       # @param group_id [String] id of the group in which jobs we're interested.

data/lib/karafka/processing/worker.rb

@@ -47,9 +47,11 @@ module Karafka
         job = @jobs_queue.pop
 
         if job
-          Karafka.monitor.instrument('worker.process', caller: self, job: job)
+          instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
 
-          Karafka.monitor.instrument('worker.processed', caller: self, job: job) do
+          Karafka.monitor.instrument('worker.process', instrument_details)
+
+          Karafka.monitor.instrument('worker.processed', instrument_details) do
             job.before_call
 
             # If a job is marked as non blocking, we can run a tick in the job queue and if there

data/lib/karafka/server.rb

@@ -104,6 +104,9 @@ module Karafka
         # We're done waiting, lets kill them!
         workers.each(&:terminate)
         listeners.each(&:terminate)
+        # We always need to shutdown clients to make sure we do not force the GC to close consumer.
+        # This can cause memory leaks and crashes.
+        listeners.each(&:shutdown)
 
         Karafka::App.producer.close
 

data/lib/karafka/setup/config.rb

@@ -12,7 +12,7 @@ module Karafka
     #   enough and will still keep the code simple
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
-      extend ::WaterDrop::Configurable
+      extend ::Karafka::Core::Configurable
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
@@ -60,9 +60,9 @@ module Karafka
       # option [Boolean] should we leave offset management to the user
       setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
-      setting :max_messages, default: 1_000
+      setting :max_messages, default: 100
       # option [Integer] number of milliseconds we can wait while fetching data
-      setting :max_wait_time, default: 5_000
+      setting :max_wait_time, default: 1_000
       # option shutdown_timeout [Integer] the number of milliseconds after which Karafka no
       #   longer waits for the consumers to stop gracefully but instead we force terminate
       #   everything.

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.0.rc2'
+  VERSION = '2.0.0.rc5'
 end