karafka 2.0.23 → 2.0.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84d8130c528081b283889f9f1ebe89b8829b800a2c5d1f4ca99f6d6ce9b4c9df
-  data.tar.gz: 3cc30f65586226bcb6d8ed4fdac912a72c1a4a7eb5691862b378629d9de1347f
+  metadata.gz: e1e0356b40b0812dc321a825731c6fcd7413f9d1d30a23288597d2f832c6b5d3
+  data.tar.gz: f887c3bc93f945329fa4bba4bcc81a8f95a9d5153fae64aa5b0936935502909d
 SHA512:
-  metadata.gz: 8e1f3fc0a3c73035fdb38093eb0594c999d0a715752f41ed95079b1d8f2e89a5a0fa0abdf054272891e89ffa4dd8d010dd2225540d4480067d8bc0359bf2b7b8
-  data.tar.gz: 7c1f3f958ef52682e46efb11e8a3ddcbe641ba85c62956fefb84ec6eacadbe9d28ee509bdfd61fcd73ea8f52b1ab867de4909990503024cc5c603788931264ca
+  metadata.gz: 32b9c06212b90bb49232931a06e9ae445b76c2c8780c0774923945458dd425233c37167825c5d1f39118cdebb8352f6a014a8e547947367fa712e46fe07c96d5
+  data.tar.gz: 2047da1a400328dd13465f9c94b033a9841675b6e7274fc9d94a1677866d0feb9611858a02296a20e6e7b8ea589c0b903ae35e572c6c6ad6c8a3040300692703

data/.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.1
+          ruby-version: 3.2
           bundler-cache: true
 
       - name: Install Diffend plugin
@@ -34,6 +34,22 @@ jobs:
       - name: Bundle Secure
         run: bundle secure
 
+  karafka-checksum:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Run Karafka license checksum verification
+        env:
+          KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
+          KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
+          KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
+          KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
+        run: bin/verify_license_integrity
+
   coditsu:
     runs-on: ubuntu-latest
     strategy:
@@ -46,12 +62,14 @@ jobs:
         run: \curl -sSL https://api.coditsu.io/run/ci | bash
 
   specs:
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     needs: diffend
     strategy:
       fail-fast: false
       matrix:
         ruby:
+          - '3.2'
           # We run it against the oldest and the newest of a given major to make sure, that there
           # are no syntax-sugars that we would use that were introduced down the road
           - '3.1'
@@ -61,7 +79,7 @@ jobs:
           - '2.7'
           - '2.7.0'
         include:
-          - ruby: '3.1'
+          - ruby: '3.2'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v3
@@ -84,17 +102,19 @@ jobs:
         run: bin/rspecs
 
   integrations:
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     needs: diffend
     strategy:
       fail-fast: false
       matrix:
         ruby:
+          - '3.2'
           - '3.1'
           - '3.0'
           - '2.7'
         include:
-          - ruby: '3.1'
+          - ruby: '3.2'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v3
@@ -130,5 +150,6 @@ jobs:
           KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
           KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
           KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
+          KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bin/integrations

data/.ruby-version

@@ -1 +1 @@
-3.1.3
+3.2.0

data/CHANGELOG.md

@@ -1,5 +1,56 @@
 # Karafka framework changelog
 
+## 2.0.26 (2023-01-10)
+- **[Feature]** Allow for disabling given topics by setting `active` to false. It will exclude them from consumption but will allow to have their definitions for using admin APIs, etc.
+- [Improvement] Early terminate on `read_topic` when reaching the last offset available on the request time.
+- [Improvement] Introduce a `quiet` state that indicates that Karafka is not only moving to quiet mode but actually that it reached it and no work will happen anymore in any of the consumer groups.
+- [Improvement] Use Karafka defined routes topics when possible for `read_topic` admin API.
+- [Improvement] Introduce `client.pause` and `client.resume` instrumentation hooks for tracking client topic partition pausing and resuming. This is alongside of `consumer.consuming.pause` that can be used to track both manual and automatic pausing with more granular consumer related details. The `client.*` should be used for low level tracking.
+- [Improvement] Replace `LoggerListener` pause notification with one based on `client.pause` instead of `consumer.consuming.pause`.
+- [Improvement] Expand `LoggerListener` with `client.resume` notification.
+- [Improvement] Replace random anonymous subscription groups ids with stable once.
+- [Improvement] Add `consumer.consume`, `consumer.revoke` and `consumer.shutting_down` notification events and move the revocation logic calling to strategies.
+- [Change] Rename job queue statistics `processing` key to `busy`. No changes needed because naming in the DataDog listener stays the same. 
+- [Fix] Fix proctitle listener state changes reporting on new states.
+- [Fix] Make sure all files descriptors are closed in the integration specs.
+- [Fix] Fix a case where empty subscription groups could leak into the execution flow.
+- [Fix] Fix `LoggerListener` reporting so it does not end with `.`.
+- [Fix] Run previously defined (if any) signal traps created prior to Karafka signals traps.
+
+## 2.0.25 (2023-01-10)
+- Release yanked due to accidental release with local changes.
+
+## 2.0.24 (2022-12-19)
+- **[Feature]** Provide out of the box encryption support for Pro.
+- [Improvement] Add instrumentation upon `#pause`.
+- [Improvement] Add instrumentation upon retries.
+- [Improvement] Assign `#id` to consumers similar to other entities for ease of debugging.
+- [Improvement] Add retries and pausing to the default `LoggerListener`.
+- [Improvement] Introduce a new final `terminated` state that will kick in prior to exit but after all the instrumentation and other things are done.
+- [Improvement] Ensure that state transitions are thread-safe and ensure state transitions can occur in one direction.
+- [Improvement] Optimize status methods proxying to `Karafka::App`.
+- [Improvement] Allow for easier state usage by introducing explicit `#to_s` for reporting.
+- [Improvement] Change auto-generated id from `SecureRandom#uuid` to `SecureRandom#hex(6)`
+- [Improvement] Emit statistic every 5 seconds by default.
+- [Improvement] Introduce general messages parser that can be swapped when needed.
+- [Fix] Do not trigger code reloading when `consumer_persistence` is enabled.
+- [Fix] Shutdown producer after all the consumer components are down and the status is stopped. This will ensure, that any instrumentation related Kafka messaging can still operate.
+
+### Upgrade notes
+
+If you want to disable `librdkafka` statistics because you do not use them at all, update the `kafka` `statistics.interval.ms` setting and set it to `0`:
+
+```ruby
+class KarafkaApp < Karafka::App
+  setup do |config|
+    # Other settings...
+    config.kafka = {
+      'statistics.interval.ms': 0
+    }
+  end
+end
+```
+
 ## 2.0.23 (2022-12-07)
 - [Maintenance] Align with `waterdrop` and `karafka-core`
 - [Improvement] Provide `Admin#read_topic` API to get topic data without subscribing.
@@ -425,7 +476,7 @@ There are several things in the plan already for 2.1 and beyond, including a web
 - Small integration specs refactoring + specs for pausing scenarios
 
 ## 2.0.0-alpha6 (2022-04-17)
-- Fix a bug, where upon missing boot file and Rails, railtie would fail with a generic exception (#818) 
+- Fix a bug, where upon missing boot file and Rails, railtie would fail with a generic exception (#818)
 - Fix an issue with parallel pristine specs colliding with each other during `bundle install` (#820)
 - Replace `consumer.consume` with `consumer.consumed` event to match the behaviour
 - Make sure, that offset committing happens before the `consumer.consumed` event is propagated

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.23)
-      karafka-core (>= 2.0.6, < 3.0.0)
+    karafka (2.0.26)
+      karafka-core (>= 2.0.8, < 3.0.0)
       thor (>= 0.20)
-      waterdrop (>= 2.4.3, < 3.0.0)
+      waterdrop (>= 2.4.7, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -29,11 +29,11 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.6)
+    karafka-core (2.0.8)
       concurrent-ruby (>= 1.1)
       rdkafka (>= 0.12)
-    mini_portile2 (2.8.0)
-    minitest (5.16.3)
+    mini_portile2 (2.8.1)
+    minitest (5.17.0)
     rake (13.0.6)
     rdkafka (0.12.0)
       ffi (~> 1.15)
@@ -45,14 +45,14 @@ GEM
       rspec-mocks (~> 3.12.0)
     rspec-core (3.12.0)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.0)
+    rspec-expectations (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.0)
+    rspec-mocks (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    simplecov (0.21.2)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
@@ -61,12 +61,14 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.3)
-      karafka-core (>= 2.0.6, < 3.0.0)
+    waterdrop (2.4.7)
+      karafka-core (>= 2.0.7, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.6)
 
 PLATFORMS
+  arm64-darwin-21
+  x86_64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -78,4 +80,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.26
+   2.4.2

data/README.md

@@ -4,7 +4,7 @@
 [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
-**Note**: Upgrade notes for migration from Karafka `1.4` to Karafka `2.0` can be found [here](https://karafka.io/docs/Upgrades-2.0/).
+**Note**: Upgrade instructions for migration from Karafka `1.4` to Karafka `2.0` can be found [here](https://karafka.io/docs/Upgrades-2.0/).
 
 ## About Karafka
 
@@ -40,6 +40,8 @@ Karafka **uses** threads to handle many messages simultaneously in the same proc
 
 ## Getting started
 
+![karafka web ui](https://raw.githubusercontent.com/karafka/misc/master/printscreens/web-ui.png)
+
 If you're entirely new to the subject, you can start with our "Kafka on Rails" articles series, which will get you up and running with the terminology and basic ideas behind using Kafka:
 
 - [Kafka on Rails: Using Kafka with Ruby on Rails – Part 1 – Kafka basics and its advantages](https://mensfeld.pl/2017/11/kafka-on-rails-using-kafka-with-ruby-on-rails-part-1-kafka-basics-and-its-advantages/)
@@ -72,10 +74,10 @@ Karafka.producer.produce_sync(topic: 'example', payload: { 'ping' => 'pong' }.to
 ```bash
 bundle exec karafka server
 
-[7616dc24-505a-417f-b87b-6bf8fc2d98c5] Polled 1 message in 1000ms
-[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example started
+[86d47f0b92f7] Polled 1 message in 1000ms
+[3732873c8a74] Consume job for ExampleConsumer on example started
 {"ping"=>"pong"}
-[dcf3a8d8-0bd9-433a-8f63-b70a0cdb0732] Consume job for ExampleConsumer on example finished in 0ms
+[3732873c8a74] Consume job for ExampleConsumer on example finished in 0ms
 ```
 
 ## Want to Upgrade? LGPL is not for you? Want to help?

data/bin/integrations

@@ -152,6 +152,13 @@ class Scenario
     end
   end
 
+  # Close all the files that are open, so they do not pile up
+  def close
+    @stdin.close
+    @stdout.close
+    @stderr.close
+  end
+
   private
 
   # Sets up a proper environment for a given spec to run and returns the run command
@@ -248,6 +255,7 @@ while finished_scenarios.size < scenarios.size
   active_scenarios.select(&:finished?).each do |exited|
     scenario = active_scenarios.delete(exited)
     scenario.report
+    scenario.close
     finished_scenarios << scenario
   end
 

data/bin/verify_license_integrity

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# This script verifies integrity of the Pro license
+# Run it before bundle install to ensure, that what you are fetching is what you expect
+# Run it after bundle install to ensure that the local artefact was not compromised
+
+#!/usr/bin/env bash
+
+set -e
+
+if [ "$MODE" != "after" ]; then
+  # Check the remote license prior to bundle installing
+  curl \
+    --fail \
+    -u $KARAFKA_PRO_USERNAME:$KARAFKA_PRO_PASSWORD \
+    https://gems.karafka.io/gems/karafka-license-$KARAFKA_PRO_VERSION.gem \
+    -o ./karafka-license.gem
+else
+  # Check the local cached one after bundle install
+  cache_path=`ruby -e 'puts "#{Gem.dir}/cache/"'`
+  cp "$cache_path/karafka-license-$KARAFKA_PRO_VERSION.gem" ./karafka-license.gem
+fi
+
+detected=`sha256sum ./karafka-license.gem | awk '{ print $1 }'`
+
+rm ./karafka-license.gem
+
+echo -n "Karafka Pro license artifact checksum verification result: "
+
+if [ "$detected" = "$KARAFKA_PRO_LICENSE_CHECKSUM" ]; then
+  echo "Success"
+else
+  echo -e "\033[0;31mFailure!\033[0m"
+  exit 1
+fi

data/config/{errors.yml → locales/errors.yml}

@@ -44,6 +44,7 @@ en:
       dead_letter_queue.max_retries_format: needs to be equal or bigger than 0
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
+      active_format: needs to be either true or false
 
     consumer_group:
       missing: needs to be present
@@ -63,7 +64,7 @@ en:
 
     pro_topic:
       virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
-      virtual_partitions.max_partitions_format: needs to be equl or more than 1
+      virtual_partitions.max_partitions_format: needs to be equal or more than 1
       manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
       long_running_job.active_format: needs to be either true or false
       dead_letter_queue_not_with_virtual_partitions: cannot be used together with Virtual Partitions

data/config/locales/pro_errors.yml

@@ -0,0 +1,18 @@
+en:
+  validations:
+    topic:
+      virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+      virtual_partitions.max_partitions_format: needs to be equal or more than 1
+      manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
+      long_running_job.active_format: needs to be either true or false
+      dead_letter_queue_not_with_virtual_partitions: cannot be used together with Virtual Partitions
+
+    config:
+      encryption.active_format: 'needs to be either true or false'
+      encryption.public_key_invalid: 'is not a valid public RSA key'
+      encryption.public_key_needs_to_be_public: 'is a private RSA key not a public one'
+      encryption.private_keys_format: 'needs to be a hash of version and private key value'
+      encryption.private_keys_need_to_be_private: 'all keys need to be private'
+      encryption.version_format: must be a non-empty string
+      encryption.public_key_format: 'is not a valid public RSA key'
+      encryption.private_keys_invalid: 'contains an invalid private RSA key string'

data/docker-compose.yml

@@ -3,8 +3,10 @@ services:
   zookeeper:
     container_name: karafka_20_zookeeper
     image: wurstmeister/zookeeper
+    restart: on-failure
     ports:
       - '2181:2181'
+
   kafka:
     container_name: karafka_20_kafka
     image: wurstmeister/kafka
@@ -22,3 +24,4 @@ services:
          benchmarks_00_10:10:1"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    restart: on-failure

data/karafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[contact@karafka.io]
   spec.homepage    = 'https://karafka.io'
-  spec.licenses    = ['LGPL-3.0', 'Commercial']
+  spec.licenses    = %w[LGPL-3.0 Commercial]
   spec.summary     = 'Karafka is Ruby and Rails efficient Kafka processing framework.'
   spec.description = <<-DESC
     Karafka is Ruby and Rails efficient Kafka processing framework.
@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.0.6', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.8', '< 3.0.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.4.3', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.4.7', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.7.0'

data/lib/karafka/active_job/job_options_contract.rb

@@ -10,7 +10,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('job_options')
       end

data/lib/karafka/admin.rb

@@ -30,29 +30,30 @@ module Karafka
       # @param name [String, Symbol] topic name
       # @param partition [Integer] partition
       # @param count [Integer] how many messages we want to get at most
-      # @param offset [Integer] offset from which we should start. If -1 is provided (default) we
-      #   will start from the latest offset
+      # @param start_offset [Integer] offset from which we should start. If -1 is provided
+      #   (default) we will start from the latest offset
       #
       # @return [Array<Karafka::Messages::Message>] array with messages
-      def read_topic(name, partition, count, offset = -1)
+      def read_topic(name, partition, count, start_offset = -1)
         messages = []
         tpl = Rdkafka::Consumer::TopicPartitionList.new
 
         with_consumer do |consumer|
-          if offset.negative?
-            offsets = consumer.query_watermark_offsets(name, partition)
-            offset = offsets.last - count
-          end
+          offsets = consumer.query_watermark_offsets(name, partition)
+          end_offset = offsets.last
 
-          offset = offset.negative? ? 0 : offset
+          start_offset = [0, offsets.last - count].max if start_offset.negative?
 
-          tpl.add_topic_and_partitions_with_offsets(name, partition => offset)
+          tpl.add_topic_and_partitions_with_offsets(name, partition => start_offset)
           consumer.assign(tpl)
 
           # We should poll as long as we don't have all the messages that we need or as long as
           # we do not read all the messages from the topic
           loop do
+            # If we've got as many messages as we've wanted stop
             break if messages.size >= count
+            # If we've reached end of the topic messages, don't process more
+            break if !messages.empty? && end_offset <= messages.last.offset
 
             message = consumer.poll(200)
             messages << message if message
@@ -67,7 +68,10 @@ module Karafka
         messages.map do |message|
           Messages::Builders::Message.call(
             message,
-            Topic.new(name, Karafka::App.config.deserializer),
+            # Use topic from routes if we can match it or create a dummy one
+            # Dummy one is used in case we cannot match the topic with routes. This can happen
+            # when admin API is used to read topics that are not part of the routing
+            Routing::Router.find_by(name: name) || Topic.new(name, App.config.deserializer),
             Time.now
           )
         end
@@ -116,8 +120,7 @@ module Karafka
       # Creates admin instance and yields it. After usage it closes the admin instance
       def with_admin
         admin = config(:producer).admin
-        result = yield(admin)
-        result
+        yield(admin)
       ensure
         admin&.close
       end
@@ -125,8 +128,7 @@ module Karafka
       # Creates consumer instance and yields it. After usage it closes the consumer instance
       def with_consumer
         consumer = config(:consumer).consumer
-        result = yield(consumer)
-        result
+        yield(consumer)
       ensure
         consumer&.close
       end

data/lib/karafka/app.rb

@@ -29,16 +29,28 @@ module Karafka
           .delete_if { |_, sgs| sgs.empty? }
           .each { |_, sgs| sgs.each { |sg| sg.topics.delete_if { |top| !top.active? } } }
           .each { |_, sgs| sgs.delete_if { |sg| sg.topics.empty? } }
+          .reject { |cg, _| cg.subscription_groups.empty? }
           .to_h
       end
 
       # Just a nicer name for the consumer groups
       alias routes consumer_groups
 
-      Status.instance_methods(false).each do |delegated|
-        define_method(delegated) do
-          App.config.internal.status.send(delegated)
-        end
+      # Allow for easier status management via `Karafka::App` by aliasing status methods here
+      Status::STATES.each do |state, transition|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          def #{state}
+            App.config.internal.status.#{state}
+          end
+
+          def #{state}?
+            App.config.internal.status.#{state}?
+          end
+
+          def #{transition}
+            App.config.internal.status.#{transition}
+          end
+        RUBY
       end
 
       # Methods that should be delegated to Karafka module

data/lib/karafka/base_consumer.rb

@@ -4,6 +4,8 @@
 module Karafka
   # Base consumer from which all Karafka consumers should inherit
   class BaseConsumer
+    # @return [String] id of the current consumer
+    attr_reader :id
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
     attr_accessor :topic
     # @return [Karafka::Messages::Messages] current messages batch
@@ -15,6 +17,11 @@ module Karafka
     # @return [Waterdrop::Producer] producer instance
     attr_accessor :producer
 
+    # Creates new consumer and assigns it an id
+    def initialize
+      @id = SecureRandom.hex(6)
+    end
+
     # Can be used to run preparation code prior to the job being enqueued
     #
     # @private
@@ -92,10 +99,6 @@ module Karafka
     # @private
     def on_revoked
       handle_revoked
-
-      Karafka.monitor.instrument('consumer.revoked', caller: self) do
-        revoked
-      end
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -109,9 +112,7 @@ module Karafka
     #
     # @private
     def on_shutdown
-      Karafka.monitor.instrument('consumer.shutdown', caller: self) do
-        shutdown
-      end
+      handle_shutdown
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -204,6 +205,17 @@ module Karafka
 
       # Indicate, that user took a manual action of pausing
       coordinator.manual_pause if manual_pause
+
+      Karafka.monitor.instrument(
+        'consumer.consuming.pause',
+        caller: self,
+        manual: manual_pause,
+        topic: messages.metadata.topic,
+        partition: messages.metadata.partition,
+        offset: offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
     end
 
     # Resumes processing of the current topic partition
@@ -232,5 +244,23 @@ module Karafka
     def revoked?
       coordinator.revoked?
     end
+
+    # Pauses the processing from the last offset to retry on given message
+    # @private
+    def retry_after_pause
+      pause(coordinator.seek_offset, nil, false)
+
+      # Instrumentation needs to run **after** `#pause` invocation because we rely on the states
+      # set by `#pause`
+      Karafka.monitor.instrument(
+        'consumer.consuming.retry',
+        caller: self,
+        topic: messages.metadata.topic,
+        partition: messages.metadata.partition,
+        offset: coordinator.seek_offset,
+        timeout: coordinator.pause_tracker.current_timeout,
+        attempt: coordinator.pause_tracker.attempt
+      )
+    end
   end
 end

data/lib/karafka/connection/client.rb

@@ -14,6 +14,9 @@ module Karafka
       # @note Consumer name may change in case we regenerate it
       attr_reader :name
 
+      # @return [String] id of the client
+      attr_reader :id
+
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
@@ -29,6 +32,7 @@ module Karafka
       #   with all the configuration details needed for us to create a client
       # @return [Karafka::Connection::Rdk::Consumer]
       def initialize(subscription_group)
+        @id = SecureRandom.hex(6)
         # Name is set when we build consumer
         @name = ''
         @mutex = Mutex.new
@@ -165,6 +169,15 @@ module Karafka
 
         return unless tpl
 
+        Karafka.monitor.instrument(
+          'client.pause',
+          caller: self,
+          subscription_group: @subscription_group,
+          topic: topic,
+          partition: partition,
+          offset: offset
+        )
+
         @paused_tpls[topic][partition] = tpl
 
         @kafka.pause(tpl)
@@ -195,6 +208,14 @@ module Karafka
         # happen in the first place
         return unless @paused_tpls[topic].delete(partition)
 
+        Karafka.monitor.instrument(
+          'client.resume',
+          caller: self,
+          subscription_group: @subscription_group,
+          topic: topic,
+          partition: partition
+        )
+
         @kafka.resume(tpl)
       ensure
         @mutex.unlock

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -21,10 +21,16 @@ module Karafka
         @finished = Set.new
       end
 
+      # @return [Boolean] true if all the subscription groups from a given consumer group are
+      #   finished
+      def finished?
+        @finished.size == @group_size
+      end
+
       # @return [Boolean] can we start shutdown on a given listener
       # @note If true, will also obtain a lock so no-one else will be closing the same time we do
       def shutdown?
-        @finished.size == @group_size && @shutdown_lock.try_lock
+        finished? && @shutdown_lock.try_lock
       end
 
       # Unlocks the shutdown lock