karafka 2.0.24 → 2.0.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25eb837923f78f1bf35402bf24469228d8f5af12090b2d406718a0077ce42598
-  data.tar.gz: 23520ecab430080061046e683c0dcb47316d1216d1ad9843894caefff76e24cc
+  metadata.gz: 5f881664edff95f6b495f8e641e01558212b1d68b52904cf756a70a330306be5
+  data.tar.gz: cb808ac7a665f8156938b3ec21d284f7d685ff1ed40eadb13029714ce6f3c277
 SHA512:
-  metadata.gz: 3e84c4c4dedd7a160ebcbd6cff53e0dfcf601a31eb19469d1a86eabb3f33507f3b1871c8f4489b743651118ed47e52925f12dba4fc67e462bd8be04d71dfa3bb
-  data.tar.gz: 8b9754f5566c5ef213c4803cfba2a4263553a5112e23fd12e42a963fd493f8f20c1409b12303628519ba7ef5bcf07aa09fabc6dd4ffc2a2114a6bad8f94cbc72
+  metadata.gz: 3f787ce67115542c3ccd01c993f6ed8abf3747a5ee0db7b8b30921a4445c7db4f5c4f054bb921efc4eaf0459abfce69f1f58f01ca650c023ad7fe09aafc3b0e3
+  data.tar.gz: 0e27a60cfe476342b0a7757e4f9c885b9c47323f8beb6d6bf4cdfe5880d9d8f3c548c5ad0ea2c0263e67c637057cd39a66575cd7b902a1c5ee82d8a4cd14e3b7

data/.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.1
+          ruby-version: 3.2
           bundler-cache: true
 
       - name: Install Diffend plugin
@@ -34,6 +34,22 @@ jobs:
       - name: Bundle Secure
         run: bundle secure
 
+  karafka-checksum:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Run Karafka license checksum verification
+        env:
+          KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
+          KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
+          KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
+          KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
+        run: bin/verify_license_integrity
+
   coditsu:
     runs-on: ubuntu-latest
     strategy:
@@ -53,6 +69,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.2'
           # We run it against the oldest and the newest of a given major to make sure, that there
           # are no syntax-sugars that we would use that were introduced down the road
           - '3.1'
@@ -62,7 +79,7 @@ jobs:
           - '2.7'
           - '2.7.0'
         include:
-          - ruby: '3.1'
+          - ruby: '3.2'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v3
@@ -92,11 +109,12 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.2'
           - '3.1'
           - '3.0'
           - '2.7'
         include:
-          - ruby: '3.1'
+          - ruby: '3.2'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v3
@@ -132,5 +150,6 @@ jobs:
           KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
           KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
           KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
+          KARAFKA_PRO_LICENSE_CHECKSUM: ${{ secrets.KARAFKA_PRO_LICENSE_CHECKSUM }}
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bin/integrations

data/.ruby-version

@@ -1 +1 @@
-3.1.3
+3.2.0

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Karafka framework changelog
 
+## 2.0.27 (2022-01-11)
+- Do not lock Ruby version in Karafka in favour of `karafka-core`.
+- Make sure `karafka-core` version is at least `2.0.9` to make sure we run `karafka-rdkafka`.
+
+## 2.0.26 (2023-01-10)
+- **[Feature]** Allow for disabling given topics by setting `active` to false. It will exclude them from consumption but will allow to have their definitions for using admin APIs, etc.
+- [Improvement] Early terminate on `read_topic` when reaching the last offset available on the request time.
+- [Improvement] Introduce a `quiet` state that indicates that Karafka is not only moving to quiet mode but actually that it reached it and no work will happen anymore in any of the consumer groups.
+- [Improvement] Use Karafka defined routes topics when possible for `read_topic` admin API.
+- [Improvement] Introduce `client.pause` and `client.resume` instrumentation hooks for tracking client topic partition pausing and resuming. This is alongside of `consumer.consuming.pause` that can be used to track both manual and automatic pausing with more granular consumer related details. The `client.*` should be used for low level tracking.
+- [Improvement] Replace `LoggerListener` pause notification with one based on `client.pause` instead of `consumer.consuming.pause`.
+- [Improvement] Expand `LoggerListener` with `client.resume` notification.
+- [Improvement] Replace random anonymous subscription groups ids with stable once.
+- [Improvement] Add `consumer.consume`, `consumer.revoke` and `consumer.shutting_down` notification events and move the revocation logic calling to strategies.
+- [Change] Rename job queue statistics `processing` key to `busy`. No changes needed because naming in the DataDog listener stays the same. 
+- [Fix] Fix proctitle listener state changes reporting on new states.
+- [Fix] Make sure all files descriptors are closed in the integration specs.
+- [Fix] Fix a case where empty subscription groups could leak into the execution flow.
+- [Fix] Fix `LoggerListener` reporting so it does not end with `.`.
+- [Fix] Run previously defined (if any) signal traps created prior to Karafka signals traps.
+
+## 2.0.25 (2023-01-10)
+- Release yanked due to accidental release with local changes.
+
 ## 2.0.24 (2022-12-19)
 - **[Feature]** Provide out of the box encryption support for Pro.
 - [Improvement] Add instrumentation upon `#pause`.

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.24)
-      karafka-core (>= 2.0.7, < 3.0.0)
+    karafka (2.0.27)
+      karafka-core (>= 2.0.9, < 3.0.0)
       thor (>= 0.20)
-      waterdrop (>= 2.4.7, < 3.0.0)
+      waterdrop (>= 2.4.9, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -29,30 +29,30 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.7)
+    karafka-core (2.0.9)
       concurrent-ruby (>= 1.1)
-      rdkafka (>= 0.12)
-    mini_portile2 (2.8.0)
-    minitest (5.16.3)
-    rake (13.0.6)
-    rdkafka (0.12.0)
+      karafka-rdkafka (>= 0.12)
+    karafka-rdkafka (0.12.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
+    mini_portile2 (2.8.1)
+    minitest (5.17.0)
+    rake (13.0.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
     rspec-core (3.12.0)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.0)
+    rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.1)
+    rspec-mocks (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    simplecov (0.21.2)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
@@ -61,13 +61,12 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.7)
-      karafka-core (>= 2.0.7, < 3.0.0)
+    waterdrop (2.4.9)
+      karafka-core (>= 2.0.9, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.6)
 
 PLATFORMS
-  x86_64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -79,4 +78,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.26
+   2.4.2

data/README.md

@@ -4,7 +4,7 @@
 [![Gem Version](https://badge.fury.io/rb/karafka.svg)](http://badge.fury.io/rb/karafka)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
-**Note**: Upgrade notes for migration from Karafka `1.4` to Karafka `2.0` can be found [here](https://karafka.io/docs/Upgrades-2.0/).
+**Note**: Upgrade instructions for migration from Karafka `1.4` to Karafka `2.0` can be found [here](https://karafka.io/docs/Upgrades-2.0/).
 
 ## About Karafka
 
@@ -40,6 +40,8 @@ Karafka **uses** threads to handle many messages simultaneously in the same proc
 
 ## Getting started
 
+![karafka web ui](https://raw.githubusercontent.com/karafka/misc/master/printscreens/web-ui.png)
+
 If you're entirely new to the subject, you can start with our "Kafka on Rails" articles series, which will get you up and running with the terminology and basic ideas behind using Kafka:
 
 - [Kafka on Rails: Using Kafka with Ruby on Rails – Part 1 – Kafka basics and its advantages](https://mensfeld.pl/2017/11/kafka-on-rails-using-kafka-with-ruby-on-rails-part-1-kafka-basics-and-its-advantages/)

data/bin/integrations

@@ -152,6 +152,13 @@ class Scenario
     end
   end
 
+  # Close all the files that are open, so they do not pile up
+  def close
+    @stdin.close
+    @stdout.close
+    @stderr.close
+  end
+
   private
 
   # Sets up a proper environment for a given spec to run and returns the run command
@@ -248,6 +255,7 @@ while finished_scenarios.size < scenarios.size
   active_scenarios.select(&:finished?).each do |exited|
     scenario = active_scenarios.delete(exited)
     scenario.report
+    scenario.close
     finished_scenarios << scenario
   end
 

data/bin/verify_license_integrity

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# This script verifies integrity of the Pro license
+# Run it before bundle install to ensure, that what you are fetching is what you expect
+# Run it after bundle install to ensure that the local artefact was not compromised
+
+#!/usr/bin/env bash
+
+set -e
+
+if [ "$MODE" != "after" ]; then
+  # Check the remote license prior to bundle installing
+  curl \
+    --fail \
+    -u $KARAFKA_PRO_USERNAME:$KARAFKA_PRO_PASSWORD \
+    https://gems.karafka.io/gems/karafka-license-$KARAFKA_PRO_VERSION.gem \
+    -o ./karafka-license.gem
+else
+  # Check the local cached one after bundle install
+  cache_path=`ruby -e 'puts "#{Gem.dir}/cache/"'`
+  cp "$cache_path/karafka-license-$KARAFKA_PRO_VERSION.gem" ./karafka-license.gem
+fi
+
+detected=`sha256sum ./karafka-license.gem | awk '{ print $1 }'`
+
+rm ./karafka-license.gem
+
+echo -n "Karafka Pro license artifact checksum verification result: "
+
+if [ "$detected" = "$KARAFKA_PRO_LICENSE_CHECKSUM" ]; then
+  echo "Success"
+else
+  echo -e "\033[0;31mFailure!\033[0m"
+  exit 1
+fi

data/config/locales/errors.yml

@@ -44,6 +44,7 @@ en:
       dead_letter_queue.max_retries_format: needs to be equal or bigger than 0
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
+      active_format: needs to be either true or false
 
     consumer_group:
       missing: needs to be present

data/karafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[contact@karafka.io]
   spec.homepage    = 'https://karafka.io'
-  spec.licenses    = ['LGPL-3.0', 'Commercial']
+  spec.licenses    = %w[LGPL-3.0 Commercial]
   spec.summary     = 'Karafka is Ruby and Rails efficient Kafka processing framework.'
   spec.description = <<-DESC
     Karafka is Ruby and Rails efficient Kafka processing framework.
@@ -21,13 +21,11 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.0.7', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.9', '< 3.0.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.4.7', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.4.9', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
-  spec.required_ruby_version = '>= 2.7.0'
-
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end

data/lib/karafka/admin.rb

@@ -30,29 +30,30 @@ module Karafka
       # @param name [String, Symbol] topic name
       # @param partition [Integer] partition
       # @param count [Integer] how many messages we want to get at most
-      # @param offset [Integer] offset from which we should start. If -1 is provided (default) we
-      #   will start from the latest offset
+      # @param start_offset [Integer] offset from which we should start. If -1 is provided
+      #   (default) we will start from the latest offset
       #
       # @return [Array<Karafka::Messages::Message>] array with messages
-      def read_topic(name, partition, count, offset = -1)
+      def read_topic(name, partition, count, start_offset = -1)
         messages = []
         tpl = Rdkafka::Consumer::TopicPartitionList.new
 
         with_consumer do |consumer|
-          if offset.negative?
-            offsets = consumer.query_watermark_offsets(name, partition)
-            offset = offsets.last - count
-          end
+          offsets = consumer.query_watermark_offsets(name, partition)
+          end_offset = offsets.last
 
-          offset = offset.negative? ? 0 : offset
+          start_offset = [0, offsets.last - count].max if start_offset.negative?
 
-          tpl.add_topic_and_partitions_with_offsets(name, partition => offset)
+          tpl.add_topic_and_partitions_with_offsets(name, partition => start_offset)
           consumer.assign(tpl)
 
           # We should poll as long as we don't have all the messages that we need or as long as
           # we do not read all the messages from the topic
           loop do
+            # If we've got as many messages as we've wanted stop
             break if messages.size >= count
+            # If we've reached end of the topic messages, don't process more
+            break if !messages.empty? && end_offset <= messages.last.offset
 
             message = consumer.poll(200)
             messages << message if message
@@ -67,7 +68,10 @@ module Karafka
         messages.map do |message|
           Messages::Builders::Message.call(
             message,
-            Topic.new(name, Karafka::App.config.deserializer),
+            # Use topic from routes if we can match it or create a dummy one
+            # Dummy one is used in case we cannot match the topic with routes. This can happen
+            # when admin API is used to read topics that are not part of the routing
+            Routing::Router.find_by(name: name) || Topic.new(name, App.config.deserializer),
             Time.now
           )
         end

data/lib/karafka/app.rb

@@ -29,6 +29,7 @@ module Karafka
           .delete_if { |_, sgs| sgs.empty? }
           .each { |_, sgs| sgs.each { |sg| sg.topics.delete_if { |top| !top.active? } } }
           .each { |_, sgs| sgs.delete_if { |sg| sg.topics.empty? } }
+          .reject { |cg, _| cg.subscription_groups.empty? }
           .to_h
       end
 

data/lib/karafka/base_consumer.rb

@@ -99,10 +99,6 @@ module Karafka
     # @private
     def on_revoked
       handle_revoked
-
-      Karafka.monitor.instrument('consumer.revoked', caller: self) do
-        revoked
-      end
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',
@@ -116,9 +112,7 @@ module Karafka
     #
     # @private
     def on_shutdown
-      Karafka.monitor.instrument('consumer.shutdown', caller: self) do
-        shutdown
-      end
+      handle_shutdown
     rescue StandardError => e
       Karafka.monitor.instrument(
         'error.occurred',

data/lib/karafka/connection/client.rb

@@ -14,6 +14,9 @@ module Karafka
       # @note Consumer name may change in case we regenerate it
       attr_reader :name
 
+      # @return [String] id of the client
+      attr_reader :id
+
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
@@ -29,6 +32,7 @@ module Karafka
       #   with all the configuration details needed for us to create a client
       # @return [Karafka::Connection::Rdk::Consumer]
       def initialize(subscription_group)
+        @id = SecureRandom.hex(6)
         # Name is set when we build consumer
         @name = ''
         @mutex = Mutex.new
@@ -165,6 +169,15 @@ module Karafka
 
         return unless tpl
 
+        Karafka.monitor.instrument(
+          'client.pause',
+          caller: self,
+          subscription_group: @subscription_group,
+          topic: topic,
+          partition: partition,
+          offset: offset
+        )
+
         @paused_tpls[topic][partition] = tpl
 
         @kafka.pause(tpl)
@@ -195,6 +208,14 @@ module Karafka
         # happen in the first place
         return unless @paused_tpls[topic].delete(partition)
 
+        Karafka.monitor.instrument(
+          'client.resume',
+          caller: self,
+          subscription_group: @subscription_group,
+          topic: topic,
+          partition: partition
+        )
+
         @kafka.resume(tpl)
       ensure
         @mutex.unlock

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -21,10 +21,16 @@ module Karafka
         @finished = Set.new
       end
 
+      # @return [Boolean] true if all the subscription groups from a given consumer group are
+      #   finished
+      def finished?
+        @finished.size == @group_size
+      end
+
       # @return [Boolean] can we start shutdown on a given listener
       # @note If true, will also obtain a lock so no-one else will be closing the same time we do
       def shutdown?
-        @finished.size == @group_size && @shutdown_lock.try_lock
+        finished? && @shutdown_lock.try_lock
       end
 
       # Unlocks the shutdown lock

data/lib/karafka/connection/listener.rb

@@ -85,7 +85,7 @@ module Karafka
       #   propagate this far.
       def fetch_loop
         # Run the main loop as long as we are not stopping or moving into quiet mode
-        until Karafka::App.stopping? || Karafka::App.quieting?
+        until Karafka::App.stopping? || Karafka::App.quieting? || Karafka::App.quiet?
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop',
             caller: self,
@@ -156,8 +156,9 @@ module Karafka
         # within this consumer group
         @consumer_group_coordinator.finish_work(id)
 
-        # Wait if we're in the quiet mode
-        wait_pinging(wait_until: -> { !Karafka::App.quieting? })
+        # Wait if we're in the process of finishing started work or finished all the work and
+        # just sitting and being quiet
+        wait_pinging(wait_until: -> { !(Karafka::App.quieting? || Karafka::App.quiet?) })
 
         # We need to wait until all the work in the whole consumer group (local to the process)
         # is done. Otherwise we may end up with locks and `Timed out LeaveGroupRequest in flight`

data/lib/karafka/connection/listeners_batch.rb

@@ -6,14 +6,20 @@ module Karafka
     class ListenersBatch
       include Enumerable
 
+      attr_reader :coordinators
+
       # @param jobs_queue [JobsQueue]
       # @return [ListenersBatch]
       def initialize(jobs_queue)
+        @coordinators = []
+
         @batch = App.subscription_groups.flat_map do |_consumer_group, subscription_groups|
           consumer_group_coordinator = Connection::ConsumerGroupCoordinator.new(
             subscription_groups.size
           )
 
+          @coordinators << consumer_group_coordinator
+
           subscription_groups.map do |subscription_group|
             Connection::Listener.new(
               consumer_group_coordinator,

data/lib/karafka/contracts/server_cli_options.rb

@@ -80,6 +80,7 @@ module Karafka
       # Makes sure we have anything to subscribe to when we start the server
       virtual do |_, errors|
         next unless errors.empty?
+
         next unless Karafka::App.subscription_groups.empty?
 
         [[%i[topics], :topics_missing]]

data/lib/karafka/contracts/topic.rb

@@ -12,7 +12,6 @@ module Karafka
         ).fetch('en').fetch('validations').fetch('topic')
       end
 
-      required(:consumer) { |val| !val.nil? }
       required(:deserializer) { |val| !val.nil? }
       required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
@@ -20,8 +19,20 @@ module Karafka
       required(:initial_offset) { |val| %w[earliest latest].include?(val) }
       required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
       required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+      required(:active) { |val| [true, false].include?(val) }
       required(:subscription_group) { |val| val.is_a?(String) && !val.empty? }
 
+      # Consumer needs to be present only if topic is active
+      # We allow not to define consumer for non-active because they may be only used via admin
+      # api or other ways and not consumed with consumer
+      virtual do |data, errors|
+        next unless errors.empty?
+        next if data.fetch(:consumer)
+        next unless data.fetch(:active)
+
+        [[%w[consumer], :missing]]
+      end
+
       virtual do |data, errors|
         next unless errors.empty?
 

data/lib/karafka/instrumentation/logger_listener.rb

@@ -63,19 +63,30 @@ module Karafka
         info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
       end
 
-      # Prints info about a pause occurrence. Irrelevant if user or system initiated.
+      # Prints info about a consumer pause occurrence. Irrelevant if user or system initiated.
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_consumer_consuming_pause(event)
+      def on_client_pause(event)
         topic = event[:topic]
         partition = event[:partition]
         offset = event[:offset]
-        consumer = event[:caller]
-        timeout = event[:timeout]
+        client = event[:caller]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{client.id}] Pausing partition #{partition} of topic #{topic} on offset #{offset}
+        MSG
+      end
+
+      # Prints information about resuming of processing of a given topic partition
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_client_resume(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        client = event[:caller]
 
         info <<~MSG.tr("\n", ' ').strip!
-          [#{consumer.id}] Pausing partition #{partition} of topic #{topic}
-          on offset #{offset} for #{timeout} ms.
+          [#{client.id}] Resuming partition #{partition} of topic #{topic}
         MSG
       end
 
@@ -128,12 +139,17 @@ module Karafka
 
         return if Karafka.pro?
 
-        info 'See LICENSE and the LGPL-3.0 for licensing details.'
+        info 'See LICENSE and the LGPL-3.0 for licensing details'
       end
 
       # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_quieting(_event)
-        info 'Switching to quiet mode. New messages will not be processed.'
+        info 'Switching to quiet mode. New messages will not be processed'
+      end
+
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quiet(_event)
+        info 'Reached quiet mode. No messages will be processed anymore'
       end
 
       # Logs info that we're going to stop the Karafka server.

data/lib/karafka/instrumentation/notifications.rb

@@ -20,30 +20,37 @@ module Karafka
         app.initialized
         app.running
         app.quieting
+        app.quiet
         app.stopping
         app.stopped
         app.terminated
 
+        client.pause
+        client.resume
+
+        connection.listener.before_fetch_loop
+        connection.listener.fetch_loop
+        connection.listener.fetch_loop.received
+
+        consumer.consume
         consumer.consumed
         consumer.consuming.pause
         consumer.consuming.retry
+        consumer.revoke
         consumer.revoked
+        consumer.shutting_down
         consumer.shutdown
 
-        process.notice_signal
+        dead_letter_queue.dispatched
 
-        connection.listener.before_fetch_loop
-        connection.listener.fetch_loop
-        connection.listener.fetch_loop.received
+        process.notice_signal
 
-        dead_letter_queue.dispatched
+        statistics.emitted
 
         worker.process
         worker.processed
         worker.completed
 
-        statistics.emitted
-
         error.occurred
       ].freeze
 

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -4,22 +4,13 @@ module Karafka
   module Instrumentation
     # Listener that sets a proc title with a nice descriptive value
     class ProctitleListener
-      # Updates proc title to an initializing one
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_initializing(_event)
-        setproctitle('initializing')
-      end
-
-      # Updates proc title to a running one
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_running(_event)
-        setproctitle('running')
-      end
-
-      # Updates proc title to a stopping one
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_stopping(_event)
-        setproctitle('stopping')
+      Status::STATES.each_key do |state|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          # Updates proc title to an appropriate state
+          def on_app_#{state}(_event)
+            setproctitle('#{state}')
+          end
+        RUBY
       end
 
       private

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -148,7 +148,7 @@ module Karafka
             jq_stats = event[:jobs_queue].statistics
 
             gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
-            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
             histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
           end
 
@@ -158,7 +158,7 @@ module Karafka
           def on_worker_processed(event)
             jq_stats = event[:jobs_queue].statistics
 
-            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
           end
 
           private

data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb

@@ -61,6 +61,11 @@ module Karafka
             coordinator.on_revoked do
               coordinator.revoke
             end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -46,6 +46,7 @@ module Karafka
             # This can happen primarily when an LRJ job gets to the internal worker queue and
             # this partition is revoked prior processing.
             unless revoked?
+              Karafka.monitor.instrument('consumer.consume', caller: self)
               Karafka.monitor.instrument('consumer.consumed', caller: self) do
                 consume
               end
@@ -91,6 +92,11 @@ module Karafka
 
               coordinator.revoke
             end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/lrj.rb

@@ -70,6 +70,11 @@ module Karafka
               # a failure. Double non-blocking resume could cause problems in coordination.
               coordinator.revoke
             end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/lrj_mom.rb

@@ -60,6 +60,11 @@ module Karafka
             coordinator.on_revoked do
               coordinator.revoke
             end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
           end
         end
       end

data/lib/karafka/process.rb

@@ -53,12 +53,14 @@ module Karafka
     # trap context s some things may not work there as expected, that is why we spawn a separate
     # thread to handle the signals process
     def trap_signal(signal)
-      trap(signal) do
+      previous_handler = ::Signal.trap(signal) do
         Thread.new do
           notice_signal(signal)
 
           (@callbacks[signal] || []).each(&:call)
         end
+
+        previous_handler.call if previous_handler.respond_to?(:call)
       end
     end
 

data/lib/karafka/processing/jobs_queue.rb

@@ -125,13 +125,13 @@ module Karafka
         @semaphores[group_id].pop while wait?(group_id)
       end
 
-      # - `processing` - number of jobs that are currently being processed (active work)
+      # - `busy` - number of jobs that are currently being processed (active work)
       # - `enqueued` - number of jobs in the queue that are waiting to be picked up by a worker
       #
       # @return [Hash] hash with basic usage statistics of this queue.
       def statistics
         {
-          processing: size - @queue.size,
+          busy: size - @queue.size,
           enqueued: @queue.size
         }.freeze
       end

data/lib/karafka/processing/strategies/base.rb

@@ -36,6 +36,11 @@ module Karafka
         def handle_revoked
           raise NotImplementedError, 'Implement in a subclass'
         end
+
+        # Shutdown handling
+        def handle_shutdown
+          raise NotImplementedError, 'Implement in a subclass'
+        end
       end
     end
   end

data/lib/karafka/processing/strategies/default.rb

@@ -25,6 +25,7 @@ module Karafka
 
         # Run the user consumption code
         def handle_consume
+          Karafka.monitor.instrument('consumer.consume', caller: self)
           Karafka.monitor.instrument('consumer.consumed', caller: self) do
             consume
           end
@@ -70,6 +71,19 @@ module Karafka
           resume
 
           coordinator.revoke
+
+          Karafka.monitor.instrument('consumer.revoke', caller: self)
+          Karafka.monitor.instrument('consumer.revoked', caller: self) do
+            revoked
+          end
+        end
+
+        # Runs the shutdown code
+        def handle_shutdown
+          Karafka.monitor.instrument('consumer.shutting_down', caller: self)
+          Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+            shutdown
+          end
         end
       end
     end

data/lib/karafka/processing/worker.rb

@@ -76,6 +76,8 @@ module Karafka
         Karafka.monitor.instrument(
           'error.occurred',
           caller: self,
+          job: job,
+          jobs_queue: @jobs_queue,
           error: e,
           type: 'worker.process.error'
         )

data/lib/karafka/routing/builder.rb

@@ -80,7 +80,7 @@ module Karafka
       # @param subscription_group_name [String, Symbol] subscription group id. When not provided,
       #   a random uuid will be used
       # @param block [Proc] further topics definitions
-      def subscription_group(subscription_group_name = SecureRandom.hex(6), &block)
+      def subscription_group(subscription_group_name = SubscriptionGroup.id, &block)
         consumer_group('app') do
           target.public_send(:subscription_group=, subscription_group_name.to_s, &block)
         end

data/lib/karafka/routing/consumer_group.rb

@@ -26,7 +26,7 @@ module Karafka
         @topics = Topics.new([])
         # Initialize the subscription group so there's always a value for it, since even if not
         # defined directly, a subscription group will be created
-        @current_subscription_group_id = SecureRandom.hex(6)
+        @current_subscription_group_id = SubscriptionGroup.id
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
@@ -55,7 +55,7 @@ module Karafka
       # topic definition
       # @param name [String, Symbol] name of the current subscription group
       # @param block [Proc] block that may include topics definitions
-      def subscription_group=(name = SecureRandom.hex(6), &block)
+      def subscription_group=(name = SubscriptionGroup.id, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
         @current_subscription_group_id = name
@@ -64,7 +64,7 @@ module Karafka
 
         # We need to reset the current subscription group after it is used, so it won't leak
         # outside to other topics that would be defined without a defined subscription group
-        @current_subscription_group_id = SecureRandom.hex(6)
+        @current_subscription_group_id = SubscriptionGroup.id
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on

data/lib/karafka/routing/consumer_mapper.rb

@@ -12,16 +12,6 @@ module Karafka
     #       raw_consumer_group_name
     #     end
     #   end
-    #
-    # @example Mapper for replacing "_" with "." in topic names
-    #   class MyMapper
-    #     def call(raw_consumer_group_name)
-    #       [
-    #         Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s),
-    #         raw_consumer_group_name
-    #       ].join('_').gsub('_', '.')
-    #     end
-    #   end
     class ConsumerMapper
       # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
       # @return [String] remapped final consumer group name

data/lib/karafka/routing/router.rb

@@ -14,16 +14,26 @@ module Karafka
       # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
       #   any route defined by user using routes.draw
       def find(topic_id)
+        find_by(id: topic_id) || raise(Errors::NonMatchingRouteError, topic_id)
+      end
+
+      # Finds first reference of a given topic based on provided lookup attribute
+      # @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
+      # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
+      def find_by(lookup)
         App.consumer_groups.each do |consumer_group|
           consumer_group.topics.each do |topic|
-            return topic if topic.id == topic_id
+            return topic if lookup.all? do |attribute, value|
+              topic.public_send(attribute) == value
+            end
           end
         end
 
-        raise(Errors::NonMatchingRouteError, topic_id)
+        nil
       end
 
       module_function :find
+      module_function :find_by
     end
   end
 end

data/lib/karafka/routing/subscription_group.rb

@@ -8,7 +8,23 @@ module Karafka
     # @note One subscription group will always belong to one consumer group, but one consumer
     #   group can have multiple subscription groups.
     class SubscriptionGroup
-      attr_reader :id, :name, :topics, :kafka
+      attr_reader :id, :name, :topics, :kafka, :consumer_group
+
+      # Numeric for counting groups
+      GROUP_COUNT = Concurrent::AtomicFixnum.new
+
+      private_constant :GROUP_COUNT
+
+      class << self
+        # Generates new subscription group id that will be used in case of anonymous subscription
+        #   groups
+        # @return [String] hex(6) compatible reproducible id
+        def id
+          ::Digest::MD5.hexdigest(
+            GROUP_COUNT.increment.to_s
+          )[0..11]
+        end
+      end
 
       # @param position [Integer] position of this subscription group in all the subscriptions
       #   groups array. We need to have this value for sake of static group memberships, where
@@ -17,6 +33,7 @@ module Karafka
       # @return [SubscriptionGroup] built subscription group
       def initialize(position, topics)
         @name = topics.first.subscription_group
+        @consumer_group = topics.first.consumer_group
         @id = "#{@name}_#{position}"
         @position = position
         @topics = topics

data/lib/karafka/routing/topic.rb

@@ -27,6 +27,7 @@ module Karafka
         @name = name.to_s
         @consumer_group = consumer_group
         @attributes = {}
+        @active = true
         # @note We use identifier related to the consumer group that owns a topic, because from
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
         #   have same topic name across multiple consumer groups
@@ -66,6 +67,12 @@ module Karafka
         end
       end
 
+      # Allows to disable topic by invoking this method and setting it to `false`.
+      # @param active [Boolean] should this topic be consumed or not
+      def active(active)
+        @active = active
+      end
+
       # @return [Class] consumer class that we should use
       # @note This is just an alias to the `#consumer` method. We however want to use it internally
       #   instead of referencing the `#consumer`. We use this to indicate that this method returns
@@ -77,6 +84,9 @@ module Karafka
 
       # @return [Boolean] should this topic be in use
       def active?
+        # Never active if disabled via routing
+        return false unless @active
+
         topics = Karafka::App.config.internal.routing.active.topics
 
         # When empty it means no topics were specified, hence all should be used
@@ -93,6 +103,7 @@ module Karafka
         Hash[map].merge!(
           id: id,
           name: name,
+          active: active?,
           consumer: consumer,
           consumer_group_id: consumer_group.id,
           subscription_group: subscription_group

data/lib/karafka/runner.rb

@@ -19,6 +19,7 @@ module Karafka
       # We aggregate threads here for a supervised shutdown process
       Karafka::Server.workers = workers
       Karafka::Server.listeners = listeners
+      Karafka::Server.jobs_queue = jobs_queue
 
       # All the listener threads need to finish
       listeners.each(&:join)

data/lib/karafka/server.rb

@@ -20,6 +20,9 @@ module Karafka
       # Set of workers
       attr_accessor :workers
 
+      # Jobs queue
+      attr_accessor :jobs_queue
+
       # Method which runs app
       def run
         self.listeners = []
@@ -45,9 +48,9 @@ module Karafka
 
         # We always need to wait for Karafka to stop here since we should wait for the stop running
         # in a separate thread (or trap context) to indicate everything is closed
-        # Since `#start` is blocking, we were get here only after the runner is done. This will
+        # Since `#start` is blocking, we will get here only after the runner is done. This will
         # not add any performance degradation because of that.
-        Thread.pass until Karafka::App.terminated?
+        sleep(0.1) until Karafka::App.terminated?
       # Try its best to shutdown underlying components before re-raising
       # rubocop:disable Lint/RescueException
       rescue Exception => e
@@ -106,8 +109,6 @@ module Karafka
         # This can cause memory leaks and crashes.
         listeners.each(&:shutdown)
 
-        Karafka::App.producer.close
-
         # We also do not forcefully terminate everything when running in the embedded mode,
         # otherwise we would overwrite the shutdown process of the process that started Karafka
         return unless process.supervised?
@@ -129,12 +130,20 @@ module Karafka
       end
 
       # Quiets the Karafka server.
-      # Karafka will stop processing but won't quiet to consumer group, so no rebalance will be
+      #
+      # Karafka will stop processing but won't quit the consumer group, so no rebalance will be
       # triggered until final shutdown.
       def quiet
         # We don't have to safe-guard it with check states as the state transitions work only
         # in one direction
         Karafka::App.quiet!
+
+        # We need one more thread to monitor the process and move to quieted once everything
+        # is quiet and no processing is happening anymore
+        Thread.new do
+          sleep(0.1) until listeners.coordinators.all?(&:finished?)
+          Karafka::App.quieted!
+        end
       end
 
       private

data/lib/karafka/status.rb

@@ -8,9 +8,15 @@ module Karafka
       initializing: :initialize!,
       initialized: :initialized!,
       running: :run!,
+      # will no longer pickup any work, but current work will be finished
       quieting: :quiet!,
+      # no work is happening but we keep process with the assignments running
+      quiet: :quieted!,
+      # shutdown started
       stopping: :stop!,
+      # all things are done and most of the things except critical are closed
       stopped: :stopped!,
+      # immediately after this process exists
       terminated: :terminate!
     }.freeze
 

data/lib/karafka/time_trackers/base.rb

@@ -8,12 +8,7 @@ module Karafka
   module TimeTrackers
     # Base class for all the time-trackers.
     class Base
-      private
-
-      # @return [Float] current time in milliseconds
-      def now
-        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1000
-      end
+      include ::Karafka::Core::Helpers::Time
     end
   end
 end

data/lib/karafka/time_trackers/pause.rb

@@ -58,7 +58,7 @@ module Karafka
       def pause(timeout = backoff_interval)
         @mutex.synchronize do
           @current_timeout = timeout
-          @started_at = now
+          @started_at = monotonic_now
           @ends_at = @started_at + timeout
         end
       end
@@ -95,7 +95,7 @@ module Karafka
       # @return [Boolean] did the pause expire
       def expired?
         @mutex.synchronize do
-          @ends_at ? now >= @ends_at : true
+          @ends_at ? monotonic_now >= @ends_at : true
         end
       end
 

data/lib/karafka/time_trackers/poll.rb

@@ -34,12 +34,12 @@ module Karafka
       # Starts time tracking.
       def start
         @attempts += 1
-        @started_at = now
+        @started_at = monotonic_now
       end
 
       # Stops time tracking of a given piece of code and updates the remaining time.
       def checkpoint
-        @remaining -= (now - @started_at)
+        @remaining -= (monotonic_now - @started_at)
       end
 
       # @return [Boolean] If anything went wrong, can we retry after a backoff period or not

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.24'
+  VERSION = '2.0.27'
 end

data/lib/karafka.rb

@@ -14,7 +14,9 @@
   base64
   date
   singleton
+  digest
   zeitwerk
+  concurrent/atomic/atomic_fixnum
 ].each(&method(:require))
 
 # Karafka framework main namespace

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.24
+  version: 2.0.27
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-12-19 00:00:00.000000000 Z
+date: 2023-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.7
+        version: 2.0.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.7
+        version: 2.0.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.7
+        version: 2.4.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.7
+        version: 2.4.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -145,6 +145,7 @@ files:
 - bin/scenario
 - bin/stress_many
 - bin/stress_one
+- bin/verify_license_integrity
 - certs/cert_chain.pem
 - certs/karafka-pro.pem
 - config/locales/errors.yml
@@ -342,14 +343,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.1
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.