karafka 2.0.17 → 2.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d15d4c803dc84a009e4a1ab02e90a9f034a396f718f742060c545f00422ffd5
-  data.tar.gz: fd69d2a4dcb11a9ea94b8a9f7a4dbb505034a304e94b1bd5e9a0a9fe44a666f0
+  metadata.gz: be91c3848b757c6af4c25f905df2b081629532bd29dbcea23ed2ef0af2e4e4a2
+  data.tar.gz: 6823d4335e4b395546642101d6754b97958c86810cbcd12819559acff74bd90d
 SHA512:
-  metadata.gz: fe75bb62fecbca6b541d1c1737a596b0a937a02a294ec62a13220915194807568be84c0ebb338de6de8a124714164ed7f7f060cdb573f5606357cce861ba364f
-  data.tar.gz: 6c622f4d2ce80b86807e0e05b3108f7fab66ba2696cd2b662a3412986bc98ba954d03bf70141c3a5526e8e4954495efa34c846a0ec594e2e05b05ef2fa01291a
+  metadata.gz: fce0259ee987e37c01ea037f81ea91b4eb770ea8eabcb9f93c66aa1a1960c903030648b5441945ef28f43a88660d18240e6db61f6885a169d70eb46174543616
+  data.tar.gz: f93985c98daba5965f8f0597da4744d1dae0603f24a6a44f4b337462f66c8b0f08d0c22dd734205a0aebe7c3dbdb73237ff568f7f1ff842b38c05a7f4b5ce463

data/.github/workflows/ci.yml

@@ -109,7 +109,6 @@ jobs:
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
-          bundler-cache: true
 
       - name: Install latest Bundler
         run: |

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Karafka framework changelog
 
+## 2.0.19 (2022-11-20)
+- **[Feature]** Provide ability to skip failing messages without dispatching them to an alternative topic (DLQ).
+- [Improvement] Improve the integration with Ruby on Rails by preventing double-require of components.
+- [Improvement] Improve stability of the shutdown process upon critical errors.
+- [Improvement] Improve stability of the integrations spec suite.
+- [Fix] Fix an issue where upon fast startup of multiple subscription groups from the same consumer group, a ghost queue would be created due to problems in `Concurrent::Hash`.
+
+## 2.0.18 (2022-11-18)
+- **[Feature]** Support quiet mode via `TSTP` signal. When used, Karafka will finish processing current messages, run `shutdown` jobs, and switch to a quiet mode where no new work is being accepted. At the same time, it will keep the consumer group quiet, and thus no rebalance will be triggered. This can be particularly useful during deployments.
+- [Improvement] Trigger `#revoked` for jobs in case revocation would happen during shutdown when jobs are still running. This should ensure, we get a notion of revocation for Pro LRJ jobs even when revocation happening upon shutdown (#1150).
+- [Improvement] Stabilize the shutdown procedure for consumer groups with many subscription groups that have non-aligned processing cost per batch.
+- [Improvement] Remove double loading of Karafka via Rails railtie.
+- [Fix] Fix invalid class references in YARD docs.
+- [Fix] prevent parallel closing of many clients.
+- [Fix] fix a case where information about revocation for a combination of LRJ + VP would not be dispatched until all VP work is done.
+
 ## 2.0.17 (2022-11-10)
 - [Fix] Few typos around DLQ and Pro DLQ Dispatch original metadata naming.
 - [Fix] Narrow the components lookup to the appropriate scope (#1114)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.17)
+    karafka (2.0.19)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/karafka.gemspec

@@ -34,7 +34,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w[lib]
 
   spec.metadata = {
+    'funding_uri' => 'https://karafka.io/#become-pro',
+    'homepage_uri' => 'https://karafka.io',
+    'changelog_uri' => 'https://github.com/karafka/karafka/blob/master/CHANGELOG.md',
+    'bug_tracker_uri' => 'https://github.com/karafka/karafka/issues',
     'source_code_uri' => 'https://github.com/karafka/karafka',
+    'documentation_uri' => 'https://karafka.io/docs',
     'rubygems_mfa_required' => 'true'
   }
 end

data/lib/active_job/karafka.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 begin
-  require 'active_job'
+  # Do not load active job if already loaded
+  require 'active_job' unless Object.const_defined?('ActiveJob')
+
   require_relative 'queue_adapters/karafka_adapter'
 
   module ActiveJob

data/lib/karafka/app.rb

@@ -14,11 +14,12 @@ module Karafka
           .builder
       end
 
-      # @return [Array<Karafka::Routing::SubscriptionGroup>] active subscription groups
+      # @return [Hash] active subscription groups grouped based on consumer group in a hash
       def subscription_groups
         consumer_groups
           .active
-          .flat_map(&:subscription_groups)
+          .map { |consumer_group| [consumer_group, consumer_group.subscription_groups] }
+          .to_h
       end
 
       # Just a nicer name for the consumer groups

data/lib/karafka/connection/client.rb

@@ -17,7 +17,11 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
-      private_constant :MAX_POLL_RETRIES
+      # We want to make sure we never close several clients in the same moment to prevent
+      # potential race conditions and other issues
+      SHUTDOWN_MUTEX = Mutex.new
+
+      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX
 
       # Creates a new consumer instance.
       #
@@ -237,6 +241,17 @@ module Karafka
         end
       end
 
+      # Runs a single poll ignoring all the potential errors
+      # This is used as a keep-alive in the shutdown stage and any errors that happen here are
+      # irrelevant from the shutdown process perspective
+      #
+      # This is used only to trigger rebalance callbacks
+      def ping
+        poll(100)
+      rescue Rdkafka::RdkafkaError
+        nil
+      end
+
       private
 
       # When we cannot store an offset, it means we no longer own the partition
@@ -281,24 +296,26 @@ module Karafka
 
       # Commits the stored offsets in a sync way and closes the consumer.
       def close
-        @mutex.synchronize do
-          # Once client is closed, we should not close it again
-          # This could only happen in case of a race-condition when forceful shutdown happens
-          # and triggers this from a different thread
-          return if @closed
-
-          @closed = true
-
-          internal_commit_offsets(async: false)
-
-          # Remove callbacks runners that were registered
-          ::Karafka::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
-          ::Karafka::Instrumentation.error_callbacks.delete(@subscription_group.id)
-
-          @kafka.close
-          @buffer.clear
-          # @note We do not clear rebalance manager here as we may still have revocation info here
-          # that we want to consider valid prior to running another reconnection
+        # Allow only one client to be closed at the same time
+        SHUTDOWN_MUTEX.synchronize do
+          # Make sure that no other operations are happening on this client when we close it
+          @mutex.synchronize do
+            # Once client is closed, we should not close it again
+            # This could only happen in case of a race-condition when forceful shutdown happens
+            # and triggers this from a different thread
+            return if @closed
+
+            @closed = true
+
+            # Remove callbacks runners that were registered
+            ::Karafka::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
+            ::Karafka::Instrumentation.error_callbacks.delete(@subscription_group.id)
+
+            @kafka.close
+            @buffer.clear
+            # @note We do not clear rebalance manager here as we may still have revocation info
+            # here that we want to consider valid prior to running another reconnection
+          end
         end
       end
 

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # This object represents a collective status of execution of group of listeners running inside
+    # of one consumer group but in separate subscription groups.
+    #
+    # There are cases when we do not want to close a given client when others from the same
+    # consumer group are running because it can cause instabilities due to early shutdown of some
+    # of the clients out of same consumer group.
+    #
+    # We also want to make sure, we close one consumer at a time while others can continue polling.
+    #
+    # This prevents a scenario, where a rebalance is not acknowledged and we loose assignment
+    # without having a chance to commit changes.
+    class ConsumerGroupCoordinator
+      # @param group_size [Integer] number of separate subscription groups in a consumer group
+      def initialize(group_size)
+        # We need two locks here:
+        # - first one is to decrement the number of listeners doing work
+        # - second to ensure only one client is being closed the same time and that others can
+        #   wait actively (not locked)
+        @work_mutex = Mutex.new
+        @shutdown_lock = Mutex.new
+        @group_size = group_size
+        @finished = Set.new
+      end
+
+      # @return [Boolean] can we start shutdown on a given listener
+      # @note If true, will also obtain a lock so no-one else will be closing the same time we do
+      def shutdown?
+        @finished.size == @group_size && @shutdown_lock.try_lock
+      end
+
+      # Unlocks the shutdown lock
+      def unlock
+        @shutdown_lock.unlock if @shutdown_lock.owned?
+      end
+
+      # Marks given listener as finished
+      # @param listener_id [String]
+      def finish_work(listener_id)
+        @finished << listener_id
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -14,13 +14,15 @@ module Karafka
       # @return [String] id of this listener
       attr_reader :id
 
+      # @param consumer_group_coordinator [Karafka::Connection::ConsumerGroupCoordinator]
       # @param subscription_group [Karafka::Routing::SubscriptionGroup]
       # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
-      def initialize(subscription_group, jobs_queue)
+      def initialize(consumer_group_coordinator, subscription_group, jobs_queue)
         proc_config = ::Karafka::App.config.internal.processing
 
         @id = SecureRandom.uuid
+        @consumer_group_coordinator = consumer_group_coordinator
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
         @coordinators = Processing::CoordinatorsBuffer.new
@@ -82,7 +84,8 @@ module Karafka
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far.
       def fetch_loop
-        until Karafka::App.stopping?
+        # Run the main loop as long as we are not stopping or moving into quiet mode
+        until Karafka::App.stopping? || Karafka::App.quieting?
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop',
             caller: self,
@@ -122,7 +125,7 @@ module Karafka
           wait
         end
 
-        # If we are stopping we will no longer schedule any jobs despite polling.
+        # If we are stopping we will no longer schedule any regular jobs despite polling.
         # We need to keep polling not to exceed the `max.poll.interval` for long-running
         # non-blocking jobs and we need to allow them to finish. We however do not want to
         # enqueue any new jobs. It's worth keeping in mind that it is the end user responsibility
@@ -131,7 +134,14 @@ module Karafka
         #
         # We do not care about resuming any partitions or lost jobs as we do not plan to do
         # anything with them as we're in the shutdown phase.
-        wait_with_poll
+        #
+        # What we do care however is the ability to still run revocation jobs in case anything
+        # would change in the cluster. We still want to notify the long-running jobs about changes
+        # that occurred in the cluster.
+        wait_pinging(
+          wait_until: -> { @jobs_queue.empty?(@subscription_group.id) },
+          after_ping: -> { build_and_schedule_revoke_lost_partitions_jobs }
+        )
 
         # We do not want to schedule the shutdown jobs prior to finishing all the jobs
         # (including non-blocking) as there might be a long-running job with a shutdown and then
@@ -139,7 +149,24 @@ module Karafka
         # as it could create a race-condition.
         build_and_schedule_shutdown_jobs
 
-        wait_with_poll
+        # Wait until all the shutdown jobs are done
+        wait_pinging(wait_until: -> { @jobs_queue.empty?(@subscription_group.id) })
+
+        # Once all the work is done, we need to decrement counter of active subscription groups
+        # within this consumer group
+        @consumer_group_coordinator.finish_work(id)
+
+        # Wait if we're in the quiet mode
+        wait_pinging(wait_until: -> { !Karafka::App.quieting? })
+
+        # We need to wait until all the work in the whole consumer group (local to the process)
+        # is done. Otherwise we may end up with locks and `Timed out LeaveGroupRequest in flight`
+        # warning notifications.
+        wait_pinging(wait_until: -> { @consumer_group_coordinator.shutdown? })
+
+        # This extra ping will make sure we've refreshed the rebalance state after other instances
+        # potentially shutdown. This will prevent us from closing with a dangling callback
+        @client.ping
 
         shutdown
 
@@ -157,6 +184,8 @@ module Karafka
         restart
 
         sleep(1) && retry
+      ensure
+        @consumer_group_coordinator.unlock
       end
 
       # Resumes processing of partitions that were paused due to an error.
@@ -256,10 +285,21 @@ module Karafka
       end
 
       # Waits without blocking the polling
-      # This should be used only when we no longer plan to use any incoming data and we can safely
-      # discard it
-      def wait_with_poll
-        @client.batch_poll until @jobs_queue.empty?(@subscription_group.id)
+      #
+      # This should be used only when we no longer plan to use any incoming messages data and we
+      # can safely discard it. We can however use the rebalance information if needed.
+      #
+      # @param wait_until [Proc] until this evaluates to true, we will poll data
+      # @param after_ping [Proc] code that we want to run after each ping (if any)
+      #
+      # @note Performance of this is not relevant (in regards to blocks) because it is used only
+      #   on shutdown and quiet, hence not in the running mode
+      def wait_pinging(wait_until:, after_ping: -> {})
+        until wait_until.call
+          @client.ping
+          after_ping.call
+          sleep(0.2)
+        end
       end
 
       # We can stop client without a problem, as it will reinitialize itself when running the

data/lib/karafka/connection/listeners_batch.rb

@@ -9,8 +9,18 @@ module Karafka
       # @param jobs_queue [JobsQueue]
       # @return [ListenersBatch]
       def initialize(jobs_queue)
-        @batch = App.subscription_groups.map do |subscription_group|
-          Connection::Listener.new(subscription_group, jobs_queue)
+        @batch = App.subscription_groups.flat_map do |_consumer_group, subscription_groups|
+          consumer_group_coordinator = Connection::ConsumerGroupCoordinator.new(
+            subscription_groups.size
+          )
+
+          subscription_groups.map do |subscription_group|
+            Connection::Listener.new(
+              consumer_group_coordinator,
+              subscription_group,
+              jobs_queue
+            )
+          end
         end
       end
 

data/lib/karafka/embedded.rb

@@ -18,6 +18,16 @@ module Karafka
         # Stop needs to be blocking to wait for all the things to finalize
         Karafka::Server.stop
       end
+
+      # Quiets Karafka upon any event
+      #
+      # @note This method is not blocking and will not wait for Karafka to fully quiet.
+      # It will trigger the quiet procedure but won't wait.
+      #
+      # @note Please keep in mind you need to `#stop` to actually stop the server anyhow.
+      def quiet
+        Karafka::Server.quiet
+      end
     end
   end
 end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -18,7 +18,7 @@ module Karafka
 
       # Logs each messages fetching attempt
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_connection_listener_fetch_loop(event)
         listener = event[:caller]
         debug "[#{listener.id}] Polling messages..."
@@ -26,7 +26,7 @@ module Karafka
 
       # Logs about messages that we've received from Kafka
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_connection_listener_fetch_loop_received(event)
         listener = event[:caller]
         time = event[:time]
@@ -42,7 +42,7 @@ module Karafka
 
       # Prints info about the fact that a given job has started
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_worker_process(event)
         job = event[:job]
         job_type = job.class.to_s.split('::').last
@@ -53,7 +53,7 @@ module Karafka
 
       # Prints info about the fact that a given job has finished
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_worker_processed(event)
         job = event[:job]
         time = event[:time]
@@ -66,7 +66,7 @@ module Karafka
       # Logs info about system signals that Karafka received and prints backtrace for threads in
       # case of ttin
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_process_notice_signal(event)
         info "Received #{event[:signal]} system signal"
 
@@ -89,7 +89,7 @@ module Karafka
 
       # Logs info that we're running Karafka app.
       #
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_running(_event)
         info "Running in #{RUBY_DESCRIPTION}"
         info "Running Karafka #{Karafka::VERSION} server"
@@ -99,23 +99,28 @@ module Karafka
         info 'See LICENSE and the LGPL-3.0 for licensing details.'
       end
 
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quieting(_event)
+        info 'Switching to quiet mode. New messages will not be processed.'
+      end
+
       # Logs info that we're going to stop the Karafka server.
       #
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_stopping(_event)
         info 'Stopping Karafka server'
       end
 
       # Logs info that we stopped the Karafka server.
       #
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_stopped(_event)
         info 'Stopped Karafka server'
       end
 
       # Logs info when we have dispatched a message the the DLQ
       #
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_dead_letter_queue_dispatched(event)
         message = event[:message]
         offset = message.offset
@@ -128,7 +133,7 @@ module Karafka
 
       # There are many types of errors that can occur in many places, but we provide a single
       # handler for all of them to simplify error instrumentation.
-      # @param event [Dry::Events::Event] event details including payload
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_error_occurred(event)
         type = event[:type]
         error = event[:error]

data/lib/karafka/instrumentation/notifications.rb

@@ -19,6 +19,7 @@ module Karafka
       EVENTS = %w[
         app.initialized
         app.running
+        app.quieting
         app.stopping
         app.stopped
 

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -42,7 +42,7 @@ module Karafka
 
           # Prints info about the fact that a given job has started
           #
-          # @param event [Dry::Events::Event] event details including payload
+          # @param event [Karafka::Core::Monitoring::Event] event details including payload
           def on_worker_process(event)
             current_span = client.trace('karafka.consumer')
             push_tags
@@ -60,7 +60,7 @@ module Karafka
 
           # Prints info about the fact that a given job has finished
           #
-          # @param event [Dry::Events::Event] event details including payload
+          # @param event [Karafka::Core::Monitoring::Event] event details including payload
           def on_worker_processed(event)
             push_tags
 
@@ -80,7 +80,7 @@ module Karafka
 
           # There are many types of errors that can occur in many places, but we provide a single
           # handler for all of them to simplify error instrumentation.
-          # @param event [Dry::Events::Event] event details including payload
+          # @param event [Karafka::Core::Monitoring::Event] event details including payload
           def on_error_occurred(event)
             push_tags
 

data/lib/karafka/pro/performance_tracker.rb

@@ -25,11 +25,13 @@ module Karafka
 
       # Builds up nested concurrent hash for data tracking
       def initialize
-        @processing_times = Concurrent::Hash.new do |topics_hash, topic|
-          topics_hash[topic] = Concurrent::Hash.new do |partitions_hash, partition|
-            # This array does not have to be concurrent because we always access single partition
-            # data via instrumentation that operates in a single thread via consumer
-            partitions_hash[partition] = []
+        @processing_times = Concurrent::Map.new do |topics_hash, topic|
+          topics_hash.compute_if_absent(topic) do
+            Concurrent::Map.new do |partitions_hash, partition|
+              # This array does not have to be concurrent because we always access single
+              # partition data via instrumentation that operates in a single thread via consumer
+              partitions_hash.compute_if_absent(partition) { [] }
+            end
           end
         end
       end

data/lib/karafka/pro/processing/coordinator.rb

@@ -20,10 +20,8 @@ module Karafka
         # @param args [Object] anything the base coordinator accepts
         def initialize(*args)
           super
-          @on_enqueued_invoked = false
-          @on_started_invoked = false
-          @on_finished_invoked = false
-          @on_revoked_invoked = false
+
+          @executed = []
           @flow_lock = Mutex.new
         end
 
@@ -34,9 +32,7 @@ module Karafka
           super
 
           @mutex.synchronize do
-            @on_enqueued_invoked = false
-            @on_started_invoked = false
-            @on_finished_invoked = false
+            @executed.clear
             @last_message = messages.last
           end
         end
@@ -50,9 +46,7 @@ module Karafka
         # enqueued
         def on_enqueued
           @flow_lock.synchronize do
-            return if @on_enqueued_invoked
-
-            @on_enqueued_invoked = true
+            return unless executable?(:on_enqueued)
 
             yield(@last_message)
           end
@@ -61,9 +55,7 @@ module Karafka
         # Runs given code only once per all the coordinated jobs upon starting first of them
         def on_started
           @flow_lock.synchronize do
-            return if @on_started_invoked
-
-            @on_started_invoked = true
+            return unless executable?(:on_started)
 
             yield(@last_message)
           end
@@ -75,25 +67,36 @@ module Karafka
         def on_finished
           @flow_lock.synchronize do
             return unless finished?
-            return if @on_finished_invoked
-
-            @on_finished_invoked = true
+            return unless executable?(:on_finished)
 
             yield(@last_message)
           end
         end
 
-        # Runs once when a partition is revoked
+        # Runs once after a partition is revoked
         def on_revoked
           @flow_lock.synchronize do
-            return unless finished?
-            return if @on_revoked_invoked
-
-            @on_revoked_invoked = true
+            return unless executable?(:on_revoked)
 
             yield(@last_message)
           end
         end
+
+        private
+
+        # Checks if given action is executable once. If it is and true is returned, this method
+        # will return false next time it is used.
+        #
+        # @param action [Symbol] what action we want to perform
+        # @return [Boolean] true if we can
+        # @note This method needs to run behind a mutex.
+        def executable?(action)
+          return false if @executed.include?(action)
+
+          @executed << action
+
+          true
+        end
       end
     end
   end

data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb

@@ -21,7 +21,7 @@ module Karafka
         #
         # AJ has manual offset management on by default and the offset management is delegated to
         # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
-        # consumed when we skip given job upon errors. In all the other scenarions marking as
+        # consumed when we skip given job upon errors. In all the other scenarios marking as
         # consumed needs to happen in the AJ consumer on a per job basis.
         module AjDlqMom
           include DlqMom
@@ -46,7 +46,7 @@ module Karafka
               else
                 coordinator.pause_tracker.reset
                 skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message)
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                 # We can commit the offset here because we know that we skip it "forever" and
                 # since AJ consumer commits the offset after each job, we also know that the
                 # previous job was successful

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -42,7 +42,7 @@ module Karafka
                 # We reset the pause to indicate we will now consider it as "ok".
                 coordinator.pause_tracker.reset
                 skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message)
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                 mark_as_consumed(skippable_message)
                 pause(coordinator.seek_offset)
               end
@@ -59,7 +59,6 @@ module Karafka
 
           # Moves the broken message into a separate queue defined via the settings
           #
-          # @private
           # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
           #   dispatch to DLQ
           def dispatch_to_dlq(skippable_message)
@@ -81,6 +80,13 @@ module Karafka
               message: skippable_message
             )
           end
+
+          # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch topic
+          #   is set to false, we will skip the dispatch, effectively ignoring the broken message
+          #   without taking any action.
+          def dispatch_to_dlq?
+            topic.dead_letter_queue.topic
+          end
         end
       end
     end

data/lib/karafka/pro/processing/strategies/dlq_lrj.rb

@@ -43,10 +43,9 @@ module Karafka
               else
                 coordinator.pause_tracker.reset
 
-                skippable_message = find_skippable_message
-
                 unless revoked?
-                  dispatch_to_dlq(skippable_message)
+                  skippable_message = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                   mark_as_consumed(skippable_message)
                 end
 

data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb

@@ -42,10 +42,12 @@ module Karafka
               else
                 coordinator.pause_tracker.reset
 
-                skippable_message = find_skippable_message
-
                 unless revoked?
-                  dispatch_to_dlq(skippable_message)
+                  if dispatch_to_dlq?
+                    skippable_message = find_skippable_message
+                    dispatch_to_dlq(skippable_message)
+                  end
+
                   seek(coordinator.seek_offset)
                 end
 

data/lib/karafka/pro/processing/strategies/dlq_mom.rb

@@ -45,8 +45,12 @@ module Karafka
               else
                 # We reset the pause to indicate we will now consider it as "ok".
                 coordinator.pause_tracker.reset
-                skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message)
+
+                if dispatch_to_dlq?
+                  skippable_message = find_skippable_message
+                  dispatch_to_dlq(skippable_message)
+                end
+
                 pause(coordinator.seek_offset)
               end
             end

data/lib/karafka/process.rb

@@ -10,6 +10,7 @@ module Karafka
       SIGQUIT
       SIGTERM
       SIGTTIN
+      SIGTSTP
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|
@@ -48,21 +49,23 @@ module Karafka
 
     # Traps a single signal and performs callbacks (if any) or just ignores this signal
     # @param [Symbol] signal type that we want to catch
+    # @note Since we do a lot of threading and queuing, we don't want to handle signals from the
+    # trap context s some things may not work there as expected, that is why we spawn a separate
+    # thread to handle the signals process
     def trap_signal(signal)
       trap(signal) do
-        notice_signal(signal)
-        (@callbacks[signal] || []).each(&:call)
+        Thread.new do
+          notice_signal(signal)
+
+          (@callbacks[signal] || []).each(&:call)
+        end
       end
     end
 
     # Informs monitoring about trapped signal
     # @param [Symbol] signal type that we received
-    # @note We cannot perform logging from trap context, that's why
-    #   we have to spin up a new thread to do this
     def notice_signal(signal)
-      Thread.new do
-        Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
-      end
+      Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
     end
   end
 end

data/lib/karafka/processing/jobs_queue.rb

@@ -20,8 +20,12 @@ module Karafka
         # scheduled by Ruby hundreds of thousands of times per group.
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
-        @semaphores = Hash.new { |h, k| h[k] = Queue.new }
+        @semaphores = Concurrent::Map.new do |h, k|
+          h.compute_if_absent(k) { Queue.new }
+        end
+
         @in_processing = Hash.new { |h, k| h[k] = [] }
+
         @mutex = Mutex.new
       end
 
@@ -47,9 +51,9 @@ module Karafka
           raise(Errors::JobsQueueSynchronizationError, job.group_id) if group.include?(job)
 
           group << job
-        end
 
-        @queue << job
+          @queue << job
+        end
       end
 
       # @return [Jobs::Base, nil] waits for a job from the main queue and returns it once available
@@ -105,7 +109,9 @@ module Karafka
       # @return [Boolean] tell us if we have anything in the processing (or for processing) from
       # a given group.
       def empty?(group_id)
-        @in_processing[group_id].empty?
+        @mutex.synchronize do
+          @in_processing[group_id].empty?
+        end
       end
 
       # Blocks when there are things in the queue in a given group and waits until all the blocking

data/lib/karafka/railtie.rb

@@ -5,7 +5,8 @@
 rails = false
 
 begin
-  require 'rails'
+  # Do not load Rails again if already loaded
+  Object.const_defined?('Rails::Railtie') || require('rails')
 
   rails = true
 rescue LoadError
@@ -17,9 +18,6 @@ rescue LoadError
 end
 
 if rails
-  # Load Karafka
-  require 'karafka'
-
   # Load ActiveJob adapter
   require 'active_job/karafka'
 

data/lib/karafka/routing/features/dead_letter_queue/contract.rb

@@ -29,6 +29,8 @@ module Karafka
 
             topic = dead_letter_queue[:topic]
 
+            # When topic is set to false, it means we just want to skip dispatch on DLQ
+            next if topic == false
             next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
 
             [[%i[dead_letter_queue topic], :format]]

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -12,7 +12,8 @@ module Karafka
           private_constant :DEFAULT_MAX_RETRIES
 
           # @param max_retries [Integer] after how many retries should we move data to dlq
-          # @param topic [String] where the messages should be moved if failing
+          # @param topic [String, false] where the messages should be moved if failing or false
+          #   if we do not want to move it anywhere and just skip
           # @return [Config] defined config
           def dead_letter_queue(max_retries: DEFAULT_MAX_RETRIES, topic: nil)
             @dead_letter_queue ||= Config.new(

data/lib/karafka/server.rb

@@ -25,12 +25,10 @@ module Karafka
 
       # Method which runs app
       def run
-        # Since we do a lot of threading and queuing, we don't want to stop from the trap context
-        # as some things may not work there as expected, that is why we spawn a separate thread to
-        # handle the stopping process
-        process.on_sigint { Thread.new { stop } }
-        process.on_sigquit { Thread.new { stop } }
-        process.on_sigterm { Thread.new { stop } }
+        process.on_sigint { stop }
+        process.on_sigquit { stop }
+        process.on_sigterm { stop }
+        process.on_sigtstp { quiet }
         process.supervise
 
         # Start is blocking until stop is called and when we stop, it will wait until
@@ -74,7 +72,8 @@ module Karafka
       #   please start a separate thread to do so.
       def stop
         # Initialize the stopping process only if Karafka was running
-        return if Karafka::App.stopping? || Karafka::App.stopped?
+        return if Karafka::App.stopping?
+        return if Karafka::App.stopped?
 
         Karafka::App.stop!
 
@@ -125,6 +124,18 @@ module Karafka
         Karafka::App.stopped! if timeout
       end
 
+      # Quiets the Karafka server.
+      # Karafka will stop processing but won't quiet to consumer group, so no rebalance will be
+      # triggered until final shutdown.
+      def quiet
+        # If we are already quieting or in the stop procedures, we should not do it again.
+        return if Karafka::App.quieting?
+        return if Karafka::App.stopping?
+        return if Karafka::App.stopped?
+
+        Karafka::App.quiet!
+      end
+
       private
 
       # @return [Karafka::Process] process wrapper instance used to catch system signal calls

data/lib/karafka/setup/attributes_map.rb

@@ -84,6 +84,7 @@ module Karafka
         reconnect.backoff.jitter.ms
         reconnect.backoff.max.ms
         reconnect.backoff.ms
+        resolve_cb
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd
         sasl.kerberos.min.time.before.relogin
@@ -215,6 +216,7 @@ module Karafka
         reconnect.backoff.ms
         request.required.acks
         request.timeout.ms
+        resolve_cb
         retries
         retry.backoff.ms
         sasl.kerberos.keytab

data/lib/karafka/status.rb

@@ -8,6 +8,7 @@ module Karafka
       initializing: :initialize!,
       initialized: :initialized!,
       running: :run!,
+      quieting: :quiet!,
       stopping: :stop!,
       stopped: :stopped!
     }.freeze

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.17'
+  VERSION = '2.0.19'
 end

data/lib/karafka.rb

@@ -86,6 +86,9 @@ end
 loader = Zeitwerk::Loader.for_gem
 # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
 loader.ignore(Karafka.gem_root.join('lib/active_job'))
+# Do not load Railtie. It will load if after everything is ready, so we don't have to load any
+# Karafka components when we require this railtie. Railtie needs to be loaded last.
+loader.ignore(Karafka.gem_root.join('lib/karafka/railtie'))
 # Do not load pro components as they will be loaded if needed and allowed
 loader.ignore(Karafka.core_root.join('pro/'))
 # Do not load vendors instrumentation components. Those need to be required manually if needed
@@ -96,3 +99,6 @@ loader.eager_load
 # This will load features but since Pro are not loaded automatically, they will not be visible
 # nor included here
 ::Karafka::Routing::Features::Base.load_all
+
+# Load railtie after everything else is ready so we know we can rely on it.
+require 'karafka/railtie'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.17
+  version: 2.0.19
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-11-10 00:00:00.000000000 Z
+date: 2022-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -176,6 +176,7 @@ files:
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
 - lib/karafka/connection/client.rb
+- lib/karafka/connection/consumer_group_coordinator.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/connection/listeners_batch.rb
 - lib/karafka/connection/messages_buffer.rb
@@ -326,7 +327,12 @@ licenses:
 - LGPL-3.0
 - Commercial
 metadata:
+  funding_uri: https://karafka.io/#become-pro
+  homepage_uri: https://karafka.io
+  changelog_uri: https://github.com/karafka/karafka/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/karafka/karafka/issues
   source_code_uri: https://github.com/karafka/karafka
+  documentation_uri: https://karafka.io/docs
   rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []