karafka 2.0.18 → 2.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17800cf4d60529269b31340d76245ed2035b463f03daceeb482241528eb5d681
-  data.tar.gz: 9caed11b4f7221d9c5e9c33607ba965a0fd9c75c7ce037d40e38df119a8255ea
+  metadata.gz: be91c3848b757c6af4c25f905df2b081629532bd29dbcea23ed2ef0af2e4e4a2
+  data.tar.gz: 6823d4335e4b395546642101d6754b97958c86810cbcd12819559acff74bd90d
 SHA512:
-  metadata.gz: 5de87acf0d5dbef037c2e980d1354c398fe47bf9a14312f772b346f59bd6f638aa0d4444b9d32984f494cd66dc3cb752f843dc72534ad9ec7df367ecde100dfc
-  data.tar.gz: 8577247eeb4eae1d5abc9e74de9c2c95787535662ed8a2a664824871fe1907abb7aa6e14619926cde218cfb06db5496ec0c38dd6fc3fc3ad0efe0b5702fe067e
+  metadata.gz: fce0259ee987e37c01ea037f81ea91b4eb770ea8eabcb9f93c66aa1a1960c903030648b5441945ef28f43a88660d18240e6db61f6885a169d70eb46174543616
+  data.tar.gz: f93985c98daba5965f8f0597da4744d1dae0603f24a6a44f4b337462f66c8b0f08d0c22dd734205a0aebe7c3dbdb73237ff568f7f1ff842b38c05a7f4b5ce463

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Karafka framework changelog
 
+## 2.0.19 (2022-11-20)
+- **[Feature]** Provide ability to skip failing messages without dispatching them to an alternative topic (DLQ).
+- [Improvement] Improve the integration with Ruby on Rails by preventing double-require of components.
+- [Improvement] Improve stability of the shutdown process upon critical errors.
+- [Improvement] Improve stability of the integrations spec suite.
+- [Fix] Fix an issue where upon fast startup of multiple subscription groups from the same consumer group, a ghost queue would be created due to problems in `Concurrent::Hash`.
+
 ## 2.0.18 (2022-11-18)
 - **[Feature]** Support quiet mode via `TSTP` signal. When used, Karafka will finish processing current messages, run `shutdown` jobs, and switch to a quiet mode where no new work is being accepted. At the same time, it will keep the consumer group quiet, and thus no rebalance will be triggered. This can be particularly useful during deployments.
 - [Improvement] Trigger `#revoked` for jobs in case revocation would happen during shutdown when jobs are still running. This should ensure, we get a notion of revocation for Pro LRJ jobs even when revocation happening upon shutdown (#1150).

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.18)
+    karafka (2.0.19)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/karafka.gemspec

@@ -34,7 +34,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w[lib]
 
   spec.metadata = {
+    'funding_uri' => 'https://karafka.io/#become-pro',
+    'homepage_uri' => 'https://karafka.io',
+    'changelog_uri' => 'https://github.com/karafka/karafka/blob/master/CHANGELOG.md',
+    'bug_tracker_uri' => 'https://github.com/karafka/karafka/issues',
     'source_code_uri' => 'https://github.com/karafka/karafka',
+    'documentation_uri' => 'https://karafka.io/docs',
     'rubygems_mfa_required' => 'true'
   }
 end

data/lib/active_job/karafka.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 begin
-  require 'active_job'
+  # Do not load active job if already loaded
+  require 'active_job' unless Object.const_defined?('ActiveJob')
+
   require_relative 'queue_adapters/karafka_adapter'
 
   module ActiveJob

data/lib/karafka/connection/client.rb

@@ -241,6 +241,17 @@ module Karafka
         end
       end
 
+      # Runs a single poll ignoring all the potential errors
+      # This is used as a keep-alive in the shutdown stage and any errors that happen here are
+      # irrelevant from the shutdown process perspective
+      #
+      # This is used only to trigger rebalance callbacks
+      def ping
+        poll(100)
+      rescue Rdkafka::RdkafkaError
+        nil
+      end
+
       private
 
       # When we cannot store an offset, it means we no longer own the partition
@@ -302,8 +313,8 @@ module Karafka
 
             @kafka.close
             @buffer.clear
-            # @note We do not clear rebalance manager here as we may still have revocation info here
-            # that we want to consider valid prior to running another reconnection
+            # @note We do not clear rebalance manager here as we may still have revocation info
+            # here that we want to consider valid prior to running another reconnection
           end
         end
       end

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # This object represents a collective status of execution of group of listeners running inside
+    # of one consumer group but in separate subscription groups.
+    #
+    # There are cases when we do not want to close a given client when others from the same
+    # consumer group are running because it can cause instabilities due to early shutdown of some
+    # of the clients out of same consumer group.
+    #
+    # We also want to make sure, we close one consumer at a time while others can continue polling.
+    #
+    # This prevents a scenario, where a rebalance is not acknowledged and we loose assignment
+    # without having a chance to commit changes.
+    class ConsumerGroupCoordinator
+      # @param group_size [Integer] number of separate subscription groups in a consumer group
+      def initialize(group_size)
+        # We need two locks here:
+        # - first one is to decrement the number of listeners doing work
+        # - second to ensure only one client is being closed the same time and that others can
+        #   wait actively (not locked)
+        @work_mutex = Mutex.new
+        @shutdown_lock = Mutex.new
+        @group_size = group_size
+        @finished = Set.new
+      end
+
+      # @return [Boolean] can we start shutdown on a given listener
+      # @note If true, will also obtain a lock so no-one else will be closing the same time we do
+      def shutdown?
+        @finished.size == @group_size && @shutdown_lock.try_lock
+      end
+
+      # Unlocks the shutdown lock
+      def unlock
+        @shutdown_lock.unlock if @shutdown_lock.owned?
+      end
+
+      # Marks given listener as finished
+      # @param listener_id [String]
+      def finish_work(listener_id)
+        @finished << listener_id
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -14,20 +14,15 @@ module Karafka
       # @return [String] id of this listener
       attr_reader :id
 
-      # Mutex for things we do not want to do in parallel even in different consumer groups
-      MUTEX = Mutex.new
-
-      private_constant :MUTEX
-
-      # @param consumer_group_status [Karafka::Connection::ConsumerGroupStatus]
+      # @param consumer_group_coordinator [Karafka::Connection::ConsumerGroupCoordinator]
       # @param subscription_group [Karafka::Routing::SubscriptionGroup]
       # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
-      def initialize(consumer_group_status, subscription_group, jobs_queue)
+      def initialize(consumer_group_coordinator, subscription_group, jobs_queue)
         proc_config = ::Karafka::App.config.internal.processing
 
         @id = SecureRandom.uuid
-        @consumer_group_status = consumer_group_status
+        @consumer_group_coordinator = consumer_group_coordinator
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
         @coordinators = Processing::CoordinatorsBuffer.new
@@ -68,17 +63,13 @@ module Karafka
       #
       # @note We wrap it with a mutex exactly because of the above case of forceful shutdown
       def shutdown
-        # We want to make sure that we never close two librdkafka clients at the same time. I'm not
-        # particularly fond of it's shutdown API being fully thread-safe
-        MUTEX.synchronize do
-          return if @stopped
-
-          @mutex.synchronize do
-            @stopped = true
-            @executors.clear
-            @coordinators.reset
-            @client.stop
-          end
+        return if @stopped
+
+        @mutex.synchronize do
+          @stopped = true
+          @executors.clear
+          @coordinators.reset
+          @client.stop
         end
       end
 
@@ -147,9 +138,9 @@ module Karafka
         # What we do care however is the ability to still run revocation jobs in case anything
         # would change in the cluster. We still want to notify the long-running jobs about changes
         # that occurred in the cluster.
-        wait_polling(
+        wait_pinging(
           wait_until: -> { @jobs_queue.empty?(@subscription_group.id) },
-          after_poll: -> { build_and_schedule_revoke_lost_partitions_jobs }
+          after_ping: -> { build_and_schedule_revoke_lost_partitions_jobs }
         )
 
         # We do not want to schedule the shutdown jobs prior to finishing all the jobs
@@ -159,19 +150,23 @@ module Karafka
         build_and_schedule_shutdown_jobs
 
         # Wait until all the shutdown jobs are done
-        wait_polling(wait_until: -> { @jobs_queue.empty?(@subscription_group.id) })
+        wait_pinging(wait_until: -> { @jobs_queue.empty?(@subscription_group.id) })
 
         # Once all the work is done, we need to decrement counter of active subscription groups
         # within this consumer group
-        @consumer_group_status.finish
+        @consumer_group_coordinator.finish_work(id)
 
         # Wait if we're in the quiet mode
-        wait_polling(wait_until: -> { !Karafka::App.quieting? })
+        wait_pinging(wait_until: -> { !Karafka::App.quieting? })
 
         # We need to wait until all the work in the whole consumer group (local to the process)
         # is done. Otherwise we may end up with locks and `Timed out LeaveGroupRequest in flight`
         # warning notifications.
-        wait_polling(wait_until: -> { !@consumer_group_status.working? })
+        wait_pinging(wait_until: -> { @consumer_group_coordinator.shutdown? })
+
+        # This extra ping will make sure we've refreshed the rebalance state after other instances
+        # potentially shutdown. This will prevent us from closing with a dangling callback
+        @client.ping
 
         shutdown
 
@@ -189,6 +184,8 @@ module Karafka
         restart
 
         sleep(1) && retry
+      ensure
+        @consumer_group_coordinator.unlock
       end
 
       # Resumes processing of partitions that were paused due to an error.
@@ -293,15 +290,15 @@ module Karafka
       # can safely discard it. We can however use the rebalance information if needed.
       #
       # @param wait_until [Proc] until this evaluates to true, we will poll data
-      # @param after_poll [Proc] code that we want to run after each batch poll (if any)
+      # @param after_ping [Proc] code that we want to run after each ping (if any)
       #
       # @note Performance of this is not relevant (in regards to blocks) because it is used only
       #   on shutdown and quiet, hence not in the running mode
-      def wait_polling(wait_until:, after_poll: -> {})
+      def wait_pinging(wait_until:, after_ping: -> {})
         until wait_until.call
-          @client.batch_poll
-
-          after_poll.call
+          @client.ping
+          after_ping.call
+          sleep(0.2)
         end
       end
 

data/lib/karafka/connection/listeners_batch.rb

@@ -10,13 +10,13 @@ module Karafka
       # @return [ListenersBatch]
       def initialize(jobs_queue)
         @batch = App.subscription_groups.flat_map do |_consumer_group, subscription_groups|
-          consumer_group_status = Connection::ConsumerGroupStatus.new(
+          consumer_group_coordinator = Connection::ConsumerGroupCoordinator.new(
             subscription_groups.size
           )
 
           subscription_groups.map do |subscription_group|
             Connection::Listener.new(
-              consumer_group_status,
+              consumer_group_coordinator,
               subscription_group,
               jobs_queue
             )

data/lib/karafka/errors.rb

@@ -44,9 +44,6 @@ module Karafka
     # This should never happen. Please open an issue if it does.
     InvalidCoordinatorStateError = Class.new(BaseError)
 
-    # This should never happen. Please open an issue if it does.
-    InvalidConsumerGroupStatusError = Class.new(BaseError)
-
     # This should never happen. Please open an issue if it does.
     StrategyNotFoundError = Class.new(BaseError)
 

data/lib/karafka/pro/performance_tracker.rb

@@ -25,11 +25,13 @@ module Karafka
 
       # Builds up nested concurrent hash for data tracking
       def initialize
-        @processing_times = Concurrent::Hash.new do |topics_hash, topic|
-          topics_hash[topic] = Concurrent::Hash.new do |partitions_hash, partition|
-            # This array does not have to be concurrent because we always access single partition
-            # data via instrumentation that operates in a single thread via consumer
-            partitions_hash[partition] = []
+        @processing_times = Concurrent::Map.new do |topics_hash, topic|
+          topics_hash.compute_if_absent(topic) do
+            Concurrent::Map.new do |partitions_hash, partition|
+              # This array does not have to be concurrent because we always access single
+              # partition data via instrumentation that operates in a single thread via consumer
+              partitions_hash.compute_if_absent(partition) { [] }
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb

@@ -21,7 +21,7 @@ module Karafka
         #
         # AJ has manual offset management on by default and the offset management is delegated to
         # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
-        # consumed when we skip given job upon errors. In all the other scenarions marking as
+        # consumed when we skip given job upon errors. In all the other scenarios marking as
         # consumed needs to happen in the AJ consumer on a per job basis.
         module AjDlqMom
           include DlqMom
@@ -46,7 +46,7 @@ module Karafka
               else
                 coordinator.pause_tracker.reset
                 skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message)
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                 # We can commit the offset here because we know that we skip it "forever" and
                 # since AJ consumer commits the offset after each job, we also know that the
                 # previous job was successful

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -42,7 +42,7 @@ module Karafka
                 # We reset the pause to indicate we will now consider it as "ok".
                 coordinator.pause_tracker.reset
                 skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message)
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                 mark_as_consumed(skippable_message)
                 pause(coordinator.seek_offset)
               end
@@ -59,7 +59,6 @@ module Karafka
 
           # Moves the broken message into a separate queue defined via the settings
           #
-          # @private
           # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
           #   dispatch to DLQ
           def dispatch_to_dlq(skippable_message)
@@ -81,6 +80,13 @@ module Karafka
               message: skippable_message
             )
           end
+
+          # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch topic
+          #   is set to false, we will skip the dispatch, effectively ignoring the broken message
+          #   without taking any action.
+          def dispatch_to_dlq?
+            topic.dead_letter_queue.topic
+          end
         end
       end
     end

data/lib/karafka/pro/processing/strategies/dlq_lrj.rb

@@ -43,10 +43,9 @@ module Karafka
               else
                 coordinator.pause_tracker.reset
 
-                skippable_message = find_skippable_message
-
                 unless revoked?
-                  dispatch_to_dlq(skippable_message)
+                  skippable_message = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                   mark_as_consumed(skippable_message)
                 end
 

data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb

@@ -42,10 +42,12 @@ module Karafka
               else
                 coordinator.pause_tracker.reset
 
-                skippable_message = find_skippable_message
-
                 unless revoked?
-                  dispatch_to_dlq(skippable_message)
+                  if dispatch_to_dlq?
+                    skippable_message = find_skippable_message
+                    dispatch_to_dlq(skippable_message)
+                  end
+
                   seek(coordinator.seek_offset)
                 end
 

data/lib/karafka/pro/processing/strategies/dlq_mom.rb

@@ -45,8 +45,12 @@ module Karafka
               else
                 # We reset the pause to indicate we will now consider it as "ok".
                 coordinator.pause_tracker.reset
-                skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message)
+
+                if dispatch_to_dlq?
+                  skippable_message = find_skippable_message
+                  dispatch_to_dlq(skippable_message)
+                end
+
                 pause(coordinator.seek_offset)
               end
             end

data/lib/karafka/processing/jobs_queue.rb

@@ -20,8 +20,12 @@ module Karafka
         # scheduled by Ruby hundreds of thousands of times per group.
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
-        @semaphores = Concurrent::Map.new { |h, k| h[k] = Queue.new }
+        @semaphores = Concurrent::Map.new do |h, k|
+          h.compute_if_absent(k) { Queue.new }
+        end
+
         @in_processing = Hash.new { |h, k| h[k] = [] }
+
         @mutex = Mutex.new
       end
 

data/lib/karafka/railtie.rb

@@ -5,7 +5,8 @@
 rails = false
 
 begin
-  require 'rails'
+  # Do not load Rails again if already loaded
+  Object.const_defined?('Rails::Railtie') || require('rails')
 
   rails = true
 rescue LoadError

data/lib/karafka/routing/features/dead_letter_queue/contract.rb

@@ -29,6 +29,8 @@ module Karafka
 
             topic = dead_letter_queue[:topic]
 
+            # When topic is set to false, it means we just want to skip dispatch on DLQ
+            next if topic == false
             next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
 
             [[%i[dead_letter_queue topic], :format]]

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -12,7 +12,8 @@ module Karafka
           private_constant :DEFAULT_MAX_RETRIES
 
           # @param max_retries [Integer] after how many retries should we move data to dlq
-          # @param topic [String] where the messages should be moved if failing
+          # @param topic [String, false] where the messages should be moved if failing or false
+          #   if we do not want to move it anywhere and just skip
           # @return [Config] defined config
           def dead_letter_queue(max_retries: DEFAULT_MAX_RETRIES, topic: nil)
             @dead_letter_queue ||= Config.new(

data/lib/karafka/setup/attributes_map.rb

@@ -84,6 +84,7 @@ module Karafka
         reconnect.backoff.jitter.ms
         reconnect.backoff.max.ms
         reconnect.backoff.ms
+        resolve_cb
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd
         sasl.kerberos.min.time.before.relogin
@@ -215,6 +216,7 @@ module Karafka
         reconnect.backoff.ms
         request.required.acks
         request.timeout.ms
+        resolve_cb
         retries
         retry.backoff.ms
         sasl.kerberos.keytab

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.18'
+  VERSION = '2.0.19'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.18
+  version: 2.0.19
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-11-18 00:00:00.000000000 Z
+date: 2022-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -176,7 +176,7 @@ files:
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
 - lib/karafka/connection/client.rb
-- lib/karafka/connection/consumer_group_status.rb
+- lib/karafka/connection/consumer_group_coordinator.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/connection/listeners_batch.rb
 - lib/karafka/connection/messages_buffer.rb
@@ -327,7 +327,12 @@ licenses:
 - LGPL-3.0
 - Commercial
 metadata:
+  funding_uri: https://karafka.io/#become-pro
+  homepage_uri: https://karafka.io
+  changelog_uri: https://github.com/karafka/karafka/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/karafka/karafka/issues
   source_code_uri: https://github.com/karafka/karafka
+  documentation_uri: https://karafka.io/docs
   rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []

metadata.gz.sig

@@ -1,2 +1,2 @@
-��7j�]��Q�"�:|fO�l�qpA�lhe�+��ᜄfK�X񳒗�
-�]#E�_?�� �k��;qeN猅�9���ح�f���BɣY��8՟�x̗-֞�ݢ�fگ݆#�i����ÔFw@�u���/�U���BC�-��w���j_�{��������/�^r�h|��
��x�
+&ޤ�|.݇ځ�a=�8�aۍ_o2Y�W��f1x|ľn�=�M�q���'k���q��6�� ��w�KK�v
WM�V�])�JƜA�;�����yԟ��=��IB+,;���
+b�7��<

data/lib/karafka/connection/consumer_group_status.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # This object represents a collective status of execution of group of listeners running inside
-    # of one consumer group but potentially in separate subscription groups.
-    #
-    # There are cases when we do not want to close a given client when others from the same
-    # consumer group are running because it can cause instabilities due to early shutdown of some
-    # of the clients out of same consumer group.
-    #
-    # Here we can track it and only shutdown listeners when all work in a group is done.
-    class ConsumerGroupStatus
-      # @param group_size [Integer] number of separate subscription groups in a consumer group
-      def initialize(group_size)
-        @mutex = Mutex.new
-        @active_size = group_size
-      end
-
-      # @return [Boolean] Are there any listeners that are still doing any type of work. If not,
-      #   it means a consumer group is safe to be shutdown fully.
-      def working?
-        @active_size.positive?
-      end
-
-      # Decrements number of working listeners in the group by one until there's none
-      def finish
-        @mutex.synchronize do
-          @active_size -= 1
-
-          return if @active_size >= 0
-
-          raise Errors::InvalidConsumerGroupStatusError, @active_size
-        end
-      end
-    end
-  end
-end