karafka 2.0.27 → 2.0.29

data/Gemfile.lock

@@ -1,31 +1,31 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.27)
+    karafka (2.0.29)
       karafka-core (>= 2.0.9, < 3.0.0)
       thor (>= 0.20)
-      waterdrop (>= 2.4.9, < 3.0.0)
+      waterdrop (>= 2.4.10, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.4)
-      activesupport (= 7.0.4)
+    activejob (7.0.4.2)
+      activesupport (= 7.0.4.2)
       globalid (>= 0.3.6)
-    activesupport (7.0.4)
+    activesupport (7.0.4.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.0)
     diff-lcs (1.5.0)
     docile (1.4.0)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
-    globalid (1.0.0)
+    globalid (1.0.1)
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
@@ -48,7 +48,7 @@ GEM
     rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.2)
+    rspec-mocks (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
@@ -61,12 +61,13 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.9)
+    waterdrop (2.4.10)
       karafka-core (>= 2.0.9, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.6)
 
 PLATFORMS
+  arm64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -78,4 +79,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.4.2
+   2.4.5

data/README.md

@@ -58,7 +58,8 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
 1. Add and install Karafka:
 
 ```bash
-bundle add karafka
+# Make sure to install Karafka 2.0
+bundle add karafka --version ">= 2.0.28"
 
 bundle exec karafka install
 ```
@@ -86,7 +87,7 @@ I also sell Karafka Pro subscriptions. It includes a commercial-friendly license
 
 **20%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
 
-Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io) for more details.
+Help me provide high-quality open-source software. Please see the Karafka [homepage](https://karafka.io/#become-pro) for more details.
 
 ## Support
 

data/bin/karafka

@@ -6,24 +6,6 @@ require 'karafka'
 # our bin/karafka cli
 ENV['KARAFKA_CLI'] = 'true'
 
-# If there is a boot file, we need to require it as we expect it to contain
-# Karafka app setup, routes, etc
-if File.exist?(Karafka.boot_file)
-  rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
-
-  # Load Rails environment file that starts Rails, so we can reference consumers and other things
-  # from `karafka.rb` file. This will work only for Rails, for non-rails a manual setup is needed
-  require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
-
-  require Karafka.boot_file.to_s
-else
-  # However when it is unavailable, we still want to be able to run help command
-  # and install command as they don't require configured app itself to run
-  raise(
-    Karafka::Errors::MissingBootFileError,
-    Karafka.boot_file
-  ) unless %w[-h install].include?(ARGV[0])
-end
-
+Karafka::Cli::Base.load
 Karafka::Cli.prepare
 Karafka::Cli.start

data/bin/verify_license_integrity

@@ -11,6 +11,8 @@ set -e
 if [ "$MODE" != "after" ]; then
   # Check the remote license prior to bundle installing
   curl \
+    --retry 5 \
+    --retry-delay 1 \
     --fail \
     -u $KARAFKA_PRO_USERNAME:$KARAFKA_PRO_PASSWORD \
     https://gems.karafka.io/gems/karafka-license-$KARAFKA_PRO_VERSION.gem \

data/config/locales/pro_errors.yml

@@ -5,7 +5,7 @@ en:
       virtual_partitions.max_partitions_format: needs to be equal or more than 1
       manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
       long_running_job.active_format: needs to be either true or false
-      dead_letter_queue_not_with_virtual_partitions: cannot be used together with Virtual Partitions
+      dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.
 
     config:
       encryption.active_format: 'needs to be either true or false'

data/karafka.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'karafka-core', '>= 2.0.9', '< 3.0.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.4.9', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.4.10', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/cli/base.rb

@@ -34,6 +34,26 @@ module Karafka
       end
 
       class << self
+        # Loads proper environment with what is needed to run the CLI
+        def load
+          # If there is a boot file, we need to require it as we expect it to contain
+          # Karafka app setup, routes, etc
+          if File.exist?(::Karafka.boot_file)
+            rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
+
+            # Load Rails environment file that starts Rails, so we can reference consumers and
+            # other things from `karafka.rb` file. This will work only for Rails, for non-rails
+            # a manual setup is needed
+            require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
+
+            require Karafka.boot_file.to_s
+          # However when it is unavailable, we still want to be able to run help command
+          # and install command as they don't require configured app itself to run
+          elsif %w[-h install].none? { |cmd| cmd == ARGV[0] }
+            raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
+          end
+        end
+
         # Allows to set options for Thor cli
         # @see https://github.com/erikhuda/thor
         # @param option Single option details

data/lib/karafka/cli/console.rb

@@ -9,22 +9,27 @@ module Karafka
       option aliases: 'c'
 
       class << self
-        # @return [String] Console executing command
+        # @return [String] Console executing command for non-Rails setup
         # @example
         #   Karafka::Cli::Console.command #=> 'KARAFKA_CONSOLE=true bundle exec irb...'
-        def command
-          envs = [
-            "IRBRC='#{Karafka.gem_root}/.console_irbrc'",
-            'KARAFKA_CONSOLE=true'
-          ]
-          "#{envs.join(' ')} bundle exec irb -r #{Karafka.boot_file}"
+        def console
+          "IRBRC='#{Karafka.gem_root}/.console_irbrc' bundle exec irb -r #{Karafka.boot_file}"
+        end
+
+        # @return [String] Console executing command for Rails setup
+        # @note In case of Rails, it has its own console, hence we can just defer to it
+        def rails_console
+          'bundle exec rails console'
         end
       end
 
       # Start the Karafka console
       def call
         cli.info
-        exec self.class.command
+
+        command = ::Karafka.rails? ? self.class.rails_console : self.class.console
+
+        exec "KARAFKA_CONSOLE=true #{command}"
       end
     end
   end

data/lib/karafka/connection/listener.rb

@@ -267,7 +267,7 @@ module Karafka
           # Start work coordination for this topic partition
           coordinator.start(messages)
 
-          @partitioner.call(topic, messages) do |group_id, partition_messages|
+          @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
             # Count the job we're going to create here
             coordinator.increment
             executor = @executors.find_or_create(topic, partition, group_id)

data/lib/karafka/instrumentation/callbacks/error.rb

@@ -28,6 +28,7 @@ module Karafka
 
           @monitor.instrument(
             'error.occurred',
+            caller: self,
             subscription_group_id: @subscription_group_id,
             consumer_group_id: @consumer_group_id,
             type: 'librdkafka.error',

data/lib/karafka/pro/active_job/consumer.rb

@@ -36,8 +36,9 @@ module Karafka
             )
 
             # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
-            # this could create random markings
-            next if topic.virtual_partitions?
+            # this could create random markings.
+            # The exception here is the collapsed state where we can move one after another
+            next if topic.virtual_partitions? && !collapsed?
 
             mark_as_consumed(message)
           end

data/lib/karafka/pro/processing/collapser.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Manages the collapse of virtual partitions
+      # Since any non-virtual partition is actually a virtual partition of size one, we can use
+      # it in a generic manner without having to distinguish between those cases.
+      #
+      # We need to have notion of the offset until we want to collapse because upon pause and retry
+      # rdkafka may purge the buffer. This means, that we may end up with smaller or bigger
+      # (different) dataset and without tracking the end of collapse, there would be a chance for
+      # things to flicker. Tracking allows us to ensure, that collapse is happening until all the
+      # messages from the corrupted batch are processed.
+      class Collapser
+        # When initialized, nothing is collapsed
+        def initialize
+          @collapsed = false
+          @until_offset = -1
+          @mutex = Mutex.new
+        end
+
+        # @return [Boolean] Should we collapse into a single consumer
+        def collapsed?
+          @collapsed
+        end
+
+        # Collapse until given offset. Until given offset is encountered or offset bigger than that
+        # we keep collapsing.
+        # @param offset [Integer] offset until which we keep the collapse
+        def collapse_until!(offset)
+          @mutex.synchronize do
+            # We check it here in case after a pause and re-fetch we would get less messages and
+            # one of them would cause an error. We do not want to overwrite the offset here unless
+            # it is bigger.
+            @until_offset = offset if offset > @until_offset
+          end
+        end
+
+        # Sets the collapse state based on the first collective offset that we are going to process
+        # and makes the decision whether or not we need to still keep the collapse.
+        # @param first_offset [Integer] first offset from a collective batch
+        def refresh!(first_offset)
+          @mutex.synchronize do
+            @collapsed = first_offset < @until_offset
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/coordinator.rb

@@ -23,6 +23,7 @@ module Karafka
 
           @executed = []
           @flow_lock = Mutex.new
+          @collapser = Collapser.new
         end
 
         # Starts the coordination process
@@ -31,12 +32,28 @@ module Karafka
         def start(messages)
           super
 
+          @collapser.refresh!(messages.first.offset)
+
           @mutex.synchronize do
             @executed.clear
             @last_message = messages.last
           end
         end
 
+        # Sets the consumer failure status and additionally starts the collapse until
+        #
+        # @param consumer [Karafka::BaseConsumer] consumer that failed
+        # @param error [StandardError] error from the failure
+        def failure!(consumer, error)
+          super
+          @collapser.collapse_until!(@last_message.offset + 1)
+        end
+
+        # @return [Boolean] are we in a collapsed state at the moment
+        def collapsed?
+          @collapser.collapsed?
+        end
+
         # @return [Boolean] is the coordinated work finished or not
         def finished?
           @running_jobs.zero?

data/lib/karafka/pro/processing/partitioner.rb

@@ -18,15 +18,29 @@ module Karafka
       class Partitioner < ::Karafka::Processing::Partitioner
         # @param topic [String] topic name
         # @param messages [Array<Karafka::Messages::Message>] karafka messages
+        # @param coordinator [Karafka::Pro::Processing::Coordinator] processing coordinator that
+        #   will be used with those messages
         # @yieldparam [Integer] group id
         # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
-        def call(topic, messages)
+        def call(topic, messages, coordinator)
           ktopic = @subscription_group.topics.find(topic)
 
-          # We only partition work if we have a virtual partitioner and more than one thread to
-          # process the data. With one thread it is not worth partitioning the work as the work
-          # itself will be assigned to one thread (pointless work)
-          if ktopic.virtual_partitions? && ktopic.virtual_partitions.max_partitions > 1
+          # We only partition work if we have:
+          # - a virtual partitioner
+          # - more than one thread to process the data
+          # - collective is not collapsed via coordinator
+          #
+          # With one thread it is not worth partitioning the work as the work itself will be
+          # assigned to one thread (pointless work)
+          #
+          # We collapse the partitioning on errors because we "regain" full ordering on a batch
+          # that potentially contains the data that caused the error.
+          #
+          # This is great because it allows us to run things without the parallelization that adds
+          # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
+          if ktopic.virtual_partitions? &&
+             ktopic.virtual_partitions.max_partitions > 1 &&
+             !coordinator.collapsed?
             # We need to reduce it to the max concurrency, so the group_id is not a direct effect
             # of the end user action. Otherwise the persistence layer for consumers would cache
             # it forever and it would cause memory leaks

data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # DLQ enabled
+        # Long-Running Job enabled
+        # Manual offset management enabled
+        # Virtual Partitions enabled
+        #
+        # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
+        # offset management works in this case.
+        module AjDlqLrjMomVp
+          include AjDlqMomVp
+          include AjLrjMom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            long_running_job
+            manual_offset_management
+            dead_letter_queue
+            virtual_partitions
+          ].freeze
+
+          # This strategy is pretty much as non VP one because of the collapse
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                return if revoked?
+                return if Karafka::App.stopping?
+
+                # Since we have VP here we do not commit intermediate offsets and need to commit
+                # them here. We do commit in collapsed mode but this is generalized.
+                mark_as_consumed(last_group_message)
+
+                seek(coordinator.seek_offset) unless revoked?
+
+                resume
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                retry_after_pause
+              else
+                coordinator.pause_tracker.reset
+                skippable_message = find_skippable_message
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                mark_as_consumed(skippable_message)
+                pause(coordinator.seek_offset, nil, false)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Manual offset management enabled
+        # Virtual Partitions enabled
+        module AjDlqMomVp
+          include Dlq
+          include Vp
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            dead_letter_queue
+            manual_offset_management
+            virtual_partitions
+          ].freeze
+
+          # Flow including moving to DLQ in the collapsed mode
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                # When this is an ActiveJob running via Pro with virtual partitions, we cannot mark
+                # intermediate jobs as processed not to mess up with the ordering.
+                # Only when all the jobs are processed and we did not loose the partition
+                # assignment and we are not stopping (Pro ActiveJob has an early break) we can
+                # commit offsets on this as only then we can be sure, that all the jobs were
+                # processed.
+                # For a non virtual partitions case, the flow is regular and state is marked after
+                # each successfully processed job
+                return if revoked?
+                return if Karafka::App.stopping?
+
+                mark_as_consumed(last_group_message)
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                retry_after_pause
+              else
+                # Here we are in a collapsed state, hence we can apply the same logic as AjDlqMom
+                coordinator.pause_tracker.reset
+                skippable_message = find_skippable_message
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                mark_as_consumed(skippable_message)
+                pause(coordinator.seek_offset, nil, false)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb

@@ -21,6 +21,7 @@ module Karafka
         # Virtual Partitions enabled
         module AjLrjMomVp
           include Default
+          include Vp
 
           # Features for this strategy
           FEATURES = %i[

data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb

@@ -19,6 +19,7 @@ module Karafka
         # Manual offset management enabled
         # Virtual Partitions enabled
         module AjMomVp
+          include Vp
           include Default
 
           # Features for this strategy
@@ -42,6 +43,9 @@ module Karafka
                 # processed.
                 # For a non virtual partitions case, the flow is regular and state is marked after
                 # each successfully processed job
+                #
+                # We can mark and we do mark intermediate jobs in the collapsed mode when running
+                # VPs
                 return if revoked?
                 return if Karafka::App.stopping?
 

data/lib/karafka/pro/processing/strategies/default.rb

@@ -53,10 +53,10 @@ module Karafka
             end
 
             # Mark job as successful
-            coordinator.consumption(self).success!
+            coordinator.success!(self)
           rescue StandardError => e
             # If failed, mark as failed
-            coordinator.consumption(self).failure!(e)
+            coordinator.failure!(self, e)
 
             # Re-raise so reported in the consumer
             raise e

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -65,13 +65,8 @@ module Karafka
           #   dispatch to DLQ
           def dispatch_to_dlq(skippable_message)
             producer.produce_async(
-              topic: topic.dead_letter_queue.topic,
-              payload: skippable_message.raw_payload,
-              key: skippable_message.partition.to_s,
-              headers: skippable_message.headers.merge(
-                'original_topic' => topic.name,
-                'original_partition' => skippable_message.partition.to_s,
-                'original_offset' => skippable_message.offset.to_s
+              build_dlq_message(
+                skippable_message
               )
             )
 
@@ -83,6 +78,35 @@ module Karafka
             )
           end
 
+          # @param skippable_message [Array<Karafka::Messages::Message>]
+          # @return [Hash] dispatch DLQ message
+          def build_dlq_message(skippable_message)
+            original_partition = skippable_message.partition.to_s
+
+            dlq_message = {
+              topic: topic.dead_letter_queue.topic,
+              key: original_partition,
+              payload: skippable_message.raw_payload,
+              headers: skippable_message.headers.merge(
+                'original_topic' => topic.name,
+                'original_partition' => original_partition,
+                'original_offset' => skippable_message.offset.to_s,
+                'original_consumer_group' => topic.consumer_group.id
+              )
+            }
+
+            # Optional method user can define in consumer to enhance the dlq message hash with
+            # some extra details if needed or to replace payload, etc
+            if respond_to?(:enhance_dlq_message, true)
+              enhance_dlq_message(
+                dlq_message,
+                skippable_message
+              )
+            end
+
+            dlq_message
+          end
+
           # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch topic
           #   is set to false, we will skip the dispatch, effectively ignoring the broken message
           #   without taking any action.

data/lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Dead-Letter Queue enabled
+        # Long-Running Job enabled
+        # Virtual Partitions enabled
+        module DlqLrjVp
+          # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
+          include Vp
+          include DlqLrj
+
+          # Features for this strategy
+          FEATURES = %i[
+            dead_letter_queue
+            long_running_job
+            virtual_partitions
+          ].freeze
+        end
+      end
+    end
+  end
+end