karafka 2.4.18 → 2.5.0.beta1

data/lib/karafka/pro/processing/piping/consumer.rb

@@ -20,16 +20,16 @@ module Karafka
 
           # Pipes given message to the provided topic with expected details. Useful for
           # pass-through operations where deserialization is not needed. Upon usage it will include
-          # all the original headers + meta headers about the source of message.
+          # all the source headers + meta headers about the source of message.
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] original message to pipe
+          # @param message [Karafka::Messages::Message] source message to pipe
           #
           # @note It will NOT deserialize the payload so it is fast
           #
           # @note We assume that there can be different number of partitions in the target topic,
-          #   this is why we use `key` based on the original topic key and not the partition id.
-          #   This will not utilize partitions beyond the number of partitions of original topic,
+          #   this is why we use `key` based on the source topic key and not the partition id.
+          #   This will not utilize partitions beyond the number of partitions of source topic,
           #   but will accommodate for topics with less partitions.
           def pipe_async(topic:, message:)
             produce_async(
@@ -40,7 +40,7 @@ module Karafka
           # Sync version of pipe for one message
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] original message to pipe
+          # @param message [Karafka::Messages::Message] source message to pipe
           # @see [#pipe_async]
           def pipe_sync(topic:, message:)
             produce_sync(
@@ -51,7 +51,7 @@ module Karafka
           # Async multi-message pipe
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param messages [Array<Karafka::Messages::Message>] original messages to pipe
+          # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
           #
           # @note If transactional producer in use and dispatch is not wrapped with a transaction,
           #   it will automatically wrap the dispatch with a transaction
@@ -66,7 +66,7 @@ module Karafka
           # Sync multi-message pipe
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param messages [Array<Karafka::Messages::Message>] original messages to pipe
+          # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
           #
           # @note If transactional producer in use and dispatch is not wrapped with a transaction,
           #   it will automatically wrap the dispatch with a transaction
@@ -81,7 +81,7 @@ module Karafka
           private
 
           # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] original message to pipe
+          # @param message [Karafka::Messages::Message] source message to pipe
           # @return [Hash] hash with message to pipe.
           #
           # @note If you need to alter this, please define the `#enhance_pipe_message` method
@@ -90,17 +90,17 @@ module Karafka
               topic: topic,
               payload: message.raw_payload,
               headers: message.raw_headers.merge(
-                'original_topic' => message.topic,
-                'original_partition' => message.partition.to_s,
-                'original_offset' => message.offset.to_s,
-                'original_consumer_group' => self.topic.consumer_group.id
+                'source_topic' => message.topic,
+                'source_partition' => message.partition.to_s,
+                'source_offset' => message.offset.to_s,
+                'source_consumer_group' => self.topic.consumer_group.id
               )
             }
 
             # Use a key only if key was provided
             if message.raw_key
               pipe_message[:key] = message.raw_key
-            # Otherwise pipe creating a key that will assign it based on the original partition
+            # Otherwise pipe creating a key that will assign it based on the source partition
             # number
             else
               pipe_message[:key] = message.partition.to_s

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -38,7 +38,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -44,7 +44,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -36,7 +36,7 @@ module Karafka
 
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(seek_offset, false) unless revoked?
+                  seek(seek_offset, false, reset_offset: false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -40,7 +40,7 @@ module Karafka
                   mark_as_consumed(last_group_message) unless revoked?
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(seek_offset, false) unless revoked?
+                  seek(seek_offset, false, reset_offset: false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb

@@ -40,7 +40,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -43,7 +43,7 @@ module Karafka
 
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(seek_offset, false) unless revoked?
+                  seek(seek_offset, false, reset_offset: false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/default.rb

@@ -55,14 +55,19 @@ module Karafka
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
               return true if seek_offset.nil?
-              # Ignore earlier offsets than the one we already committed
-              return true if seek_offset > message.offset
+              # Ignore if it is the same offset as the one that is marked currently
+              # We ignore second marking because it changes nothing and in case of people using
+              # metadata storage but with automatic offset marking, this would cause metadata to be
+              # erased by automatic marking
+              return true if (seek_offset - 1) == message.offset
               return false if revoked?
 
               # If we are not inside a transaction but this is a transactional topic, we mark with
               # artificially created transaction
               stored = if producer.transactional?
                          mark_with_transaction(message, offset_metadata, true)
+                       elsif @_transactional_marking
+                         raise Errors::NonTransactionalMarkingAttemptError
                        else
                          client.mark_as_consumed(message, offset_metadata)
                        end
@@ -92,14 +97,19 @@ module Karafka
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
               return true if seek_offset.nil?
-              # Ignore earlier offsets than the one we already committed
-              return true if seek_offset > message.offset
+              # Ignore if it is the same offset as the one that is marked currently
+              # We ignore second marking because it changes nothing and in case of people using
+              # metadata storage but with automatic offset marking, this would cause metadata to be
+              # erased by automatic marking
+              return true if (seek_offset - 1) == message.offset
               return false if revoked?
 
               # If we are not inside a transaction but this is a transactional topic, we mark with
               # artificially created transaction
               stored = if producer.transactional?
                          mark_with_transaction(message, offset_metadata, false)
+                       elsif @_transactional_marking
+                         raise Errors::NonTransactionalMarkingAttemptError
                        else
                          client.mark_as_consumed!(message, offset_metadata)
                        end
@@ -143,6 +153,7 @@ module Karafka
               self.producer = active_producer
 
               transaction_started = false
+              transaction_completed = false
 
               # Prevent from nested transactions. It would not make any sense
               raise Errors::TransactionAlreadyInitializedError if @_in_transaction
@@ -159,6 +170,12 @@ module Karafka
                 # transaction. We do it only for transactions that contain offset management as for
                 # producer only, this is not relevant.
                 raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
+
+                # If we do not reach this, we should not move seek offsets because it means that
+                # either an error occured or transaction was aborted.
+                # In case of error, it will bubble up so no issue but in case of abort, while we
+                # do not reach this place, the code will continue
+                transaction_completed = true
               end
 
               @_in_transaction = false
@@ -180,8 +197,13 @@ module Karafka
               #     to mimic this
               #   - Complex strategies like VPs can use this in VPs to mark in parallel without
               #     having to redefine the transactional flow completely
-              @_transaction_marked.each do |marking|
-                marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+              #
+              # @note This should be applied only if transaction did not error and if it was not
+              #   aborted.
+              if transaction_completed
+                @_transaction_marked.each do |marking|
+                  marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+                end
               end
 
               true
@@ -213,6 +235,9 @@ module Karafka
               offset_metadata
             )
 
+            # This one is long lived and used to make sure, that users do not mix transactional
+            # marking with non-transactional. When this happens we should raise error
+            @_transactional_marking = true
             @_in_transaction_marked = true
             @_transaction_marked ||= []
             @_transaction_marked << [message, offset_metadata, async]
@@ -252,8 +277,11 @@ module Karafka
             # seek offset can be nil only in case `#seek` was invoked with offset reset request
             # In case like this we ignore marking
             return true if seek_offset.nil?
-            # Ignore earlier offsets than the one we already committed
-            return true if seek_offset > message.offset
+            # Ignore if it is the same offset as the one that is marked currently
+            # We ignore second marking because it changes nothing and in case of people using
+            # metadata storage but with automatic offset marking, this would cause metadata to be
+            # erased by automatic marking
+            return true if (seek_offset - 1) == message.offset
             return false if revoked?
 
             # If we have already marked this successfully in a transaction that was running

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -145,19 +145,19 @@ module Karafka
             # @param skippable_message [Array<Karafka::Messages::Message>]
             # @return [Hash] dispatch DLQ message
             def build_dlq_message(skippable_message)
-              original_partition = skippable_message.partition.to_s
+              source_partition = skippable_message.partition.to_s
 
               dlq_message = {
-                topic: topic.dead_letter_queue.topic,
-                key: original_partition,
+                topic: @_dispatch_to_dlq_topic || topic.dead_letter_queue.topic,
+                key: source_partition,
                 payload: skippable_message.raw_payload,
                 headers: skippable_message.raw_headers.merge(
-                  'original_topic' => topic.name,
-                  'original_partition' => original_partition,
-                  'original_offset' => skippable_message.offset.to_s,
-                  'original_consumer_group' => topic.consumer_group.id,
-                  'original_key' => skippable_message.raw_key.to_s,
-                  'original_attempts' => attempt.to_s
+                  'source_topic' => topic.name,
+                  'source_partition' => source_partition,
+                  'source_offset' => skippable_message.offset.to_s,
+                  'source_consumer_group' => topic.consumer_group.id,
+                  'source_key' => skippable_message.raw_key.to_s,
+                  'source_attempts' => attempt.to_s
                 )
               }
 
@@ -205,7 +205,7 @@ module Karafka
             # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
             # In case of `:retry` always `#retry_after_pause` is applied
             def apply_dlq_flow
-              flow = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
+              flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
 
               case flow
               when :retry
@@ -216,6 +216,8 @@ module Karafka
                 @_dispatch_to_dlq = false
               when :dispatch
                 @_dispatch_to_dlq = true
+                # Use custom topic if it was returned from the strategy
+                @_dispatch_to_dlq_topic = target_topic || topic.dead_letter_queue.topic
               else
                 raise Karafka::UnsupportedCaseError, flow
               end
@@ -227,6 +229,8 @@ module Karafka
 
               # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
               pause(seek_offset, nil, false)
+            ensure
+              @_dispatch_to_dlq_topic = nil
             end
 
             # Marks message that went to DLQ (if applicable) based on the requested method

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -40,7 +40,7 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked? && !coordinator.manual_seek?
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -35,7 +35,7 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked? && !coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -31,7 +31,9 @@ module Karafka
 
                   mark_as_consumed(last_group_message) unless revoked?
                   # We should not overwrite user manual seel request with our seek
-                  seek(seek_offset, false) unless revoked? || coordinator.manual_seek?
+                  unless revoked? || coordinator.manual_seek?
+                    seek(seek_offset, false, reset_offset: false)
+                  end
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -30,7 +30,7 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   unless revoked? || coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                   end
 
                   resume

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -67,7 +67,7 @@ module Karafka
               if filter.mark_as_consumed?
                 send(
                   filter.marking_method,
-                  filter.cursor
+                  filter.marking_cursor
                 )
               end
 

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -43,7 +43,10 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   mark_as_consumed(last_group_message) unless revoked?
-                  seek(seek_offset, false) unless revoked? || coordinator.manual_seek?
+
+                  unless revoked? || coordinator.manual_seek?
+                    seek(seek_offset, false, reset_offset: false)
+                  end
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/lrj/ftr.rb

@@ -40,7 +40,7 @@ module Karafka
                   elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb

@@ -38,7 +38,7 @@ module Karafka
                   elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -40,7 +40,7 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   unless revoked? || coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                   end
 
                   resume

data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module VirtualPartitions
+        module Distributors
+          # Balanced distributor that groups messages by partition key
+          # and processes larger groups first while maintaining message order within groups
+          class Balanced < Base
+            # @param messages [Array<Karafka::Messages::Message>] messages to distribute
+            # @return [Hash<Integer, Array<Karafka::Messages::Message>>] hash with group ids as
+            #   keys and message groups as values
+            def call(messages)
+              # Group messages by partition key
+              key_groupings = messages.group_by { |msg| config.partitioner.call(msg) }
+
+              worker_loads = Array.new(config.max_partitions, 0)
+              worker_assignments = Array.new(config.max_partitions) { [] }
+
+              # Sort keys by workload in descending order
+              sorted_keys = key_groupings.keys.sort_by { |key| -key_groupings[key].size }
+
+              # Assign each key to the worker with the least current load
+              sorted_keys.each do |key|
+                # Find worker with minimum current load
+                min_load_worker = worker_loads.each_with_index.min_by { |load, _| load }[1]
+                messages = key_groupings[key]
+
+                # Assign this key to that worker
+                worker_assignments[min_load_worker] += messages
+                worker_loads[min_load_worker] += messages.size
+              end
+
+              # Combine messages for each worker and sort by offset
+              worker_assignments
+                .each_with_index
+                .reject { |group_messages, _| group_messages.empty? }
+                .map! { |group_messages, index| [index, group_messages.sort_by!(&:offset)] }
+                .to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/virtual_partitions/distributors/base.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      # Processing components for virtual partitions
+      module VirtualPartitions
+        # Distributors for virtual partitions
+        module Distributors
+          # Base class for all virtual partition distributors
+          class Base
+            # @param config [Karafka::Pro::Routing::Features::VirtualPartitions::Config]
+            def initialize(config)
+              @config = config
+            end
+
+            private
+
+            # @return [Karafka::Pro::Routing::Features::VirtualPartitions::Config]
+            attr_reader :config
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module VirtualPartitions
+        module Distributors
+          # Consistent distributor that ensures messages with the same partition key
+          # are always processed in the same virtual partition
+          class Consistent < Base
+            # @param messages [Array<Karafka::Messages::Message>] messages to distribute
+            # @return [Hash<Integer, Array<Karafka::Messages::Message>>] hash with group ids as
+            #   keys and message groups as values
+            def call(messages)
+              messages
+                .group_by { |msg| config.reducer.call(config.partitioner.call(msg)) }
+                .to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/contracts/config.rb

@@ -29,12 +29,16 @@ module Karafka
             end
 
             nested(:topics) do
-              required(:schedules) do |val|
-                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              nested(:schedules) do
+                required(:name) do |val|
+                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                end
               end
 
-              required(:logs) do |val|
-                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              nested(:logs) do
+                required(:name) do |val|
+                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                end
               end
             end
           end

data/lib/karafka/pro/recurring_tasks/dispatcher.rb

@@ -12,7 +12,7 @@ module Karafka
           # Snapshots to Kafka current schedule state
           def schedule
             produce(
-              topics.schedules,
+              topics.schedules.name,
               'state:schedule',
               serializer.schedule(::Karafka::Pro::RecurringTasks.schedule)
             )
@@ -25,7 +25,7 @@ module Karafka
           #   because in the web ui we work with the full name and it is easier. Since
           def command(name, task_id)
             produce(
-              topics.schedules,
+              topics.schedules.name,
               "command:#{name}:#{task_id}",
               serializer.command(name, task_id)
             )
@@ -35,7 +35,7 @@ module Karafka
           # @param event [Karafka::Core::Monitoring::Event]
           def log(event)
             produce(
-              topics.logs,
+              topics.logs.name,
               event[:task].id,
               serializer.log(event)
             )

data/lib/karafka/pro/recurring_tasks/setup/config.rb

@@ -32,8 +32,13 @@ module Karafka
           )
 
           setting(:topics) do
-            setting(:schedules, default: 'karafka_recurring_tasks_schedules')
-            setting(:logs, default: 'karafka_recurring_tasks_logs')
+            setting(:schedules) do
+              setting(:name, default: 'karafka_recurring_tasks_schedules')
+            end
+
+            setting(:logs) do
+              setting(:name, default: 'karafka_recurring_tasks_logs')
+            end
           end
 
           configure

data/lib/karafka/pro/recurring_tasks.rb

@@ -73,6 +73,19 @@ module Karafka
 
           Karafka.monitor.subscribe(Listener.new)
         end
+
+        # Basically since we may have custom producers configured that are not the same as the
+        # default one, we hold a reference to old pre-fork producer. This means, that when we
+        # initialize it again in post-fork, as long as user uses defaults we should re-inherit
+        # it from the default config.
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        # @param pre_fork_producer [WaterDrop::Producer]
+        def post_fork(config, pre_fork_producer)
+          return unless config.recurring_tasks.producer == pre_fork_producer
+
+          config.recurring_tasks.producer = config.producer
+        end
       end
     end
   end

data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb

@@ -12,7 +12,7 @@ module Karafka
           module Topic
             # @param strategy [#call, nil] Strategy we want to use or nil if a default strategy
             # (same as in OSS) should be applied
-            # @param args [Hash] OSS DLQ arguments
+            # @param args [Hash] Pro DLQ arguments
             def dead_letter_queue(strategy: nil, **args)
               return @dead_letter_queue if @dead_letter_queue
 

data/lib/karafka/pro/routing/features/multiplexing/config.rb

@@ -14,6 +14,7 @@ module Karafka
             :min,
             :max,
             :boot,
+            :scale_delay,
             keyword_init: true
           ) do
             alias_method :active?, :active

data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb

@@ -28,6 +28,7 @@ module Karafka
                 optional(:multiplexing_min) { |val| val.is_a?(Integer) && val >= 1 }
                 optional(:multiplexing_max) { |val| val.is_a?(Integer) && val >= 1 }
                 optional(:multiplexing_boot) { |val| val.is_a?(Integer) && val >= 1 }
+                optional(:multiplexing_scale_delay) { |val| val.is_a?(Integer) && val >= 1_000 }
               end
 
               # Makes sure min is not more than max
@@ -78,6 +79,22 @@ module Karafka
                 [[%w[subscription_group_details], :multiplexing_boot_not_dynamic]]
               end
 
+              # Makes sure we do not run multiplexing with 1 always which does not make much sense
+              # because then it behaves like without multiplexing and can create problems for
+              # users running multiplexed subscription groups with multiple topics
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless min(data)
+                next unless max(data)
+
+                min = min(data)
+                max = max(data)
+
+                next unless min == 1 && max == 1
+
+                [[%w[subscription_group_details], :multiplexing_one_not_enough]]
+              end
+
               class << self
                 # @param data [Hash] topic details
                 # @return [Integer, false] min or false if missing