karafka 2.4.18 → 2.5.0

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb

@@ -40,7 +40,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -43,7 +43,7 @@ module Karafka
 
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(seek_offset, false) unless revoked?
+                  seek(seek_offset, false, reset_offset: false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/default.rb

@@ -55,14 +55,19 @@ module Karafka
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
               return true if seek_offset.nil?
-              # Ignore earlier offsets than the one we already committed
-              return true if seek_offset > message.offset
+              # Ignore if it is the same offset as the one that is marked currently
+              # We ignore second marking because it changes nothing and in case of people using
+              # metadata storage but with automatic offset marking, this would cause metadata to be
+              # erased by automatic marking
+              return true if (seek_offset - 1) == message.offset
               return false if revoked?
 
               # If we are not inside a transaction but this is a transactional topic, we mark with
               # artificially created transaction
               stored = if producer.transactional?
                          mark_with_transaction(message, offset_metadata, true)
+                       elsif @_transactional_marking
+                         raise Errors::NonTransactionalMarkingAttemptError
                        else
                          client.mark_as_consumed(message, offset_metadata)
                        end
@@ -92,14 +97,19 @@ module Karafka
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
               return true if seek_offset.nil?
-              # Ignore earlier offsets than the one we already committed
-              return true if seek_offset > message.offset
+              # Ignore if it is the same offset as the one that is marked currently
+              # We ignore second marking because it changes nothing and in case of people using
+              # metadata storage but with automatic offset marking, this would cause metadata to be
+              # erased by automatic marking
+              return true if (seek_offset - 1) == message.offset
               return false if revoked?
 
               # If we are not inside a transaction but this is a transactional topic, we mark with
               # artificially created transaction
               stored = if producer.transactional?
                          mark_with_transaction(message, offset_metadata, false)
+                       elsif @_transactional_marking
+                         raise Errors::NonTransactionalMarkingAttemptError
                        else
                          client.mark_as_consumed!(message, offset_metadata)
                        end
@@ -143,6 +153,7 @@ module Karafka
               self.producer = active_producer
 
               transaction_started = false
+              transaction_completed = false
 
               # Prevent from nested transactions. It would not make any sense
               raise Errors::TransactionAlreadyInitializedError if @_in_transaction
@@ -159,6 +170,12 @@ module Karafka
                 # transaction. We do it only for transactions that contain offset management as for
                 # producer only, this is not relevant.
                 raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
+
+                # If we do not reach this, we should not move seek offsets because it means that
+                # either an error occured or transaction was aborted.
+                # In case of error, it will bubble up so no issue but in case of abort, while we
+                # do not reach this place, the code will continue
+                transaction_completed = true
               end
 
               @_in_transaction = false
@@ -180,8 +197,13 @@ module Karafka
               #     to mimic this
               #   - Complex strategies like VPs can use this in VPs to mark in parallel without
               #     having to redefine the transactional flow completely
-              @_transaction_marked.each do |marking|
-                marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+              #
+              # @note This should be applied only if transaction did not error and if it was not
+              #   aborted.
+              if transaction_completed
+                @_transaction_marked.each do |marking|
+                  marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+                end
               end
 
               true
@@ -213,6 +235,9 @@ module Karafka
               offset_metadata
             )
 
+            # This one is long lived and used to make sure, that users do not mix transactional
+            # marking with non-transactional. When this happens we should raise error
+            @_transactional_marking = true
             @_in_transaction_marked = true
             @_transaction_marked ||= []
             @_transaction_marked << [message, offset_metadata, async]
@@ -252,8 +277,11 @@ module Karafka
             # seek offset can be nil only in case `#seek` was invoked with offset reset request
             # In case like this we ignore marking
             return true if seek_offset.nil?
-            # Ignore earlier offsets than the one we already committed
-            return true if seek_offset > message.offset
+            # Ignore if it is the same offset as the one that is marked currently
+            # We ignore second marking because it changes nothing and in case of people using
+            # metadata storage but with automatic offset marking, this would cause metadata to be
+            # erased by automatic marking
+            return true if (seek_offset - 1) == message.offset
             return false if revoked?
 
             # If we have already marked this successfully in a transaction that was running

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -145,19 +145,20 @@ module Karafka
             # @param skippable_message [Array<Karafka::Messages::Message>]
             # @return [Hash] dispatch DLQ message
             def build_dlq_message(skippable_message)
-              original_partition = skippable_message.partition.to_s
+              source_partition = skippable_message.partition.to_s
 
               dlq_message = {
-                topic: topic.dead_letter_queue.topic,
-                key: original_partition,
+                topic: @_dispatch_to_dlq_topic || topic.dead_letter_queue.topic,
+                key: skippable_message.raw_key,
+                partition_key: source_partition,
                 payload: skippable_message.raw_payload,
                 headers: skippable_message.raw_headers.merge(
-                  'original_topic' => topic.name,
-                  'original_partition' => original_partition,
-                  'original_offset' => skippable_message.offset.to_s,
-                  'original_consumer_group' => topic.consumer_group.id,
-                  'original_key' => skippable_message.raw_key.to_s,
-                  'original_attempts' => attempt.to_s
+                  'source_topic' => topic.name,
+                  'source_partition' => source_partition,
+                  'source_offset' => skippable_message.offset.to_s,
+                  'source_consumer_group' => topic.consumer_group.id,
+                  'source_attempts' => attempt.to_s,
+                  'source_trace_id' => errors_tracker.trace_id
                 )
               }
 
@@ -205,7 +206,7 @@ module Karafka
             # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
             # In case of `:retry` always `#retry_after_pause` is applied
             def apply_dlq_flow
-              flow = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
+              flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
 
               case flow
               when :retry
@@ -216,6 +217,8 @@ module Karafka
                 @_dispatch_to_dlq = false
               when :dispatch
                 @_dispatch_to_dlq = true
+                # Use custom topic if it was returned from the strategy
+                @_dispatch_to_dlq_topic = target_topic || topic.dead_letter_queue.topic
               else
                 raise Karafka::UnsupportedCaseError, flow
               end
@@ -227,6 +230,8 @@ module Karafka
 
               # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
               pause(seek_offset, nil, false)
+            ensure
+              @_dispatch_to_dlq_topic = nil
             end
 
             # Marks message that went to DLQ (if applicable) based on the requested method

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -40,7 +40,7 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked? && !coordinator.manual_seek?
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -35,7 +35,7 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked? && !coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -31,7 +31,9 @@ module Karafka
 
                   mark_as_consumed(last_group_message) unless revoked?
                   # We should not overwrite user manual seel request with our seek
-                  seek(seek_offset, false) unless revoked? || coordinator.manual_seek?
+                  unless revoked? || coordinator.manual_seek?
+                    seek(seek_offset, false, reset_offset: false)
+                  end
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -30,7 +30,7 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   unless revoked? || coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                   end
 
                   resume

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -67,7 +67,7 @@ module Karafka
               if filter.mark_as_consumed?
                 send(
                   filter.marking_method,
-                  filter.cursor
+                  filter.marking_cursor
                 )
               end
 

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -43,7 +43,10 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   mark_as_consumed(last_group_message) unless revoked?
-                  seek(seek_offset, false) unless revoked? || coordinator.manual_seek?
+
+                  unless revoked? || coordinator.manual_seek?
+                    seek(seek_offset, false, reset_offset: false)
+                  end
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/lrj/ftr.rb

@@ -40,7 +40,7 @@ module Karafka
                   elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb

@@ -38,7 +38,7 @@ module Karafka
                   elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -40,7 +40,7 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   unless revoked? || coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                   end
 
                   resume

data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module VirtualPartitions
+        module Distributors
+          # Balanced distributor that groups messages by partition key
+          # and processes larger groups first while maintaining message order within groups
+          class Balanced < Base
+            # @param messages [Array<Karafka::Messages::Message>] messages to distribute
+            # @return [Hash<Integer, Array<Karafka::Messages::Message>>] hash with group ids as
+            #   keys and message groups as values
+            def call(messages)
+              # Group messages by partition key
+              key_groupings = messages.group_by { |msg| config.partitioner.call(msg) }
+
+              worker_loads = Array.new(config.max_partitions, 0)
+              worker_assignments = Array.new(config.max_partitions) { [] }
+
+              # Sort keys by workload in descending order
+              sorted_keys = key_groupings.keys.sort_by { |key| -key_groupings[key].size }
+
+              # Assign each key to the worker with the least current load
+              sorted_keys.each do |key|
+                # Find worker with minimum current load
+                min_load_worker = worker_loads.each_with_index.min_by { |load, _| load }[1]
+                messages = key_groupings[key]
+
+                # Assign this key to that worker
+                worker_assignments[min_load_worker] += messages
+                worker_loads[min_load_worker] += messages.size
+              end
+
+              # Combine messages for each worker and sort by offset
+              worker_assignments
+                .each_with_index
+                .reject { |group_messages, _| group_messages.empty? }
+                .map! { |group_messages, index| [index, group_messages.sort_by!(&:offset)] }
+                .to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/virtual_partitions/distributors/base.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      # Processing components for virtual partitions
+      module VirtualPartitions
+        # Distributors for virtual partitions
+        module Distributors
+          # Base class for all virtual partition distributors
+          class Base
+            # @param config [Karafka::Pro::Routing::Features::VirtualPartitions::Config]
+            def initialize(config)
+              @config = config
+            end
+
+            private
+
+            # @return [Karafka::Pro::Routing::Features::VirtualPartitions::Config]
+            attr_reader :config
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module VirtualPartitions
+        module Distributors
+          # Consistent distributor that ensures messages with the same partition key
+          # are always processed in the same virtual partition
+          class Consistent < Base
+            # @param messages [Array<Karafka::Messages::Message>] messages to distribute
+            # @return [Hash<Integer, Array<Karafka::Messages::Message>>] hash with group ids as
+            #   keys and message groups as values
+            def call(messages)
+              messages
+                .group_by { |msg| config.reducer.call(config.partitioner.call(msg)) }
+                .to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/contracts/config.rb

@@ -29,12 +29,16 @@ module Karafka
             end
 
             nested(:topics) do
-              required(:schedules) do |val|
-                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              nested(:schedules) do
+                required(:name) do |val|
+                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                end
               end
 
-              required(:logs) do |val|
-                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              nested(:logs) do
+                required(:name) do |val|
+                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                end
               end
             end
           end

data/lib/karafka/pro/recurring_tasks/dispatcher.rb

@@ -12,7 +12,7 @@ module Karafka
           # Snapshots to Kafka current schedule state
           def schedule
             produce(
-              topics.schedules,
+              topics.schedules.name,
               'state:schedule',
               serializer.schedule(::Karafka::Pro::RecurringTasks.schedule)
             )
@@ -25,7 +25,7 @@ module Karafka
           #   because in the web ui we work with the full name and it is easier. Since
           def command(name, task_id)
             produce(
-              topics.schedules,
+              topics.schedules.name,
               "command:#{name}:#{task_id}",
               serializer.command(name, task_id)
             )
@@ -35,7 +35,7 @@ module Karafka
           # @param event [Karafka::Core::Monitoring::Event]
           def log(event)
             produce(
-              topics.logs,
+              topics.logs.name,
               event[:task].id,
               serializer.log(event)
             )

data/lib/karafka/pro/recurring_tasks/setup/config.rb

@@ -32,8 +32,13 @@ module Karafka
           )
 
           setting(:topics) do
-            setting(:schedules, default: 'karafka_recurring_tasks_schedules')
-            setting(:logs, default: 'karafka_recurring_tasks_logs')
+            setting(:schedules) do
+              setting(:name, default: 'karafka_recurring_tasks_schedules')
+            end
+
+            setting(:logs) do
+              setting(:name, default: 'karafka_recurring_tasks_logs')
+            end
           end
 
           configure

data/lib/karafka/pro/recurring_tasks.rb

@@ -29,7 +29,10 @@ module Karafka
           @schedule.instance_exec(&block)
 
           @schedule.each do |task|
-            Contracts::Task.new.validate!(task.to_h)
+            Contracts::Task.new.validate!(
+              task.to_h,
+              scope: ['recurring_tasks', task.id]
+            )
           end
 
           @schedule
@@ -59,7 +62,10 @@ module Karafka
 
         # @param config [Karafka::Core::Configurable::Node] root node config
         def post_setup(config)
-          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+          RecurringTasks::Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
 
           # Published after task is successfully executed
           Karafka.monitor.notifications_bus.register_event('recurring_tasks.task.executed')
@@ -73,6 +79,19 @@ module Karafka
 
           Karafka.monitor.subscribe(Listener.new)
         end
+
+        # Basically since we may have custom producers configured that are not the same as the
+        # default one, we hold a reference to old pre-fork producer. This means, that when we
+        # initialize it again in post-fork, as long as user uses defaults we should re-inherit
+        # it from the default config.
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        # @param pre_fork_producer [WaterDrop::Producer]
+        def post_fork(config, pre_fork_producer)
+          return unless config.recurring_tasks.producer == pre_fork_producer
+
+          config.recurring_tasks.producer = config.producer
+        end
       end
     end
   end

data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb

@@ -12,7 +12,7 @@ module Karafka
           module Topic
             # @param strategy [#call, nil] Strategy we want to use or nil if a default strategy
             # (same as in OSS) should be applied
-            # @param args [Hash] OSS DLQ arguments
+            # @param args [Hash] Pro DLQ arguments
             def dead_letter_queue(strategy: nil, **args)
               return @dead_letter_queue if @dead_letter_queue
 

data/lib/karafka/pro/routing/features/multiplexing/config.rb

@@ -14,6 +14,7 @@ module Karafka
             :min,
             :max,
             :boot,
+            :scale_delay,
             keyword_init: true
           ) do
             alias_method :active?, :active

data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb

@@ -28,6 +28,7 @@ module Karafka
                 optional(:multiplexing_min) { |val| val.is_a?(Integer) && val >= 1 }
                 optional(:multiplexing_max) { |val| val.is_a?(Integer) && val >= 1 }
                 optional(:multiplexing_boot) { |val| val.is_a?(Integer) && val >= 1 }
+                optional(:multiplexing_scale_delay) { |val| val.is_a?(Integer) && val >= 1_000 }
               end
 
               # Makes sure min is not more than max
@@ -78,6 +79,22 @@ module Karafka
                 [[%w[subscription_group_details], :multiplexing_boot_not_dynamic]]
               end
 
+              # Makes sure we do not run multiplexing with 1 always which does not make much sense
+              # because then it behaves like without multiplexing and can create problems for
+              # users running multiplexed subscription groups with multiple topics
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless min(data)
+                next unless max(data)
+
+                min = min(data)
+                max = max(data)
+
+                next unless min == 1 && max == 1
+
+                [[%w[subscription_group_details], :multiplexing_one_not_enough]]
+              end
+
               class << self
                 # @param data [Hash] topic details
                 # @return [Integer, false] min or false if missing

data/lib/karafka/pro/routing/features/multiplexing/proxy.rb

@@ -14,12 +14,15 @@ module Karafka
             #   disabling dynamic multiplexing
             # @param max [Integer] max multiplexing count
             # @param boot [Integer] how many listeners should we start during boot by default
-            def multiplexing(min: nil, max: 1, boot: nil)
+            # @param scale_delay [Integer] number of ms of delay before applying any scale
+            #   operation to a consumer group
+            def multiplexing(min: nil, max: 1, boot: nil, scale_delay: 60_000)
               @target.current_subscription_group_details.merge!(
                 multiplexing_min: min || max,
                 multiplexing_max: max,
                 # Picks half of max by default as long as possible. Otherwise goes with min
-                multiplexing_boot: boot || [min || max, (max / 2)].max
+                multiplexing_boot: boot || [min || max, (max / 2)].max,
+                multiplexing_scale_delay: scale_delay
               )
             end
           end

data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb

@@ -16,9 +16,16 @@ module Karafka
                 max = @details.fetch(:multiplexing_max, 1)
                 min = @details.fetch(:multiplexing_min, max)
                 boot = @details.fetch(:multiplexing_boot, max / 2)
+                scale_delay = @details.fetch(:multiplexing_scale_delay, 60_000)
                 active = max > 1
 
-                Config.new(active: active, min: min, max: max, boot: boot)
+                Config.new(
+                  active: active,
+                  min: min,
+                  max: max,
+                  boot: boot,
+                  scale_delay: scale_delay
+                )
               end
             end
 

data/lib/karafka/pro/routing/features/parallel_segments/builder.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Expansions for the routing builder
+          module Builder
+            # Builds and saves given consumer group
+            # @param group_id [String, Symbol] name for consumer group
+            # @param block [Proc] proc that should be executed in the proxy context
+            def consumer_group(group_id, &block)
+              consumer_group = find { |cg| cg.name == group_id.to_s }
+
+              # Re-opening a CG should not change its parallel setup
+              if consumer_group
+                super
+              else
+                # We build a temp consumer group and a target to check if it has parallel segments
+                # enabled and if so, we do not add it to the routing but instead we build the
+                # appropriate number of parallel segment groups
+                temp_consumer_group = ::Karafka::Routing::ConsumerGroup.new(group_id.to_s)
+                temp_target = Karafka::Routing::Proxy.new(temp_consumer_group, &block).target
+                config = temp_target.parallel_segments
+
+                if config.active?
+                  config.count.times do |i|
+                    sub_name = [group_id, config.merge_key, i.to_s].join
+                    sub_consumer_group = Karafka::Routing::ConsumerGroup.new(sub_name)
+                    self << Karafka::Routing::Proxy.new(sub_consumer_group, &block).target
+                  end
+                # If not parallel segments are not active we go with the default flow
+                else
+                  super
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end