karafka 2.4.18 → 2.5.0.beta2

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb

@@ -38,7 +38,7 @@ module Karafka
                   elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -40,7 +40,7 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   unless revoked? || coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                   end
 
                   resume

data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module VirtualPartitions
+        module Distributors
+          # Balanced distributor that groups messages by partition key
+          # and processes larger groups first while maintaining message order within groups
+          class Balanced < Base
+            # @param messages [Array<Karafka::Messages::Message>] messages to distribute
+            # @return [Hash<Integer, Array<Karafka::Messages::Message>>] hash with group ids as
+            #   keys and message groups as values
+            def call(messages)
+              # Group messages by partition key
+              key_groupings = messages.group_by { |msg| config.partitioner.call(msg) }
+
+              worker_loads = Array.new(config.max_partitions, 0)
+              worker_assignments = Array.new(config.max_partitions) { [] }
+
+              # Sort keys by workload in descending order
+              sorted_keys = key_groupings.keys.sort_by { |key| -key_groupings[key].size }
+
+              # Assign each key to the worker with the least current load
+              sorted_keys.each do |key|
+                # Find worker with minimum current load
+                min_load_worker = worker_loads.each_with_index.min_by { |load, _| load }[1]
+                messages = key_groupings[key]
+
+                # Assign this key to that worker
+                worker_assignments[min_load_worker] += messages
+                worker_loads[min_load_worker] += messages.size
+              end
+
+              # Combine messages for each worker and sort by offset
+              worker_assignments
+                .each_with_index
+                .reject { |group_messages, _| group_messages.empty? }
+                .map! { |group_messages, index| [index, group_messages.sort_by!(&:offset)] }
+                .to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/virtual_partitions/distributors/base.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      # Processing components for virtual partitions
+      module VirtualPartitions
+        # Distributors for virtual partitions
+        module Distributors
+          # Base class for all virtual partition distributors
+          class Base
+            # @param config [Karafka::Pro::Routing::Features::VirtualPartitions::Config]
+            def initialize(config)
+              @config = config
+            end
+
+            private
+
+            # @return [Karafka::Pro::Routing::Features::VirtualPartitions::Config]
+            attr_reader :config
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module VirtualPartitions
+        module Distributors
+          # Consistent distributor that ensures messages with the same partition key
+          # are always processed in the same virtual partition
+          class Consistent < Base
+            # @param messages [Array<Karafka::Messages::Message>] messages to distribute
+            # @return [Hash<Integer, Array<Karafka::Messages::Message>>] hash with group ids as
+            #   keys and message groups as values
+            def call(messages)
+              messages
+                .group_by { |msg| config.reducer.call(config.partitioner.call(msg)) }
+                .to_h
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/recurring_tasks/contracts/config.rb

@@ -29,12 +29,16 @@ module Karafka
             end
 
             nested(:topics) do
-              required(:schedules) do |val|
-                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              nested(:schedules) do
+                required(:name) do |val|
+                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                end
               end
 
-              required(:logs) do |val|
-                val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+              nested(:logs) do
+                required(:name) do |val|
+                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                end
               end
             end
           end

data/lib/karafka/pro/recurring_tasks/dispatcher.rb

@@ -12,7 +12,7 @@ module Karafka
           # Snapshots to Kafka current schedule state
           def schedule
             produce(
-              topics.schedules,
+              topics.schedules.name,
               'state:schedule',
               serializer.schedule(::Karafka::Pro::RecurringTasks.schedule)
             )
@@ -25,7 +25,7 @@ module Karafka
           #   because in the web ui we work with the full name and it is easier. Since
           def command(name, task_id)
             produce(
-              topics.schedules,
+              topics.schedules.name,
               "command:#{name}:#{task_id}",
               serializer.command(name, task_id)
             )
@@ -35,7 +35,7 @@ module Karafka
           # @param event [Karafka::Core::Monitoring::Event]
           def log(event)
             produce(
-              topics.logs,
+              topics.logs.name,
               event[:task].id,
               serializer.log(event)
             )

data/lib/karafka/pro/recurring_tasks/setup/config.rb

@@ -32,8 +32,13 @@ module Karafka
           )
 
           setting(:topics) do
-            setting(:schedules, default: 'karafka_recurring_tasks_schedules')
-            setting(:logs, default: 'karafka_recurring_tasks_logs')
+            setting(:schedules) do
+              setting(:name, default: 'karafka_recurring_tasks_schedules')
+            end
+
+            setting(:logs) do
+              setting(:name, default: 'karafka_recurring_tasks_logs')
+            end
           end
 
           configure

data/lib/karafka/pro/recurring_tasks.rb

@@ -73,6 +73,19 @@ module Karafka
 
           Karafka.monitor.subscribe(Listener.new)
         end
+
+        # Basically since we may have custom producers configured that are not the same as the
+        # default one, we hold a reference to old pre-fork producer. This means, that when we
+        # initialize it again in post-fork, as long as user uses defaults we should re-inherit
+        # it from the default config.
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        # @param pre_fork_producer [WaterDrop::Producer]
+        def post_fork(config, pre_fork_producer)
+          return unless config.recurring_tasks.producer == pre_fork_producer
+
+          config.recurring_tasks.producer = config.producer
+        end
       end
     end
   end

data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb

@@ -12,7 +12,7 @@ module Karafka
           module Topic
             # @param strategy [#call, nil] Strategy we want to use or nil if a default strategy
             # (same as in OSS) should be applied
-            # @param args [Hash] OSS DLQ arguments
+            # @param args [Hash] Pro DLQ arguments
             def dead_letter_queue(strategy: nil, **args)
               return @dead_letter_queue if @dead_letter_queue
 

data/lib/karafka/pro/routing/features/multiplexing/config.rb

@@ -14,6 +14,7 @@ module Karafka
             :min,
             :max,
             :boot,
+            :scale_delay,
             keyword_init: true
           ) do
             alias_method :active?, :active

data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb

@@ -28,6 +28,7 @@ module Karafka
                 optional(:multiplexing_min) { |val| val.is_a?(Integer) && val >= 1 }
                 optional(:multiplexing_max) { |val| val.is_a?(Integer) && val >= 1 }
                 optional(:multiplexing_boot) { |val| val.is_a?(Integer) && val >= 1 }
+                optional(:multiplexing_scale_delay) { |val| val.is_a?(Integer) && val >= 1_000 }
               end
 
               # Makes sure min is not more than max
@@ -78,6 +79,22 @@ module Karafka
                 [[%w[subscription_group_details], :multiplexing_boot_not_dynamic]]
               end
 
+              # Makes sure we do not run multiplexing with 1 always which does not make much sense
+              # because then it behaves like without multiplexing and can create problems for
+              # users running multiplexed subscription groups with multiple topics
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless min(data)
+                next unless max(data)
+
+                min = min(data)
+                max = max(data)
+
+                next unless min == 1 && max == 1
+
+                [[%w[subscription_group_details], :multiplexing_one_not_enough]]
+              end
+
               class << self
                 # @param data [Hash] topic details
                 # @return [Integer, false] min or false if missing

data/lib/karafka/pro/routing/features/multiplexing/proxy.rb

@@ -14,12 +14,15 @@ module Karafka
             #   disabling dynamic multiplexing
             # @param max [Integer] max multiplexing count
             # @param boot [Integer] how many listeners should we start during boot by default
-            def multiplexing(min: nil, max: 1, boot: nil)
+            # @param scale_delay [Integer] number of ms of delay before applying any scale
+            #   operation to a consumer group
+            def multiplexing(min: nil, max: 1, boot: nil, scale_delay: 60_000)
               @target.current_subscription_group_details.merge!(
                 multiplexing_min: min || max,
                 multiplexing_max: max,
                 # Picks half of max by default as long as possible. Otherwise goes with min
-                multiplexing_boot: boot || [min || max, (max / 2)].max
+                multiplexing_boot: boot || [min || max, (max / 2)].max,
+                multiplexing_scale_delay: scale_delay
               )
             end
           end

data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb

@@ -16,9 +16,16 @@ module Karafka
                 max = @details.fetch(:multiplexing_max, 1)
                 min = @details.fetch(:multiplexing_min, max)
                 boot = @details.fetch(:multiplexing_boot, max / 2)
+                scale_delay = @details.fetch(:multiplexing_scale_delay, 60_000)
                 active = max > 1
 
-                Config.new(active: active, min: min, max: max, boot: boot)
+                Config.new(
+                  active: active,
+                  min: min,
+                  max: max,
+                  boot: boot,
+                  scale_delay: scale_delay
+                )
               end
             end
 

data/lib/karafka/pro/routing/features/parallel_segments/builder.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Expansions for the routing builder
+          module Builder
+            # Builds and saves given consumer group
+            # @param group_id [String, Symbol] name for consumer group
+            # @param block [Proc] proc that should be executed in the proxy context
+            def consumer_group(group_id, &block)
+              consumer_group = find { |cg| cg.name == group_id.to_s }
+
+              # Re-opening a CG should not change its parallel setup
+              if consumer_group
+                super
+              else
+                # We build a temp consumer group and a target to check if it has parallel segments
+                # enabled and if so, we do not add it to the routing but instead we build the
+                # appropriate number of parallel segment groups
+                temp_consumer_group = ::Karafka::Routing::ConsumerGroup.new(group_id.to_s)
+                temp_target = Karafka::Routing::Proxy.new(temp_consumer_group, &block).target
+                config = temp_target.parallel_segments
+
+                if config.active?
+                  config.count.times do |i|
+                    sub_name = [group_id, config.merge_key, i.to_s].join
+                    sub_consumer_group = Karafka::Routing::ConsumerGroup.new(sub_name)
+                    self << Karafka::Routing::Proxy.new(sub_consumer_group, &block).target
+                  end
+                # If not parallel segments are not active we go with the default flow
+                else
+                  super
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Config for parallel segments.
+          # @note Used on the consumer level, not per topic
+          Config = Struct.new(
+            :active,
+            :count,
+            :partitioner,
+            :reducer,
+            :merge_key,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Parallel segments are defined on the consumer group (since it creates many), thus we
+          # define them on the consumer group.
+          # This module adds extra methods needed there to make it work
+          module ConsumerGroup
+            # @return [Config] parallel segments config
+            def parallel_segments
+              # We initialize it as disabled if not configured by the user
+              public_send(:parallel_segments=, count: 1)
+            end
+
+            # Allows setting parallel segments configuration
+            #
+            # @param count [Integer] number of parallel segments (number of parallel consumer
+            #   groups that will be created)
+            # @param partitioner [nil, #call] nil or callable partitioner
+            # @param reducer [nil, #call] reducer for parallel key. It allows for using a custom
+            #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+            # @param merge_key [String] key used to build the parallel segment consumer groups
+            #
+            # @note This method is an assignor but the API is actually via the `#parallel_segments`
+            #   method. Our `Routing::Proxy` normalizes that the way we want to have it exposed
+            #   for the end users.
+            def parallel_segments=(
+              count: 1,
+              partitioner: nil,
+              reducer: nil,
+              merge_key: '-parallel-'
+            )
+              @parallel_segments ||= Config.new(
+                active: count > 1,
+                count: count,
+                partitioner: partitioner,
+                reducer: reducer || ->(parallel_key) { parallel_key.to_s.sum % count },
+                merge_key: merge_key
+              )
+            end
+
+            # @return [Boolean] are parallel segments active
+            def parallel_segments?
+              parallel_segments.active?
+            end
+
+            # @return [Integer] id of the segment (0 or bigger) or -1 if parallel segments are not
+            #   active
+            def segment_id
+              return @segment_id if @segment_id
+
+              @segment_id = if parallel_segments?
+                              name.split(parallel_segments.merge_key).last.to_i
+                            else
+                              -1
+                            end
+            end
+
+            # @return [String] original segment consumer group name
+            def segment_origin
+              name.split(parallel_segments.merge_key).first
+            end
+
+            # @return [Hash] consumer group setup with the parallel segments definition in it
+            def to_h
+              super.merge(
+                parallel_segments: parallel_segments.to_h.merge(
+                  segment_id: segment_id
+                )
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Namespace for parallel segments contracts
+          module Contracts
+            # Contract to validate configuration of the parallel segments feature
+            class ConsumerGroup < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('consumer_group')
+
+                nested(:parallel_segments) do
+                  required(:active) { |val| [true, false].include?(val) }
+                  required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                  required(:reducer) { |val| val.respond_to?(:call) }
+                  required(:count) { |val| val.is_a?(Integer) && val >= 1 }
+                  required(:merge_key) { |val| val.is_a?(String) && val.size >= 1 }
+                end
+
+                # When parallel segments are defined, partitioner needs to respond to `#call` and
+                # it cannot be nil
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  parallel_segments = data[:parallel_segments]
+
+                  next unless parallel_segments[:active]
+                  next if parallel_segments[:partitioner].respond_to?(:call)
+
+                  [[%i[parallel_segments partitioner], :respond_to_call]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/topic.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Parallel segments related expansions to the topic building flow
+          module Topic
+            # Injects the parallel segments filter as the first filter during building of each of
+            # the topics in case parallel segments are enabled.
+            #
+            # @param args [Object] anything accepted by the topic initializer
+            def initialize(*args)
+              super
+
+              return unless consumer_group.parallel_segments?
+
+              builder = lambda do |topic, _partition|
+                mom = topic.manual_offset_management?
+
+                # We have two filters for mom and non-mom scenario not to mix this logic
+                filter_scope = Karafka::Pro::Processing::ParallelSegments::Filters
+                filter_class = mom ? filter_scope::Mom : filter_scope::Default
+
+                filter_class.new(
+                  segment_id: consumer_group.segment_id,
+                  partitioner: consumer_group.parallel_segments.partitioner,
+                  reducer: consumer_group.parallel_segments.reducer
+                )
+              end
+
+              filter(builder)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature that allows parallelizing message processing within a single consumer group by
+        # creating multiple consumer group instances. It enables processing messages from each
+        # partition in parallel by distributing them to separate consumer group instances based on
+        # a partitioning key. Useful for both CPU and IO bound operations.
+        #
+        # Each parallel segment operates as an independent consumer group instance, processing
+        # messages that are assigned to it based on the configured partitioner and reducer.
+        # This allows for better resource utilization and increased processing throughput without
+        # requiring changes to the topic's partition count.
+        class ParallelSegments < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/pattern.rb

@@ -45,7 +45,7 @@ module Karafka
               # topic but this minimizes simple mistakes
               #
               # This sub-part of sh1 should be unique enough and short-enough to use it here
-              digest = Digest::SHA1.hexdigest(safe_regexp.source)[8..16]
+              digest = Digest::SHA256.hexdigest(safe_regexp.source)[8..16]
               @name = name ? name.to_s : "karafka-pattern-#{digest}"
               @config = config
             end

data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb

@@ -29,7 +29,7 @@ module Karafka
               consumer_group tasks_cfg.group_id do
                 # Registers the primary topic that we use to control schedules execution. This is
                 # the one that we use to trigger recurring tasks.
-                schedules_topic = topic(topics_cfg.schedules) do
+                schedules_topic = topic(topics_cfg.schedules.name) do
                   consumer tasks_cfg.consumer_class
                   deserializer tasks_cfg.deserializer
                   # Because the topic method name as well as builder proxy method name is the same
@@ -83,7 +83,7 @@ module Karafka
 
                 # This topic is to store logs that we can then inspect either from the admin or via
                 # the Web UI
-                logs_topic = topic(topics_cfg.logs) do
+                logs_topic = topic(topics_cfg.logs.name) do
                   active(false)
                   deserializer tasks_cfg.deserializer
                   target.recurring_tasks(true)

data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb

@@ -12,14 +12,14 @@ module Karafka
           module Builder
             # Enabled scheduled messages operations and adds needed topics and other stuff.
             #
-            # @param group_name [String, false] name for scheduled messages topic that is also used
+            # @param topic_name [String, false] name for scheduled messages topic that is also used
             #   as a group identifier. Users can have multiple schedule topics flows to prevent key
             #   collisions, prioritize and do other stuff. `false` if not active.
             # @param block [Proc] optional reconfiguration of the topics definitions.
             # @note Namespace for topics should include the divider as it is not automatically
             #   added.
-            def scheduled_messages(group_name = false, &block)
-              return unless group_name
+            def scheduled_messages(topic_name = false, &block)
+              return unless topic_name
 
               # Load zlib only if user enables scheduled messages
               require 'zlib'
@@ -32,7 +32,7 @@ module Karafka
               consumer_group msg_cfg.group_id do
                 # Registers the primary topic that we use to control schedules execution. This is
                 # the one that we use to trigger scheduled messages.
-                messages_topic = topic(group_name) do
+                messages_topic = topic(topic_name) do
                   instance_eval(&block) if block && block.arity.zero?
 
                   consumer msg_cfg.consumer_class
@@ -54,7 +54,11 @@ module Karafka
                   consumer_persistence(true)
 
                   # This needs to be enabled for the eof to work correctly
-                  kafka('enable.partition.eof': true, inherit: true)
+                  kafka(
+                    'enable.partition.eof': true,
+                    'auto.offset.reset': 'earliest',
+                    inherit: true
+                  )
                   eofed(true)
 
                   # Since this is a topic that gets replayed because of schedule management, we do
@@ -96,7 +100,7 @@ module Karafka
                 # Holds states of scheduler per each of the partitions since they tick
                 # independently. We only hold future statistics not to have to deal with
                 # any type of state restoration
-                states_topic = topic("#{group_name}#{msg_cfg.states_postfix}") do
+                states_topic = topic("#{topic_name}#{msg_cfg.states_postfix}") do
                   active(false)
                   target.scheduled_messages(true)
                   config(