karafka 2.4.18 → 2.5.0.beta1

data/lib/karafka/pro/routing/features/multiplexing/proxy.rb

@@ -14,12 +14,15 @@ module Karafka
             #   disabling dynamic multiplexing
             # @param max [Integer] max multiplexing count
             # @param boot [Integer] how many listeners should we start during boot by default
-            def multiplexing(min: nil, max: 1, boot: nil)
+            # @param scale_delay [Integer] number of ms of delay before applying any scale
+            #   operation to a consumer group
+            def multiplexing(min: nil, max: 1, boot: nil, scale_delay: 60_000)
               @target.current_subscription_group_details.merge!(
                 multiplexing_min: min || max,
                 multiplexing_max: max,
                 # Picks half of max by default as long as possible. Otherwise goes with min
-                multiplexing_boot: boot || [min || max, (max / 2)].max
+                multiplexing_boot: boot || [min || max, (max / 2)].max,
+                multiplexing_scale_delay: scale_delay
               )
             end
           end

data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb

@@ -16,9 +16,16 @@ module Karafka
                 max = @details.fetch(:multiplexing_max, 1)
                 min = @details.fetch(:multiplexing_min, max)
                 boot = @details.fetch(:multiplexing_boot, max / 2)
+                scale_delay = @details.fetch(:multiplexing_scale_delay, 60_000)
                 active = max > 1
 
-                Config.new(active: active, min: min, max: max, boot: boot)
+                Config.new(
+                  active: active,
+                  min: min,
+                  max: max,
+                  boot: boot,
+                  scale_delay: scale_delay
+                )
               end
             end
 

data/lib/karafka/pro/routing/features/parallel_segments/builder.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Expansions for the routing builder
+          module Builder
+            # Builds and saves given consumer group
+            # @param group_id [String, Symbol] name for consumer group
+            # @param block [Proc] proc that should be executed in the proxy context
+            def consumer_group(group_id, &block)
+              consumer_group = find { |cg| cg.name == group_id.to_s }
+
+              # Re-opening a CG should not change its parallel setup
+              if consumer_group
+                super
+              else
+                # We build a temp consumer group and a target to check if it has parallel segments
+                # enabled and if so, we do not add it to the routing but instead we build the
+                # appropriate number of parallel segment groups
+                temp_consumer_group = ::Karafka::Routing::ConsumerGroup.new(group_id.to_s)
+                temp_target = Karafka::Routing::Proxy.new(temp_consumer_group, &block).target
+                config = temp_target.parallel_segments
+
+                if config.active?
+                  config.count.times do |i|
+                    sub_name = [group_id, config.merge_key, i.to_s].join
+                    sub_consumer_group = Karafka::Routing::ConsumerGroup.new(sub_name)
+                    self << Karafka::Routing::Proxy.new(sub_consumer_group, &block).target
+                  end
+                # If not parallel segments are not active we go with the default flow
+                else
+                  super
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Config for parallel segments.
+          # @note Used on the consumer level, not per topic
+          Config = Struct.new(
+            :active,
+            :count,
+            :partitioner,
+            :reducer,
+            :merge_key,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Parallel segments are defined on the consumer group (since it creates many), thus we
+          # define them on the consumer group.
+          # This module adds extra methods needed there to make it work
+          module ConsumerGroup
+            # @return [Config] parallel segments config
+            def parallel_segments
+              # We initialize it as disabled if not configured by the user
+              public_send(:parallel_segments=, count: 1)
+            end
+
+            # Allows setting parallel segments configuration
+            #
+            # @param count [Integer] number of parallel segments (number of parallel consumer
+            #   groups that will be created)
+            # @param partitioner [nil, #call] nil or callable partitioner
+            # @param reducer [nil, #call] reducer for parallel key. It allows for using a custom
+            #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+            # @param merge_key [String] key used to build the parallel segment consumer groups
+            #
+            # @note This method is an assignor but the API is actually via the `#parallel_segments`
+            #   method. Our `Routing::Proxy` normalizes that the way we want to have it exposed
+            #   for the end users.
+            def parallel_segments=(
+              count: 1,
+              partitioner: nil,
+              reducer: nil,
+              merge_key: '-parallel-'
+            )
+              @parallel_segments ||= Config.new(
+                active: count > 1,
+                count: count,
+                partitioner: partitioner,
+                reducer: reducer || ->(parallel_key) { parallel_key.to_s.sum % count },
+                merge_key: merge_key
+              )
+            end
+
+            # @return [Boolean] are parallel segments active
+            def parallel_segments?
+              parallel_segments.active?
+            end
+
+            # @return [Integer] id of the segment (0 or bigger) or -1 if parallel segments are not
+            #   active
+            def segment_id
+              return @segment_id if @segment_id
+
+              @segment_id = if parallel_segments?
+                              name.split(parallel_segments.merge_key).last.to_i
+                            else
+                              -1
+                            end
+            end
+
+            # @return [String] original segment consumer group name
+            def segment_origin
+              name.split(parallel_segments.merge_key).first
+            end
+
+            # @return [Hash] consumer group setup with the parallel segments definition in it
+            def to_h
+              super.merge(
+                parallel_segments: parallel_segments.to_h.merge(
+                  segment_id: segment_id
+                )
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Namespace for parallel segments contracts
+          module Contracts
+            # Contract to validate configuration of the parallel segments feature
+            class ConsumerGroup < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('consumer_group')
+
+                nested(:parallel_segments) do
+                  required(:active) { |val| [true, false].include?(val) }
+                  required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                  required(:reducer) { |val| val.respond_to?(:call) }
+                  required(:count) { |val| val.is_a?(Integer) && val >= 1 }
+                  required(:merge_key) { |val| val.is_a?(String) && val.size >= 1 }
+                end
+
+                # When parallel segments are defined, partitioner needs to respond to `#call` and
+                # it cannot be nil
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  parallel_segments = data[:parallel_segments]
+
+                  next unless parallel_segments[:active]
+                  next if parallel_segments[:partitioner].respond_to?(:call)
+
+                  [[%i[parallel_segments partitioner], :respond_to_call]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/topic.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Parallel segments related expansions to the topic building flow
+          module Topic
+            # Injects the parallel segments filter as the first filter during building of each of
+            # the topics in case parallel segments are enabled.
+            #
+            # @param args [Object] anything accepted by the topic initializer
+            def initialize(*args)
+              super
+
+              return unless consumer_group.parallel_segments?
+
+              builder = lambda do |topic, _partition|
+                mom = topic.manual_offset_management?
+
+                # We have two filters for mom and non-mom scenario not to mix this logic
+                filter_scope = Karafka::Pro::Processing::ParallelSegments::Filters
+                filter_class = mom ? filter_scope::Mom : filter_scope::Default
+
+                filter_class.new(
+                  segment_id: consumer_group.segment_id,
+                  partitioner: consumer_group.parallel_segments.partitioner,
+                  reducer: consumer_group.parallel_segments.reducer
+                )
+              end
+
+              filter(builder)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature that allows parallelizing message processing within a single consumer group by
+        # creating multiple consumer group instances. It enables processing messages from each
+        # partition in parallel by distributing them to separate consumer group instances based on
+        # a partitioning key. Useful for both CPU and IO bound operations.
+        #
+        # Each parallel segment operates as an independent consumer group instance, processing
+        # messages that are assigned to it based on the configured partitioner and reducer.
+        # This allows for better resource utilization and increased processing throughput without
+        # requiring changes to the topic's partition count.
+        class ParallelSegments < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/pattern.rb

@@ -45,7 +45,7 @@ module Karafka
               # topic but this minimizes simple mistakes
               #
               # This sub-part of sh1 should be unique enough and short-enough to use it here
-              digest = Digest::SHA1.hexdigest(safe_regexp.source)[8..16]
+              digest = Digest::SHA256.hexdigest(safe_regexp.source)[8..16]
               @name = name ? name.to_s : "karafka-pattern-#{digest}"
               @config = config
             end

data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb

@@ -29,7 +29,7 @@ module Karafka
               consumer_group tasks_cfg.group_id do
                 # Registers the primary topic that we use to control schedules execution. This is
                 # the one that we use to trigger recurring tasks.
-                schedules_topic = topic(topics_cfg.schedules) do
+                schedules_topic = topic(topics_cfg.schedules.name) do
                   consumer tasks_cfg.consumer_class
                   deserializer tasks_cfg.deserializer
                   # Because the topic method name as well as builder proxy method name is the same
@@ -83,7 +83,7 @@ module Karafka
 
                 # This topic is to store logs that we can then inspect either from the admin or via
                 # the Web UI
-                logs_topic = topic(topics_cfg.logs) do
+                logs_topic = topic(topics_cfg.logs.name) do
                   active(false)
                   deserializer tasks_cfg.deserializer
                   target.recurring_tasks(true)

data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb

@@ -12,14 +12,14 @@ module Karafka
           module Builder
             # Enabled scheduled messages operations and adds needed topics and other stuff.
             #
-            # @param group_name [String, false] name for scheduled messages topic that is also used
+            # @param topic_name [String, false] name for scheduled messages topic that is also used
             #   as a group identifier. Users can have multiple schedule topics flows to prevent key
             #   collisions, prioritize and do other stuff. `false` if not active.
             # @param block [Proc] optional reconfiguration of the topics definitions.
             # @note Namespace for topics should include the divider as it is not automatically
             #   added.
-            def scheduled_messages(group_name = false, &block)
-              return unless group_name
+            def scheduled_messages(topic_name = false, &block)
+              return unless topic_name
 
               # Load zlib only if user enables scheduled messages
               require 'zlib'
@@ -32,7 +32,7 @@ module Karafka
               consumer_group msg_cfg.group_id do
                 # Registers the primary topic that we use to control schedules execution. This is
                 # the one that we use to trigger scheduled messages.
-                messages_topic = topic(group_name) do
+                messages_topic = topic(topic_name) do
                   instance_eval(&block) if block && block.arity.zero?
 
                   consumer msg_cfg.consumer_class
@@ -54,7 +54,11 @@ module Karafka
                   consumer_persistence(true)
 
                   # This needs to be enabled for the eof to work correctly
-                  kafka('enable.partition.eof': true, inherit: true)
+                  kafka(
+                    'enable.partition.eof': true,
+                    'auto.offset.reset': 'earliest',
+                    inherit: true
+                  )
                   eofed(true)
 
                   # Since this is a topic that gets replayed because of schedule management, we do
@@ -96,7 +100,7 @@ module Karafka
                 # Holds states of scheduler per each of the partitions since they tick
                 # independently. We only hold future statistics not to have to deal with
                 # any type of state restoration
-                states_topic = topic("#{group_name}#{msg_cfg.states_postfix}") do
+                states_topic = topic("#{topic_name}#{msg_cfg.states_postfix}") do
                   active(false)
                   target.scheduled_messages(true)
                   config(

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -8,15 +8,33 @@ module Karafka
     module Routing
       module Features
         class VirtualPartitions < Base
-          # Config for virtual partitions
+          # Configuration for virtual partitions feature
           Config = Struct.new(
             :active,
             :partitioner,
             :max_partitions,
             :offset_metadata_strategy,
             :reducer,
+            :distribution,
             keyword_init: true
-          ) { alias_method :active?, :active }
+          ) do
+            # @return [Boolean] is this feature active
+            def active?
+              active
+            end
+
+            # @return [Object] distributor instance for the current distribution
+            def distributor
+              @distributor ||= case distribution
+                               when :balanced
+                                 Processing::VirtualPartitions::Distributors::Balanced.new(self)
+                               when :consistent
+                                 Processing::VirtualPartitions::Distributors::Consistent.new(self)
+                               else
+                                 raise Karafka::Errors::UnsupportedCaseError, distribution
+                               end
+            end
+          end
         end
       end
     end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -26,6 +26,7 @@ module Karafka
                 required(:reducer) { |val| val.respond_to?(:call) }
                 required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
                 required(:offset_metadata_strategy) { |val| %i[exact current].include?(val) }
+                required(:distribution) { |val| %i[consistent balanced].include?(val) }
               end
 
               # When virtual partitions are defined, partitioner needs to respond to `#call` and it

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -20,13 +20,18 @@ module Karafka
             #   the most recently reported metadata
             # @param reducer [nil, #call] reducer for VPs key. It allows for using a custom
             #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+            # @param distribution [Symbol] the strategy to use for virtual partitioning. Can be
+            #   either `:consistent` or `:balanced`. The `:balanced` strategy ensures balanced
+            #   distribution of work across available workers while maintaining message order
+            #   within groups.
             # @return [VirtualPartitions] method that allows to set the virtual partitions details
             #   during the routing configuration and then allows to retrieve it
             def virtual_partitions(
               max_partitions: Karafka::App.config.concurrency,
               partitioner: nil,
               offset_metadata_strategy: :current,
-              reducer: nil
+              reducer: nil,
+              distribution: :consistent
             )
               @virtual_partitions ||= Config.new(
                 active: !partitioner.nil?,
@@ -35,7 +40,8 @@ module Karafka
                 offset_metadata_strategy: offset_metadata_strategy,
                 # If no reducer provided, we use this one. It just runs a modulo on the sum of
                 # a stringified version, providing fairly good distribution.
-                reducer: reducer || ->(virtual_key) { virtual_key.to_s.sum % max_partitions }
+                reducer: reducer || ->(virtual_key) { virtual_key.to_s.sum % max_partitions },
+                distribution: distribution
               )
             end
 

data/lib/karafka/pro/scheduled_messages/consumer.rb

@@ -52,6 +52,9 @@ module Karafka
 
           # If end of the partition is reached, it always means all data is loaded
           @state.loaded!
+
+          tags.add(:state, @state.to_s)
+
           @states_reporter.call
         end
 
@@ -64,7 +67,6 @@ module Karafka
           return unless @state.loaded?
 
           keys = []
-          epochs = []
 
           # We first collect all the data for dispatch and then dispatch and **only** after
           # dispatch that is sync is successful we remove those messages from the daily buffer
@@ -72,16 +74,13 @@ module Karafka
           # with timeouts, etc, we need to be sure it wen through prior to deleting those messages
           # from the daily buffer. That way we ensure the at least once delivery and in case of
           # a transactional producer, exactly once delivery.
-          @daily_buffer.for_dispatch do |epoch, message|
-            epochs << epoch
+          @daily_buffer.for_dispatch do |message|
             keys << message.key
             @dispatcher << message
           end
 
           @dispatcher.flush
 
-          @max_epoch.update(epochs.max)
-
           keys.each { |key| @daily_buffer.delete(key) }
 
           @states_reporter.call
@@ -93,14 +92,6 @@ module Karafka
         # accumulator and time related per-message operations.
         # @param message [Karafka::Messages::Message]
         def process_message(message)
-          # If we started to receive messages younger than the moment we created the consumer for
-          # the given day, it means we have loaded all the history and we are no longer in the
-          # loading phase.
-          if message.timestamp.to_i > @today.created_at
-            @state.loaded!
-            tags.add(:state, @state.to_s)
-          end
-
           # If this is a schedule message we need to check if this is for today. Tombstone events
           # are always considered immediate as they indicate, that a message with a given key
           # was already dispatched or that user decided not to dispatch and cancelled the dispatch
@@ -120,6 +111,14 @@ module Karafka
             end
           end
 
+          # Tombstone events are only published after we have dispatched given message. This means
+          # that we've got that far in the dispatching time. This allows us (with a certain buffer)
+          # to quickly reject older messages (older in sense of being scheduled for previous times)
+          # instead of loading them into memory until they are expired
+          if message.headers['schedule_source_type'] == 'tombstone'
+            @max_epoch.update(message.headers['schedule_target_epoch'])
+          end
+
           # Add to buffer all tombstones and messages for the same day
           @daily_buffer << message
         end
@@ -129,7 +128,7 @@ module Karafka
           # If this is a new assignment we always need to seek from beginning to load the data
           if @state.fresh?
             clear!
-            seek(0)
+            seek(:earliest)
 
             return true
           end
@@ -140,7 +139,7 @@ module Karafka
           # If day has ended we reload and start new day with new schedules
           if @today.ended?
             clear!
-            seek(0)
+            seek(:earliest)
 
             return true
           end

data/lib/karafka/pro/scheduled_messages/daily_buffer.rb

@@ -45,19 +45,22 @@ module Karafka
 
         # Yields messages that should be dispatched (sent) to Kafka
         #
-        # @yieldparam [Integer, Karafka::Messages::Message] epoch of the message and the message
-        #   itself
-        #
-        # @note We yield epoch alongside of the message so we do not have to extract it several
-        #   times later on. This simplifies the API
+        # @yieldparam [Karafka::Messages::Message] messages to be dispatched sorted from the once
+        #   that are the oldest (lowest epoch)
         def for_dispatch
           dispatch = Time.now.to_i
 
+          selected = []
+
           @accu.each_value do |epoch, message|
             next unless epoch <= dispatch
 
-            yield(epoch, message)
+            selected << [epoch, message]
           end
+
+          selected
+            .sort_by!(&:first)
+            .each { |_, message| yield(message) }
         end
 
         # Removes given key from the accumulator

data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb

@@ -10,6 +10,12 @@ module Karafka
       module Deserializers
         # Converts certain pieces of headers into their integer form for messages
         class Headers
+          # We only directly operate on epoch and other details for schedules and tombstones.
+          # cancel requests don't have to be deserialized that way since they don't have epoch
+          WORKABLE_TYPES = %w[schedule tombstone].freeze
+
+          private_constant :WORKABLE_TYPES
+
           # @param metadata [Karafka::aMessages::Metadata]
           # @return [Hash] headers
           def call(metadata)
@@ -19,7 +25,7 @@ module Karafka
 
             # tombstone and cancellation events are not operable, thus we do not have to cast any
             # of the headers pieces
-            return raw_headers unless type == 'schedule'
+            return raw_headers unless WORKABLE_TYPES.include?(type)
 
             headers = raw_headers.dup
             headers['schedule_target_epoch'] = headers['schedule_target_epoch'].to_i

data/lib/karafka/pro/scheduled_messages/max_epoch.rb

@@ -10,22 +10,31 @@ module Karafka
       # until which messages were dispatched by us. This allows us to quickly skip those messages
       # during recovery, because we do know, they were dispatched.
       class MaxEpoch
+        # We always give a bit of a buffer when using the max dispatch epoch because while we
+        # are dispatching messages, we could also later receive data for time close to our
+        # dispatch times. This is why when reloading days we give ourselves one hour of a window
+        # that we will keep until tombstones expire them. This prevents edge cases race-conditions
+        # when multiple scheduled events scheduled close to each other would bump epoch in such a
+        # way, that it would end up ignoring certain events.
+        GRACE_PERIOD = 60 * 60
+
+        private_constant :GRACE_PERIOD
+
+        # @return [Integer] max epoch recorded
+        attr_reader :to_i
+
         def initialize
           @max = -1
+          @to_i = @max
         end
 
         # Updates epoch if bigger than current max
         # @param new_max [Integer] potential new max epoch
         def update(new_max)
-          return unless new_max
           return unless new_max > @max
 
           @max = new_max
-        end
-
-        # @return [Integer] max epoch recorded
-        def to_i
-          @max
+          @to_i = @max - GRACE_PERIOD
         end
       end
     end

data/lib/karafka/pro/scheduled_messages.rb

@@ -53,6 +53,19 @@ module Karafka
         def post_setup(config)
           RecurringTasks::Contracts::Config.new.validate!(config.to_h)
         end
+
+        # Basically since we may have custom producers configured that are not the same as the
+        # default one, we hold a reference to old pre-fork producer. This means, that when we
+        # initialize it again in post-fork, as long as user uses defaults we should re-inherit
+        # it from the default config.
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        # @param pre_fork_producer [WaterDrop::Producer]
+        def post_fork(config, pre_fork_producer)
+          return unless config.scheduled_messages.producer == pre_fork_producer
+
+          config.scheduled_messages.producer = config.producer
+        end
       end
     end
   end