karafka 2.4.18 → 2.5.0

data/lib/karafka/pro/routing/features/parallel_segments/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Config for parallel segments.
+          # @note Used on the consumer level, not per topic
+          Config = Struct.new(
+            :active,
+            :count,
+            :partitioner,
+            :reducer,
+            :merge_key,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Parallel segments are defined on the consumer group (since it creates many), thus we
+          # define them on the consumer group.
+          # This module adds extra methods needed there to make it work
+          module ConsumerGroup
+            # @return [Config] parallel segments config
+            def parallel_segments
+              # We initialize it as disabled if not configured by the user
+              public_send(:parallel_segments=, count: 1)
+            end
+
+            # Allows setting parallel segments configuration
+            #
+            # @param count [Integer] number of parallel segments (number of parallel consumer
+            #   groups that will be created)
+            # @param partitioner [nil, #call] nil or callable partitioner
+            # @param reducer [nil, #call] reducer for parallel key. It allows for using a custom
+            #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+            # @param merge_key [String] key used to build the parallel segment consumer groups
+            #
+            # @note This method is an assignor but the API is actually via the `#parallel_segments`
+            #   method. Our `Routing::Proxy` normalizes that the way we want to have it exposed
+            #   for the end users.
+            def parallel_segments=(
+              count: 1,
+              partitioner: nil,
+              reducer: nil,
+              merge_key: '-parallel-'
+            )
+              @parallel_segments ||= Config.new(
+                active: count > 1,
+                count: count,
+                partitioner: partitioner,
+                reducer: reducer || ->(parallel_key) { parallel_key.to_s.sum % count },
+                merge_key: merge_key
+              )
+            end
+
+            # @return [Boolean] are parallel segments active
+            def parallel_segments?
+              parallel_segments.active?
+            end
+
+            # @return [Integer] id of the segment (0 or bigger) or -1 if parallel segments are not
+            #   active
+            def segment_id
+              return @segment_id if @segment_id
+
+              @segment_id = if parallel_segments?
+                              name.split(parallel_segments.merge_key).last.to_i
+                            else
+                              -1
+                            end
+            end
+
+            # @return [String] original segment consumer group name
+            def segment_origin
+              name.split(parallel_segments.merge_key).first
+            end
+
+            # @return [Hash] consumer group setup with the parallel segments definition in it
+            def to_h
+              super.merge(
+                parallel_segments: parallel_segments.to_h.merge(
+                  segment_id: segment_id
+                )
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Namespace for parallel segments contracts
+          module Contracts
+            # Contract to validate configuration of the parallel segments feature
+            class ConsumerGroup < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('consumer_group')
+
+                nested(:parallel_segments) do
+                  required(:active) { |val| [true, false].include?(val) }
+                  required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                  required(:reducer) { |val| val.respond_to?(:call) }
+                  required(:count) { |val| val.is_a?(Integer) && val >= 1 }
+                  required(:merge_key) { |val| val.is_a?(String) && val.size >= 1 }
+                end
+
+                # When parallel segments are defined, partitioner needs to respond to `#call` and
+                # it cannot be nil
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  parallel_segments = data[:parallel_segments]
+
+                  next unless parallel_segments[:active]
+                  next if parallel_segments[:partitioner].respond_to?(:call)
+
+                  [[%i[parallel_segments partitioner], :respond_to_call]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments/topic.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ParallelSegments < Base
+          # Parallel segments related expansions to the topic building flow
+          module Topic
+            # Injects the parallel segments filter as the first filter during building of each of
+            # the topics in case parallel segments are enabled.
+            #
+            # @param args [Object] anything accepted by the topic initializer
+            def initialize(*args)
+              super
+
+              return unless consumer_group.parallel_segments?
+
+              builder = lambda do |topic, _partition|
+                mom = topic.manual_offset_management?
+
+                # We have two filters for mom and non-mom scenario not to mix this logic
+                filter_scope = Karafka::Pro::Processing::ParallelSegments::Filters
+                filter_class = mom ? filter_scope::Mom : filter_scope::Default
+
+                filter_class.new(
+                  segment_id: consumer_group.segment_id,
+                  partitioner: consumer_group.parallel_segments.partitioner,
+                  reducer: consumer_group.parallel_segments.reducer
+                )
+              end
+
+              filter(builder)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/parallel_segments.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature that allows parallelizing message processing within a single consumer group by
+        # creating multiple consumer group instances. It enables processing messages from each
+        # partition in parallel by distributing them to separate consumer group instances based on
+        # a partitioning key. Useful for both CPU and IO bound operations.
+        #
+        # Each parallel segment operates as an independent consumer group instance, processing
+        # messages that are assigned to it based on the configured partitioner and reducer.
+        # This allows for better resource utilization and increased processing throughput without
+        # requiring changes to the topic's partition count.
+        class ParallelSegments < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/pattern.rb

@@ -45,7 +45,7 @@ module Karafka
               # topic but this minimizes simple mistakes
               #
               # This sub-part of sh1 should be unique enough and short-enough to use it here
-              digest = Digest::SHA1.hexdigest(safe_regexp.source)[8..16]
+              digest = Digest::SHA256.hexdigest(safe_regexp.source)[8..16]
               @name = name ? name.to_s : "karafka-pattern-#{digest}"
               @config = config
             end

data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb

@@ -29,7 +29,7 @@ module Karafka
               consumer_group tasks_cfg.group_id do
                 # Registers the primary topic that we use to control schedules execution. This is
                 # the one that we use to trigger recurring tasks.
-                schedules_topic = topic(topics_cfg.schedules) do
+                schedules_topic = topic(topics_cfg.schedules.name) do
                   consumer tasks_cfg.consumer_class
                   deserializer tasks_cfg.deserializer
                   # Because the topic method name as well as builder proxy method name is the same
@@ -83,7 +83,7 @@ module Karafka
 
                 # This topic is to store logs that we can then inspect either from the admin or via
                 # the Web UI
-                logs_topic = topic(topics_cfg.logs) do
+                logs_topic = topic(topics_cfg.logs.name) do
                   active(false)
                   deserializer tasks_cfg.deserializer
                   target.recurring_tasks(true)

data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb

@@ -12,14 +12,14 @@ module Karafka
           module Builder
             # Enabled scheduled messages operations and adds needed topics and other stuff.
             #
-            # @param group_name [String, false] name for scheduled messages topic that is also used
+            # @param topic_name [String, false] name for scheduled messages topic that is also used
             #   as a group identifier. Users can have multiple schedule topics flows to prevent key
             #   collisions, prioritize and do other stuff. `false` if not active.
             # @param block [Proc] optional reconfiguration of the topics definitions.
             # @note Namespace for topics should include the divider as it is not automatically
             #   added.
-            def scheduled_messages(group_name = false, &block)
-              return unless group_name
+            def scheduled_messages(topic_name = false, &block)
+              return unless topic_name
 
               # Load zlib only if user enables scheduled messages
               require 'zlib'
@@ -32,7 +32,7 @@ module Karafka
               consumer_group msg_cfg.group_id do
                 # Registers the primary topic that we use to control schedules execution. This is
                 # the one that we use to trigger scheduled messages.
-                messages_topic = topic(group_name) do
+                messages_topic = topic(topic_name) do
                   instance_eval(&block) if block && block.arity.zero?
 
                   consumer msg_cfg.consumer_class
@@ -54,7 +54,11 @@ module Karafka
                   consumer_persistence(true)
 
                   # This needs to be enabled for the eof to work correctly
-                  kafka('enable.partition.eof': true, inherit: true)
+                  kafka(
+                    'enable.partition.eof': true,
+                    'auto.offset.reset': 'earliest',
+                    inherit: true
+                  )
                   eofed(true)
 
                   # Since this is a topic that gets replayed because of schedule management, we do
@@ -96,7 +100,7 @@ module Karafka
                 # Holds states of scheduler per each of the partitions since they tick
                 # independently. We only hold future statistics not to have to deal with
                 # any type of state restoration
-                states_topic = topic("#{group_name}#{msg_cfg.states_postfix}") do
+                states_topic = topic("#{topic_name}#{msg_cfg.states_postfix}") do
                   active(false)
                   target.scheduled_messages(true)
                   config(

data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb

@@ -28,7 +28,8 @@ module Karafka
               # Validates that each node has at least one assignment.
               #
               # @param builder [Karafka::Routing::Builder]
-              def validate!(builder)
+              # @param scope [Array<String>]
+              def validate!(builder, scope: [])
                 nodes_setup = Hash.new do |h, node_id|
                   h[node_id] = { active: false, node_id: node_id }
                 end
@@ -49,7 +50,7 @@ module Karafka
                 end
 
                 nodes_setup.each_value do |details|
-                  super(details)
+                  super(details, scope: scope)
                 end
               end
 

data/lib/karafka/pro/routing/features/swarm.rb

@@ -17,7 +17,10 @@ module Karafka
             # @param config [Karafka::Core::Configurable::Node] app config
             def post_setup(config)
               config.monitor.subscribe('app.before_warmup') do
-                Contracts::Routing.new.validate!(config.internal.routing.builder)
+                Contracts::Routing.new.validate!(
+                  config.internal.routing.builder,
+                  scope: %w[swarm]
+                )
               end
             end
           end

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -8,15 +8,33 @@ module Karafka
     module Routing
       module Features
         class VirtualPartitions < Base
-          # Config for virtual partitions
+          # Configuration for virtual partitions feature
           Config = Struct.new(
             :active,
             :partitioner,
             :max_partitions,
             :offset_metadata_strategy,
             :reducer,
+            :distribution,
             keyword_init: true
-          ) { alias_method :active?, :active }
+          ) do
+            # @return [Boolean] is this feature active
+            def active?
+              active
+            end
+
+            # @return [Object] distributor instance for the current distribution
+            def distributor
+              @distributor ||= case distribution
+                               when :balanced
+                                 Processing::VirtualPartitions::Distributors::Balanced.new(self)
+                               when :consistent
+                                 Processing::VirtualPartitions::Distributors::Consistent.new(self)
+                               else
+                                 raise Karafka::Errors::UnsupportedCaseError, distribution
+                               end
+            end
+          end
         end
       end
     end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -26,6 +26,7 @@ module Karafka
                 required(:reducer) { |val| val.respond_to?(:call) }
                 required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
                 required(:offset_metadata_strategy) { |val| %i[exact current].include?(val) }
+                required(:distribution) { |val| %i[consistent balanced].include?(val) }
               end
 
               # When virtual partitions are defined, partitioner needs to respond to `#call` and it

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -20,13 +20,18 @@ module Karafka
             #   the most recently reported metadata
             # @param reducer [nil, #call] reducer for VPs key. It allows for using a custom
             #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+            # @param distribution [Symbol] the strategy to use for virtual partitioning. Can be
+            #   either `:consistent` or `:balanced`. The `:balanced` strategy ensures balanced
+            #   distribution of work across available workers while maintaining message order
+            #   within groups.
             # @return [VirtualPartitions] method that allows to set the virtual partitions details
             #   during the routing configuration and then allows to retrieve it
             def virtual_partitions(
               max_partitions: Karafka::App.config.concurrency,
               partitioner: nil,
               offset_metadata_strategy: :current,
-              reducer: nil
+              reducer: nil,
+              distribution: :consistent
             )
               @virtual_partitions ||= Config.new(
                 active: !partitioner.nil?,
@@ -35,7 +40,8 @@ module Karafka
                 offset_metadata_strategy: offset_metadata_strategy,
                 # If no reducer provided, we use this one. It just runs a modulo on the sum of
                 # a stringified version, providing fairly good distribution.
-                reducer: reducer || ->(virtual_key) { virtual_key.to_s.sum % max_partitions }
+                reducer: reducer || ->(virtual_key) { virtual_key.to_s.sum % max_partitions },
+                distribution: distribution
               )
             end
 

data/lib/karafka/pro/scheduled_messages/consumer.rb

@@ -8,13 +8,27 @@ module Karafka
     module ScheduledMessages
       # Consumer that coordinates scheduling of messages when the time comes
       class Consumer < ::Karafka::BaseConsumer
+        include Helpers::ConfigImporter.new(
+          dispatcher_class: %i[scheduled_messages dispatcher_class]
+        )
+
+        # In case there is an extremely high turnover of messages, EOF may never kick in,
+        # effectively not changing status from loading to loaded. We use the time consumer instance
+        # was created + a buffer time to detect such a case (loading + messages from the time it
+        # was already running) to switch the state despite no EOF
+        # This is in seconds
+        GRACE_PERIOD = 15
+
+        private_constant :GRACE_PERIOD
+
         # Prepares the initial state of all stateful components
         def initialized
           clear!
           # Max epoch is always moving forward with the time. Never backwards, hence we do not
           # reset it at all.
           @max_epoch = MaxEpoch.new
-          @state = State.new(nil)
+          @state = State.new
+          @reloads = 0
         end
 
         # Processes messages and runs dispatch (via tick) if needed
@@ -23,11 +37,25 @@ module Karafka
 
           messages.each do |message|
             SchemaValidator.call(message)
+
+            # We always track offsets of messages, even if they would be later on skipped or
+            # ignored for any reason. That way we have debug info that is useful once in a while.
+            @tracker.offsets(message)
+
             process_message(message)
           end
 
           @states_reporter.call
 
+          recent_timestamp = messages.last.timestamp.to_i
+          post_started_timestamp = @tracker.started_at + GRACE_PERIOD
+
+          # If we started getting messages that are beyond the current time, it means we have
+          # loaded enough to start scheduling. The upcoming messages are from the future looking
+          # from perspective of the current consumer start. We add a bit of grace period not to
+          # deal with edge cases
+          loaded! if @state.loading? && recent_timestamp > post_started_timestamp
+
           eofed if eofed?
 
           # Unless given day data is fully loaded we should not dispatch any notifications nor
@@ -51,8 +79,7 @@ module Karafka
           return if reload!
 
           # If end of the partition is reached, it always means all data is loaded
-          @state.loaded!
-          @states_reporter.call
+          loaded!
         end
 
         # Performs periodic operations when no new data is provided to the topic partition
@@ -64,7 +91,6 @@ module Karafka
           return unless @state.loaded?
 
           keys = []
-          epochs = []
 
           # We first collect all the data for dispatch and then dispatch and **only** after
           # dispatch that is sync is successful we remove those messages from the daily buffer
@@ -72,35 +98,30 @@ module Karafka
           # with timeouts, etc, we need to be sure it wen through prior to deleting those messages
           # from the daily buffer. That way we ensure the at least once delivery and in case of
           # a transactional producer, exactly once delivery.
-          @daily_buffer.for_dispatch do |epoch, message|
-            epochs << epoch
+          @daily_buffer.for_dispatch do |message|
             keys << message.key
             @dispatcher << message
           end
 
           @dispatcher.flush
 
-          @max_epoch.update(epochs.max)
-
           keys.each { |key| @daily_buffer.delete(key) }
 
           @states_reporter.call
         end
 
+        # Move the state to shutdown and publish immediately
+        def shutdown
+          @state.stopped!
+          @states_reporter.call!
+        end
+
         private
 
         # Takes each message and adds it to the daily accumulator if needed or performs other
         # accumulator and time related per-message operations.
         # @param message [Karafka::Messages::Message]
         def process_message(message)
-          # If we started to receive messages younger than the moment we created the consumer for
-          # the given day, it means we have loaded all the history and we are no longer in the
-          # loading phase.
-          if message.timestamp.to_i > @today.created_at
-            @state.loaded!
-            tags.add(:state, @state.to_s)
-          end
-
           # If this is a schedule message we need to check if this is for today. Tombstone events
           # are always considered immediate as they indicate, that a message with a given key
           # was already dispatched or that user decided not to dispatch and cancelled the dispatch
@@ -109,7 +130,7 @@ module Karafka
             time = message.headers['schedule_target_epoch']
 
             # Do not track historical below today as those will be reflected in the daily buffer
-            @tracker.track(message) if time >= @today.starts_at
+            @tracker.future(message) if time >= @today.starts_at
 
             if time > @today.ends_at || time < @max_epoch.to_i
               # Clean the message immediately when not needed (won't be scheduled) to preserve
@@ -120,6 +141,14 @@ module Karafka
             end
           end
 
+          # Tombstone events are only published after we have dispatched given message. This means
+          # that we've got that far in the dispatching time. This allows us (with a certain buffer)
+          # to quickly reject older messages (older in sense of being scheduled for previous times)
+          # instead of loading them into memory until they are expired
+          if message.headers['schedule_source_type'] == 'tombstone'
+            @max_epoch.update(message.headers['schedule_target_epoch'])
+          end
+
           # Add to buffer all tombstones and messages for the same day
           @daily_buffer << message
         end
@@ -129,7 +158,8 @@ module Karafka
           # If this is a new assignment we always need to seek from beginning to load the data
           if @state.fresh?
             clear!
-            seek(0)
+            @reloads += 1
+            seek(:earliest)
 
             return true
           end
@@ -140,7 +170,8 @@ module Karafka
           # If day has ended we reload and start new day with new schedules
           if @today.ended?
             clear!
-            seek(0)
+            @reloads += 1
+            seek(:earliest)
 
             return true
           end
@@ -148,6 +179,13 @@ module Karafka
           false
         end
 
+        # Moves the state to loaded and publishes the state update
+        def loaded!
+          @state.loaded!
+          tags.add(:state, @state.to_s)
+          @states_reporter.call!
+        end
+
         # Resets all buffers and states so we can start a new day with a clean slate
         # We can fully recreate the dispatcher because any undispatched messages will be dispatched
         # with the new day dispatcher after it is reloaded.
@@ -155,22 +193,19 @@ module Karafka
           @daily_buffer = DailyBuffer.new
           @today = Day.new
           @tracker = Tracker.new
-          @state = State.new(false)
-          @dispatcher = config.dispatcher_class.new(topic.name, partition)
+          @state = State.new
+          @state.loading!
+          @dispatcher = dispatcher_class.new(topic.name, partition)
           @states_reporter = Helpers::IntervalRunner.new do
             @tracker.today = @daily_buffer.size
             @tracker.state = @state.to_s
+            @tracker.reloads = @reloads
 
             @dispatcher.state(@tracker)
           end
 
           tags.add(:state, @state.to_s)
         end
-
-        # @return [Karafka::Core::Configurable::Node] Schedules config node
-        def config
-          @config ||= Karafka::App.config.scheduled_messages
-        end
       end
     end
   end

data/lib/karafka/pro/scheduled_messages/daily_buffer.rb

@@ -45,19 +45,22 @@ module Karafka
 
         # Yields messages that should be dispatched (sent) to Kafka
         #
-        # @yieldparam [Integer, Karafka::Messages::Message] epoch of the message and the message
-        #   itself
-        #
-        # @note We yield epoch alongside of the message so we do not have to extract it several
-        #   times later on. This simplifies the API
+        # @yieldparam [Karafka::Messages::Message] messages to be dispatched sorted from the once
+        #   that are the oldest (lowest epoch)
         def for_dispatch
           dispatch = Time.now.to_i
 
+          selected = []
+
           @accu.each_value do |epoch, message|
             next unless epoch <= dispatch
 
-            yield(epoch, message)
+            selected << [epoch, message]
           end
+
+          selected
+            .sort_by!(&:first)
+            .each { |_, message| yield(message) }
         end
 
         # Removes given key from the accumulator

data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb

@@ -10,6 +10,12 @@ module Karafka
       module Deserializers
         # Converts certain pieces of headers into their integer form for messages
         class Headers
+          # We only directly operate on epoch and other details for schedules and tombstones.
+          # cancel requests don't have to be deserialized that way since they don't have epoch
+          WORKABLE_TYPES = %w[schedule tombstone].freeze
+
+          private_constant :WORKABLE_TYPES
+
           # @param metadata [Karafka::aMessages::Metadata]
           # @return [Hash] headers
           def call(metadata)
@@ -19,7 +25,7 @@ module Karafka
 
             # tombstone and cancellation events are not operable, thus we do not have to cast any
             # of the headers pieces
-            return raw_headers unless type == 'schedule'
+            return raw_headers unless WORKABLE_TYPES.include?(type)
 
             headers = raw_headers.dup
             headers['schedule_target_epoch'] = headers['schedule_target_epoch'].to_i