karafka 2.6.0.beta1 → 2.6.0.rc2

data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb

@@ -50,12 +50,13 @@ module Karafka
               #   groups that will be created)
               # @param partitioner [nil, #call] nil or callable partitioner
               # @param reducer [nil, #call] reducer for parallel key. It allows for using a custom
-              #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+              #   reducer to achieve enhanced parallelization when the default reducer is not
+              #   enough.
               # @param merge_key [String] key used to build the parallel segment consumer groups
               #
-              # @note This method is an assignor but the API is actually via the `#parallel_segments`
-              #   method. Our `Routing::Proxy` normalizes that the way we want to have it exposed
-              #   for the end users.
+              # @note This method is an assignor but the API is actually via the
+              #   `#parallel_segments` method. Our `Routing::Proxy` normalizes that the way we want
+              #   to have it exposed for the end users.
               def parallel_segments=(
                 count: 1,
                 partitioner: nil,

data/lib/karafka/pro/routing/features/consumer_groups/patterns/config.rb

@@ -33,12 +33,6 @@ module Karafka
     module Routing
       module Features
         module ConsumerGroups
-          # Holds pattern info reference
-          # Type is set to:
-          #   `:regular` - in case patterns are not used and topic is just a regular existing topic
-          #                matched directly based on the name
-          #   `:discovered` - in case it is a real topic on which we started to listed
-          #   `:matcher` - represents a regular expression used by librdkafka
           class Patterns < Base
             # Config for pattern based topic Only pattern related topics are active in this context
             Config = Struct.new(

data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb

@@ -42,7 +42,8 @@ module Karafka
                 @patterns = Patterns.new([])
               end
 
-              # @return [::Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Patterns] created patterns
+              # @return [::Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Patterns]
+              #   created patterns
               def patterns
                 @patterns
               end

data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/pattern.rb

@@ -34,7 +34,6 @@ module Karafka
       module Features
         module ConsumerGroups
           class Patterns < Base
-            # Namespace for patterns related contracts
             module Contracts
               # Contract used to validate pattern data
               class Pattern < Karafka::Contracts::Base

data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb

@@ -71,7 +71,8 @@ module Karafka
 
               # Adds the discovered topic into the routing
               #
-              # @param pattern [Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Pattern] matched pattern
+              # @param pattern [Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Pattern]
+              #   matched pattern
               # @param discovered_topic [String] topic that we discovered that should be part of the
               #   routing from now on.
               # @param sg_topics [Array<Karafka::Routing::Topic>]

data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb

@@ -100,7 +100,8 @@ module Karafka
               # prevent it from running but internally in this component we need to ensure, that
               # prior to the validations we operate on a regexp
               #
-              # @return [Regexp] returns a regexp always even if what we've received was not a regexp
+              # @return [Regexp] returns a regexp always even if what we've received was not a
+              #   regexp
               def safe_regexp
                 # This regexp will never match anything
                 regexp.is_a?(Regexp) ? regexp : /$a/

data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb

@@ -47,15 +47,15 @@ module Karafka
               #
               # @param topic_name [String] topic name
               # @return [Karafka::Routing::Topic]
-              # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
-              #   please create an issue.
+              # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see
+              #   it, please create an issue.
               #
-              # @note This method should not be used in context of finding multiple missing topics in
-              #   loops because it catches exceptions and attempts to expand routes. If this is used
-              #   in a loop for lookups on thousands of topics with detector expansion, this may
-              #   be slow. It should be used in the context where newly discovered topics are found
-              #   and should by design match a pattern. For quick lookups on batches of topics, it
-              #   is recommended to use a custom built lookup with conditional expander.
+              # @note This method should not be used in context of finding multiple missing topics
+              #   in loops because it catches exceptions and attempts to expand routes. If this is
+              #   used in a loop for lookups on thousands of topics with detector expansion, this
+              #   may be slow. It should be used in the context where newly discovered topics are
+              #   found and should by design match a pattern. For quick lookups on batches of
+              #   topics, it is recommended to use a custom built lookup with conditional expander.
               def find(topic_name)
                 super
               rescue Karafka::Errors::TopicNotFoundError

data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb

@@ -49,12 +49,12 @@ module Karafka
               # poll where messages were not received.
               # @param active [Boolean] should ticking happen for this topic assignments.
               # @param interval [Integer] minimum interval to run periodic jobs on given topic.
-              # @param during_pause [Boolean, nil] Should periodic jobs run when partition is paused.
-              #   It is set to `nil` by default allowing for detection when this value is not
-              #   configured but should be built dynamically based on LRJ status.
+              # @param during_pause [Boolean, nil] Should periodic jobs run when partition is
+              #   paused. It is set to `nil` by default allowing for detection when this value is
+              #   not configured but should be built dynamically based on LRJ status.
               # @param during_retry [Boolean, nil] Should we run when there was an error and we are
-              #   in a retry flow. Please note that for this to work, `during_pause` also needs to be
-              #   set to true as errors retry happens after pause.
+              #   in a retry flow. Please note that for this to work, `during_pause` also needs to
+              #   be set to true as errors retry happens after pause.
               def periodic_job(
                 active = false,
                 interval: nil,

data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb

@@ -52,6 +52,39 @@ module Karafka
                 tasks_cfg = App.config.recurring_tasks
                 topics_cfg = tasks_cfg.topics
 
+                # Declarative topic definitions live independently of routing. We declare the
+                # broker-side structure here so it is managed by the CLI topics commands without
+                # relying on the (to be retired) routing `config(...)` bridge.
+                #
+                # We guard each declaration with `find_topic` because `Declaratives::Builder#topic`
+                # re-applies its block on an already existing declaration (additive semantics),
+                # unlike the old bridge which used `||=` and kept the first declaration. Without the
+                # guard, our framework defaults would clobber a topic the user may have declared
+                # themselves via `App.declaratives.draw`. Skipping when already present preserves
+                # first-declaration-wins.
+                App.declaratives.draw do
+                  # Keep older data for a day and compact to the last state available
+                  unless find_topic(topics_cfg.schedules.name)
+                    topic(topics_cfg.schedules.name) do
+                      config(
+                        "cleanup.policy": "compact,delete",
+                        "retention.ms": 86_400_000
+                      )
+                    end
+                  end
+
+                  # Keep cron logs of executions for a week and after that remove. Week should be
+                  # enough and should not produce too much data.
+                  unless find_topic(topics_cfg.logs.name)
+                    topic(topics_cfg.logs.name) do
+                      config(
+                        "cleanup.policy": "delete",
+                        "retention.ms": 604_800_000
+                      )
+                    end
+                  end
+                end
+
                 consumer_group tasks_cfg.group_id do
                   # Registers the primary topic that we use to control schedules execution. This is
                   # the one that we use to trigger recurring tasks.
@@ -84,14 +117,7 @@ module Karafka
                     # Since the execution of particular tasks is isolated and guarded, it should not
                     # leak. This means, that this is to handle errors like schedule version
                     # incompatibility and other errors that will not go away without a redeployment
-                    pause_timeout(60 * 1_000)
-                    pause_max_timeout(60 * 1_000)
-
-                    # Keep older data for a day and compact to the last state available
-                    config(
-                      "cleanup.policy": "compact,delete",
-                      "retention.ms": 86_400_000
-                    )
+                    pause(timeout: 60 * 1_000, max_timeout: 60 * 1_000)
 
                     # This is the core of execution. Since we're producers of states, we need a way
                     # to tick without having new data
@@ -113,13 +139,6 @@ module Karafka
                     active(false)
                     deserializer tasks_cfg.deserializer
                     target.recurring_tasks(true)
-
-                    # Keep cron logs of executions for a week and after that remove. Week should be
-                    # enough and should not produce too much data.
-                    config(
-                      "cleanup.policy": "delete",
-                      "retention.ms": 604_800_000
-                    )
                   end
 
                   yield(schedules_topic, logs_topic) if block && block.arity.positive?

data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb

@@ -38,9 +38,9 @@ module Karafka
             module Builder
               # Enabled scheduled messages operations and adds needed topics and other stuff.
               #
-              # @param topic_name [String, false] name for scheduled messages topic that is also used
-              #   as a group identifier. Users can have multiple schedule topics flows to prevent key
-              #   collisions, prioritize and do other stuff. `false` if not active.
+              # @param topic_name [String, false] name for scheduled messages topic that is also
+              #   used as a group identifier. Users can have multiple schedule topics flows to
+              #   prevent key collisions, prioritize and do other stuff. `false` if not active.
               # @param block [Proc] optional reconfiguration of the topics definitions.
               # @note Namespace for topics should include the divider as it is not automatically
               #   added.
@@ -54,6 +54,54 @@ module Karafka
                 # should have same partition count.
                 default_partitions = 5
                 msg_cfg = App.config.scheduled_messages
+                states_topic_name = "#{topic_name}#{msg_cfg.states_postfix}"
+
+                # Declarative topic definitions live independently of routing. We declare the
+                # broker-side structure here so it is managed by the CLI topics commands without
+                # relying on the (to be retired) routing `config(...)` bridge.
+                #
+                # We guard each declaration with `find_topic` because `Declaratives::Builder#topic`
+                # re-applies its block on an already existing declaration (additive semantics),
+                # unlike the old bridge which used `||=` and kept the first declaration. Without the
+                # guard, our framework defaults would clobber a topic the user may have declared
+                # themselves via `App.declaratives.draw`. Skipping when already present preserves
+                # first-declaration-wins.
+                App.declaratives.draw do
+                  # This is a setup that should allow messages to be compacted fairly fast. Since
+                  # each dispatched message should be removed via tombstone, they do not have to
+                  # be present in the topic for too long.
+                  unless find_topic(topic_name)
+                    topic(topic_name) do
+                      partitions default_partitions
+                      config(
+                        # Will ensure, that after tombstone is present, given scheduled message,
+                        # that has been dispatched is removed by Kafka
+                        "cleanup.policy": "compact",
+                        # When 10% or more dispatches are done, compact data
+                        "min.cleanable.dirty.ratio": 0.1,
+                        # Frequent segment rotation to support intense compaction
+                        "segment.ms": 3_600_000,
+                        "delete.retention.ms": 3_600_000,
+                        "segment.bytes": 52_428_800
+                      )
+                    end
+                  end
+
+                  # Holds states of scheduler per each of the partitions. Same partition count as
+                  # the messages topic since they tick independently per partition.
+                  unless find_topic(states_topic_name)
+                    topic(states_topic_name) do
+                      partitions default_partitions
+                      config(
+                        "cleanup.policy": "compact",
+                        "min.cleanable.dirty.ratio": 0.1,
+                        "segment.ms": 3_600_000,
+                        "delete.retention.ms": 3_600_000,
+                        "segment.bytes": 52_428_800
+                      )
+                    end
+                  end
+                end
 
                 consumer_group msg_cfg.group_id do
                   # Registers the primary topic that we use to control schedules execution. This is
@@ -93,22 +141,6 @@ module Karafka
 
                     max_wait_time(1_000)
 
-                    # This is a setup that should allow messages to be compacted fairly fast. Since
-                    # each dispatched message should be removed via tombstone, they do not have to
-                    # be present in the topic for too long.
-                    config(
-                      partitions: default_partitions,
-                      # Will ensure, that after tombstone is present, given scheduled message, that
-                      # has been dispatched is removed by Kafka
-                      "cleanup.policy": "compact",
-                      # When 10% or more dispatches are done, compact data
-                      "min.cleanable.dirty.ratio": 0.1,
-                      # Frequent segment rotation to support intense compaction
-                      "segment.ms": 3_600_000,
-                      "delete.retention.ms": 3_600_000,
-                      "segment.bytes": 52_428_800
-                    )
-
                     # This is the core of execution. Since we dispatch data in time intervals, we
                     # need to be able to do this even when no new data is coming
                     periodic(
@@ -126,17 +158,9 @@ module Karafka
                   # Holds states of scheduler per each of the partitions since they tick
                   # independently. We only hold future statistics not to have to deal with
                   # any type of state restoration
-                  states_topic = topic("#{topic_name}#{msg_cfg.states_postfix}") do
+                  states_topic = topic(states_topic_name) do
                     active(false)
                     target.scheduled_messages(true)
-                    config(
-                      partitions: default_partitions,
-                      "cleanup.policy": "compact",
-                      "min.cleanable.dirty.ratio": 0.1,
-                      "segment.ms": 3_600_000,
-                      "delete.retention.ms": 3_600_000,
-                      "segment.bytes": 52_428_800
-                    )
                     deserializers(
                       payload: msg_cfg.deserializers.payload
                     )

data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb

@@ -46,7 +46,8 @@ module Karafka
                 super
               end
 
-              # @param active [Boolean] should this topic be considered related to scheduled messages
+              # @param active [Boolean] should this topic be considered related to scheduled
+              #   messages
               def scheduled_messages(active = false)
                 @scheduled_messages ||= Config.new(active: active)
               end

data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb

@@ -45,16 +45,17 @@ module Karafka
                 super
               end
 
-              # @param max_partitions [Integer] max number of virtual partitions that can come out of
-              #   the single distribution flow. When set to more than the Karafka threading, will
+              # @param max_partitions [Integer] max number of virtual partitions that can come out
+              #   of the single distribution flow. When set to more than the Karafka threading, will
               #   create more work than workers. When less, can ensure we have spare resources to
               #   process other things in parallel.
               # @param partitioner [nil, #call] nil or callable partitioner
               # @param offset_metadata_strategy [Symbol] how we should match the metadata for the
-              #   offset. `:exact` will match the offset matching metadata and `:current` will select
-              #   the most recently reported metadata
+              #   offset. `:exact` will match the offset matching metadata and `:current` will
+              #   select the most recently reported metadata
               # @param reducer [nil, #call] reducer for VPs key. It allows for using a custom
-              #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+              #   reducer to achieve enhanced parallelization when the default reducer is not
+              #   enough.
               # @param distribution [Symbol] the strategy to use for virtual partitioning. Can be
               #   either `:consistent` or `:balanced`. The `:balanced` strategy ensures balanced
               #   distribution of work across available workers while maintaining message order

data/lib/karafka/pro/routing/features/pausing/config.rb

@@ -33,17 +33,13 @@ module Karafka
     module Routing
       module Features
         class Pausing < Base
-          # Config for pausing feature
-          Config = Struct.new(
-            :active,
-            :timeout,
-            :max_timeout,
-            :with_exponential_backoff,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
-            alias_method :with_exponential_backoff?, :with_exponential_backoff
-          end
+          # Config for pausing feature.
+          #
+          # By convention each routing feature exposes a `Config`. The pause value object is defined
+          # in OSS (`Karafka::Routing::Features::Pausing::Config`) because the per-topic `#pause`
+          # reader defaults to the global config there, so the feature points its `Config` at that
+          # shared class rather than redefining it.
+          Config = Karafka::Routing::Features::Pausing::Config
         end
       end
     end

data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb

@@ -42,10 +42,8 @@ module Karafka
                   File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
                 ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
 
-                # Validate the nested pausing configuration
-                # Both old setters and new pause() method update the pausing config,
-                # so we only need to validate this one format
-                nested :pausing do
+                # Validate the nested pause configuration
+                nested :pause do
                   required(:active) { |val| [true, false].include?(val) }
                   required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
                   required(:max_timeout) { |val| val.is_a?(Integer) && val.positive? }
@@ -56,13 +54,13 @@ module Karafka
                 virtual do |data, errors|
                   next unless errors.empty?
 
-                  pausing = data.fetch(:pausing)
-                  timeout = pausing.fetch(:timeout)
-                  max_timeout = pausing.fetch(:max_timeout)
+                  pause = data.fetch(:pause)
+                  timeout = pause.fetch(:timeout)
+                  max_timeout = pause.fetch(:max_timeout)
 
                   next if timeout <= max_timeout
 
-                  [[%i[pausing timeout], :max_timeout_vs_pause_max_timeout]]
+                  [[%i[pause timeout], :max_timeout_vs_pause_max_timeout]]
                 end
               end
             end

data/lib/karafka/pro/routing/features/pausing/topic.rb

@@ -35,84 +35,39 @@ module Karafka
         class Pausing < Base
           # Expansion allowing for a per topic pause strategy definitions
           module Topic
-            # This method sets up the extra instance variable to nil before calling
-            # the parent class initializer. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              @pausing = nil
-              super
-            end
-
-            # Allows for per-topic pausing strategy setting
+            # Allows for per-topic pausing strategy setting.
+            #
+            # Overrides the OSS `#pause` reader (this module is prepended onto `Routing::Topic`).
+            # With no arguments it returns the current configuration, defaulting to the global
+            # `config.pause.*` settings via `super`. With arguments it overrides the pausing
+            # strategy for this topic and marks it as active.
             #
             # @param timeout [Integer] how long should we wait upon processing error (milliseconds)
             # @param max_timeout [Integer] what is the max timeout in case of an exponential
             #   backoff (milliseconds)
             # @param with_exponential_backoff [Boolean] should we use exponential backoff
-            # @return [Config] pausing config object
+            # @return [Karafka::Routing::Features::Pausing::Config] pausing config object
             def pause(timeout: nil, max_timeout: nil, with_exponential_backoff: nil)
-              # If no arguments provided, just return or initialize the config
-              return pausing if timeout.nil? && max_timeout.nil? && with_exponential_backoff.nil?
+              config = super()
 
-              # Update instance variables for backwards compatibility
-              # This ensures code reading @pause_timeout directly or via the inherited getter
-              # will get the correct values
-              @pause_timeout = timeout if timeout
-              @pause_max_timeout = max_timeout if max_timeout
-
-              unless with_exponential_backoff.nil?
-                @pause_with_exponential_backoff = with_exponential_backoff
-              end
+              # If no arguments provided, just return the current (default or overridden) config
+              return config if timeout.nil? && max_timeout.nil? && with_exponential_backoff.nil?
 
-              # Create or update the config
-              @pausing ||= Config.new(
-                active: false,
-                timeout: @pause_timeout || Karafka::App.config.pause.timeout,
-                max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
-                with_exponential_backoff: if @pause_with_exponential_backoff.nil?
-                                            Karafka::App.config.pause.with_exponential_backoff
-                                          else
-                                            @pause_with_exponential_backoff
-                                          end
-              )
-
-              @pausing.timeout = timeout if timeout
-              @pausing.max_timeout = max_timeout if max_timeout
+              config.timeout = timeout if timeout
+              config.max_timeout = max_timeout if max_timeout
 
               unless with_exponential_backoff.nil?
-                @pausing.with_exponential_backoff = with_exponential_backoff
+                config.with_exponential_backoff = with_exponential_backoff
               end
 
-              @pausing.active = true
-
-              @pausing
-            end
-
-            # @return [Config] pausing configuration object
-            def pausing
-              @pausing ||= Config.new(
-                active: false,
-                timeout: @pause_timeout || Karafka::App.config.pause.timeout,
-                max_timeout: @pause_max_timeout || Karafka::App.config.pause.max_timeout,
-                with_exponential_backoff: if @pause_with_exponential_backoff.nil?
-                                            Karafka::App.config.pause.with_exponential_backoff
-                                          else
-                                            @pause_with_exponential_backoff
-                                          end
-              )
-            end
+              config.active = true
 
-            # @return [Boolean] is pausing explicitly configured
-            def pausing?
-              pausing.active?
+              config
             end
 
-            # @return [Hash] topic with all its native configuration options plus pausing settings
-            def to_h
-              super.merge(
-                pausing: pausing.to_h
-              ).freeze
+            # @return [Boolean] is pausing explicitly configured on a per-topic basis
+            def pause?
+              pause.active?
             end
           end
         end

data/lib/karafka/pro/scheduled_messages/consumer.rb

@@ -72,7 +72,13 @@ module Karafka
 
           @states_reporter.call
 
-          recent_timestamp = messages.last.timestamp.to_i
+          # Clamp to the wall clock: a future-dated message timestamp (producer clock skew or an
+          # explicitly set future timestamp) must not be treated as proof of catch-up. Otherwise a
+          # single future-dated message read mid-backlog would flip `loaded!` while older tombstones
+          # at higher offsets are still unread, dispatching schedules whose cancelling tombstones
+          # have not been loaded yet (double dispatch). Mirrors the `[timestamp, now].min` clamp in
+          # the batch metadata builder.
+          recent_timestamp = [messages.last.timestamp.to_i, Time.now.to_i].min
           post_started_timestamp = @tracker.started_at + GRACE_PERIOD
 
           # If we started getting messages that are beyond the current time, it means we have

data/lib/karafka/pro/scheduled_messages/schema_validator.rb

@@ -43,7 +43,12 @@ module Karafka
           def call(message)
             message_version = message.headers["schedule_schema_version"]
 
-            return if message_version <= ScheduledMessages::SCHEMA_VERSION
+            # Versions are semver, so they need to be compared segment-wise and not as strings
+            # ("1.2.0" is older than "1.10.0" but lexicographically greater)
+            message_gem_version = Gem::Version.new(message_version)
+            current_gem_version = Gem::Version.new(ScheduledMessages::SCHEMA_VERSION)
+
+            return if message_gem_version <= current_gem_version
 
             raise Errors::IncompatibleSchemaError, message_version
           end