karafka 2.5.0.rc2 → 2.5.1.beta1

data/config/locales/pro_errors.yml

@@ -1,105 +1,108 @@
 en:
   validations:
-    topic:
-      virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
-      virtual_partitions.max_partitions_format: needs to be equal or more than 1
-      virtual_partitions.offset_metadata_strategy_format: needs to be either :exact or :current
-      virtual_partitions.reducer_format: "needs to respond to `#call`"
-      virtual_partitions.distribution_format: "needs to be either :consistent or :balanced"
-
-      long_running_job.active_format: needs to be either true or false
-
-      dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.
-      dead_letter_queue.strategy_format: 'needs to respond to #call'
-      dead_letter_queue.strategy_missing: needs to be present
-
-      throttling.active_format: needs to be either true or false
-      throttling.limit_format: needs to be equal or more than 1
-      throttling.interval_format: needs to be equal or more than 1
-
-      filtering.active_missing: needs to be present
-      filtering.factory_format: 'needs to respond to #call'
-      filtering.factories_format: 'needs to contain only factories responding to #call'
-      filtering.active_format: 'needs to be boolean'
-
-      expiring.ttl_format: 'needs to be equal or more than 0 and an integer'
-      expiring.active_format: 'needs to be boolean'
-
-      delaying.delay_format: 'needs to be equal or more than 0 and an integer'
-      delaying.active_format: 'needs to be boolean'
-
-      pause_timeout_format: needs to be an integer bigger than 0
-      pause_max_timeout_format: needs to be an integer bigger than 0
-      pause_with_exponential_backoff_format: needs to be either true or false
-      pause_timeout_max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
-
-      patterns.active_format: 'needs to be boolean'
-      patterns.type_format: 'needs to be :matcher, :discovered or :regular'
-
-      periodic_job.active_missing: needs to be present
-      periodic_job.active_format: 'needs to be boolean'
-      periodic_job.interval_missing: 'needs to be present'
-      periodic_job.interval_format: 'needs to be an integer equal or more than 100'
-      periodic_job.during_pause_format: 'needs to be boolean'
-      periodic_job.during_retry_format: 'needs to be boolean'
-      periodic_job.materialized_format: 'needs to be boolean'
-      periodic_job.materialized_missing: 'needs to be present'
-
-      inline_insights.active_format: 'needs to be boolean'
-      inline_insights.required_format: 'needs to be boolean'
-
-      offset_metadata.active_format: 'needs to be boolean'
-      offset_metadata.cache_format: 'needs to be boolean'
-      offset_metadata.deserializer_missing: needs to be present
-      offset_metadata.deserializer_format: 'needs to respond to #call'
-
-      subscription_group_details.multiplexing_min_format: 'needs to be an integer equal or more than 1'
-      subscription_group_details.multiplexing_max_format: 'needs to be an integer equal or more than 1'
-      subscription_group_details_multiplexing_min_max_mismatch: 'min needs to be equal or less than max'
-      subscription_group_details_multiplexing_boot_mismatch: 'boot needs to be between min and max'
-      subscription_group_details.multiplexing_boot_format: 'needs to be an integer equal or more than 1'
-      subscription_group_details.multiplexing_boot_not_dynamic: 'needs to be equal to max when not in dynamic mode'
-      subscription_group_details_multiplexing_one_not_enough: 'min and max cannot equal 1'
-      subscription_group_details.multiplexing_scale_delay_format: 'needs to be an integer equal or more than 1000'
-
-      swarm.active_format: needs to be true
-      swarm.nodes_format: needs to be a range, array of nodes ids or a hash with direct assignments
-      swarm_nodes_with_non_existent_nodes: includes unreachable nodes ids
-
-      recurring_tasks.active_format: 'needs to be boolean'
-      scheduled_messages.active_format: 'needs to be boolean'
-      scheduled_messages.active_missing: 'needs to be boolean'
-
-      direct_assignments.active_missing: needs to be present
-      direct_assignments.active_format: 'needs to be boolean'
-      direct_assignments.partitions_missing: 'needs to be present'
-      direct_assignments.partitions_format: 'needs to be true, list of partitions or a range of partitions (finite)'
-      direct_assignments_active_but_empty: 'cannot be empty and active at the same time'
-      direct_assignments_swarm_not_complete: 'cannot have partitions that are assigned but not allocated'
-      direct_assignments_swarm_overbooked: 'cannot allocate partitions in swarm that were not assigned'
-      direct_assignments_patterns_active: 'patterns cannot be used with direct assignments'
-
-      adaptive_iterator.active_missing: needs to be present
-      adaptive_iterator.active_format: 'needs to be boolean'
-      adaptive_iterator.marking_method_format: 'needs to be either #mark_as_consumed or #mark_as_consumed!'
-      adaptive_iterator.clean_after_yielding_format: 'needs to be boolean'
-      adaptive_iterator.safety_margin_format: 'needs to be between 1 and 99'
-      adaptive_iterator_with_virtual_partitions: 'cannot be used with virtual partitions'
-      adaptive_iterator_with_long_running_job: 'cannot be used with long running jobs'
-
-    consumer_group:
-      patterns_format: must be an array with hashes
-      patterns_missing: needs to be present
-      patterns_regexps_not_unique: 'must be unique within consumer group'
-
-      direct_assignments_homogenous: 'single consumer group cannot mix regular and direct assignments'
-
-      parallel_segments.partitioner_format: needs to be defined and needs to respond to `#call`
-      parallel_segments.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
-      parallel_segments.count_format: needs to be equal or more than 1
-      parallel_segments.active_format: needs to be boolean
-      parallel_segments.reducer_format: "needs to respond to `#call`"
-      parallel_segments.merge_key_format: "needs to be a non-empty string"
+    routing:
+      swarm_nodes_not_used: 'At least one of the nodes has no assignments'
+
+      topic:
+        virtual_partitions.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+        virtual_partitions.max_partitions_format: needs to be equal or more than 1
+        virtual_partitions.offset_metadata_strategy_format: needs to be either :exact or :current
+        virtual_partitions.reducer_format: "needs to respond to `#call`"
+        virtual_partitions.distribution_format: "needs to be either :consistent or :balanced"
+
+        long_running_job.active_format: needs to be either true or false
+
+        dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.
+        dead_letter_queue.strategy_format: 'needs to respond to #call'
+        dead_letter_queue.strategy_missing: needs to be present
+
+        throttling.active_format: needs to be either true or false
+        throttling.limit_format: needs to be equal or more than 1
+        throttling.interval_format: needs to be equal or more than 1
+
+        filtering.active_missing: needs to be present
+        filtering.factory_format: 'needs to respond to #call'
+        filtering.factories_format: 'needs to contain only factories responding to #call'
+        filtering.active_format: 'needs to be boolean'
+
+        expiring.ttl_format: 'needs to be equal or more than 0 and an integer'
+        expiring.active_format: 'needs to be boolean'
+
+        delaying.delay_format: 'needs to be equal or more than 0 and an integer'
+        delaying.active_format: 'needs to be boolean'
+
+        pause_timeout_format: needs to be an integer bigger than 0
+        pause_max_timeout_format: needs to be an integer bigger than 0
+        pause_with_exponential_backoff_format: needs to be either true or false
+        pause_timeout_max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
+
+        patterns.active_format: 'needs to be boolean'
+        patterns.type_format: 'needs to be :matcher, :discovered or :regular'
+
+        periodic_job.active_missing: needs to be present
+        periodic_job.active_format: 'needs to be boolean'
+        periodic_job.interval_missing: 'needs to be present'
+        periodic_job.interval_format: 'needs to be an integer equal or more than 100'
+        periodic_job.during_pause_format: 'needs to be boolean'
+        periodic_job.during_retry_format: 'needs to be boolean'
+        periodic_job.materialized_format: 'needs to be boolean'
+        periodic_job.materialized_missing: 'needs to be present'
+
+        inline_insights.active_format: 'needs to be boolean'
+        inline_insights.required_format: 'needs to be boolean'
+
+        offset_metadata.active_format: 'needs to be boolean'
+        offset_metadata.cache_format: 'needs to be boolean'
+        offset_metadata.deserializer_missing: needs to be present
+        offset_metadata.deserializer_format: 'needs to respond to #call'
+
+        subscription_group_details.multiplexing_min_format: 'needs to be an integer equal or more than 1'
+        subscription_group_details.multiplexing_max_format: 'needs to be an integer equal or more than 1'
+        subscription_group_details_multiplexing_min_max_mismatch: 'min needs to be equal or less than max'
+        subscription_group_details_multiplexing_boot_mismatch: 'boot needs to be between min and max'
+        subscription_group_details.multiplexing_boot_format: 'needs to be an integer equal or more than 1'
+        subscription_group_details.multiplexing_boot_not_dynamic: 'needs to be equal to max when not in dynamic mode'
+        subscription_group_details_multiplexing_one_not_enough: 'min and max cannot equal 1'
+        subscription_group_details.multiplexing_scale_delay_format: 'needs to be an integer equal or more than 1000'
+
+        swarm.active_format: needs to be true
+        swarm.nodes_format: needs to be a range, array of nodes ids or a hash with direct assignments
+        swarm_nodes_with_non_existent_nodes: includes unreachable nodes ids
+
+        recurring_tasks.active_format: 'needs to be boolean'
+        scheduled_messages.active_format: 'needs to be boolean'
+        scheduled_messages.active_missing: 'needs to be boolean'
+
+        direct_assignments.active_missing: needs to be present
+        direct_assignments.active_format: 'needs to be boolean'
+        direct_assignments.partitions_missing: 'needs to be present'
+        direct_assignments.partitions_format: 'needs to be true, list of partitions or a range of partitions (finite)'
+        direct_assignments_active_but_empty: 'cannot be empty and active at the same time'
+        direct_assignments_swarm_not_complete: 'cannot have partitions that are assigned but not allocated'
+        direct_assignments_swarm_overbooked: 'cannot allocate partitions in swarm that were not assigned'
+        direct_assignments_patterns_active: 'patterns cannot be used with direct assignments'
+
+        adaptive_iterator.active_missing: needs to be present
+        adaptive_iterator.active_format: 'needs to be boolean'
+        adaptive_iterator.marking_method_format: 'needs to be either #mark_as_consumed or #mark_as_consumed!'
+        adaptive_iterator.clean_after_yielding_format: 'needs to be boolean'
+        adaptive_iterator.safety_margin_format: 'needs to be between 1 and 99'
+        adaptive_iterator_with_virtual_partitions: 'cannot be used with virtual partitions'
+        adaptive_iterator_with_long_running_job: 'cannot be used with long running jobs'
+
+      consumer_group:
+        patterns_format: must be an array with hashes
+        patterns_missing: needs to be present
+        patterns_regexps_not_unique: 'must be unique within consumer group'
+
+        direct_assignments_homogenous: 'single consumer group cannot mix regular and direct assignments'
+
+        parallel_segments.partitioner_format: needs to be defined and needs to respond to `#call`
+        parallel_segments.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+        parallel_segments.count_format: needs to be equal or more than 1
+        parallel_segments.active_format: needs to be boolean
+        parallel_segments.reducer_format: "needs to respond to `#call`"
+        parallel_segments.merge_key_format: "needs to be a non-empty string"
 
     pattern:
       regexp_format: must be a regular expression
@@ -107,40 +110,38 @@ en:
       regexp_string_format: 'needs to be a string and start with ^'
       missing: needs to be present
 
-    config:
-      encryption.active_format: 'needs to be either true or false'
-      encryption.public_key_invalid: 'is not a valid public RSA key'
-      encryption.public_key_needs_to_be_public: 'is a private RSA key not a public one'
-      encryption.private_keys_format: 'needs to be a hash of version and private key value'
-      encryption.private_keys_need_to_be_private: 'all keys need to be private'
-      encryption.version_format: must be a non-empty string
-      encryption.public_key_format: 'is not a valid public RSA key'
-      encryption.private_keys_invalid: 'contains an invalid private RSA key string'
-      encryption.fingerprinter_missing: 'needs to be false or respond to #hexdigest method'
-      encryption.fingerprinter_format: 'needs to be false or respond to #hexdigest method'
-
-      patterns.ttl_format: needs to be an integer bigger than 0
-      patterns.ttl_missing: needs to be present
-
-      recurring_tasks.consumer_class_format: 'needs to inherit from Karafka::BaseConsumer'
-      recurring_tasks.group_id_format: 'needs to be a string with a Kafka accepted format'
-      recurring_tasks.topics.schedules.name_format: 'needs to be a string with a Kafka accepted format'
-      recurring_tasks.topics.logs.name_format: 'needs to be a string with a Kafka accepted format'
-      recurring_tasks.interval_format: 'needs to be equal or more than 1000 and an integer'
-      recurring_tasks.deserializer_format: 'needs to be configured'
-      recurring_tasks.logging_format: needs to be a boolean
-
-      scheduled_messages.consumer_class_format: 'must be a class'
-      scheduled_messages.dispatcher_class_format: 'must be a class'
-      scheduled_messages.flush_batch_size_format: needs to be an integer bigger than 0
-      scheduled_messages.interval_format: needs to be an integer bigger or equal to 1000
-      scheduled_messages.deserializers.headers_format: cannot be nil
-      scheduled_messages.deserializers.payload_format: cannot be nil
-      scheduled_messages.group_id_format: 'needs to be a string with a Kafka accepted format'
-      scheduled_messages.states_postfix_format: 'needs to be a string with a Kafka accepted format'
-
-    routing:
-      swarm_nodes_not_used: 'At least one of the nodes has no assignments'
+    setup:
+      config:
+        encryption.active_format: 'needs to be either true or false'
+        encryption.public_key_invalid: 'is not a valid public RSA key'
+        encryption.public_key_needs_to_be_public: 'is a private RSA key not a public one'
+        encryption.private_keys_format: 'needs to be a hash of version and private key value'
+        encryption.private_keys_need_to_be_private: 'all keys need to be private'
+        encryption.version_format: must be a non-empty string
+        encryption.public_key_format: 'is not a valid public RSA key'
+        encryption.private_keys_invalid: 'contains an invalid private RSA key string'
+        encryption.fingerprinter_missing: 'needs to be false or respond to #hexdigest method'
+        encryption.fingerprinter_format: 'needs to be false or respond to #hexdigest method'
+
+        patterns.ttl_format: needs to be an integer bigger than 0
+        patterns.ttl_missing: needs to be present
+
+        recurring_tasks.consumer_class_format: 'needs to inherit from Karafka::BaseConsumer'
+        recurring_tasks.group_id_format: 'needs to be a string with a Kafka accepted format'
+        recurring_tasks.topics.schedules.name_format: 'needs to be a string with a Kafka accepted format'
+        recurring_tasks.topics.logs.name_format: 'needs to be a string with a Kafka accepted format'
+        recurring_tasks.interval_format: 'needs to be equal or more than 1000 and an integer'
+        recurring_tasks.deserializer_format: 'needs to be configured'
+        recurring_tasks.logging_format: needs to be a boolean
+
+        scheduled_messages.consumer_class_format: 'must be a class'
+        scheduled_messages.dispatcher_class_format: 'must be a class'
+        scheduled_messages.flush_batch_size_format: needs to be an integer bigger than 0
+        scheduled_messages.interval_format: needs to be an integer bigger or equal to 1000
+        scheduled_messages.deserializers.headers_format: cannot be nil
+        scheduled_messages.deserializers.payload_format: cannot be nil
+        scheduled_messages.group_id_format: 'needs to be a string with a Kafka accepted format'
+        scheduled_messages.states_postfix_format: 'needs to be a string with a Kafka accepted format'
 
     recurring_tasks:
       id_format: 'can include only alphanumeric characters (a-z, A-Z, 0-9), hyphens (-), and underscores (_)'

data/karafka.gemspec

@@ -22,12 +22,12 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'base64', '~> 0.2'
-  spec.add_dependency 'karafka-core', '>= 2.5.2', '< 2.6.0'
-  spec.add_dependency 'karafka-rdkafka', '>= 0.19.5'
+  spec.add_dependency 'karafka-core', '>= 2.5.6', '< 2.6.0'
+  spec.add_dependency 'karafka-rdkafka', '>= 0.21.0'
   spec.add_dependency 'waterdrop', '>= 2.8.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
-  spec.required_ruby_version = '>= 3.0.0'
+  spec.required_ruby_version = '>= 3.1.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = %w[karafka]

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -4,9 +4,33 @@
 module ActiveJob
   # ActiveJob queue adapters
   module QueueAdapters
+    # Determine the appropriate base class for the Karafka adapter.
+    #
+    # This complex inheritance logic addresses a Rails 7.1 compatibility issue where
+    # ActiveJob::QueueAdapters::AbstractAdapter is not properly autoloaded during
+    # early initialization phases, causing "uninitialized constant" errors.
+    #
+    # The issue occurs because:
+    # 1. AbstractAdapter is autoloaded, not directly required in Rails 7+
+    # 2. Rails 7.1 has specific timing issues during the boot process
+    # 3. Queue adapters may be loaded before Rails completes initialization
+    #
+    # Inheritance strategy:
+    # - Rails 7.1: Inherit from Object (avoids AbstractAdapter autoloading issues)
+    # - Other Rails versions: Inherit from AbstractAdapter (normal behavior)
+    # - No Rails: Inherit from Object (standalone ActiveJob usage)
+    #
+    # @see https://github.com/sidekiq/sidekiq/issues/6746 Similar issue in Sidekiq
+    base = if defined?(Rails) && defined?(Rails::VERSION)
+             (Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR < 2 ? Object : AbstractAdapter)
+           else
+             # Fallback when Rails is not loaded
+             Object
+           end
+
     # Karafka adapter for enqueuing jobs
     # This is here for ease of integration with ActiveJob.
-    class KarafkaAdapter
+    class KarafkaAdapter < base
       include Karafka::Helpers::ConfigImporter.new(
         dispatcher: %i[internal active_job dispatcher]
       )
@@ -40,6 +64,11 @@ module ActiveJob
       def enqueue_after_transaction_commit?
         true
       end
+
+      # @return [Boolean] should we stop the job. Used by the ActiveJob continuation feature
+      def stopping?
+        Karafka::App.done?
+      end
     end
   end
 end

data/lib/karafka/active_job/dispatcher.rb

@@ -46,17 +46,27 @@ module Karafka
         end
       end
 
-      # Raises info, that Karafka backend does not support scheduling jobs
+      # Raises info, that Karafka backend does not support scheduling jobs if someone wants to
+      # schedule jobs in the future. It works for past and present because we want to support
+      # things like continuation and `#retry_on` API with no wait and no jitter
       #
-      # @param _job [Object] job we cannot enqueue
-      # @param _timestamp [Time] time when job should run
+      # @param job [Object] job we cannot enqueue
+      # @param timestamp [Time] time when job should run
       #
-      # @note Karafka Pro supports this feature
-      def dispatch_at(_job, _timestamp)
-        raise NotImplementedError, <<~ERROR_MESSAGE
-          This queueing backend does not support scheduling jobs.
-          Consider using Karafka Pro, which supports this via the Scheduled Messages feature.
-        ERROR_MESSAGE
+      # @note Karafka Pro supports future jobs
+      #
+      # @note In order for jobs to work with this you need to set jitter to false and no wait
+      def dispatch_at(job, timestamp)
+        # Dispatch at is used by some of the ActiveJob features that actually do not back-off
+        # but things go via this API nonetheless.
+        if timestamp.to_f <= Time.now.to_f
+          dispatch(job)
+        else
+          raise NotImplementedError, <<~ERROR_MESSAGE
+            This queueing backend does not support scheduling future jobs.
+            Consider using Karafka Pro, which supports this via the Scheduled Messages feature.
+          ERROR_MESSAGE
+        end
       end
 
       private

data/lib/karafka/admin/acl.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Admin
+  class Admin
     # Struct and set of operations for ACLs management that simplifies their usage.
     # It allows to use Ruby symbol based definitions instead of usage of librdkafka types
     # (it allows to use rdkafka numerical types as well out of the box)
@@ -10,11 +10,7 @@ module Karafka
     #
     # This API works based on ability to create a `Karafka:Admin::Acl` object that can be then used
     # using `#create`, `#delete` and `#describe` class API.
-    class Acl
-      extend Helpers::ConfigImporter.new(
-        max_wait_time: %i[admin max_wait_time]
-      )
-
+    class Acl < Admin
       # Types of resources for which we can assign permissions.
       #
       # Resource refers to any entity within the Kafka ecosystem for which access control can be
@@ -31,7 +27,9 @@ module Karafka
         # use when you want to assign acl to a given consumer group
         consumer_group: Rdkafka::Bindings::RD_KAFKA_RESOURCE_GROUP,
         # use when you want to assign acl to a given broker
-        broker: Rdkafka::Bindings::RD_KAFKA_RESOURCE_BROKER
+        broker: Rdkafka::Bindings::RD_KAFKA_RESOURCE_BROKER,
+        # use when you want to assign acl to a transactional id
+        transactional_id: Rdkafka::Bindings::RD_KAFKA_RESOURCE_TRANSACTIONAL_ID
       }.freeze
 
       # Resource pattern types define how ACLs (Access Control Lists) are applied to resources,
@@ -165,7 +163,7 @@ module Karafka
         # Yields admin instance, allows to run Acl operations and awaits on the final result
         # Makes sure that admin is closed afterwards.
         def with_admin_wait
-          Admin.with_admin do |admin|
+          with_admin do |admin|
             yield(admin).wait(max_wait_timeout: max_wait_time)
           end
         end
@@ -229,6 +227,7 @@ module Karafka
         @host = host
         @operation = map(operation, OPERATIONS_MAP)
         @permission_type = map(permission_type, PERMISSION_TYPES_MAP)
+        super()
         freeze
       end
 

data/lib/karafka/admin/configs/config.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Admin
-    module Configs
+  class Admin
+    class Configs
       # Represents a single config entry that is related to a resource
       class Config
         attr_reader :name, :value, :synonyms

data/lib/karafka/admin/configs/resource.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Admin
-    module Configs
+  class Admin
+    class Configs
       # Represents a single resource in the context of configuration management
       class Resource
         # Types of resources that have workable configs.

data/lib/karafka/admin/configs.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Admin
+  class Admin
     # Namespace for admin operations related to configuration management
     #
     # At the moment Karafka supports configuration management for brokers and topics
@@ -9,11 +9,7 @@ module Karafka
     # You can describe configuration as well as alter it.
     #
     # Altering is done in the incremental way.
-    module Configs
-      extend Helpers::ConfigImporter.new(
-        max_wait_time: %i[admin max_wait_time]
-      )
-
+    class Configs < Admin
       class << self
         # Fetches given resources configurations from Kafka
         #
@@ -97,7 +93,7 @@ module Karafka
         # Yields admin instance, allows to run Acl operations and awaits on the final result
         # Makes sure that admin is closed afterwards.
         def with_admin_wait
-          Admin.with_admin do |admin|
+          with_admin do |admin|
             yield(admin).wait(max_wait_timeout: max_wait_time)
           end
         end