karafka 2.5.0.rc1 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a66089d998c0dabb1070e4e8f1895a068e8f2aa8e752fb38ef9da1633b9704d
-  data.tar.gz: 188ea36894e0a32168303654510ef2e072d19f8ad39e5f0155547b6c96dbfdb2
+  metadata.gz: 03a08ef42e32f92069ef95b4380b744f7188dd2248f296abc752e5cee9d12c7f
+  data.tar.gz: e23896dcf66e16cddf193ee1b412fb0560dd82e50dcf01b31f3bb93d451afcc3
 SHA512:
-  metadata.gz: d3ee8f86dd3b26dea69f9e03972ac3aced8b76d8156a9c359cb2b50114f3156306281a7f147935fa4d566e2d398f0a343205255a96f9560528b9dc2d21ca166c
-  data.tar.gz: 843e553470b78b107080df06ae7f7bd716d5d08363fb0d381709522aed397bb5da6d46c0d1aee06a72c6ba063b112118a669ff3f079fef4c53f672cf08ed1ee0
+  metadata.gz: 52356bcb5a97f121a6e383bccc7530ef3e5252442ff0a13122b3cfebc8579a00a396576e4965f74473f950b28c820d898d259cffd90b26ff80c40701527cf97f
+  data.tar.gz: 8fe1550960de8de921e21a45b170a0b1282fdd6d9c5f81f2285cfc5dc958c8d54dbfcb953a997d87891f2ab29f010d8b418987a55825f7d25bb124c717876fab

data/CHANGELOG.md

@@ -1,11 +1,14 @@
 # Karafka Framework Changelog
 
-## 2.5.0 (Unreleased)
+## 2.5.0 (2025-06-15)
 - **[Breaking]** Change how consistency of DLQ dispatches works in Pro (`partition_key` vs. direct partition id mapping).
 - **[Breaking]** Remove the headers `source_key` from the Pro DLQ dispatched messages as the original key is now fully preserved.
 - **[Breaking]** Use DLQ and Piping prefix `source_` instead of `original_` to align with naming convention of Kafka Streams and Apache Flink for future usage.
 - **[Breaking]** Rename scheduled jobs topics names in their config (Pro).
+- **[Breaking]** Change K8s listener response from `204` to `200` and include JSON body with reasons.
+- **[Breaking]** Replace admin config `max_attempts` with `max_retries_duration` and 
 - **[Feature]** Parallel Segments for concurrent processing of the same partition with more than partition count of processes (Pro).
+- [Enhancement] Normalize topic + partition logs format.
 - [Enhancement] Support KIP-82 (header values of arrays).
 - [Enhancement] Enhance errors tracker with `#counts` that contains per-error class specific counters for granular flow handling.
 - [Enhancement] Provide explicit `Karafka::Admin.copy_consumer_group` API.
@@ -41,7 +44,9 @@
 - [Enhancement] Enrich scheduled messages state reporter with debug data.
 - [Enhancement] Introduce a new state called `stopped` to the scheduled messages.
 - [Enhancement] Do not overwrite the `key` in the Pro DLQ dispatched messages for routing reasons.
-- [Enhancement] Introduce `errors_tracker.trace_id` for distributed error details correlation with the Web UI. 
+- [Enhancement] Introduce `errors_tracker.trace_id` for distributed error details correlation with the Web UI.
+- [Enhancement] Improve contracts validations reporting.
+- [Enhancement] Optimize topic creation and repartitioning admin operations for topics with hundreds of partitions.
 - [Refactor] Introduce a `bin/verify_kafka_warnings` script to clean Kafka from temporary test-suite topics.
 - [Refactor] Introduce a `bin/verify_topics_naming` script to ensure proper test topics naming convention.
 - [Refactor] Make sure all temporary topics have a `it-` prefix in their name.
@@ -66,6 +71,8 @@
 - [Fix] Scheduled Messages re-seek moves to `latest` on inheritance of initial offset when `0` offset is compacted.
 - [Fix] Seek to `:latest` without `topic_partition_position` (-1) will not seek at all.
 - [Fix] Extremely high turn over of scheduled messages can cause them not to reach EOF/Loaded state.
+- [Fix] Fix incorrectly passed `max_wait_time` to rdkafka (ms instead of seconds) causing too long wait.
+- [Fix] Remove aggresive requerying of the Kafka cluster on topic creation/removal/altering.
 - [Change] Move to trusted-publishers and remove signing since no longer needed.
 
 ## 2.4.18 (2025-04-09)

data/Gemfile

@@ -18,7 +18,7 @@ end
 group :integrations do
   gem 'activejob', require: false
   gem 'karafka-testing', '>= 2.5.0', require: false
-  gem 'karafka-web', '>= 0.11.0.beta1', require: false
+  gem 'karafka-web', '>= 0.11.0.rc2', require: false
 end
 
 group :test do

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka (2.5.0.rc1)
+    karafka (2.5.0)
       base64 (~> 0.2)
-      karafka-core (>= 2.5.0, < 2.6.0)
+      karafka-core (>= 2.5.2, < 2.6.0)
       karafka-rdkafka (>= 0.19.5)
       waterdrop (>= 2.8.3, < 3.0.0)
       zeitwerk (~> 2.3)
@@ -27,9 +27,9 @@ GEM
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
       uri (>= 0.13.1)
-    base64 (0.2.0)
+    base64 (0.3.0)
     benchmark (0.4.1)
-    bigdecimal (3.1.9)
+    bigdecimal (3.2.2)
     byebug (12.0.0)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.3)
@@ -39,7 +39,7 @@ GEM
     erubi (1.13.1)
     et-orbi (1.2.11)
       tzinfo
-    factory_bot (6.5.2)
+    factory_bot (6.5.4)
       activesupport (>= 6.1.0)
     ffi (1.17.2)
     ffi (1.17.2-aarch64-linux-gnu)
@@ -59,7 +59,7 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.5.1)
+    karafka-core (2.5.2)
       karafka-rdkafka (>= 0.19.2, < 0.21.0)
       logger (>= 1.6.0)
     karafka-rdkafka (0.19.5)
@@ -69,9 +69,9 @@ GEM
     karafka-testing (2.5.1)
       karafka (>= 2.5.0.beta1, < 2.6.0)
       waterdrop (>= 2.8.0)
-    karafka-web (0.11.0.beta3)
+    karafka-web (0.11.0)
       erubi (~> 1.4)
-      karafka (>= 2.5.0.beta1, < 2.6.0)
+      karafka (>= 2.5.0.rc2, < 2.6.0)
       karafka-core (>= 2.5.0, < 2.6.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -80,9 +80,9 @@ GEM
     minitest (5.25.5)
     ostruct (0.6.1)
     raabro (1.4.0)
-    rack (3.1.15)
+    rack (3.1.16)
     rake (13.3.0)
-    roda (3.92.0)
+    roda (3.93.0)
       rack
     rspec (3.13.1)
       rspec-core (~> 3.13.0)
@@ -113,7 +113,7 @@ GEM
       karafka-core (>= 2.4.9, < 3.0.0)
       karafka-rdkafka (>= 0.19.2)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.18)
+    zeitwerk (2.7.3)
 
 PLATFORMS
   aarch64-linux-gnu
@@ -135,7 +135,7 @@ DEPENDENCIES
   fugit
   karafka!
   karafka-testing (>= 2.5.0)
-  karafka-web (>= 0.11.0.beta1)
+  karafka-web (>= 0.11.0.rc2)
   ostruct
   rspec
   simplecov

data/bin/integrations

@@ -45,6 +45,7 @@ class Scenario
     'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
     'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze,
     'swarm/forceful_shutdown_of_hanging_spec.rb' => [2].freeze,
+    'swarm/with_blocking_at_exit_spec.rb' => [2].freeze,
     'instrumentation/post_errors_instrumentation_error_spec.rb' => [1].freeze,
     'cli/declaratives/delete/existing_with_exit_code_spec.rb' => [2].freeze,
     'cli/declaratives/create/new_with_exit_code_spec.rb' => [2].freeze,

data/config/locales/errors.yml

@@ -84,7 +84,8 @@ en:
       admin.kafka_format: needs to be a hash
       admin.group_id_format: 'needs to be a string with a Kafka accepted format'
       admin.max_wait_time_format: 'needs to be an integer bigger than 0'
-      admin.max_attempts_format: 'needs to be an integer bigger than 0'
+      admin.retry_backoff_format: 'needs to be an integer bigger than 100'
+      admin.max_retries_duration_format: 'needs to be an integer bigger than 1000'
 
       swarm.nodes_format: 'needs to be an integer bigger than 0'
       swarm.node_format: needs to be false or node instance

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.9.1
+    image: confluentinc/cp-kafka:8.0.0
 
     ports:
       - 9092:9092

data/karafka.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'base64', '~> 0.2'
-  spec.add_dependency 'karafka-core', '>= 2.5.0', '< 2.6.0'
+  spec.add_dependency 'karafka-core', '>= 2.5.2', '< 2.6.0'
   spec.add_dependency 'karafka-rdkafka', '>= 0.19.5'
   spec.add_dependency 'waterdrop', '>= 2.8.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'

data/lib/karafka/active_job/job_extensions.rb

@@ -21,7 +21,10 @@ module Karafka
 
         # Make sure, that karafka options that someone wants to use are valid before assigning
         # them
-        App.config.internal.active_job.job_options_contract.validate!(new_options)
+        App.config.internal.active_job.job_options_contract.validate!(
+          new_options,
+          scope: %w[active_job]
+        )
 
         # We need to modify this hash because otherwise we would modify parent hash.
         self._karafka_options = _karafka_options.dup

data/lib/karafka/admin.rb

@@ -10,10 +10,13 @@ module Karafka
   #   Cluster on which operations are performed can be changed via `admin.kafka` config, however
   #   there is no multi-cluster runtime support.
   module Admin
+    extend Core::Helpers::Time
+
     extend Helpers::ConfigImporter.new(
       max_wait_time: %i[admin max_wait_time],
       poll_timeout: %i[admin poll_timeout],
-      max_attempts: %i[admin max_attempts],
+      max_retries_duration: %i[admin max_retries_duration],
+      retry_backoff: %i[admin retry_backoff],
       group_id: %i[admin group_id],
       app_kafka: %i[kafka],
       admin_kafka: %i[admin kafka]
@@ -122,7 +125,7 @@ module Karafka
           handler = admin.create_topic(name, partitions, replication_factor, topic_config)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: max_wait_time) },
+            -> { handler.wait(max_wait_timeout: max_wait_time_seconds) },
             -> { topics_names.include?(name) }
           )
         end
@@ -136,7 +139,7 @@ module Karafka
           handler = admin.delete_topic(name)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: max_wait_time) },
+            -> { handler.wait(max_wait_timeout: max_wait_time_seconds) },
             -> { !topics_names.include?(name) }
           )
         end
@@ -151,7 +154,7 @@ module Karafka
           handler = admin.create_partitions(name, partitions)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: max_wait_time) },
+            -> { handler.wait(max_wait_timeout: max_wait_time_seconds) },
             -> { topic_info(name).fetch(:partition_count) >= partitions }
           )
         end
@@ -362,7 +365,7 @@ module Karafka
       def delete_consumer_group(consumer_group_id)
         with_admin do |admin|
           handler = admin.delete_group(consumer_group_id)
-          handler.wait(max_wait_timeout: max_wait_time)
+          handler.wait(max_wait_timeout: max_wait_time_seconds)
         end
       end
 
@@ -564,6 +567,12 @@ module Karafka
 
       private
 
+      # @return [Integer] number of seconds to wait. `rdkafka` requires this value
+      #   (`max_wait_time`) to be provided in seconds while we define it in ms hence the conversion
+      def max_wait_time_seconds
+        max_wait_time / 1_000.0
+      end
+
       # Adds a new callback for given rdkafka instance for oauth token refresh (if needed)
       #
       # @param id [String, Symbol] unique (for the lifetime of instance) id that we use for
@@ -602,20 +611,23 @@ module Karafka
       # @param handler [Proc] the wait handler operation
       # @param breaker [Proc] extra condition upon timeout that indicates things were finished ok
       def with_re_wait(handler, breaker)
-        attempt ||= 0
-        attempt += 1
+        start_time = monotonic_now
+        # Convert milliseconds to seconds for sleep
+        sleep_time = retry_backoff / 1000.0
 
-        handler.call
+        loop do
+          handler.call
 
-        # If breaker does not operate, it means that the requested change was applied but is still
-        # not visible and we need to wait
-        raise(Errors::ResultNotVisibleError) unless breaker.call
-      rescue Rdkafka::AbstractHandle::WaitTimeoutError, Errors::ResultNotVisibleError
-        return if breaker.call
+          sleep(sleep_time)
 
-        retry if attempt <= max_attempts
+          return if breaker.call
+        rescue Rdkafka::AbstractHandle::WaitTimeoutError
+          return if breaker.call
 
-        raise
+          next if monotonic_now - start_time < max_retries_duration
+
+          raise(Errors::ResultNotVisibleError)
+        end
       end
 
       # @param type [Symbol] type of config we want

data/lib/karafka/contracts/base.rb

@@ -5,12 +5,13 @@ module Karafka
     # Base contract for all Karafka contracts
     class Base < ::Karafka::Core::Contractable::Contract
       # @param data [Hash] data for validation
+      # @param scope [Array<String>] nested scope if in use
       # @return [Boolean] true if all good
       # @raise [Errors::InvalidConfigurationError] invalid configuration error
       # @note We use contracts only in the config validation context, so no need to add support
       #   for multiple error classes. It will be added when it will be needed.
-      def validate!(data)
-        super(data, Errors::InvalidConfigurationError)
+      def validate!(data, scope: [])
+        super(data, Errors::InvalidConfigurationError, scope: scope)
       end
     end
   end

data/lib/karafka/contracts/config.rb

@@ -53,7 +53,8 @@ module Karafka
         required(:kafka) { |val| val.is_a?(Hash) }
         required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
         required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
-        required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+        required(:retry_backoff) { |val| val.is_a?(Integer) && val >= 100 }
+        required(:max_retries_duration) { |val| val.is_a?(Integer) && val >= 1_000 }
       end
 
       # We validate internals just to be sure, that they are present and working

data/lib/karafka/instrumentation/logger_listener.rb

@@ -76,7 +76,7 @@ module Karafka
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
         partition = job.executor.partition
-        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic}/#{partition} started"
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic}-#{partition} started"
       end
 
       # Prints info about the fact that a given job has finished
@@ -91,7 +91,7 @@ module Karafka
         partition = job.executor.partition
         info <<~MSG.tr("\n", ' ').strip!
           [#{job.id}] #{job_type} job for #{consumer}
-          on #{topic}/#{partition} finished in #{time} ms
+          on #{topic}-#{partition} finished in #{time} ms
         MSG
       end
 
@@ -108,7 +108,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{client.id}]
-          Pausing on topic #{topic}/#{partition}
+          Pausing on topic #{topic}-#{partition}
           on #{offset ? "offset #{offset}" : 'the consecutive offset'}
         MSG
       end
@@ -122,7 +122,7 @@ module Karafka
         client = event[:caller]
 
         info <<~MSG.tr("\n", ' ').strip!
-          [#{client.id}] Resuming on topic #{topic}/#{partition}
+          [#{client.id}] Resuming on topic #{topic}-#{partition}
         MSG
       end
 
@@ -138,7 +138,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Retrying of #{consumer.class} after #{timeout} ms
-          on topic #{topic}/#{partition} from offset #{offset}
+          on topic #{topic}-#{partition} from offset #{offset}
         MSG
       end
 
@@ -153,7 +153,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Seeking from #{consumer.class}
-          on topic #{topic}/#{partition} to offset #{seek_offset}
+          on topic #{topic}-#{partition} to offset #{seek_offset}
         MSG
       end
 
@@ -233,7 +233,7 @@ module Karafka
           info "#{group_prefix}: No partitions revoked"
         else
           revoked_partitions.each do |topic, partitions|
-            info "#{group_prefix}: Partition(s) #{partitions.join(', ')} of #{topic} revoked"
+            info "#{group_prefix}: #{topic}-[#{partitions.join(',')}] revoked"
           end
         end
       end
@@ -251,7 +251,7 @@ module Karafka
           info "#{group_prefix}: No partitions assigned"
         else
           assigned_partitions.each do |topic, partitions|
-            info "#{group_prefix}: Partition(s) #{partitions.join(', ')} of #{topic} assigned"
+            info "#{group_prefix}: #{topic}-[#{partitions.join(',')}] assigned"
           end
         end
       end
@@ -269,7 +269,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Dispatched message #{offset}
-          from #{topic}/#{partition}
+          from #{topic}-#{partition}
           to DLQ topic: #{dlq_topic}
         MSG
       end
@@ -288,7 +288,7 @@ module Karafka
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Throttled and will resume
           from message #{offset}
-          on #{topic}/#{partition}
+          on #{topic}-#{partition}
         MSG
       end
 
@@ -303,7 +303,7 @@ module Karafka
 
         info <<~MSG.tr("\n", ' ').strip!
           [#{consumer.id}] Post-filtering seeking to message #{offset}
-          on #{topic}/#{partition}
+          on #{topic}-#{partition}
         MSG
       end
 

data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb

@@ -8,11 +8,12 @@ module Karafka
       # Namespace for instrumentation related with Kubernetes
       module Kubernetes
         # Base Kubernetes Listener providing basic HTTP server capabilities to respond with health
+        # statuses
         class BaseListener
           include ::Karafka::Core::Helpers::Time
 
           # All good with Karafka
-          OK_CODE = '204 No Content'
+          OK_CODE = '200 OK'
 
           # Some timeouts, fail
           FAIL_CODE = '500 Internal Server Error'
@@ -38,11 +39,15 @@ module Karafka
 
           # Responds to a HTTP request with the process liveness status
           def respond
+            body = JSON.generate(status_body)
+
             client = @server.accept
             client.gets
             client.print "HTTP/1.1 #{healthy? ? OK_CODE : FAIL_CODE}\r\n"
-            client.print "Content-Type: text/plain\r\n"
+            client.print "Content-Type: application/json\r\n"
+            client.print "Content-Length: #{body.bytesize}\r\n"
             client.print "\r\n"
+            client.print body
             client.close
 
             true
@@ -50,6 +55,16 @@ module Karafka
             !@server.closed?
           end
 
+          # @return [Hash] hash that will be the response body
+          def status_body
+            {
+              status: healthy? ? 'healthy' : 'unhealthy',
+              timestamp: Time.now.to_i,
+              port: @port,
+              process_id: ::Process.pid
+            }
+          end
+
           # Starts background thread with micro-http monitoring
           def start
             @server = TCPServer.new(*[@hostname, @port].compact)

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -53,7 +53,7 @@ module Karafka
             consuming_ttl: 5 * 60 * 1_000,
             polling_ttl: 5 * 60 * 1_000
           )
-            # If this is set to true, it indicates unrecoverable error like fencing
+            # If this is set to a symbol, it indicates unrecoverable error like fencing
             # While fencing can be partial (for one of the SGs), we still should consider this
             # as an undesired state for the whole process because it halts processing in a
             # non-recoverable manner forever
@@ -116,7 +116,7 @@ module Karafka
             # We mark as unrecoverable only on certain errors that will not be fixed by retrying
             return unless UNRECOVERABLE_RDKAFKA_ERRORS.include?(error.code)
 
-            @unrecoverable = true
+            @unrecoverable = error.code
           end
 
           # Deregister the polling tracker for given listener
@@ -142,17 +142,29 @@ module Karafka
           # Did we exceed any of the ttls
           # @return [String] 204 string if ok, 500 otherwise
           def healthy?
-            time = monotonic_now
-
             return false if @unrecoverable
-            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
-            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+            return false if polling_ttl_exceeded?
+            return false if consuming_ttl_exceeded?
 
             true
           end
 
           private
 
+          # @return [Boolean] true if the consumer exceeded the polling ttl
+          def polling_ttl_exceeded?
+            time = monotonic_now
+
+            @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+          end
+
+          # @return [Boolean] true if the consumer exceeded the consuming ttl
+          def consuming_ttl_exceeded?
+            time = monotonic_now
+
+            @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+          end
+
           # Wraps the logic with a mutex
           # @param block [Proc] code we want to run in mutex
           def synchronize(&block)
@@ -191,6 +203,17 @@ module Karafka
               @consumptions.delete(thread_id)
             end
           end
+
+          # @return [Hash] response body status
+          def status_body
+            super.merge!(
+              errors: {
+                polling_ttl_exceeded: polling_ttl_exceeded?,
+                consumption_ttl_exceeded: consuming_ttl_exceeded?,
+                unrecoverable: @unrecoverable
+              }
+            )
+          end
         end
       end
     end

data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb

@@ -47,6 +47,15 @@ module Karafka
           def healthy?
             (monotonic_now - @controlling) < @controlling_ttl
           end
+
+          # @return [Hash] response body status
+          def status_body
+            super.merge!(
+              errors: {
+                controlling_ttl_exceeded: !healthy?
+              }
+            )
+          end
         end
       end
     end

data/lib/karafka/pro/encryption.rb

@@ -22,7 +22,10 @@ module Karafka
 
         # @param config [Karafka::Core::Configurable::Node] root node config
         def post_setup(config)
-          Encryption::Contracts::Config.new.validate!(config.to_h)
+          Encryption::Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
 
           # Don't inject extra components if encryption is not active
           return unless config.encryption.active

data/lib/karafka/pro/recurring_tasks.rb

@@ -29,7 +29,10 @@ module Karafka
           @schedule.instance_exec(&block)
 
           @schedule.each do |task|
-            Contracts::Task.new.validate!(task.to_h)
+            Contracts::Task.new.validate!(
+              task.to_h,
+              scope: ['recurring_tasks', task.id]
+            )
           end
 
           @schedule
@@ -59,7 +62,10 @@ module Karafka
 
         # @param config [Karafka::Core::Configurable::Node] root node config
         def post_setup(config)
-          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+          RecurringTasks::Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
 
           # Published after task is successfully executed
           Karafka.monitor.notifications_bus.register_event('recurring_tasks.task.executed')

data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb

@@ -28,7 +28,8 @@ module Karafka
               # Validates that each node has at least one assignment.
               #
               # @param builder [Karafka::Routing::Builder]
-              def validate!(builder)
+              # @param scope [Array<String>]
+              def validate!(builder, scope: [])
                 nodes_setup = Hash.new do |h, node_id|
                   h[node_id] = { active: false, node_id: node_id }
                 end
@@ -49,7 +50,7 @@ module Karafka
                 end
 
                 nodes_setup.each_value do |details|
-                  super(details)
+                  super(details, scope: scope)
                 end
               end
 

data/lib/karafka/pro/routing/features/swarm.rb

@@ -17,7 +17,10 @@ module Karafka
             # @param config [Karafka::Core::Configurable::Node] app config
             def post_setup(config)
               config.monitor.subscribe('app.before_warmup') do
-                Contracts::Routing.new.validate!(config.internal.routing.builder)
+                Contracts::Routing.new.validate!(
+                  config.internal.routing.builder,
+                  scope: %w[swarm]
+                )
               end
             end
           end

data/lib/karafka/pro/scheduled_messages/proxy.rb

@@ -60,7 +60,11 @@ module Karafka
             # We need to ensure that the message we want to proxy is fully legit. Otherwise, since
             # we envelope details like target topic, we could end up having incorrect data to
             # schedule
-            MSG_CONTRACT.validate!(message, WaterDrop::Errors::MessageInvalidError)
+            MSG_CONTRACT.validate!(
+              message,
+              WaterDrop::Errors::MessageInvalidError,
+              scope: %w[scheduled_messages message]
+            )
 
             headers = (message[:headers] || {}).merge(
               'schedule_schema_version' => ScheduledMessages::SCHEMA_VERSION,
@@ -166,9 +170,17 @@ module Karafka
           # complies with our requirements
           # @param proxy_message [Hash] our message envelope
           def validate!(proxy_message)
-            POST_CONTRACT.validate!(proxy_message)
+            POST_CONTRACT.validate!(
+              proxy_message,
+              scope: %w[scheduled_messages message]
+            )
+
             # After proxy specific validations we also ensure, that the final form is correct
-            MSG_CONTRACT.validate!(proxy_message, WaterDrop::Errors::MessageInvalidError)
+            MSG_CONTRACT.validate!(
+              proxy_message,
+              WaterDrop::Errors::MessageInvalidError,
+              scope: %w[scheduled_messages message]
+            )
           end
         end
       end

data/lib/karafka/pro/scheduled_messages.rb

@@ -51,7 +51,10 @@ module Karafka
 
         # @param config [Karafka::Core::Configurable::Node] root node config
         def post_setup(config)
-          RecurringTasks::Contracts::Config.new.validate!(config.to_h)
+          ScheduledMessages::Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
         end
 
         # Basically since we may have custom producers configured that are not the same as the

data/lib/karafka/routing/builder.rb

@@ -50,15 +50,24 @@ module Karafka
 
           # Ensures high-level routing details consistency
           # Contains checks that require knowledge about all the consumer groups to operate
-          Contracts::Routing.new.validate!(map(&:to_h))
+          Contracts::Routing.new.validate!(
+            map(&:to_h),
+            scope: %w[routes]
+          )
 
           each do |consumer_group|
             # Validate consumer group settings
-            Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+            Contracts::ConsumerGroup.new.validate!(
+              consumer_group.to_h,
+              scope: ['routes', consumer_group.name]
+            )
 
             # and then its topics settings
             consumer_group.topics.each do |topic|
-              Contracts::Topic.new.validate!(topic.to_h)
+              Contracts::Topic.new.validate!(
+                topic.to_h,
+                scope: ['routes', consumer_group.name, topic.name]
+              )
             end
 
             # Initialize subscription groups after all the routing is done

data/lib/karafka/routing/features/base/expander.rb

@@ -38,13 +38,19 @@ module Karafka
 
                 each do |consumer_group|
                   if scope::Contracts.const_defined?('ConsumerGroup', false)
-                    scope::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+                    scope::Contracts::ConsumerGroup.new.validate!(
+                      consumer_group.to_h,
+                      scope: ['routes', consumer_group.name]
+                    )
                   end
 
                   next unless scope::Contracts.const_defined?('Topic', false)
 
                   consumer_group.topics.each do |topic|
-                    scope::Contracts::Topic.new.validate!(topic.to_h)
+                    scope::Contracts::Topic.new.validate!(
+                      topic.to_h,
+                      scope: ['routes', consumer_group.name, topic.name]
+                    )
                   end
                 end
 

data/lib/karafka/server.rb

@@ -51,7 +51,10 @@ module Karafka
         # embedded
         # We cannot validate this during the start because config needs to be populated and routes
         # need to be defined.
-        cli_contract.validate!(activity_manager.to_h)
+        cli_contract.validate!(
+          activity_manager.to_h,
+          scope: %w[cli]
+        )
 
         # We clear as we do not want parent handlers in case of working from fork
         process.clear

data/lib/karafka/setup/config.rb

@@ -131,11 +131,20 @@ module Karafka
         # option max_wait_time [Integer] We wait only for this amount of time before raising error
         # as we intercept this error and retry after checking that the operation was finished or
         # failed using external factor.
-        setting :max_wait_time, default: 1_000
+        #
+        # For async this will finish immediately but for sync operations this will wait and we
+        # will get a confirmation. 60 seconds is ok for both cases as for async, the re-wait will
+        # kick in
+        setting :max_wait_time, default: 60 * 1_000
+
+        # How long should we wait on admin operation retrying before giving up and raising an
+        # error that result is not visible
+        setting :max_retries_duration, default: 60_000
 
-        # How many times should be try. 1 000 ms x 60 => 60 seconds wait in total and then we give
-        # up on pending operations
-        setting :max_attempts, default: 60
+        # In case of fast-finished async work, this `retry_backoff` help us not re-query Kafka
+        # too fast after previous call to check the async operation results. Basically prevents
+        # us from spamming metadata requests to Kafka in a loop
+        setting :retry_backoff, default: 500
 
         # option poll_timeout [Integer] time in ms
         # How long should a poll wait before yielding on no results (rdkafka-ruby setting)
@@ -352,7 +361,10 @@ module Karafka
 
           configure(&block)
 
-          Contracts::Config.new.validate!(config.to_h)
+          Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
 
           configure_components
 

data/lib/karafka/swarm/supervisor.rb

@@ -42,7 +42,10 @@ module Karafka
       # Creates needed number of forks, installs signals and starts supervision
       def run
         # Validate the CLI provided options the same way as we do for the regular server
-        cli_contract.validate!(activity_manager.to_h)
+        cli_contract.validate!(
+          activity_manager.to_h,
+          scope: %w[swarm cli]
+        )
 
         # Close producer just in case. While it should not be used, we do not want even a
         # theoretical case since librdkafka is not thread-safe.
@@ -154,7 +157,7 @@ module Karafka
         # Run forceful kill
         manager.terminate
         # And wait until linux kills them
-        # This prevents us from existing forcefully with any dead child process still existing
+        # This prevents us from exiting forcefully with any dead child process still existing
         # Since we have sent the `KILL` signal, it must die, so we can wait until all dead
         sleep(supervision_sleep) until manager.stopped?
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.5.0.rc1'
+  VERSION = '2.5.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.5.0.rc1
+  version: 2.5.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -29,7 +29,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 2.5.2
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.6.0
@@ -39,7 +39,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 2.5.2
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.6.0