karafka 2.4.12 → 2.4.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f96e616f91d60d5054e276f52cc2ecde9d4a6f6c8e25b3d61c384936b9239e4f
-  data.tar.gz: 8dfad6cc5d0cb4cdcc885b58e129929f67b0fdc4ee7922c21f9728cbdd863f5b
+  metadata.gz: 22f45da117cf90a2ecbec04dbcaf39634b1e1b85e00f521d4880954c65268ecf
+  data.tar.gz: ce5318aaa8f52954a80981662a41ad17ab314e9706a00f2edf3e840604b87b32
 SHA512:
-  metadata.gz: bfea8217fb7ba89158b926417a0dd0cab42460a607b3bf25a62a400ab83e510806b966f81bd2ca76144e5aff62c99fb58b35be94ca1ad5c61ca113e760098243
-  data.tar.gz: ee66d2c6a11dc6baac3cc836ea5455d6c88838f1db7c0f9af9a92c3f5d3c7b8dd9d1f8c42dac4c9908042da3b192d8579466deecc687126640fc7bc5e0aeafb2
+  metadata.gz: d8ec82f91aea2bdba595fa290feec1ca0b25dbd73cd0eafac6554538d3506c47ad8db42036ed838b199a3016341bf67d6daf4f03dfced3ddcf783850c0dc97bf
+  data.tar.gz: 2b750fce3294dda031f5a97bf5343711dcf52d1dbc00c61e029b602b302a549e9a70bb6ac4e952842e1ad840d7dd7b28115ed72118aad54917497672593de133

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,51 +1,43 @@
 ---
 name: Bug Report
-about: Report an issue with Karafka you've discovered.
+about: Report an issue within the Karafka ecosystem you've discovered.
 ---
 
-*Be clear, concise and precise in your description of the problem.
-Open an issue with a descriptive title and a summary in grammatically correct,
-complete sentences.*
+To make this process smoother for everyone involved, please read the following information before filling out the template.
 
-*Use the template below when reporting bugs. Please, make sure that
-you're running the latest stable Karafka and that the problem you're reporting
-hasn't been reported (and potentially fixed) already.*
+Scope of the OSS Support
+===========
 
-*Before filing the ticket you should replace all text above the horizontal
-rule with your own words.*
+We do not provide OSS support for outdated versions of Karafka and its components.
 
---------
+Please ensure that you are using a version that is still actively supported. We cannot assist with any no longer maintained versions unless you support us with our Pro offering (https://karafka.io/docs/Pro-Support/).
 
-## Expected behavior
+We acknowledge that understanding the specifics of your application and its configuration can be essential for resolving certain issues. However, due to the extensive time and resources such analysis can require, this may fall beyond our Open Source Support scope.
 
-Describe here how you expected Karafka to behave in this particular situation.
+If Karafka or its components are critical to your infrastructure, we encourage you to consider our Pro Offering.
 
-## Actual behavior
+By backing us up, you can gain direct assistance and ensure your use case receives the dedicated attention it deserves.
 
-Describe here what actually happened.
 
-## Steps to reproduce the problem
+Important Links to Read
+===========
 
-This is extremely important! Providing us with a reliable way to reproduce
-a problem will expedite its solution.
+Please take a moment to review the following resources before submitting your report:
 
-## Your setup details
+- Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
+- Support Policy: https://karafka.io/docs/Support/
+- Versions, Lifecycle, and EOL: https://karafka.io/docs/Versions-Lifecycle-and-EOL/
 
-Please provide kafka version and the output of `karafka info` or `bundle exec karafka info` if using Bundler.
 
-Here's an example:
+Bug Report Details
+===========
 
-```
-$ [bundle exec] karafka info
-Karafka version: 2.2.10 + Pro
-Ruby version: ruby 3.2.2 (2023-03-30 revision e51014f9c0) [x86_64-linux]
-Rdkafka version: 0.13.8
-Consumer groups count: 2
-Subscription groups count: 2
-Workers count: 2
-Application client id: example_app
-Boot file: /app/karafka.rb
-Environment: development
-License: Commercial
-License entity: karafka-ci
-```
+Please provide all the details per our Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
+
+Failing to provide the required details may result in the issue being closed. Please include all necessary information to help us understand and resolve your issue effectively.
+
+
+Additional Context
+===========
+
+Add any other context about the problem here.

data/.github/workflows/ci.yml

@@ -89,6 +89,13 @@ jobs:
         run: |
           docker compose up -d || (sleep 5 && docker compose up -d)
 
+      # Newer versions of ActiveSupport and Rails do not work with Ruby 3.1 anymore.
+      # While we use newer by default we do want to resolve older and test, thus we remove
+      # Gemfile.lock and let it resolve to the most compatible version possible
+      - name: Remove Gemfile.lock if Ruby 3.1
+        if: matrix.ruby == '3.1'
+        run: rm -f Gemfile.lock
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:

data/.ruby-version

@@ -1 +1 @@
-3.3.5
+3.3.6

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Karafka Framework Changelog
 
+## 2.4.14 (2024-11-25)
+- [Enhancement] Improve low-level critical error reporting.
+- [Enhancement] Expand Kubernetes Liveness state reporting with critical errors detection.
+- [Enhancement] Save several string allocations and one array allocation on each job execution when using Datadog instrumentation.
+- [Enhancement] Support `eofed` jobs in the AppSignal instrumentation.
+- [Enhancement] Allow running bootfile-less Rails setup Karafka CLI commands where stuff is configured in the initializers.
+- [Fix] `Instrumentation::Vendors::Datadog::LoggerListener` treats eof jobs as consume jobs.
+
+## 2.4.13 (2024-10-11)
+- [Enhancement] Make declarative topics return different exit codes on migrable/non-migrable states (0 - no changes, 2 - changes) when used with `--detailed-exitcode` flag.
+- [Enhancement] Introduce `config.strict_declarative_topics` that should force declaratives on all non-pattern based topics and DLQ topics
+- [Enhancement] Report ignored repartitioning to lower number of partitions in declarative topics.
+- [Enhancement] Promote the `LivenessListener#healty?` to a public API.
+- [Fix] Fix `Karafka::Errors::MissingBootFileError` when debugging in VScode with ruby-lsp.
+- [Fix] Require `karafka-core` `>=` `2.4.4` to prevent dependencies conflicts. 
+- [Fix] Validate swarm cli and always parse options from argv (roelbondoc)
+
 ## 2.4.12 (2024-09-17)
 - **[Feature]** Provide Adaptive Iterator feature as a fast alternative to Long-Running Jobs (Pro).
 - [Enhancement] Provide `Consumer#each` as a delegation to messages batch.

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.12)
+    karafka (2.4.14)
       base64 (~> 0.2)
-      karafka-core (>= 2.4.3, < 2.5.0)
+      karafka-core (>= 2.4.4, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
       waterdrop (>= 2.7.3, < 3.0.0)
       zeitwerk (~> 2.3)
@@ -11,11 +11,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.2.1)
-      activesupport (= 7.2.1)
+    activejob (8.0.0)
+      activesupport (= 8.0.0)
       globalid (>= 0.3.6)
-    activesupport (7.2.1)
+    activesupport (8.0.0)
       base64
+      benchmark (>= 0.3)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
@@ -25,7 +26,9 @@ GEM
       minitest (>= 5.1)
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
     base64 (0.2.0)
+    benchmark (0.3.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
     concurrent-ruby (1.3.4)
@@ -44,7 +47,7 @@ GEM
       raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
-    i18n (1.14.5)
+    i18n (1.14.6)
       concurrent-ruby (~> 1.0)
     karafka-core (2.4.4)
       karafka-rdkafka (>= 0.15.0, < 0.18.0)
@@ -64,7 +67,7 @@ GEM
     logger (1.6.1)
     mini_portile2 (2.8.7)
     minitest (5.25.1)
-    ostruct (0.6.0)
+    ostruct (0.6.1)
     raabro (1.4.0)
     rack (3.1.7)
     rake (13.2.1)
@@ -93,6 +96,7 @@ GEM
     tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    uri (1.0.0)
     waterdrop (2.8.0)
       karafka-core (>= 2.4.3, < 3.0.0)
       karafka-rdkafka (>= 0.17.5)

data/bin/integrations

@@ -45,7 +45,10 @@ class Scenario
     'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
     'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze,
     'swarm/forceful_shutdown_of_hanging_spec.rb' => [2].freeze,
-    'instrumentation/post_errors_instrumentation_error_spec.rb' => [1].freeze
+    'instrumentation/post_errors_instrumentation_error_spec.rb' => [1].freeze,
+    'cli/declaratives/delete/existing_with_exit_code_spec.rb' => [2].freeze,
+    'cli/declaratives/create/new_with_exit_code_spec.rb' => [2].freeze,
+    'cli/declaratives/plan/when_changes_with_detailed_exit_code_spec.rb' => [2].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES
@@ -240,9 +243,13 @@ ARGV.each do |filter|
   end
 end
 
-# Remove Rails 7.2 specs from Ruby 3.0 because it requires 3.1
+# Remove Rails 7.2 specs from Ruby < 3.1 because it requires 3.1
+# Remove Rails 8.0 specs from Ruby < 3.2 because it requires 3.2
 specs.delete_if do |spec|
-  RUBY_VERSION < '3.1' && spec.include?('rails72')
+  next true if RUBY_VERSION < '3.1' && spec.include?('rails72')
+  next true if RUBY_VERSION < '3.2' && spec.include?('rails8')
+
+  false
 end
 
 raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?

data/config/locales/errors.yml

@@ -14,6 +14,7 @@ en:
       pause_max_timeout_format: needs to be an integer bigger than 0
       pause_with_exponential_backoff_format: needs to be either true or false
       strict_topics_namespacing_format: needs to be either true or false
+      strict_declarative_topics_format: needs to be either true or false
       shutdown_timeout_format: needs to be an integer bigger than 0
       max_wait_time_format: needs to be an integer bigger than 0
       max_wait_time_max_wait_time_vs_swarm_node_report_timeout: >
@@ -147,6 +148,9 @@ en:
         all topic names within a single consumer group must be unique considering namespacing styles
         disable this validation by setting config.strict_topics_namespacing to false
 
+    routing:
+      without_declarative_definition: lacks explicit declarative topics definition
+
     job_options:
       missing: needs to be present
       dispatch_method_format: needs to be either :produce_async or :produce_sync

data/karafka.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'base64', '~> 0.2'
-  spec.add_dependency 'karafka-core', '>= 2.4.3', '< 2.5.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.4', '< 2.5.0'
   spec.add_dependency 'karafka-rdkafka', '>= 0.17.2'
   spec.add_dependency 'waterdrop', '>= 2.7.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'

data/lib/karafka/cli/base.rb

@@ -41,22 +41,38 @@ module Karafka
       class << self
         # Loads proper environment with what is needed to run the CLI
         def load
+          rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
+          is_rails = Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
+
+          # If the boot file is disabled and this is a Rails app, we assume that user moved the
+          # karafka app configuration to initializers or other Rails loading related place.
+          # It is not recommended but some users tend to do this. In such cases we just try to load
+          # the Rails stuff hoping that it will also load Karafka stuff
+          if Karafka.boot_file.to_s == 'false' && is_rails
+            require rails_env_rb
+
+            return
+          end
+
           # If there is a boot file, we need to require it as we expect it to contain
           # Karafka app setup, routes, etc
           if File.exist?(::Karafka.boot_file)
-            rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
-
             # Load Rails environment file that starts Rails, so we can reference consumers and
             # other things from `karafka.rb` file. This will work only for Rails, for non-rails
             # a manual setup is needed
-            require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
-
+            require rails_env_rb if is_rails
             require Karafka.boot_file.to_s
+
+            return
+          end
+
           # However when it is unavailable, we still want to be able to run help command
           # and install command as they don't require configured app itself to run
-          elsif %w[-h install].none? { |cmd| cmd == ARGV[0] }
-            raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
-          end
+          return if %w[-h install].any? { |cmd| cmd == ARGV[0] }
+
+          # All other commands except help and install do require an existing boot file if it was
+          # declared
+          raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
         end
 
         # Allows to set options for Thor cli
@@ -96,7 +112,7 @@ module Karafka
                 *[names, option[2], option[1]].flatten
               ) { |value| options[option[0]] = value }
             end
-          end.parse!
+          end.parse(ARGV)
 
           options
         end

data/lib/karafka/cli/topics/plan.rb

@@ -16,7 +16,10 @@ module Karafka
             return false
           end
 
+          changes = false
+
           unless topics_to_create.empty?
+            changes = true
             puts 'Following topics will be created:'
             puts
 
@@ -33,20 +36,54 @@ module Karafka
           end
 
           unless topics_to_repartition.empty?
-            puts 'Following topics will be repartitioned:'
-            puts
+            upscale = {}
+            downscale = {}
 
             topics_to_repartition.each do |topic, partitions|
               from = partitions
               to = topic.declaratives.partitions
 
-              puts "  #{yellow('~')} #{topic.name}:"
-              puts "    #{yellow('~')} partitions: \"#{red(from)}\" #{grey('=>')} \"#{green(to)}\""
+              if from < to
+                upscale[topic] = partitions
+              else
+                downscale[topic] = partitions
+              end
+            end
+
+            unless upscale.empty?
+              changes = true
+              puts 'Following topics will be repartitioned:'
+              puts
+
+              upscale.each do |topic, partitions|
+                from = partitions
+                to = topic.declaratives.partitions
+                y = yellow('~')
+                puts "  #{y} #{topic.name}:"
+                puts "    #{y} partitions: \"#{red(from)}\" #{grey('=>')} \"#{green(to)}\""
+                puts
+              end
+            end
+
+            unless downscale.empty?
+              puts(
+                'Following topics repartitioning will be ignored as downscaling is not supported:'
+              )
               puts
+
+              downscale.each do |topic, partitions|
+                from = partitions
+                to = topic.declaratives.partitions
+
+                puts "  #{grey('~')} #{topic.name}:"
+                puts "    #{grey('~')} partitions: \"#{grey(from)}\" #{grey('=>')} \"#{grey(to)}\""
+                puts
+              end
             end
           end
 
           unless topics_to_alter.empty?
+            changes = true
             puts 'Following topics will have configuration changes:'
             puts
 
@@ -65,7 +102,7 @@ module Karafka
             end
           end
 
-          true
+          changes
         end
 
         private

data/lib/karafka/cli/topics.rb

@@ -10,26 +10,53 @@ module Karafka
       )
 
       desc 'Allows for the topics management'
+
+      option(
+        :detailed_exitcode,
+        'Exists with 0 when no changes, 1 when error and 2 when changes present or applied',
+        TrueClass,
+        %w[
+          --detailed_exitcode
+        ]
+      )
+
+      # We exit with 0 if no changes happened
+      NO_CHANGES_EXIT_CODE = 0
+
+      # When any changes happened (or could happen) we return 2 because 1 is default when Ruby
+      # crashes
+      CHANGES_EXIT_CODE = 2
+
+      private_constant :NO_CHANGES_EXIT_CODE, :CHANGES_EXIT_CODE
+
       # @param action [String] action we want to take
       def call(action = 'missing')
-        case action
-        when 'create'
-          Topics::Create.new.call
-        when 'delete'
-          Topics::Delete.new.call
-        when 'reset'
-          Topics::Reset.new.call
-        when 'repartition'
-          Topics::Repartition.new.call
-        when 'migrate'
-          Topics::Migrate.new.call
-        when 'align'
-          Topics::Align.new.call
-        when 'plan'
-          Topics::Plan.new.call
-        else
-          raise ::ArgumentError, "Invalid topics action: #{action}"
-        end
+        detailed_exit_code = options.fetch(:detailed_exitcode, false)
+
+        command = case action
+                  when 'create'
+                    Topics::Create
+                  when 'delete'
+                    Topics::Delete
+                  when 'reset'
+                    Topics::Reset
+                  when 'repartition'
+                    Topics::Repartition
+                  when 'migrate'
+                    Topics::Migrate
+                  when 'align'
+                    Topics::Align
+                  when 'plan'
+                    Topics::Plan
+                  else
+                    raise ::ArgumentError, "Invalid topics action: #{action}"
+                  end
+
+        changes = command.new.call
+
+        return unless detailed_exit_code
+
+        changes ? exit(CHANGES_EXIT_CODE) : exit(NO_CHANGES_EXIT_CODE)
       end
     end
   end

data/lib/karafka/connection/client.rb

@@ -41,6 +41,9 @@ module Karafka
         :topic_authorization_failed, # 29
         :group_authorization_failed, # 30
         :cluster_authorization_failed, # 31
+        :illegal_generation,
+        # this will not recover as fencing is permanent
+        :fenced, # -144
         # This can happen for many reasons, including issues with static membership being fenced
         :fatal # -150
       ].freeze

data/lib/karafka/contracts/config.rb

@@ -34,6 +34,7 @@ module Karafka
       required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
       required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+      required(:strict_declarative_topics) { |val| [true, false].include?(val) }
 
       nested(:swarm) do
         required(:nodes) { |val| val.is_a?(Integer) && val.positive? }

data/lib/karafka/contracts/routing.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Ensures that routing wide rules are obeyed
+    class Routing < Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('routing')
+      end
+
+      # Ensures, that when declarative topics strict requirement is on, all topics have
+      # declarative definition (including DLQ topics)
+      # @note It will ignore routing pattern topics because those topics are virtual
+      virtual do |data, errors|
+        next unless errors.empty?
+        # Do not validate declaratives unless required and explicitly enabled
+        next unless Karafka::App.config.strict_declarative_topics
+
+        # Collects declarative topics. Please note, that any topic that has a `#topic` reference,
+        # will be declarative by default unless explicitly disabled. This however does not apply
+        # to the DLQ definitions
+        dec_topics = Set.new
+        # All topics including the DLQ topics names that are marked as active
+        topics = Set.new
+
+        data.each do |consumer_group|
+          consumer_group[:topics].each do |topic|
+            pat = topic[:patterns]
+            # Ignore pattern topics because they won't exist and should not be declarative
+            # managed
+            topics << topic[:name] if !pat || !pat[:active]
+
+            dlq = topic[:dead_letter_queue]
+            topics << dlq[:topic] if dlq[:active]
+
+            dec = topic[:declaratives]
+
+            dec_topics << topic[:name] if dec[:active]
+          end
+        end
+
+        missing_dec = topics - dec_topics
+
+        next if missing_dec.empty?
+
+        missing_dec.map do |topic_name|
+          [
+            [:topics, topic_name],
+            :without_declarative_definition
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb

@@ -48,6 +48,7 @@ module Karafka
             consumer.revoked.error
             consumer.shutdown.error
             consumer.tick.error
+            consumer.eofed.error
           ].freeze
 
           private_constant :USER_CONSUMER_ERROR_TYPES
@@ -107,7 +108,8 @@ module Karafka
           [
             %i[revoke revoked revoked],
             %i[shutting_down shutdown shutdown],
-            %i[tick ticked tick]
+            %i[tick ticked tick],
+            %i[eof eofed eofed]
           ].each do |before, after, name|
             class_eval <<~RUBY, __FILE__, __LINE__ + 1
               # Keeps track of user code execution

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -35,6 +35,7 @@ module Karafka
           def initialize(&block)
             configure
             setup(&block) if block
+            @job_types_cache = {}
           end
 
           # @param block [Proc] configuration block
@@ -51,7 +52,7 @@ module Karafka
             push_tags
 
             job = event[:job]
-            job_type = job.class.to_s.split('::').last
+            job_type = fetch_job_type(job.class)
             consumer = job.executor.topic.consumer
             topic = job.executor.topic.name
 
@@ -68,8 +69,16 @@ module Karafka
                        'revoked'
                      when 'Idle'
                        'idle'
-                     else
+                     when 'Eofed'
+                       'eofed'
+                     when 'EofedNonBlocking'
+                       'eofed'
+                     when 'ConsumeNonBlocking'
                        'consume'
+                     when 'Consume'
+                       'consume'
+                     else
+                       raise Errors::UnsupportedCaseError, job_type
                      end
 
             current_span.resource = "#{consumer}##{action}"
@@ -121,6 +130,8 @@ module Karafka
               error "Consumer on shutdown failed due to an error: #{error}"
             when 'consumer.tick.error'
               error "Consumer tick failed due to an error: #{error}"
+            when 'consumer.eofed.error'
+              error "Consumer eofed failed due to an error: #{error}"
             when 'worker.process.error'
               fatal "Worker processing failed due to an error: #{error}"
             when 'connection.listener.fetch_loop.error'
@@ -169,6 +180,18 @@ module Karafka
 
             Karafka.logger.pop_tags
           end
+
+          private
+
+          # Takes the job class and extracts the job type.
+          # @param job_class [Class] job class
+          # @return [String]
+          # @note It does not have to be thread-safe despite running in multiple threads because
+          #   the assignment race condition is irrelevant here since the same value will be
+          #   assigned.
+          def fetch_job_type(job_class)
+            @job_types_cache[job_class] ||= job_class.to_s.split('::').last
+          end
         end
       end
     end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -82,10 +82,11 @@ module Karafka
             statistics = event[:statistics]
             consumer_group_id = event[:consumer_group_id]
 
-            base_tags = default_tags + ["consumer_group:#{consumer_group_id}"]
+            tags = ["consumer_group:#{consumer_group_id}"]
+            tags.concat(default_tags)
 
             rd_kafka_metrics.each do |metric|
-              report_metric(metric, statistics, base_tags)
+              report_metric(metric, statistics, tags)
             end
           end
 
@@ -93,13 +94,14 @@ module Karafka
           #
           # @param event [Karafka::Core::Monitoring::Event]
           def on_error_occurred(event)
-            extra_tags = ["type:#{event[:type]}"]
+            tags = ["type:#{event[:type]}"]
+            tags.concat(default_tags)
 
             if event.payload[:caller].respond_to?(:messages)
-              extra_tags += consumer_tags(event.payload[:caller])
+              tags.concat(consumer_tags(event.payload[:caller]))
             end
 
-            count('error_occurred', 1, tags: default_tags + extra_tags)
+            count('error_occurred', 1, tags: tags)
           end
 
           # Reports how many messages we've polled and how much time did we spend on it
@@ -111,10 +113,11 @@ module Karafka
 
             consumer_group_id = event[:subscription_group].consumer_group.id
 
-            extra_tags = ["consumer_group:#{consumer_group_id}"]
+            tags = ["consumer_group:#{consumer_group_id}"]
+            tags.concat(default_tags)
 
-            histogram('listener.polling.time_taken', time_taken, tags: default_tags + extra_tags)
-            histogram('listener.polling.messages', messages_count, tags: default_tags + extra_tags)
+            histogram('listener.polling.time_taken', time_taken, tags: tags)
+            histogram('listener.polling.messages', messages_count, tags: tags)
           end
 
           # Here we report majority of things related to processing as we have access to the
@@ -125,7 +128,8 @@ module Karafka
             messages = consumer.messages
             metadata = messages.metadata
 
-            tags = default_tags + consumer_tags(consumer)
+            tags = consumer_tags(consumer)
+            tags.concat(default_tags)
 
             count('consumer.messages', messages.count, tags: tags)
             count('consumer.batches', 1, tags: tags)
@@ -146,7 +150,8 @@ module Karafka
               #
               # @param event [Karafka::Core::Monitoring::Event]
               def on_consumer_#{after}(event)
-                tags = default_tags + consumer_tags(event.payload[:caller])
+                tags = consumer_tags(event.payload[:caller])
+                tags.concat(default_tags)
 
                 count('consumer.#{name}', 1, tags: tags)
               end
@@ -158,9 +163,10 @@ module Karafka
           def on_worker_process(event)
             jq_stats = event[:jobs_queue].statistics
 
-            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
-            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
-            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
+            tags = default_tags
+            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: tags)
+            histogram('worker.processing', jq_stats[:busy], tags: tags)
+            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: tags)
           end
 
           # We report this metric before and after processing for higher accuracy
@@ -240,11 +246,14 @@ module Karafka
                 # node ids
                 next if broker_statistics['nodeid'] == -1
 
+                tags = ["broker:#{broker_statistics['nodename']}"]
+                tags.concat(base_tags)
+
                 public_send(
                   metric.type,
                   metric.name,
                   broker_statistics.dig(*metric.key_location),
-                  tags: base_tags + ["broker:#{broker_statistics['nodename']}"]
+                  tags: tags
                 )
               end
             when :topics
@@ -259,14 +268,14 @@ module Karafka
                   next if partition_statistics['fetch_state'] == 'stopped'
                   next if partition_statistics['fetch_state'] == 'none'
 
+                  tags = ["topic:#{topic_name}", "partition:#{partition_name}"]
+                  tags.concat(base_tags)
+
                   public_send(
                     metric.type,
                     metric.name,
                     partition_statistics.dig(*metric.key_location),
-                    tags: base_tags + [
-                      "topic:#{topic_name}",
-                      "partition:#{partition_name}"
-                    ]
+                    tags: tags
                   )
                 end
               end

data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb

@@ -29,13 +29,13 @@ module Karafka
             @port = port
           end
 
-          private
-
           # @return [Boolean] true if all good, false if we should tell k8s to kill this process
           def healthy?
             raise NotImplementedError, 'Implement in a subclass'
           end
 
+          private
+
           # Responds to a HTTP request with the process liveness status
           def respond
             client = @server.accept

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -26,6 +26,19 @@ module Karafka
         #
         # @note Please use `Kubernetes::SwarmLivenessListener` when operating in the swarm mode
         class LivenessListener < BaseListener
+          # When any of those occurs, it means something went wrong in a way that cannot be
+          # recovered. In such cases we should report that the consumer process is not healthy.
+          # - `fenced` - This instance has been fenced by a newer instance and will not do any
+          #   processing at all never. Fencing most of the time means the instance.group.id has
+          #   been reused without properly terminating the previous consumer process first
+          # - `fatal` - any fatal error that halts the processing forever
+          UNRECOVERABLE_RDKAFKA_ERRORS = [
+            :fenced, # -144
+            :fatal # -150
+          ].freeze
+
+          private_constant :UNRECOVERABLE_RDKAFKA_ERRORS
+
           # @param hostname [String, nil] hostname or nil to bind on all
           # @param port [Integer] TCP port on which we want to run our HTTP status server
           # @param consuming_ttl [Integer] time in ms after which we consider consumption hanging.
@@ -40,6 +53,11 @@ module Karafka
             consuming_ttl: 5 * 60 * 1_000,
             polling_ttl: 5 * 60 * 1_000
           )
+            # If this is set to true, it indicates unrecoverable error like fencing
+            # While fencing can be partial (for one of the SGs), we still should consider this
+            # as an undesired state for the whole process because it halts processing in a
+            # non-recoverable manner forever
+            @unrecoverable = false
             @polling_ttl = polling_ttl
             @consuming_ttl = consuming_ttl
             @mutex = Mutex.new
@@ -86,10 +104,19 @@ module Karafka
             RUBY
           end
 
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_error_occurred(_event)
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(event)
             clear_consumption_tick
             clear_polling_tick
+
+            error = event[:error]
+
+            # We are only interested in the rdkafka errors
+            return unless error.is_a?(Rdkafka::RdkafkaError)
+            # We mark as unrecoverable only on certain errors that will not be fixed by retrying
+            return unless UNRECOVERABLE_RDKAFKA_ERRORS.include?(error.code)
+
+            @unrecoverable = true
           end
 
           # Deregister the polling tracker for given listener
@@ -112,6 +139,18 @@ module Karafka
             clear_polling_tick
           end
 
+          # Did we exceed any of the ttls
+          # @return [String] 204 string if ok, 500 otherwise
+          def healthy?
+            time = monotonic_now
+
+            return false if @unrecoverable
+            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+
+            true
+          end
+
           private
 
           # Wraps the logic with a mutex
@@ -152,17 +191,6 @@ module Karafka
               @consumptions.delete(thread_id)
             end
           end
-
-          # Did we exceed any of the ttls
-          # @return [String] 204 string if ok, 500 otherwise
-          def healthy?
-            time = monotonic_now
-
-            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
-            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
-
-            true
-          end
         end
       end
     end

data/lib/karafka/routing/builder.rb

@@ -48,6 +48,10 @@ module Karafka
 
           instance_eval(&block)
 
+          # Ensures high-level routing details consistency
+          # Contains checks that require knowledge about all the consumer groups to operate
+          Contracts::Routing.new.validate!(map(&:to_h))
+
           each do |consumer_group|
             # Validate consumer group settings
             Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)

data/lib/karafka/setup/config.rb

@@ -68,6 +68,11 @@ module Karafka
       setting :strict_topics_namespacing, default: true
       # option [String] default consumer group name for implicit routing
       setting :group_id, default: 'app'
+      # option [Boolean] when set to true, it will validate as part of the routing validation, that
+      # all topics and DLQ topics (even not active) have the declarative topics definitions.
+      # Really useful when you want to ensure that all topics in routing are managed via
+      # declaratives.
+      setting :strict_declarative_topics, default: false
 
       setting :oauth do
         # option [false, #call] Listener for using oauth bearer. This listener will be able to

data/lib/karafka/swarm/supervisor.rb

@@ -20,7 +20,9 @@ module Karafka
         shutdown_timeout: %i[shutdown_timeout],
         supervision_sleep: %i[internal supervision_sleep],
         forceful_exit_code: %i[internal forceful_exit_code],
-        process: %i[internal process]
+        process: %i[internal process],
+        cli_contract: %i[internal cli contract],
+        activity_manager: %i[internal routing activity_manager]
       )
 
       # How long extra should we wait on shutdown before forceful termination
@@ -39,6 +41,9 @@ module Karafka
 
       # Creates needed number of forks, installs signals and starts supervision
       def run
+        # Validate the CLI provided options the same way as we do for the regular server
+        cli_contract.validate!(activity_manager.to_h)
+
         # Close producer just in case. While it should not be used, we do not want even a
         # theoretical case since librdkafka is not thread-safe.
         # We close it prior to forking just to make sure, there is no issue with initialized
@@ -67,7 +72,11 @@ module Karafka
           lock
           control
         end
-      # If anything went wrong, signal this and die
+
+      # If the cli contract validation failed reraise immediately and stop the process
+      rescue Karafka::Errors::InvalidConfigurationError => e
+        raise e
+      # If anything went wrong during supervision, signal this and die
       # Supervisor is meant to be thin and not cause any issues. If you encounter this case
       # please report it as it should be considered critical
       rescue StandardError => e

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.12'
+  VERSION = '2.4.14'
 end

data/lib/karafka.rb

@@ -49,17 +49,29 @@ module Karafka
       @monitor ||= App.config.monitor
     end
 
-    # @return [String] root path of this gem
+    # @return [Pathname] root path of this gem
     def gem_root
       Pathname.new(File.expand_path('..', __dir__))
     end
 
-    # @return [String] Karafka app root path (user application path)
+    # @return [Pathname] Karafka app root path (user application path)
     def root
-      Pathname.new(ENV['KARAFKA_ROOT_DIR'] || File.dirname(ENV['BUNDLE_GEMFILE']))
+      # If user points to a different root explicitly, use it
+      return Pathname.new(ENV['KARAFKA_ROOT_DIR']) if ENV['KARAFKA_ROOT_DIR']
+
+      # By default we infer the project root from bundler.
+      # We cannot use the BUNDLE_GEMFILE env directly because it may be altered by things like
+      # ruby-lsp. Instead we always fallback to the most outer Gemfile. In most of the cases, it
+      # won't matter but in case of some automatic setup alterations like ruby-lsp, the location
+      # from which the project starts may not match the original Gemfile.
+      Pathname.new(
+        File.dirname(
+          Bundler.with_unbundled_env { Bundler.default_gemfile }
+        )
+      )
     end
 
-    # @return [String] path to Karafka gem root core
+    # @return [Pathname] path to Karafka gem root core
     def core_root
       Pathname.new(File.expand_path('karafka', __dir__))
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.12
+  version: 2.4.14
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-09-17 00:00:00.000000000 Z
+date: 2024-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.3
+        version: 2.4.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.3
+        version: 2.4.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -219,6 +219,7 @@ files:
 - lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb
 - lib/karafka/contracts/consumer_group.rb
+- lib/karafka/contracts/routing.rb
 - lib/karafka/contracts/server_cli_options.rb
 - lib/karafka/contracts/topic.rb
 - lib/karafka/deserializers/headers.rb
@@ -619,7 +620,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
+rubygems_version: 3.5.22
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.