karafka 2.4.12 → 2.4.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f96e616f91d60d5054e276f52cc2ecde9d4a6f6c8e25b3d61c384936b9239e4f
-  data.tar.gz: 8dfad6cc5d0cb4cdcc885b58e129929f67b0fdc4ee7922c21f9728cbdd863f5b
+  metadata.gz: 4556d1cccdb4ff51132f141f71a3c81d05dfdf71839241566bed12aa40ffcb83
+  data.tar.gz: 92af5f2db3c7614efdff4f4e0e92e012befd2b3f61bd89eb6b509cfb82a0569c
 SHA512:
-  metadata.gz: bfea8217fb7ba89158b926417a0dd0cab42460a607b3bf25a62a400ab83e510806b966f81bd2ca76144e5aff62c99fb58b35be94ca1ad5c61ca113e760098243
-  data.tar.gz: ee66d2c6a11dc6baac3cc836ea5455d6c88838f1db7c0f9af9a92c3f5d3c7b8dd9d1f8c42dac4c9908042da3b192d8579466deecc687126640fc7bc5e0aeafb2
+  metadata.gz: a1a28ea5cfc19cb7b3c2861fdc755bdb5e1c0e718b8b0244d87675e0d1cbae4cc568f8af28984a2845287e291207d4532e6c92cf87327ef17fb87f8263cd0d57
+  data.tar.gz: cc77cceea79fd8cbbe5013221fb4427daf9209aed781fff0d684ea1128a9b3984d69cb42dd14bd88c0d01968820e4f8e8ce3e1450d505614f7d12434cc46aedf

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Karafka Framework Changelog
 
+## 2.4.13 (2024-10-11)
+- [Enhancement] Make declarative topics return different exit codes on migrable/non-migrable states (0 - no changes, 2 - changes).
+- [Enhancement] Introduce `config.strict_declarative_topics` that should force declaratives on all non-pattern based topics and DLQ topics
+- [Enhancement] Report ignored repartitioning to lower number of partitions in declarative topics.
+- [Enhancement] Promote the `LivenessListener#healty?` to a public API.
+- [Fix] Fix `Karafka::Errors::MissingBootFileError` when debugging in VScode with ruby-lsp.
+- [Fix] Require `karafka-core` `>=` `2.4.4` to prevent dependencies conflicts. 
+- [Fix] Validate swarm cli and always parse options from argv (roelbondoc)
+
 ## 2.4.12 (2024-09-17)
 - **[Feature]** Provide Adaptive Iterator feature as a fast alternative to Long-Running Jobs (Pro).
 - [Enhancement] Provide `Consumer#each` as a delegation to messages batch.

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.12)
+    karafka (2.4.13)
       base64 (~> 0.2)
-      karafka-core (>= 2.4.3, < 2.5.0)
+      karafka-core (>= 2.4.4, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
       waterdrop (>= 2.7.3, < 3.0.0)
       zeitwerk (~> 2.3)

data/bin/integrations

@@ -45,7 +45,10 @@ class Scenario
     'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
     'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze,
     'swarm/forceful_shutdown_of_hanging_spec.rb' => [2].freeze,
-    'instrumentation/post_errors_instrumentation_error_spec.rb' => [1].freeze
+    'instrumentation/post_errors_instrumentation_error_spec.rb' => [1].freeze,
+    'cli/declaratives/delete/existing_with_exit_code_spec.rb' => [2].freeze,
+    'cli/declaratives/create/new_with_exit_code_spec.rb' => [2].freeze,
+    'cli/declaratives/plan/when_changes_with_detailed_exit_code_spec.rb' => [2].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES

data/config/locales/errors.yml

@@ -14,6 +14,7 @@ en:
       pause_max_timeout_format: needs to be an integer bigger than 0
       pause_with_exponential_backoff_format: needs to be either true or false
       strict_topics_namespacing_format: needs to be either true or false
+      strict_declarative_topics_format: needs to be either true or false
       shutdown_timeout_format: needs to be an integer bigger than 0
       max_wait_time_format: needs to be an integer bigger than 0
       max_wait_time_max_wait_time_vs_swarm_node_report_timeout: >
@@ -147,6 +148,9 @@ en:
         all topic names within a single consumer group must be unique considering namespacing styles
         disable this validation by setting config.strict_topics_namespacing to false
 
+    routing:
+      without_declarative_definition: lacks explicit declarative topics definition
+
     job_options:
       missing: needs to be present
       dispatch_method_format: needs to be either :produce_async or :produce_sync

data/karafka.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   DESC
 
   spec.add_dependency 'base64', '~> 0.2'
-  spec.add_dependency 'karafka-core', '>= 2.4.3', '< 2.5.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.4', '< 2.5.0'
   spec.add_dependency 'karafka-rdkafka', '>= 0.17.2'
   spec.add_dependency 'waterdrop', '>= 2.7.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'

data/lib/karafka/cli/base.rb

@@ -96,7 +96,7 @@ module Karafka
                 *[names, option[2], option[1]].flatten
               ) { |value| options[option[0]] = value }
             end
-          end.parse!
+          end.parse(ARGV)
 
           options
         end

data/lib/karafka/cli/topics/plan.rb

@@ -16,7 +16,10 @@ module Karafka
             return false
           end
 
+          changes = false
+
           unless topics_to_create.empty?
+            changes = true
             puts 'Following topics will be created:'
             puts
 
@@ -33,20 +36,54 @@ module Karafka
           end
 
           unless topics_to_repartition.empty?
-            puts 'Following topics will be repartitioned:'
-            puts
+            upscale = {}
+            downscale = {}
 
             topics_to_repartition.each do |topic, partitions|
               from = partitions
               to = topic.declaratives.partitions
 
-              puts "  #{yellow('~')} #{topic.name}:"
-              puts "    #{yellow('~')} partitions: \"#{red(from)}\" #{grey('=>')} \"#{green(to)}\""
+              if from < to
+                upscale[topic] = partitions
+              else
+                downscale[topic] = partitions
+              end
+            end
+
+            unless upscale.empty?
+              changes = true
+              puts 'Following topics will be repartitioned:'
+              puts
+
+              upscale.each do |topic, partitions|
+                from = partitions
+                to = topic.declaratives.partitions
+                y = yellow('~')
+                puts "  #{y} #{topic.name}:"
+                puts "    #{y} partitions: \"#{red(from)}\" #{grey('=>')} \"#{green(to)}\""
+                puts
+              end
+            end
+
+            unless downscale.empty?
+              puts(
+                'Following topics repartitioning will be ignored as downscaling is not supported:'
+              )
               puts
+
+              downscale.each do |topic, partitions|
+                from = partitions
+                to = topic.declaratives.partitions
+
+                puts "  #{grey('~')} #{topic.name}:"
+                puts "    #{grey('~')} partitions: \"#{grey(from)}\" #{grey('=>')} \"#{grey(to)}\""
+                puts
+              end
             end
           end
 
           unless topics_to_alter.empty?
+            changes = true
             puts 'Following topics will have configuration changes:'
             puts
 
@@ -65,7 +102,7 @@ module Karafka
             end
           end
 
-          true
+          changes
         end
 
         private

data/lib/karafka/cli/topics.rb

@@ -10,26 +10,53 @@ module Karafka
       )
 
       desc 'Allows for the topics management'
+
+      option(
+        :detailed_exitcode,
+        'Exists with 0 when no changes, 1 when error and 2 when changes present or applied',
+        TrueClass,
+        %w[
+          --detailed_exitcode
+        ]
+      )
+
+      # We exit with 0 if no changes happened
+      NO_CHANGES_EXIT_CODE = 0
+
+      # When any changes happened (or could happen) we return 2 because 1 is default when Ruby
+      # crashes
+      CHANGES_EXIT_CODE = 2
+
+      private_constant :NO_CHANGES_EXIT_CODE, :CHANGES_EXIT_CODE
+
       # @param action [String] action we want to take
       def call(action = 'missing')
-        case action
-        when 'create'
-          Topics::Create.new.call
-        when 'delete'
-          Topics::Delete.new.call
-        when 'reset'
-          Topics::Reset.new.call
-        when 'repartition'
-          Topics::Repartition.new.call
-        when 'migrate'
-          Topics::Migrate.new.call
-        when 'align'
-          Topics::Align.new.call
-        when 'plan'
-          Topics::Plan.new.call
-        else
-          raise ::ArgumentError, "Invalid topics action: #{action}"
-        end
+        detailed_exit_code = options.fetch(:detailed_exitcode, false)
+
+        command = case action
+                  when 'create'
+                    Topics::Create
+                  when 'delete'
+                    Topics::Delete
+                  when 'reset'
+                    Topics::Reset
+                  when 'repartition'
+                    Topics::Repartition
+                  when 'migrate'
+                    Topics::Migrate
+                  when 'align'
+                    Topics::Align
+                  when 'plan'
+                    Topics::Plan
+                  else
+                    raise ::ArgumentError, "Invalid topics action: #{action}"
+                  end
+
+        changes = command.new.call
+
+        return unless detailed_exit_code
+
+        changes ? exit(CHANGES_EXIT_CODE) : exit(NO_CHANGES_EXIT_CODE)
       end
     end
   end

data/lib/karafka/contracts/config.rb

@@ -34,6 +34,7 @@ module Karafka
       required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
       required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+      required(:strict_declarative_topics) { |val| [true, false].include?(val) }
 
       nested(:swarm) do
         required(:nodes) { |val| val.is_a?(Integer) && val.positive? }

data/lib/karafka/contracts/routing.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Ensures that routing wide rules are obeyed
+    class Routing < Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('routing')
+      end
+
+      # Ensures, that when declarative topics strict requirement is on, all topics have
+      # declarative definition (including DLQ topics)
+      # @note It will ignore routing pattern topics because those topics are virtual
+      virtual do |data, errors|
+        next unless errors.empty?
+        # Do not validate declaratives unless required and explicitly enabled
+        next unless Karafka::App.config.strict_declarative_topics
+
+        # Collects declarative topics. Please note, that any topic that has a `#topic` reference,
+        # will be declarative by default unless explicitly disabled. This however does not apply
+        # to the DLQ definitions
+        dec_topics = Set.new
+        # All topics including the DLQ topics names that are marked as active
+        topics = Set.new
+
+        data.each do |consumer_group|
+          consumer_group[:topics].each do |topic|
+            pat = topic[:patterns]
+            # Ignore pattern topics because they won't exist and should not be declarative
+            # managed
+            topics << topic[:name] if !pat || !pat[:active]
+
+            dlq = topic[:dead_letter_queue]
+            topics << dlq[:topic] if dlq[:active]
+
+            dec = topic[:declaratives]
+
+            dec_topics << topic[:name] if dec[:active]
+          end
+        end
+
+        missing_dec = topics - dec_topics
+
+        next if missing_dec.empty?
+
+        missing_dec.map do |topic_name|
+          [
+            [:topics, topic_name],
+            :without_declarative_definition
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb

@@ -29,13 +29,13 @@ module Karafka
             @port = port
           end
 
-          private
-
           # @return [Boolean] true if all good, false if we should tell k8s to kill this process
           def healthy?
             raise NotImplementedError, 'Implement in a subclass'
           end
 
+          private
+
           # Responds to a HTTP request with the process liveness status
           def respond
             client = @server.accept

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -112,6 +112,17 @@ module Karafka
             clear_polling_tick
           end
 
+          # Did we exceed any of the ttls
+          # @return [String] 204 string if ok, 500 otherwise
+          def healthy?
+            time = monotonic_now
+
+            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+
+            true
+          end
+
           private
 
           # Wraps the logic with a mutex
@@ -152,17 +163,6 @@ module Karafka
               @consumptions.delete(thread_id)
             end
           end
-
-          # Did we exceed any of the ttls
-          # @return [String] 204 string if ok, 500 otherwise
-          def healthy?
-            time = monotonic_now
-
-            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
-            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
-
-            true
-          end
         end
       end
     end

data/lib/karafka/routing/builder.rb

@@ -48,6 +48,10 @@ module Karafka
 
           instance_eval(&block)
 
+          # Ensures high-level routing details consistency
+          # Contains checks that require knowledge about all the consumer groups to operate
+          Contracts::Routing.new.validate!(map(&:to_h))
+
           each do |consumer_group|
             # Validate consumer group settings
             Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)

data/lib/karafka/setup/config.rb

@@ -68,6 +68,11 @@ module Karafka
       setting :strict_topics_namespacing, default: true
       # option [String] default consumer group name for implicit routing
       setting :group_id, default: 'app'
+      # option [Boolean] when set to true, it will validate as part of the routing validation, that
+      # all topics and DLQ topics (even not active) have the declarative topics definitions.
+      # Really useful when you want to ensure that all topics in routing are managed via
+      # declaratives.
+      setting :strict_declarative_topics, default: false
 
       setting :oauth do
         # option [false, #call] Listener for using oauth bearer. This listener will be able to

data/lib/karafka/swarm/supervisor.rb

@@ -20,7 +20,9 @@ module Karafka
         shutdown_timeout: %i[shutdown_timeout],
         supervision_sleep: %i[internal supervision_sleep],
         forceful_exit_code: %i[internal forceful_exit_code],
-        process: %i[internal process]
+        process: %i[internal process],
+        cli_contract: %i[internal cli contract],
+        activity_manager: %i[internal routing activity_manager]
       )
 
       # How long extra should we wait on shutdown before forceful termination
@@ -39,6 +41,9 @@ module Karafka
 
       # Creates needed number of forks, installs signals and starts supervision
       def run
+        # Validate the CLI provided options the same way as we do for the regular server
+        cli_contract.validate!(activity_manager.to_h)
+
         # Close producer just in case. While it should not be used, we do not want even a
         # theoretical case since librdkafka is not thread-safe.
         # We close it prior to forking just to make sure, there is no issue with initialized
@@ -67,7 +72,11 @@ module Karafka
           lock
           control
         end
-      # If anything went wrong, signal this and die
+
+      # If the cli contract validation failed reraise immediately and stop the process
+      rescue Karafka::Errors::InvalidConfigurationError => e
+        raise e
+      # If anything went wrong during supervision, signal this and die
       # Supervisor is meant to be thin and not cause any issues. If you encounter this case
       # please report it as it should be considered critical
       rescue StandardError => e

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.12'
+  VERSION = '2.4.13'
 end

data/lib/karafka.rb

@@ -49,17 +49,29 @@ module Karafka
       @monitor ||= App.config.monitor
     end
 
-    # @return [String] root path of this gem
+    # @return [Pathname] root path of this gem
     def gem_root
       Pathname.new(File.expand_path('..', __dir__))
     end
 
-    # @return [String] Karafka app root path (user application path)
+    # @return [Pathname] Karafka app root path (user application path)
     def root
-      Pathname.new(ENV['KARAFKA_ROOT_DIR'] || File.dirname(ENV['BUNDLE_GEMFILE']))
+      # If user points to a different root explicitly, use it
+      return Pathname.new(ENV['KARAFKA_ROOT_DIR']) if ENV['KARAFKA_ROOT_DIR']
+
+      # By default we infer the project root from bundler.
+      # We cannot use the BUNDLE_GEMFILE env directly because it may be altered by things like
+      # ruby-lsp. Instead we always fallback to the most outer Gemfile. In most of the cases, it
+      # won't matter but in case of some automatic setup alterations like ruby-lsp, the location
+      # from which the project starts may not match the original Gemfile.
+      Pathname.new(
+        File.dirname(
+          Bundler.with_unbundled_env { Bundler.default_gemfile }
+        )
+      )
     end
 
-    # @return [String] path to Karafka gem root core
+    # @return [Pathname] path to Karafka gem root core
     def core_root
       Pathname.new(File.expand_path('karafka', __dir__))
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.12
+  version: 2.4.13
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-09-17 00:00:00.000000000 Z
+date: 2024-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.3
+        version: 2.4.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.3
+        version: 2.4.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -219,6 +219,7 @@ files:
 - lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb
 - lib/karafka/contracts/consumer_group.rb
+- lib/karafka/contracts/routing.rb
 - lib/karafka/contracts/server_cli_options.rb
 - lib/karafka/contracts/topic.rb
 - lib/karafka/deserializers/headers.rb