karafka 2.5.0.rc2 → 2.5.1.beta1

data/lib/karafka/setup/contracts/config.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # Namespace for setup related contracts
+    module Contracts
+      # Contract with validation rules for Karafka configuration details.
+      #
+      # @note There are many more configuration options inside of the
+      #   `Karafka::Setup::Config` model, but we don't validate them here as they are
+      #   validated per each route (topic + consumer_group) because they can be overwritten,
+      #   so we validate all of that once all the routes are defined and ready.
+      class Config < Karafka::Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('setup').fetch('config')
+        end
+
+        # Topics regexp constant reference for easier usage
+        TOPIC_REGEXP = ::Karafka::Contracts::TOPIC_REGEXP
+
+        private_constant :TOPIC_REGEXP
+
+        # License validity happens in the licenser. Here we do only the simple consistency checks
+        nested(:license) do
+          required(:token) { |val| [true, false].include?(val) || val.is_a?(String) }
+          required(:entity) { |val| val.is_a?(String) }
+        end
+
+        required(:client_id) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+        required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
+        required(:consumer_persistence) { |val| [true, false].include?(val) }
+        required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
+        required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
+        required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
+        required(:strict_topics_namespacing) { |val| [true, false].include?(val) }
+        required(:shutdown_timeout) { |val| val.is_a?(Integer) && val.positive? }
+        required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
+        required(:group_id) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+        required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+        required(:strict_declarative_topics) { |val| [true, false].include?(val) }
+        required(:worker_thread_priority) { |val| (-3..3).to_a.include?(val) }
+
+        nested(:swarm) do
+          required(:nodes) { |val| val.is_a?(Integer) && val.positive? }
+          required(:node) { |val| val == false || val.is_a?(Karafka::Swarm::Node) }
+        end
+
+        nested(:oauth) do
+          required(:token_provider_listener) do |val|
+            val == false || val.respond_to?(:on_oauthbearer_token_refresh)
+          end
+        end
+
+        nested(:admin) do
+          # Can be empty because inherits values from the root kafka
+          required(:kafka) { |val| val.is_a?(Hash) }
+          required(:group_id) { |val| val.is_a?(String) && TOPIC_REGEXP.match?(val) }
+          required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
+          required(:retry_backoff) { |val| val.is_a?(Integer) && val >= 100 }
+          required(:max_retries_duration) { |val| val.is_a?(Integer) && val >= 1_000 }
+        end
+
+        # We validate internals just to be sure, that they are present and working
+        nested(:internal) do
+          required(:status) { |val| !val.nil? }
+          required(:process) { |val| !val.nil? }
+          # In theory this could be less than a second, however this would impact the maximum time
+          # of a single consumer queue poll, hence we prevent it
+          required(:tick_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+          required(:supervision_sleep) { |val| val.is_a?(Numeric) && val.positive? }
+          required(:forceful_exit_code) { |val| val.is_a?(Integer) && val >= 0 }
+          required(:forceful_shutdown_wait) { |val| val.is_a?(Integer) && val >= 0 }
+
+          nested(:swarm) do
+            required(:manager) { |val| !val.nil? }
+            required(:orphaned_exit_code) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:supervision_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+            required(:liveness_interval) { |val| val.is_a?(Integer) && val >= 1_000 }
+            required(:liveness_listener) { |val| !val.nil? }
+            required(:node_report_timeout) { |val| val.is_a?(Integer) && val >= 1_000 }
+            required(:node_restart_timeout) { |val| val.is_a?(Integer) && val >= 1_000 }
+          end
+
+          nested(:connection) do
+            required(:manager) { |val| !val.nil? }
+            required(:conductor) { |val| !val.nil? }
+            required(:reset_backoff) { |val| val.is_a?(Integer) && val >= 1_000 }
+            required(:listener_thread_priority) { |val| (-3..3).to_a.include?(val) }
+
+            nested(:proxy) do
+              nested(:commit) do
+                required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+                required(:wait_time) { |val| val.is_a?(Integer) && val.positive? }
+              end
+
+              # All of them have the same requirements
+              %i[
+                query_watermark_offsets
+                offsets_for_times
+                committed
+                metadata
+              ].each do |scope|
+                nested(scope) do
+                  required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
+                  required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+                  required(:wait_time) { |val| val.is_a?(Integer) && val.positive? }
+                end
+              end
+            end
+          end
+
+          nested(:routing) do
+            required(:builder) { |val| !val.nil? }
+            required(:subscription_groups_builder) { |val| !val.nil? }
+          end
+
+          nested(:processing) do
+            required(:jobs_builder) { |val| !val.nil? }
+            required(:jobs_queue_class) { |val| !val.nil? }
+            required(:scheduler_class) { |val| !val.nil? }
+            required(:coordinator_class) { |val| !val.nil? }
+            required(:errors_tracker_class) { |val| val.nil? || val.is_a?(Class) }
+            required(:partitioner_class) { |val| !val.nil? }
+            required(:strategy_selector) { |val| !val.nil? }
+            required(:expansions_selector) { |val| !val.nil? }
+            required(:executor_class) { |val| !val.nil? }
+            required(:worker_job_call_wrapper) { |val| val == false || val.respond_to?(:wrap) }
+          end
+
+          nested(:active_job) do
+            required(:dispatcher) { |val| !val.nil? }
+            required(:job_options_contract) { |val| !val.nil? }
+            required(:consumer_class) { |val| !val.nil? }
+          end
+        end
+
+        # Ensure all root kafka keys are symbols
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          detected_errors = []
+
+          data.fetch(:kafka).each_key do |key|
+            next if key.is_a?(Symbol)
+
+            detected_errors << [[:kafka, key], :key_must_be_a_symbol]
+          end
+
+          detected_errors
+        end
+
+        # Ensure all admin kafka keys are symbols
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          detected_errors = []
+
+          data.fetch(:admin).fetch(:kafka).each_key do |key|
+            next if key.is_a?(Symbol)
+
+            detected_errors << [[:admin, :kafka, key], :key_must_be_a_symbol]
+          end
+
+          detected_errors
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          pause_timeout = data.fetch(:pause_timeout)
+          pause_max_timeout = data.fetch(:pause_max_timeout)
+
+          next if pause_timeout <= pause_max_timeout
+
+          [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
+        end
+
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          shutdown_timeout = data.fetch(:shutdown_timeout)
+          max_wait_time = data.fetch(:max_wait_time)
+
+          next if max_wait_time < shutdown_timeout
+
+          [[%i[shutdown_timeout], :shutdown_timeout_vs_max_wait_time]]
+        end
+
+        # `internal.swarm.node_report_timeout` should not be close to `max_wait_time` otherwise
+        # there may be a case where node cannot report often enough because it is clogged by
+        # waiting on more data.
+        #
+        # We handle that at a config level to make sure that this is correctly configured.
+        #
+        # We do not validate this in the context of swarm usage (validate only if...) because it is
+        # often that swarm only runs on prod and we do not want to crash it surprisingly.
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          max_wait_time = data.fetch(:max_wait_time)
+          node_report_timeout = data.fetch(:internal)[:swarm][:node_report_timeout] || false
+
+          next unless node_report_timeout
+          # max wait time should be at least 20% smaller than the reporting time to have enough
+          # time for reporting
+          next if max_wait_time < node_report_timeout * 0.8
+
+          [[%i[max_wait_time], :max_wait_time_vs_swarm_node_report_timeout]]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/defaults_injector.rb

@@ -16,6 +16,7 @@ module Karafka
         # some features may use this value for computation and it is better to ensure, we do
         # always have it
         'max.poll.interval.ms': 300_000,
+        'socket.nagle.disable': true,
         'client.software.version': [
           "v#{Karafka::VERSION}",
           "rdkafka-ruby-v#{Rdkafka::VERSION}",
@@ -42,7 +43,8 @@ module Karafka
       PRODUCER_KAFKA_DEV_DEFAULTS = {
         # For all of those same reasoning as for the consumer
         'allow.auto.create.topics': 'true',
-        'topic.metadata.refresh.interval.ms': 5_000
+        'topic.metadata.refresh.interval.ms': 5_000,
+        'socket.nagle.disable': true
       }.freeze
 
       private_constant :CONSUMER_KAFKA_DEFAULTS, :CONSUMER_KAFKA_DEV_DEFAULTS,

data/lib/karafka/swarm/node.rb

@@ -45,7 +45,9 @@ module Karafka
       # @param parent_pid [Integer] parent pid for zombie fencing
       def initialize(id, parent_pid)
         @id = id
-        @parent_pidfd = Pidfd.new(parent_pid)
+        @parent_pid = parent_pid
+        @mutex = Mutex.new
+        @alive = nil
       end
 
       # Starts a new fork and:
@@ -56,6 +58,9 @@ module Karafka
       # @note Parent API
       def start
         @reader, @writer = IO.pipe
+        # Reset alive status when starting/restarting a node
+        # nil means unknown status - will check with waitpid
+        @mutex.synchronize { @alive = nil }
 
         # :nocov:
         @pid = ::Process.fork do
@@ -108,7 +113,6 @@ module Karafka
         # :nocov:
 
         @writer.close
-        @pidfd = Pidfd.new(@pid)
       end
 
       # Indicates that this node is doing well
@@ -147,13 +151,42 @@ module Karafka
       # @note Parent API
       # @note Keep in mind that the fact that process is alive does not mean it is healthy
       def alive?
-        @pidfd.alive?
+        # Don't try to waitpid on ourselves - just check if process exists
+        return true if @pid == ::Process.pid
+
+        @mutex.synchronize do
+          # Return cached result if we've already determined the process is dead
+          return false if @alive == false
+
+          begin
+            # Try to reap the process without blocking. If it returns the pid,
+            # the process has exited (zombie). If it returns nil, still running.
+            result = ::Process.waitpid(@pid, ::Process::WNOHANG)
+
+            if result
+              # Process has exited and we've reaped it
+              @alive = false
+              false
+            else
+              # Process is still running
+              true
+            end
+          rescue Errno::ECHILD
+            # Process doesn't exist or already reaped
+            @alive = false
+            false
+          rescue Errno::ESRCH
+            # Process doesn't exist
+            @alive = false
+            false
+          end
+        end
       end
 
       # @return [Boolean] true if node is orphaned or false otherwise. Used for orphans detection.
       # @note Child API
       def orphaned?
-        !@parent_pidfd.alive?
+        ::Process.ppid != @parent_pid
       end
 
       # Sends sigterm to the node
@@ -176,13 +209,40 @@ module Karafka
 
       # Sends provided signal to the node
       # @param signal [String]
+      # @return [Boolean] true if signal was sent, false if process doesn't exist
       def signal(signal)
-        @pidfd.signal(signal)
+        ::Process.kill(signal, @pid)
+        true
+      rescue Errno::ESRCH
+        # Process doesn't exist
+        false
       end
 
       # Removes the dead process from the processes table
+      # @return [Boolean] true if process was reaped, false if still running or already reaped
       def cleanup
-        @pidfd.cleanup
+        @mutex.synchronize do
+          # If we've already marked it as dead (reaped in alive?), nothing to do
+          return false if @alive == false
+
+          begin
+            # WNOHANG means don't block if process hasn't exited yet
+            result = ::Process.waitpid(@pid, ::Process::WNOHANG)
+
+            if result
+              # Process exited and was reaped
+              @alive = false
+              true
+            else
+              # Process is still running
+              false
+            end
+          rescue Errno::ECHILD
+            # Process already reaped or doesn't exist, which is fine
+            @alive = false
+            false
+          end
+        end
       end
 
       private

data/lib/karafka/swarm.rb

@@ -18,9 +18,9 @@ module Karafka
         )
       end
 
-      # @return [Boolean] true if fork API and pidfd OS API are available, otherwise false
+      # @return [Boolean] true if fork API is available, otherwise false
       def supported?
-        ::Process.respond_to?(:fork) && Swarm::Pidfd.supported?
+        ::Process.respond_to?(:fork)
       end
     end
   end

data/lib/karafka/templates/karafka.rb.erb

@@ -12,12 +12,7 @@ Bundler.require(:default, ENV['KARAFKA_ENV'])
 # Karafka framework configuration
 APP_LOADER = Zeitwerk::Loader.new
 APP_LOADER.enable_reloading
-
-%w[
-  lib
-  app/consumers
-].each { |dir| APP_LOADER.push_dir(dir) }
-
+APP_LOADER.push_dir('app/consumers')
 APP_LOADER.setup
 APP_LOADER.eager_load
 <% end -%>
@@ -25,7 +20,7 @@ APP_LOADER.eager_load
 class KarafkaApp < Karafka::App
   setup do |config|
     config.kafka = { 'bootstrap.servers': '127.0.0.1:9092' }
-    config.client_id = 'YOUR_APP_NAME'
+    config.client_id = "YOUR_APP_NAME-#{Process.pid}-#{Socket.gethostname}"
 
     # IMPORTANT: Customize this group_id with your application name.
     # The group_id should be unique per application to properly track message consumption.

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.5.0.rc2'
+  VERSION = '2.5.1.beta1'
 end

data/lib/karafka.rb

@@ -1,23 +1,22 @@
 # frozen_string_literal: true
 
-%w[
-  karafka-core
-  delegate
-  English
-  rdkafka
-  waterdrop
-  json
-  forwardable
-  fileutils
-  openssl
-  optparse
-  base64
-  date
-  singleton
-  digest
-  zeitwerk
-  logger
-].each(&method(:require))
+require 'karafka-core'
+require 'delegate'
+require 'English'
+require 'rdkafka'
+require 'waterdrop'
+require 'json'
+require 'forwardable'
+require 'fileutils'
+require 'openssl'
+require 'optparse'
+require 'socket'
+require 'base64'
+require 'date'
+require 'singleton'
+require 'digest'
+require 'zeitwerk'
+require 'logger'
 
 # Karafka framework main namespace
 module Karafka

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.5.0.rc2
+  version: 2.5.1.beta1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -29,7 +29,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.2
+        version: 2.5.6
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.6.0
@@ -39,7 +39,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.2
+        version: 2.5.6
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.6.0
@@ -49,14 +49,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.19.5
+        version: 0.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.19.5
+        version: 0.21.0
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
@@ -110,8 +110,10 @@ files:
 - ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
-- ".github/workflows/ci.yml"
+- ".github/workflows/ci_linux_ubuntu_x86_64_gnu.yml"
+- ".github/workflows/ci_macos_arm64.yml"
 - ".github/workflows/push.yml"
+- ".github/workflows/trigger-wiki-refresh.yml"
 - ".github/workflows/verify-action-pins.yml"
 - ".gitignore"
 - ".rspec"
@@ -167,11 +169,14 @@ files:
 - lib/karafka/admin/configs.rb
 - lib/karafka/admin/configs/config.rb
 - lib/karafka/admin/configs/resource.rb
+- lib/karafka/admin/consumer_groups.rb
+- lib/karafka/admin/topics.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb
 - lib/karafka/cli/base.rb
 - lib/karafka/cli/console.rb
+- lib/karafka/cli/contracts/server.rb
 - lib/karafka/cli/help.rb
 - lib/karafka/cli/info.rb
 - lib/karafka/cli/install.rb
@@ -201,11 +206,6 @@ files:
 - lib/karafka/constraints.rb
 - lib/karafka/contracts.rb
 - lib/karafka/contracts/base.rb
-- lib/karafka/contracts/config.rb
-- lib/karafka/contracts/consumer_group.rb
-- lib/karafka/contracts/routing.rb
-- lib/karafka/contracts/server_cli_options.rb
-- lib/karafka/contracts/topic.rb
 - lib/karafka/deserializers/headers.rb
 - lib/karafka/deserializers/key.rb
 - lib/karafka/deserializers/payload.rb
@@ -260,6 +260,7 @@ files:
 - lib/karafka/pro/cleaner/messages/message.rb
 - lib/karafka/pro/cleaner/messages/messages.rb
 - lib/karafka/pro/cleaner/messages/metadata.rb
+- lib/karafka/pro/cli/contracts/server.rb
 - lib/karafka/pro/cli/parallel_segments.rb
 - lib/karafka/pro/cli/parallel_segments/base.rb
 - lib/karafka/pro/cli/parallel_segments/collapse.rb
@@ -267,7 +268,6 @@ files:
 - lib/karafka/pro/connection/manager.rb
 - lib/karafka/pro/connection/multiplexing/listener.rb
 - lib/karafka/pro/contracts/base.rb
-- lib/karafka/pro/contracts/server_cli_options.rb
 - lib/karafka/pro/encryption.rb
 - lib/karafka/pro/encryption/cipher.rb
 - lib/karafka/pro/encryption/contracts/config.rb
@@ -534,6 +534,9 @@ files:
 - lib/karafka/routing/activity_manager.rb
 - lib/karafka/routing/builder.rb
 - lib/karafka/routing/consumer_group.rb
+- lib/karafka/routing/contracts/consumer_group.rb
+- lib/karafka/routing/contracts/routing.rb
+- lib/karafka/routing/contracts/topic.rb
 - lib/karafka/routing/features/active_job.rb
 - lib/karafka/routing/features/active_job/builder.rb
 - lib/karafka/routing/features/active_job/config.rb
@@ -576,6 +579,7 @@ files:
 - lib/karafka/server.rb
 - lib/karafka/setup/attributes_map.rb
 - lib/karafka/setup/config.rb
+- lib/karafka/setup/contracts/config.rb
 - lib/karafka/setup/defaults_injector.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
@@ -583,7 +587,6 @@ files:
 - lib/karafka/swarm/liveness_listener.rb
 - lib/karafka/swarm/manager.rb
 - lib/karafka/swarm/node.rb
-- lib/karafka/swarm/pidfd.rb
 - lib/karafka/swarm/supervisor.rb
 - lib/karafka/templates/application_consumer.rb.erb
 - lib/karafka/templates/example_consumer.rb.erb
@@ -613,14 +616,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.0.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.
 test_files: []