karafka 2.5.0.rc2 → 2.5.1.beta1

data/lib/karafka/connection/client.rb

@@ -49,6 +49,7 @@ module Karafka
         :illegal_generation,
         # this will not recover as fencing is permanent
         :fenced, # -144
+        :auto_offset_reset, # -140
         # This can happen for many reasons, including issues with static membership being fenced
         :fatal # -150
       ].freeze
@@ -436,6 +437,13 @@ module Karafka
         @wrapped_kafka.query_watermark_offsets(topic, partition)
       end
 
+      # @return [String] safe inspection string that is causing circular dependencies and other
+      #   issues
+      def inspect
+        state = @closed ? 'closed' : 'open'
+        "#<#{self.class.name} id=#{@id.inspect} name=#{@name.inspect} state=#{state}>"
+      end
+
       private
 
       # When we cannot store an offset, it means we no longer own the partition

data/lib/karafka/connection/listener.rb

@@ -98,6 +98,9 @@ module Karafka
           @status.public_send("#{state}?")
         end
 
+        # Defined below
+        next if transition == :start!
+
         # Moves listener to a given state
         define_method transition do
           @status.public_send(transition)
@@ -266,7 +269,8 @@ module Karafka
         reset
 
         # Ruby sleep is in seconds
-        sleep_time = reset_backoff / 10_000.0
+        sleep_time = reset_backoff / 1_000.0
+
         sleep(sleep_time) && retry
       end
 

data/lib/karafka/connection/status.rb

@@ -23,16 +23,19 @@ module Karafka
 
       STATES.each do |state, transition|
         class_eval <<~RUBY, __FILE__, __LINE__ + 1
-          # Moves status to a different state
-          def #{transition}
-            @mutex.synchronize do
-              # Do not allow reverse state transitions (we always go one way) or transition to the
-              # same state as currently
-              return if @status && STATES.keys.index(:#{state}) <= STATES.keys.index(@status)
+          # Defined below
+          if #{transition != :stop!}
+            # Moves status to a different state
+            def #{transition}
+              @mutex.synchronize do
+                # Do not allow reverse state transitions (we always go one way) or transition to the
+                # same state as currently
+                return if @status && STATES.keys.index(:#{state}) <= STATES.keys.index(@status)
 
-              @status = :#{state}
-              conductor.signal
-              monitor.instrument("connection.listener.#{state}", caller: self)
+                @status = :#{state}
+                conductor.signal
+                monitor.instrument("connection.listener.#{state}", caller: self)
+              end
             end
           end
 

data/lib/karafka/errors.rb

@@ -133,14 +133,6 @@ module Karafka
     # optional dependencies are needed but not available.
     DependencyConstraintsError = Class.new(BaseError)
 
-    # Raised when we were not able to open pidfd for given pid
-    # This should not happen. If you see it, please report.
-    PidfdOpenFailedError = Class.new(BaseError)
-
-    # Failed to send signal to a process via pidfd
-    # This should not happen. If you see it, please report.
-    PidfdSignalFailedError = Class.new(BaseError)
-
     # Raised when given option/feature is not supported on a given platform or when given option
     # is not supported in a given configuration
     UnsupportedOptionError = Class.new(BaseError)

data/lib/karafka/instrumentation/assignments_tracker.rb

@@ -50,6 +50,22 @@ module Karafka
         end
       end
 
+      # @return [String] thread-safe and lock-safe inspect implementation
+      def inspect
+        info = if @mutex.try_lock
+                 begin
+                   assignments = @assignments.dup.transform_keys(&:name).inspect
+                   "assignments=#{assignments}"
+                 ensure
+                   @mutex.unlock
+                 end
+               else
+                 'busy'
+               end
+
+        "#<#{self.class.name} #{info}>"
+      end
+
       # When client is under reset due to critical issues, remove all of its assignments as we will
       #   get a new set of assignments
       # @param event [Karafka::Core::Monitoring::Event]

data/lib/karafka/instrumentation/logger_listener.rb

@@ -172,9 +172,10 @@ module Karafka
         Thread.list.each do |thread|
           tid = (thread.object_id ^ ::Process.pid).to_s(36)
 
+          warn ''
           warn "Thread TID-#{tid} #{thread.name}"
 
-          if thread.backtrace
+          if thread.backtrace && !thread.backtrace.empty?
             warn thread.backtrace.join("\n")
           else
             warn '<no backtrace available>'
@@ -341,48 +342,50 @@ module Karafka
       def on_error_occurred(event)
         type = event[:type]
         error = event[:error]
-        details = (error.backtrace || []).join("\n")
+        backtrace = (error.backtrace || []).join("\n")
+
+        details = [error.to_s, error_details(event)].compact.join(' ')
 
         case type
         when 'consumer.initialized.error'
-          error "Consumer initialized error: #{error}"
-          error details
+          error "Consumer initialized error: #{details}"
+          error backtrace
         when 'consumer.wrap.error'
-          error "Consumer wrap failed due to an error: #{error}"
-          error details
+          error "Consumer wrap failed due to an error: #{details}"
+          error backtrace
         when 'consumer.consume.error'
-          error "Consumer consuming error: #{error}"
-          error details
+          error "Consumer consuming error: #{details}"
+          error backtrace
         when 'consumer.revoked.error'
-          error "Consumer on revoked failed due to an error: #{error}"
-          error details
+          error "Consumer on revoked failed due to an error: #{details}"
+          error backtrace
         when 'consumer.idle.error'
-          error "Consumer idle failed due to an error: #{error}"
-          error details
+          error "Consumer idle failed due to an error: #{details}"
+          error backtrace
         when 'consumer.shutdown.error'
-          error "Consumer on shutdown failed due to an error: #{error}"
-          error details
+          error "Consumer on shutdown failed due to an error: #{details}"
+          error backtrace
         when 'consumer.tick.error'
-          error "Consumer on tick failed due to an error: #{error}"
-          error details
+          error "Consumer on tick failed due to an error: #{details}"
+          error backtrace
         when 'consumer.eofed.error'
-          error "Consumer on eofed failed due to an error: #{error}"
-          error details
+          error "Consumer on eofed failed due to an error: #{details}"
+          error backtrace
         when 'consumer.after_consume.error'
-          error "Consumer on after_consume failed due to an error: #{error}"
-          error details
+          error "Consumer on after_consume failed due to an error: #{details}"
+          error backtrace
         when 'worker.process.error'
-          fatal "Worker processing failed due to an error: #{error}"
-          fatal details
+          fatal "Worker processing failed due to an error: #{details}"
+          fatal backtrace
         when 'connection.listener.fetch_loop.error'
-          error "Listener fetch loop error: #{error}"
-          error details
+          error "Listener fetch loop error: #{details}"
+          error backtrace
         when 'swarm.supervisor.error'
-          fatal "Swarm supervisor crashed due to an error: #{error}"
-          fatal details
+          fatal "Swarm supervisor crashed due to an error: #{details}"
+          fatal backtrace
         when 'runner.call.error'
-          fatal "Runner crashed due to an error: #{error}"
-          fatal details
+          fatal "Runner crashed due to an error: #{details}"
+          fatal backtrace
         when 'app.stopping.error'
           # Counts number of workers and listeners that were still active when forcing the
           # shutdown. Please note, that unless all listeners are closed, workers will not finalize
@@ -400,44 +403,44 @@ module Karafka
 
           error message
         when 'app.forceful_stopping.error'
-          error "Forceful shutdown error occurred: #{error}"
-          error details
+          error "Forceful shutdown error occurred: #{details}"
+          error backtrace
         when 'librdkafka.error'
-          error "librdkafka internal error occurred: #{error}"
-          error details
+          error "librdkafka internal error occurred: #{details}"
+          error backtrace
         # Those can occur when emitted statistics are consumed by the end user and the processing
         # of statistics fails. The statistics are emitted from librdkafka main loop thread and
         # any errors there crash the whole thread
         when 'callbacks.statistics.error'
-          error "callbacks.statistics processing failed due to an error: #{error}"
-          error details
+          error "callbacks.statistics processing failed due to an error: #{details}"
+          error backtrace
         when 'callbacks.error.error'
-          error "callbacks.error processing failed due to an error: #{error}"
-          error details
+          error "callbacks.error processing failed due to an error: #{details}"
+          error backtrace
         # Those will only occur when retries in the client fail and when they did not stop after
         # back-offs
         when 'connection.client.poll.error'
-          error "Data polling error occurred: #{error}"
-          error details
+          error "Data polling error occurred: #{details}"
+          error backtrace
         when 'connection.client.rebalance_callback.error'
-          error "Rebalance callback error occurred: #{error}"
-          error details
+          error "Rebalance callback error occurred: #{details}"
+          error backtrace
         when 'connection.client.unsubscribe.error'
-          error "Client unsubscribe error occurred: #{error}"
-          error details
+          error "Client unsubscribe error occurred: #{details}"
+          error backtrace
         when 'parallel_segments.reducer.error'
-          error "Parallel segments reducer error occurred: #{error}"
-          error details
+          error "Parallel segments reducer error occurred: #{details}"
+          error backtrace
         when 'parallel_segments.partitioner.error'
-          error "Parallel segments partitioner error occurred: #{error}"
-          error details
+          error "Parallel segments partitioner error occurred: #{details}"
+          error backtrace
         when 'virtual_partitions.partitioner.error'
-          error "Virtual partitions partitioner error occurred: #{error}"
-          error details
+          error "Virtual partitions partitioner error occurred: #{details}"
+          error backtrace
         # This handles any custom errors coming from places like Web-UI, etc
         else
-          error "#{type} error occurred: #{error.class} - #{error}"
-          error details
+          error "#{type} error occurred: #{error.class} - #{details}"
+          error backtrace
         end
       end
 
@@ -453,6 +456,62 @@ module Karafka
       def log_polling?
         @log_polling
       end
+
+      # Extracts some structural location
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      # @note It uses similar approach to the Web UI but here we collect less info because it goes
+      #   to the logs.
+      def error_details(event)
+        caller_ref = event[:caller]
+
+        # Collect extra info if it was a consumer related error.
+        # Those come from user code
+        details = case caller_ref
+                  when Karafka::BaseConsumer
+                    extract_consumer_info(caller_ref)
+                  when Karafka::Connection::Client
+                    extract_client_info(caller_ref)
+                  when Karafka::Connection::Listener
+                    extract_listener_info(caller_ref)
+                  else
+                    {}
+                  end
+
+        return nil if details.empty?
+
+        "[#{details.map { |label, value| "#{label}: #{value}" }.join(', ')}]"
+      end
+
+      # @param consumer [::Karafka::BaseConsumer]
+      # @return [Hash] hash with consumer specific info for details of error
+      def extract_consumer_info(consumer)
+        {
+          consumer_group: consumer.topic.consumer_group.id,
+          subscription_group: consumer.topic.subscription_group.id,
+          topic: consumer.topic.name,
+          partition: consumer.partition,
+          first_offset: consumer.messages.metadata.first_offset,
+          last_offset: consumer.messages.metadata.last_offset
+        }
+      end
+
+      # @param client [::Karafka::Connection::Client]
+      # @return [Hash] hash with client specific info for details of error
+      def extract_client_info(client)
+        {
+          consumer_group: client.subscription_group.consumer_group.id,
+          subscription_group: client.subscription_group.id
+        }
+      end
+
+      # @param listener [::Karafka::Connection::Listener]
+      # @return [Hash] hash with listener specific info for details of error
+      def extract_listener_info(listener)
+        {
+          consumer_group: listener.subscription_group.consumer_group.id,
+          subscription_group: listener.subscription_group.id
+        }
+      end
     end
   end
 end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -83,6 +83,11 @@ module Karafka
         # @param job [Object] job we want to enqueue
         # @param timestamp [Time] time when job should run
         def dispatch_at(job, timestamp)
+          # If request is in the past, we can dispatch it directly without any envelope wrapping
+          # and proxy pass via scheduled messages. This improves latency and also allows for things
+          # like immediate continuation to work without having to have scheduled messages
+          return dispatch(job) if timestamp.to_f <= Time.now.to_f
+
           target_message = dispatch_details(job).merge!(
             topic: job.queue_name,
             payload: ::ActiveSupport::JSON.encode(serialize_job(job))

data/lib/karafka/pro/cleaner/messages/messages.rb

@@ -9,22 +9,32 @@ module Karafka
       module Messages
         # Extensions to the messages batch allowing for automatic cleaning of each message after
         # message is processed.
+        #
+        # This module is prepended to Karafka::Messages::Messages to add cleaning functionality.
+        # The implementation calls super() to maintain compatibility with other libraries that
+        # also prepend modules to modify the #each method (e.g., DataDog tracing).
+        # See: https://github.com/DataDog/dd-trace-rb/issues/4867
         module Messages
           # @param clean [Boolean] do we want to clean each message after we're done working with
           #   it.
-          # @yield block we want to execute per each message
+          # @param block [Proc] block we want to execute per each message
           #
           # @note Cleaning messages after we're done with each of them and did not fail does not
           #   affect any other functionalities. The only thing that is crucial is to make sure,
           #   that if DLQ is used, that we mark each message as consumed when using this API as
           #   otherwise a cleaned message may be dispatched and that should never happen
-          def each(clean: false)
-            @messages_array.each do |message|
-              yield(message)
-
-              next unless clean
-
-              message.clean!
+          #
+          # @note This method calls super() to ensure compatibility with other libraries that
+          #   may have prepended modules to modify #each behavior. This preserves the method
+          #   chain and allows instrumentation libraries to function correctly.
+          def each(clean: false, &block)
+            if clean
+              super() do |message|
+                yield(message)
+                message.clean!
+              end
+            else
+              super(&block)
             end
           end
         end

data/lib/karafka/pro/cli/contracts/server.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      # CLI related contracts
+      module Contracts
+        # Contract for validating correctness of the server cli command options.
+        # It differs slightly from the OSS one because it is aware of the routing patterns
+        class Server < ::Karafka::Cli::Contracts::Server
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('cli').fetch('server')
+          end
+
+          %i[
+            include
+            exclude
+          ].each do |action|
+            optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
+            optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
+            optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
+
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              value = data.fetch(:"#{action}_consumer_groups")
+
+              # If there were no consumer_groups declared in the server cli, it means that we will
+              # run all of them and no need to validate them here at all
+              next if value.empty?
+              next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
+
+              # Found unknown consumer groups
+              [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
+            end
+
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              value = data.fetch(:"#{action}_subscription_groups")
+
+              # If there were no subscription_groups declared in the server cli, it means that we
+              # will run all of them and no need to validate them here at all
+              next if value.empty?
+
+              subscription_groups = Karafka::App
+                                    .consumer_groups
+                                    .map(&:subscription_groups)
+                                    .flatten
+                                    .map(&:name)
+
+              next if (value - subscription_groups).empty?
+
+              # Found unknown subscription groups
+              [[[:"#{action}_subscription_groups"], :subscription_groups_inclusion]]
+            end
+
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              value = data.fetch(:"#{action}_topics")
+
+              # If there were no topics declared in the server cli, it means that we will
+              # run all of them and no need to validate them here at all
+              next if value.empty?
+
+              topics = Karafka::App
+                       .consumer_groups
+                       .map(&:subscription_groups)
+                       .flatten
+                       .map(&:topics)
+                       .map { |gtopics| gtopics.map(&:name) }
+                       .flatten
+
+              next if (value - topics).empty?
+
+              # If there are any patterns defined, we cannot report on topics inclusions because
+              # topics may be added during boot or runtime. We go with simple assumption:
+              # if there are patterns defined, we do not check the inclusions at all
+              next unless Karafka::App.consumer_groups.map(&:patterns).flatten.empty?
+
+              # Found unknown topics
+              [[[:"#{action}_topics"], :topics_inclusion]]
+            end
+          end
+
+          # Makes sure we have anything to subscribe to when we start the server
+          virtual do |_, errors|
+            next unless errors.empty?
+
+            next unless Karafka::App.subscription_groups.empty?
+
+            [[%i[include_topics], :topics_missing]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -15,7 +15,7 @@ module Karafka
               File.read(
                 File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
               )
-            ).fetch('en').fetch('validations').fetch('config')
+            ).fetch('en').fetch('validations').fetch('setup').fetch('config')
           end
 
           nested(:encryption) do

data/lib/karafka/pro/loader.rb

@@ -87,7 +87,7 @@ module Karafka
         def reconfigure(config)
           icfg = config.internal
 
-          icfg.cli.contract = Contracts::ServerCliOptions.new
+          icfg.cli.contract = Cli::Contracts::Server.new
 
           # Use manager that supports multiplexing
           icfg.connection.manager = Connection::Manager.new

data/lib/karafka/pro/recurring_tasks/contracts/config.rb

@@ -15,7 +15,7 @@ module Karafka
               File.read(
                 File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
               )
-            ).fetch('en').fetch('validations').fetch('config')
+            ).fetch('en').fetch('validations').fetch('setup').fetch('config')
           end
 
           nested(:recurring_tasks) do

data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:adaptive_iterator) do

data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class AdaptiveIterator < Base
           # Topic extension allowing us to enable and configure adaptive iterator
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @adaptive_iterator = nil
+            end
+
             # @param active [Boolean] should we use the automatic adaptive iterator
             # @param safety_margin [Integer]
             #   How big of a margin we leave ourselves so we can safely communicate back with

data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:dead_letter_queue) do

data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class DeadLetterQueue < Base
           # Expansions to the topic API in DLQ
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @dead_letter_queue = nil
+            end
+
             # @param strategy [#call, nil] Strategy we want to use or nil if a default strategy
             # (same as in OSS) should be applied
             # @param args [Hash] Pro DLQ arguments

data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:delaying) do

data/lib/karafka/pro/routing/features/delaying/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class Delaying < Base
           # Topic delaying API extensions
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @delaying = nil
+            end
+
             # @param delay [Integer, nil] minimum age of a message we want to process
             def delaying(delay = nil)
               # Those settings are used for validation

data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb

@@ -16,7 +16,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('consumer_group')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('consumer_group')
 
                 virtual do |data, errors|
                   next unless errors.empty?

data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:direct_assignments) do

data/lib/karafka/pro/routing/features/direct_assignments/topic.rb

@@ -10,6 +10,15 @@ module Karafka
         class DirectAssignments < Base
           # Topic extensions for direct assignments
           module Topic
+            # This method calls the parent class initializer and then sets up the
+            # extra instance variable to nil. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              super
+              @direct_assignments = nil
+            end
+
             # Allows for direct assignment of
             # @param partitions_or_all [true, Array<Integer>] informs Karafka that we want
             #   to use direct assignments instead of automatic for this topic. It also allows us

data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb

@@ -17,7 +17,7 @@ module Karafka
                   File.read(
                     File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                   )
-                ).fetch('en').fetch('validations').fetch('topic')
+                ).fetch('en').fetch('validations').fetch('routing').fetch('topic')
               end
 
               nested(:expiring) do