karafka 2.4.0 → 2.4.18

data/lib/karafka/instrumentation/vendors/appsignal/client.rb

@@ -23,11 +23,16 @@ module Karafka
           # @param action_name [String] action name. For processing this should be equal to
           #   consumer class + method name
           def start_transaction(action_name)
-            transaction = ::Appsignal::Transaction.create(
-              SecureRandom.uuid,
-              namespace_name,
-              ::Appsignal::Transaction::GenericRequest.new({})
-            )
+            transaction =
+              if version_4_or_newer?
+                ::Appsignal::Transaction.create(namespace_name)
+              else
+                ::Appsignal::Transaction.create(
+                  SecureRandom.uuid,
+                  namespace_name,
+                  ::Appsignal::Transaction::GenericRequest.new({})
+                )
+              end
 
             transaction.set_action_if_nil(action_name)
           end
@@ -45,10 +50,10 @@ module Karafka
           def metadata=(metadata_hash)
             return unless transaction?
 
-            transaction = ::Appsignal::Transaction.current
+            current_transaction = transaction
 
             stringify_hash(metadata_hash).each do |key, value|
-              transaction.set_metadata(key, value)
+              current_transaction.set_metadata(key, value)
             end
           end
 
@@ -78,15 +83,20 @@ module Karafka
             )
           end
 
-          # Sends the error that occurred to Appsignal
+          # Report the error that occurred to Appsignal
           #
           # @param error [Object] error we want to ship to Appsignal
-          def send_error(error)
+          def report_error(error)
+            if ::Appsignal.respond_to?(:report_error)
+              # This helper will always report the error
+              ::Appsignal.report_error(error) do |transaction|
+                transaction.set_namespace(namespace_name)
+              end
             # If we have an active transaction we should use it instead of creating a generic one
             # That way proper namespace and other data may be transferred
             #
             # In case there is no transaction, a new generic background job one will be used
-            if transaction?
+            elsif transaction?
               transaction.set_error(error)
             else
               ::Appsignal.send_error(error) do |transaction|
@@ -99,7 +109,11 @@ module Karafka
           # @param name [Symbol] probe name
           # @param probe [Proc] code to run every minute
           def register_probe(name, probe)
-            ::Appsignal::Minutely.probes.register(name, probe)
+            if ::Appsignal::Probes.respond_to?(:register)
+              ::Appsignal::Probes.register(name, probe)
+            else
+              ::Appsignal::Minutely.probes.register(name, probe)
+            end
           end
 
           private
@@ -129,6 +143,13 @@ module Karafka
           def namespace_name
             @namespace_name ||= ::Appsignal::Transaction::BACKGROUND_JOB
           end
+
+          # @return [Boolean] is this v4+ version of Appsignal gem or older. Used for backwards
+          #   compatibility checks.
+          def version_4_or_newer?
+            @version_4_or_newer ||=
+              Gem::Version.new(::Appsignal::VERSION) >= Gem::Version.new('4.0.0')
+          end
         end
       end
     end

data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb

@@ -21,7 +21,7 @@ module Karafka
           #
           # @param event [Karafka::Core::Monitoring::Event]
           def on_error_occurred(event)
-            client.send_error(event[:error])
+            client.report_error(event[:error])
           end
         end
       end

data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb

@@ -48,6 +48,7 @@ module Karafka
             consumer.revoked.error
             consumer.shutdown.error
             consumer.tick.error
+            consumer.eofed.error
           ].freeze
 
           private_constant :USER_CONSUMER_ERROR_TYPES
@@ -107,7 +108,8 @@ module Karafka
           [
             %i[revoke revoked revoked],
             %i[shutting_down shutdown shutdown],
-            %i[tick ticked tick]
+            %i[tick ticked tick],
+            %i[eof eofed eofed]
           ].each do |before, after, name|
             class_eval <<~RUBY, __FILE__, __LINE__ + 1
               # Keeps track of user code execution

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -35,6 +35,7 @@ module Karafka
           def initialize(&block)
             configure
             setup(&block) if block
+            @job_types_cache = {}
           end
 
           # @param block [Proc] configuration block
@@ -51,28 +52,11 @@ module Karafka
             push_tags
 
             job = event[:job]
-            job_type = job.class.to_s.split('::').last
+            job_type = fetch_job_type(job.class)
             consumer = job.executor.topic.consumer
             topic = job.executor.topic.name
 
-            action = case job_type
-                     when 'Periodic'
-                       'tick'
-                     when 'PeriodicNonBlocking'
-                       'tick'
-                     when 'Shutdown'
-                       'shutdown'
-                     when 'Revoked'
-                       'revoked'
-                     when 'RevokedNonBlocking'
-                       'revoked'
-                     when 'Idle'
-                       'idle'
-                     else
-                       'consume'
-                     end
-
-            current_span.resource = "#{consumer}##{action}"
+            current_span.resource = "#{consumer}##{job.class.action}"
             info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
 
             pop_tags
@@ -121,6 +105,8 @@ module Karafka
               error "Consumer on shutdown failed due to an error: #{error}"
             when 'consumer.tick.error'
               error "Consumer tick failed due to an error: #{error}"
+            when 'consumer.eofed.error'
+              error "Consumer eofed failed due to an error: #{error}"
             when 'worker.process.error'
               fatal "Worker processing failed due to an error: #{error}"
             when 'connection.listener.fetch_loop.error'
@@ -169,6 +155,18 @@ module Karafka
 
             Karafka.logger.pop_tags
           end
+
+          private
+
+          # Takes the job class and extracts the job type.
+          # @param job_class [Class] job class
+          # @return [String]
+          # @note It does not have to be thread-safe despite running in multiple threads because
+          #   the assignment race condition is irrelevant here since the same value will be
+          #   assigned.
+          def fetch_job_type(job_class)
+            @job_types_cache[job_class] ||= job_class.to_s.split('::').last
+          end
         end
       end
     end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -82,10 +82,11 @@ module Karafka
             statistics = event[:statistics]
             consumer_group_id = event[:consumer_group_id]
 
-            base_tags = default_tags + ["consumer_group:#{consumer_group_id}"]
+            tags = ["consumer_group:#{consumer_group_id}"]
+            tags.concat(default_tags)
 
             rd_kafka_metrics.each do |metric|
-              report_metric(metric, statistics, base_tags)
+              report_metric(metric, statistics, tags)
             end
           end
 
@@ -93,13 +94,14 @@ module Karafka
           #
           # @param event [Karafka::Core::Monitoring::Event]
           def on_error_occurred(event)
-            extra_tags = ["type:#{event[:type]}"]
+            tags = ["type:#{event[:type]}"]
+            tags.concat(default_tags)
 
             if event.payload[:caller].respond_to?(:messages)
-              extra_tags += consumer_tags(event.payload[:caller])
+              tags.concat(consumer_tags(event.payload[:caller]))
             end
 
-            count('error_occurred', 1, tags: default_tags + extra_tags)
+            count('error_occurred', 1, tags: tags)
           end
 
           # Reports how many messages we've polled and how much time did we spend on it
@@ -111,10 +113,11 @@ module Karafka
 
             consumer_group_id = event[:subscription_group].consumer_group.id
 
-            extra_tags = ["consumer_group:#{consumer_group_id}"]
+            tags = ["consumer_group:#{consumer_group_id}"]
+            tags.concat(default_tags)
 
-            histogram('listener.polling.time_taken', time_taken, tags: default_tags + extra_tags)
-            histogram('listener.polling.messages', messages_count, tags: default_tags + extra_tags)
+            histogram('listener.polling.time_taken', time_taken, tags: tags)
+            histogram('listener.polling.messages', messages_count, tags: tags)
           end
 
           # Here we report majority of things related to processing as we have access to the
@@ -125,7 +128,8 @@ module Karafka
             messages = consumer.messages
             metadata = messages.metadata
 
-            tags = default_tags + consumer_tags(consumer)
+            tags = consumer_tags(consumer)
+            tags.concat(default_tags)
 
             count('consumer.messages', messages.count, tags: tags)
             count('consumer.batches', 1, tags: tags)
@@ -146,7 +150,8 @@ module Karafka
               #
               # @param event [Karafka::Core::Monitoring::Event]
               def on_consumer_#{after}(event)
-                tags = default_tags + consumer_tags(event.payload[:caller])
+                tags = consumer_tags(event.payload[:caller])
+                tags.concat(default_tags)
 
                 count('consumer.#{name}', 1, tags: tags)
               end
@@ -158,9 +163,10 @@ module Karafka
           def on_worker_process(event)
             jq_stats = event[:jobs_queue].statistics
 
-            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
-            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
-            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
+            tags = default_tags
+            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: tags)
+            histogram('worker.processing', jq_stats[:busy], tags: tags)
+            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: tags)
           end
 
           # We report this metric before and after processing for higher accuracy
@@ -240,11 +246,14 @@ module Karafka
                 # node ids
                 next if broker_statistics['nodeid'] == -1
 
+                tags = ["broker:#{broker_statistics['nodename']}"]
+                tags.concat(base_tags)
+
                 public_send(
                   metric.type,
                   metric.name,
                   broker_statistics.dig(*metric.key_location),
-                  tags: base_tags + ["broker:#{broker_statistics['nodename']}"]
+                  tags: tags
                 )
               end
             when :topics
@@ -259,14 +268,14 @@ module Karafka
                   next if partition_statistics['fetch_state'] == 'stopped'
                   next if partition_statistics['fetch_state'] == 'none'
 
+                  tags = ["topic:#{topic_name}", "partition:#{partition_name}"]
+                  tags.concat(base_tags)
+
                   public_send(
                     metric.type,
                     metric.name,
                     partition_statistics.dig(*metric.key_location),
-                    tags: base_tags + [
-                      "topic:#{topic_name}",
-                      "partition:#{partition_name}"
-                    ]
+                    tags: tags
                   )
                 end
               end

data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb

@@ -29,13 +29,13 @@ module Karafka
             @port = port
           end
 
-          private
-
           # @return [Boolean] true if all good, false if we should tell k8s to kill this process
           def healthy?
             raise NotImplementedError, 'Implement in a subclass'
           end
 
+          private
+
           # Responds to a HTTP request with the process liveness status
           def respond
             client = @server.accept

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -26,6 +26,19 @@ module Karafka
         #
         # @note Please use `Kubernetes::SwarmLivenessListener` when operating in the swarm mode
         class LivenessListener < BaseListener
+          # When any of those occurs, it means something went wrong in a way that cannot be
+          # recovered. In such cases we should report that the consumer process is not healthy.
+          # - `fenced` - This instance has been fenced by a newer instance and will not do any
+          #   processing at all never. Fencing most of the time means the instance.group.id has
+          #   been reused without properly terminating the previous consumer process first
+          # - `fatal` - any fatal error that halts the processing forever
+          UNRECOVERABLE_RDKAFKA_ERRORS = [
+            :fenced, # -144
+            :fatal # -150
+          ].freeze
+
+          private_constant :UNRECOVERABLE_RDKAFKA_ERRORS
+
           # @param hostname [String, nil] hostname or nil to bind on all
           # @param port [Integer] TCP port on which we want to run our HTTP status server
           # @param consuming_ttl [Integer] time in ms after which we consider consumption hanging.
@@ -40,6 +53,11 @@ module Karafka
             consuming_ttl: 5 * 60 * 1_000,
             polling_ttl: 5 * 60 * 1_000
           )
+            # If this is set to true, it indicates unrecoverable error like fencing
+            # While fencing can be partial (for one of the SGs), we still should consider this
+            # as an undesired state for the whole process because it halts processing in a
+            # non-recoverable manner forever
+            @unrecoverable = false
             @polling_ttl = polling_ttl
             @consuming_ttl = consuming_ttl
             @mutex = Mutex.new
@@ -86,10 +104,19 @@ module Karafka
             RUBY
           end
 
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_error_occurred(_event)
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(event)
             clear_consumption_tick
             clear_polling_tick
+
+            error = event[:error]
+
+            # We are only interested in the rdkafka errors
+            return unless error.is_a?(Rdkafka::RdkafkaError)
+            # We mark as unrecoverable only on certain errors that will not be fixed by retrying
+            return unless UNRECOVERABLE_RDKAFKA_ERRORS.include?(error.code)
+
+            @unrecoverable = true
           end
 
           # Deregister the polling tracker for given listener
@@ -112,6 +139,18 @@ module Karafka
             clear_polling_tick
           end
 
+          # Did we exceed any of the ttls
+          # @return [String] 204 string if ok, 500 otherwise
+          def healthy?
+            time = monotonic_now
+
+            return false if @unrecoverable
+            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+
+            true
+          end
+
           private
 
           # Wraps the logic with a mutex
@@ -152,17 +191,6 @@ module Karafka
               @consumptions.delete(thread_id)
             end
           end
-
-          # Did we exceed any of the ttls
-          # @return [String] 204 string if ok, 500 otherwise
-          def healthy?
-            time = monotonic_now
-
-            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
-            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
-
-            true
-          end
         end
       end
     end

data/lib/karafka/messages/message.rb

@@ -8,15 +8,9 @@ module Karafka
     # heavy-deserialization data without slowing down the whole application.
     class Message
       extend Forwardable
-
-      class << self
-        # @return [Object] general parser
-        # @note We cache it here for performance reasons. It is 2.5x times faster than getting it
-        #   via the config chain.
-        def parser
-          @parser ||= App.config.internal.messages.parser
-        end
-      end
+      extend Helpers::ConfigImporter.new(
+        parser: %i[internal messages parser]
+      )
 
       attr_reader :metadata
       # raw payload needs to be mutable as we want to have option to change it in the parser
@@ -51,6 +45,12 @@ module Karafka
         @deserialized
       end
 
+      # @return [Boolean] true if the message has a key and raw payload is nil, it is a tombstone
+      #   event. Otherwise it is not.
+      def tombstone?
+        !raw_key.nil? && @raw_payload.nil?
+      end
+
       private
 
       # @return [Object] deserialized data

data/lib/karafka/pro/active_job/consumer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -28,14 +20,26 @@ module Karafka
           # each job.
           partitioner: nil,
           # Allows for usage of `:key` or `:partition_key`
-          partition_key_type: :key
+          partition_key_type: :key,
+          # Topic to where this message should go when using scheduled messages. When defined,
+          # it will be used with `enqueue_at`. If not defined it will raise an error.
+          scheduled_messages_topic: nil,
+          # Allows for setting a callable producer since at the moment of defining the class,
+          # variants may not be available
+          #
+          # We do not initialize it with `-> { ::Karafka.producer }` so we do not have to call it
+          #   each time for the defaults to preserve CPU cycles.
+          #
+          # We also do **not** cache the execution of this producer lambda because we want to
+          # support job args based producer selection
+          producer: nil
         }.freeze
 
         private_constant :DEFAULTS
 
         # @param job [ActiveJob::Base] job
         def dispatch(job)
-          ::Karafka.producer.public_send(
+          producer(job).public_send(
             fetch_option(job, :dispatch_method, DEFAULTS),
             dispatch_details(job).merge!(
               topic: job.queue_name,
@@ -47,27 +51,71 @@ module Karafka
         # Bulk dispatches multiple jobs using the Rails 7.1+ API
         # @param jobs [Array<ActiveJob::Base>] jobs we want to dispatch
         def dispatch_many(jobs)
-          dispatches = Hash.new { |hash, key| hash[key] = [] }
+          # First level is type of dispatch and second is the producer we want to use to dispatch
+          dispatches = Hash.new do |hash, key|
+            hash[key] = Hash.new do |hash2, key2|
+              hash2[key2] = []
+            end
+          end
 
           jobs.each do |job|
             d_method = fetch_option(job, :dispatch_many_method, DEFAULTS)
+            producer = producer(job)
 
-            dispatches[d_method] << dispatch_details(job).merge!(
+            dispatches[d_method][producer] << dispatch_details(job).merge!(
               topic: job.queue_name,
               payload: ::ActiveSupport::JSON.encode(serialize_job(job))
             )
           end
 
-          dispatches.each do |type, messages|
-            ::Karafka.producer.public_send(
-              type,
-              messages
-            )
+          dispatches.each do |d_method, producers|
+            producers.each do |producer, messages|
+              producer.public_send(
+                d_method,
+                messages
+              )
+            end
           end
         end
 
+        # Will enqueue a job to run in the future
+        #
+        # @param job [Object] job we want to enqueue
+        # @param timestamp [Time] time when job should run
+        def dispatch_at(job, timestamp)
+          target_message = dispatch_details(job).merge!(
+            topic: job.queue_name,
+            payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+          )
+
+          proxy_message = Pro::ScheduledMessages.schedule(
+            message: target_message,
+            epoch: timestamp.to_i,
+            envelope: {
+              # Select the scheduled messages proxy topic
+              topic: fetch_option(job, :scheduled_messages_topic, DEFAULTS)
+            }
+          )
+
+          producer(job).public_send(
+            fetch_option(job, :dispatch_method, DEFAULTS),
+            proxy_message
+          )
+        end
+
         private
 
+        # Selects the producer based on options. If callable `:producer` is defined, it will use
+        # it. If not, will just use the `Karafka.producer`.
+        #
+        # @param job [ActiveJob::Base] job instance
+        # @return [WaterDrop::Producer, WaterDrop::Producer::Variant] producer or a variant
+        def producer(job)
+          dynamic_producer = fetch_option(job, :producer, DEFAULTS)
+
+          dynamic_producer ? dynamic_producer.call(job) : ::Karafka.producer
+        end
+
         # @param job [ActiveJob::Base] job instance
         # @return [Hash] hash with dispatch details to which we merge topic and payload
         def dispatch_details(job)

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -25,14 +17,24 @@ module Karafka
           ).fetch('en').fetch('validations').fetch('job_options')
         end
 
+        optional(:producer) { |val| val.nil? || val.respond_to?(:call) }
         optional(:partitioner) { |val| val.respond_to?(:call) }
         optional(:partition_key_type) { |val| %i[key partition_key partition].include?(val) }
+
+        # Whether this is a legit scheduled messages topic will be validated during the first
+        # dispatch, so we do not repeat validations here
+        optional(:scheduled_messages_topic) do |val|
+          (val.is_a?(String) || val.is_a?(Symbol)) &&
+            ::Karafka::Contracts::TOPIC_REGEXP.match?(val.to_s)
+        end
+
         optional(:dispatch_method) do |val|
           %i[
             produce_async
             produce_sync
           ].include?(val)
         end
+
         optional(:dispatch_many_method) do |val|
           %i[
             produce_many_async

data/lib/karafka/pro/base_consumer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/cleaner/errors.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro