karafka 1.4.12 → 2.2.10

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # A karafka's logger listener for Datadog
+        # It depends on the 'ddtrace' gem
+        class LoggerListener
+          include ::Karafka::Core::Configurable
+          extend Forwardable
+
+          def_delegators :config, :client, :service_name
+
+          # `Datadog::Tracing` client that we should use to trace stuff
+          setting :client
+
+          # @see https://docs.datadoghq.com/tracing/trace_collection/dd_libraries/ruby
+          setting :service_name, default: nil
+
+          configure
+
+          # Log levels that we use in this particular listener
+          USED_LOG_LEVELS = %i[
+            info
+            error
+            fatal
+          ].freeze
+
+          private_constant :USED_LOG_LEVELS
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            configure
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Prints info about the fact that a given job has started
+          #
+          # @param event [Karafka::Core::Monitoring::Event] event details including payload
+          def on_worker_process(event)
+            current_span = client.trace('karafka.consumer', service: service_name)
+            push_tags
+
+            job = event[:job]
+            job_type = job.class.to_s.split('::').last
+            consumer = job.executor.topic.consumer
+            topic = job.executor.topic.name
+
+            current_span.resource = "#{consumer}#consume"
+            info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
+
+            pop_tags
+          end
+
+          # Prints info about the fact that a given job has finished
+          #
+          # @param event [Karafka::Core::Monitoring::Event] event details including payload
+          def on_worker_processed(event)
+            push_tags
+
+            job = event[:job]
+            time = event[:time]
+            job_type = job.class.to_s.split('::').last
+            consumer = job.executor.topic.consumer
+            topic = job.executor.topic.name
+
+            info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
+
+            client.active_span&.finish
+
+            pop_tags
+          end
+
+          # There are many types of errors that can occur in many places, but we provide a single
+          # handler for all of them to simplify error instrumentation.
+          # @param event [Karafka::Core::Monitoring::Event] event details including payload
+          def on_error_occurred(event)
+            push_tags
+
+            error = event[:error]
+            client.active_span&.set_error(error)
+
+            case event[:type]
+            when 'consumer.consume.error'
+              error "Consumer consuming error: #{error}"
+            when 'consumer.revoked.error'
+              error "Consumer on revoked failed due to an error: #{error}"
+            when 'consumer.before_enqueue.error'
+              error "Consumer before enqueue failed due to an error: #{error}"
+            when 'consumer.before_consume.error'
+              error "Consumer before consume failed due to an error: #{error}"
+            when 'consumer.after_consume.error'
+              error "Consumer after consume failed due to an error: #{error}"
+            when 'consumer.shutdown.error'
+              error "Consumer on shutdown failed due to an error: #{error}"
+            when 'worker.process.error'
+              fatal "Worker processing failed due to an error: #{error}"
+            when 'connection.listener.fetch_loop.error'
+              error "Listener fetch loop error: #{error}"
+            when 'runner.call.error'
+              fatal "Runner crashed due to an error: #{error}"
+            when 'app.stopping.error'
+              error 'Forceful Karafka server stop'
+            when 'librdkafka.error'
+              error "librdkafka internal error occurred: #{error}"
+              # Those will only occur when retries in the client fail and when they did not stop
+              # after back-offs
+            when 'connection.client.poll.error'
+              error "Data polling error occurred: #{error}"
+            else
+              pop_tags
+              # This should never happen. Please contact the maintainers
+              raise Errors::UnsupportedCaseError, event
+            end
+
+            pop_tags
+          end
+
+          USED_LOG_LEVELS.each do |log_level|
+            define_method log_level do |*args|
+              Karafka.logger.send(log_level, *args)
+            end
+          end
+
+          # Pushes datadog's tags to the logger
+          # This is required when tracing log lines asynchronously to correlate logs of the same
+          # process together
+          def push_tags
+            return unless Karafka.logger.respond_to?(:push_tags)
+
+            Karafka.logger.push_tags(client.log_correlation)
+          end
+
+          # Pops datadog's tags from the logger
+          # This is required when tracing log lines asynchronously to avoid the logs of the
+          # different processes to be correlated
+          def pop_tags
+            return unless Karafka.logger.respond_to?(:pop_tags)
+
+            Karafka.logger.pop_tags
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # Listener that can be used to subscribe to Karafka to receive stats via StatsD
+        # and/or Datadog
+        #
+        # @note You need to setup the `dogstatsd-ruby` client and assign it
+        class MetricsListener
+          include ::Karafka::Core::Configurable
+          extend Forwardable
+
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          # Namespace under which the DD metrics should be published
+          setting :namespace, default: 'karafka'
+
+          # Datadog client that we should use to publish the metrics
+          setting :client
+
+          # Default tags we want to publish (for example hostname)
+          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
+          setting :default_tags, default: []
+
+          # All the rdkafka metrics we want to publish
+          #
+          # By default we publish quite a lot so this can be tuned
+          # Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
+          setting :rd_kafka_metrics, default: [
+            # Client metrics
+            RdKafkaMetric.new(:count, :root, 'messages.consumed', 'rxmsgs_d'),
+            RdKafkaMetric.new(:count, :root, 'messages.consumed.bytes', 'rxmsg_bytes'),
+
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'consume.attempts', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'consume.errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.connects', 'connects_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.disconnects', 'disconnects_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99]),
+
+            # Topics metrics
+            RdKafkaMetric.new(:gauge, :topics, 'consumer.lags', 'consumer_lag_stored'),
+            RdKafkaMetric.new(:gauge, :topics, 'consumer.lags_delta', 'consumer_lag_stored_d')
+          ].freeze
+
+          configure
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            configure
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Hooks up to Karafka instrumentation for emitted statistics
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+            consumer_group_id = event[:consumer_group_id]
+
+            base_tags = default_tags + ["consumer_group:#{consumer_group_id}"]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics, base_tags)
+            end
+          end
+
+          # Increases the errors count by 1
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(event)
+            extra_tags = ["type:#{event[:type]}"]
+
+            if event.payload[:caller].respond_to?(:messages)
+              extra_tags += consumer_tags(event.payload[:caller])
+            end
+
+            count('error_occurred', 1, tags: default_tags + extra_tags)
+          end
+
+          # Reports how many messages we've polled and how much time did we spend on it
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_fetch_loop_received(event)
+            time_taken = event[:time]
+            messages_count = event[:messages_buffer].size
+
+            consumer_group_id = event[:subscription_group].consumer_group.id
+
+            extra_tags = ["consumer_group:#{consumer_group_id}"]
+
+            histogram('listener.polling.time_taken', time_taken, tags: default_tags + extra_tags)
+            histogram('listener.polling.messages', messages_count, tags: default_tags + extra_tags)
+          end
+
+          # Here we report majority of things related to processing as we have access to the
+          # consumer
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consumed(event)
+            consumer = event.payload[:caller]
+            messages = consumer.messages
+            metadata = messages.metadata
+
+            tags = default_tags + consumer_tags(consumer)
+
+            count('consumer.messages', messages.count, tags: tags)
+            count('consumer.batches', 1, tags: tags)
+            gauge('consumer.offset', metadata.last_offset, tags: tags)
+            histogram('consumer.consumed.time_taken', event[:time], tags: tags)
+            histogram('consumer.batch_size', messages.count, tags: tags)
+            histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
+            histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
+          end
+
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoked(event)
+            tags = default_tags + consumer_tags(event.payload[:caller])
+
+            count('consumer.revoked', 1, tags: tags)
+          end
+
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutdown(event)
+            tags = default_tags + consumer_tags(event.payload[:caller])
+
+            count('consumer.shutdown', 1, tags: tags)
+          end
+
+          # Worker related metrics
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_worker_process(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
+            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
+            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
+          end
+
+          # We report this metric before and after processing for higher accuracy
+          # Without this, the utilization would not be fully reflected
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_worker_processed(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            histogram('worker.processing', jq_stats[:busy], tags: default_tags)
+          end
+
+          private
+
+          %i[
+            count
+            gauge
+            histogram
+            increment
+            decrement
+          ].each do |metric_type|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              def #{metric_type}(key, *args)
+                client.#{metric_type}(
+                  namespaced_metric(key),
+                  *args
+                )
+              end
+            METHODS
+          end
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}.#{metric_name}"
+          end
+
+          # Reports a given metric statistics to Datadog
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          # @param base_tags [Array<String>] base tags we want to start with
+          def report_metric(metric, statistics, base_tags)
+            case metric.scope
+            when :root
+              public_send(
+                metric.type,
+                metric.name,
+                statistics.fetch(*metric.key_location),
+                tags: base_tags
+              )
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  broker_statistics.dig(*metric.key_location),
+                  tags: base_tags + ["broker:#{broker_statistics['nodename']}"]
+                )
+              end
+            when :topics
+              statistics.fetch('topics').each do |topic_name, topic_values|
+                topic_values['partitions'].each do |partition_name, partition_statistics|
+                  next if partition_name == '-1'
+                  # Skip until lag info is available
+                  next if partition_statistics['consumer_lag'] == -1
+                  next if partition_statistics['consumer_lag_stored'] == -1
+
+                  # Skip if we do not own the fetch assignment
+                  next if partition_statistics['fetch_state'] == 'stopped'
+                  next if partition_statistics['fetch_state'] == 'none'
+
+                  public_send(
+                    metric.type,
+                    metric.name,
+                    partition_statistics.dig(*metric.key_location),
+                    tags: base_tags + [
+                      "topic:#{topic_name}",
+                      "partition:#{partition_name}"
+                    ]
+                  )
+                end
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+
+          # Builds basic per consumer tags for publication
+          #
+          # @param consumer [Karafka::BaseConsumer]
+          # @return [Array<String>]
+          def consumer_tags(consumer)
+            messages = consumer.messages
+            metadata = messages.metadata
+            consumer_group_id = consumer.topic.consumer_group.id
+
+            [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}",
+              "consumer_group:#{consumer_group_id}"
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'socket'
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      # Namespace for instrumentation related with Kubernetes
+      module Kubernetes
+        # Kubernetes HTTP listener that does not only reply when process is not fully hanging, but
+        # also allows to define max time of processing and looping.
+        #
+        # Processes like Karafka server can hang while still being reachable. For example, in case
+        # something would hang inside of the user code, Karafka could stop polling and no new
+        # data would be processed, but process itself would still be active. This listener allows
+        # for defining of a ttl that gets bumped on each poll loop and before and after processing
+        # of a given messages batch.
+        class LivenessListener
+          include ::Karafka::Core::Helpers::Time
+
+          # All good with Karafka
+          OK_CODE = '204 No Content'
+
+          # Some timeouts, fail
+          FAIL_CODE = '500 Internal Server Error'
+
+          private_constant :OK_CODE, :FAIL_CODE
+
+          # @param hostname [String, nil] hostname or nil to bind on all
+          # @param port [Integer] TCP port on which we want to run our HTTP status server
+          # @param consuming_ttl [Integer] time in ms after which we consider consumption hanging.
+          #   It allows us to define max consumption time after which k8s should consider given
+          #   process as hanging
+          # @param polling_ttl [Integer] max time in ms for polling. If polling (any) does not
+          #   happen that often, process should be considered dead.
+          # @note The default TTL matches the default `max.poll.interval.ms`
+          def initialize(
+            hostname: nil,
+            port: 3000,
+            consuming_ttl: 5 * 60 * 1_000,
+            polling_ttl: 5 * 60 * 1_000
+          )
+            @server = TCPServer.new(*[hostname, port].compact)
+            @polling_ttl = polling_ttl
+            @consuming_ttl = consuming_ttl
+            @mutex = Mutex.new
+            @pollings = {}
+            @consumptions = {}
+
+            Thread.new do
+              loop do
+                break unless respond
+              end
+            end
+          end
+
+          # Tick on each fetch
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_fetch_loop(_event)
+            mark_polling_tick
+          end
+
+          # Tick on starting work
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consume(_event)
+            mark_consumption_tick
+          end
+
+          # Tick on finished work
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consumed(_event)
+            clear_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoke(_event)
+            mark_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoked(_event)
+            clear_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutting_down(_event)
+            mark_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutdown(_event)
+            clear_consumption_tick
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(_event)
+            clear_consumption_tick
+            clear_polling_tick
+          end
+
+          # Stop the http server when we stop the process
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_stopped(_event)
+            @server.close
+          end
+
+          private
+
+          # Wraps the logic with a mutex
+          # @param block [Proc] code we want to run in mutex
+          def synchronize(&block)
+            @mutex.synchronize(&block)
+          end
+
+          # @return [Integer] object id of the current thread
+          def thread_id
+            Thread.current.object_id
+          end
+
+          # Update the polling tick time for current thread
+          def mark_polling_tick
+            synchronize do
+              @pollings[thread_id] = monotonic_now
+            end
+          end
+
+          # Clear current thread polling time tracker
+          def clear_polling_tick
+            synchronize do
+              @pollings.delete(thread_id)
+            end
+          end
+
+          # Update the processing tick time
+          def mark_consumption_tick
+            synchronize do
+              @consumptions[thread_id] = monotonic_now
+            end
+          end
+
+          # Clear current thread consumption time tracker
+          def clear_consumption_tick
+            synchronize do
+              @consumptions.delete(thread_id)
+            end
+          end
+
+          # Responds to a HTTP request with the process liveness status
+          def respond
+            client = @server.accept
+            client.gets
+            client.print "HTTP/1.1 #{status}\r\n"
+            client.print "Content-Type: text/plain\r\n"
+            client.print "\r\n"
+            client.close
+
+            true
+          rescue Errno::ECONNRESET, Errno::EPIPE, IOError
+            !@server.closed?
+          end
+
+          # Did we exceed any of the ttls
+          # @return [String] 204 string if ok, 500 otherwise
+          def status
+            time = monotonic_now
+
+            return FAIL_CODE if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+            return FAIL_CODE if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+
+            OK_CODE
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/licenser.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Checks the license presence for pro and loads pro components when needed (if any)
+  class Licenser
+    # Location in the gem where we store the public key
+    PUBLIC_KEY_LOCATION = File.join(Karafka.gem_root, 'certs', 'karafka-pro.pem')
+
+    private_constant :PUBLIC_KEY_LOCATION
+
+    class << self
+      # Tries to load the license and yields if successful
+      def detect
+        # If required, do not require again
+        require('karafka-license') unless const_defined?('::Karafka::License')
+
+        yield
+
+        true
+      rescue LoadError
+        false
+      end
+
+      # Tries to prepare license and verifies it
+      #
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def prepare_and_verify(license_config)
+        # If license is not loaded, nothing to do
+        return unless const_defined?('::Karafka::License')
+
+        prepare(license_config)
+        verify(license_config)
+      end
+
+      private
+
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def prepare(license_config)
+        license_config.token = Karafka::License.token
+      end
+
+      # Check license and setup license details (if needed)
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def verify(license_config)
+        public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
+
+        # We gsub and strip in case someone copy-pasted it as a multi line string
+        formatted_token = license_config.token.strip.delete("\n").delete(' ')
+        decoded_token = Base64.decode64(formatted_token)
+
+        begin
+          data = public_key.public_decrypt(decoded_token)
+        rescue OpenSSL::OpenSSLError
+          data = nil
+        end
+
+        details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
+
+        license_config.entity = details.fetch('entity')
+      end
+
+      # Raises an error with info, that used token is invalid
+      # @param license_config [Karafka::Core::Configurable::Node]
+      def raise_invalid_license_token(license_config)
+        # We set it to false so `Karafka.pro?` method behaves as expected
+        license_config.token = false
+
+        raise(
+          Errors::InvalidLicenseTokenError,
+          <<~MSG.tr("\n", ' ')
+            License key you provided is invalid.
+            Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+          MSG
+        )
+      end
+    end
+  end
+end