karafka 1.4.13 → 2.0.0

data/lib/karafka/instrumentation/callbacks/statistics.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    module Callbacks
+      # Statistics callback handler
+      # @see `WaterDrop::Instrumentation::Callbacks::Statistics` for details on why we decorate
+      #   those statistics
+      class Statistics
+        # @param subscription_group_id [String] id of the current subscription group
+        # @param consumer_group_id [String] id of the current consumer group
+        # @param client_name [String] rdkafka client name
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(subscription_group_id, consumer_group_id, client_name, monitor)
+          @subscription_group_id = subscription_group_id
+          @consumer_group_id = consumer_group_id
+          @client_name = client_name
+          @monitor = monitor
+          @statistics_decorator = ::Karafka::Core::Monitoring::StatisticsDecorator.new
+        end
+
+        # Emits decorated statistics to the monitor
+        # @param statistics [Hash] rdkafka statistics
+        def call(statistics)
+          # Emit only statistics related to our client
+          # rdkafka does not have per-instance statistics hook, thus we need to make sure that we
+          # emit only stats that are related to current producer. Otherwise we would emit all of
+          # all the time.
+          return unless @client_name == statistics['name']
+
+          @monitor.instrument(
+            'statistics.emitted',
+            subscription_group_id: @subscription_group_id,
+            consumer_group_id: @consumer_group_id,
+            statistics: @statistics_decorator.call(statistics)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default listener that hooks up to our instrumentation and uses its events for logging
+    # It can be removed/replaced or anything without any harm to the Karafka app flow.
+    class LoggerListener
+      # Log levels that we use in this particular listener
+      USED_LOG_LEVELS = %i[
+        debug
+        info
+        warn
+        error
+        fatal
+      ].freeze
+
+      # Logs each messages fetching attempt
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_listener_fetch_loop(event)
+        listener = event[:caller]
+        debug "[#{listener.id}] Polling messages..."
+      end
+
+      # Logs about messages that we've received from Kafka
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_listener_fetch_loop_received(event)
+        listener = event[:caller]
+        time = event[:time]
+        messages_count = event[:messages_buffer].size
+
+        message = "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
+
+        # We don't want the "polled 0" in dev as it would spam the log
+        # Instead we publish only info when there was anything we could poll and fail over to the
+        # zero notifications when in debug mode
+        messages_count.zero? ? debug(message) : info(message)
+      end
+
+      # Prints info about the fact that a given job has started
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_worker_process(event)
+        job = event[:job]
+        job_type = job.class.to_s.split('::').last
+        consumer = job.executor.topic.consumer
+        topic = job.executor.topic.name
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
+      end
+
+      # Prints info about the fact that a given job has finished
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_worker_processed(event)
+        job = event[:job]
+        time = event[:time]
+        job_type = job.class.to_s.split('::').last
+        consumer = job.executor.topic.consumer
+        topic = job.executor.topic.name
+        info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} finished in #{time}ms"
+      end
+
+      # Logs info about system signals that Karafka received and prints backtrace for threads in
+      # case of ttin
+      #
+      # @param event [Dry::Events::Event] event details including payload
+      def on_process_notice_signal(event)
+        info "Received #{event[:signal]} system signal"
+
+        # We print backtrace only for ttin
+        return unless event[:signal] == :SIGTTIN
+
+        # Inspired by Sidekiq
+        Thread.list.each do |thread|
+          tid = (thread.object_id ^ ::Process.pid).to_s(36)
+
+          warn "Thread TID-#{tid} #{thread['label']}"
+
+          if thread.backtrace
+            warn thread.backtrace.join("\n")
+          else
+            warn '<no backtrace available>'
+          end
+        end
+      end
+
+      # Logs info that we're initializing Karafka app.
+      #
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_initializing(_event)
+        info 'Initializing Karafka framework'
+      end
+
+      # Logs info that we're running Karafka app.
+      #
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_running(_event)
+        info 'Running Karafka server'
+      end
+
+      # Logs info that we're going to stop the Karafka server.
+      #
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping(_event)
+        info 'Stopping Karafka server'
+      end
+
+      # Logs info that we stopped the Karafka server.
+      #
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopped(_event)
+        info 'Stopped Karafka server'
+      end
+
+      # There are many types of errors that can occur in many places, but we provide a single
+      # handler for all of them to simplify error instrumentation.
+      # @param event [Dry::Events::Event] event details including payload
+      def on_error_occurred(event)
+        type = event[:type]
+        error = event[:error]
+        details = (error.backtrace || []).join("\n")
+
+        case type
+        when 'consumer.consume.error'
+          error "Consumer consuming error: #{error}"
+          error details
+        when 'consumer.revoked.error'
+          error "Consumer on revoked failed due to an error: #{error}"
+          error details
+        when 'consumer.shutdown.error'
+          error "Consumer on shutdown failed due to an error: #{error}"
+          error details
+        when 'worker.process.error'
+          fatal "Worker processing failed due to an error: #{error}"
+          fatal details
+        when 'connection.listener.fetch_loop.error'
+          error "Listener fetch loop error: #{error}"
+          error details
+        when 'licenser.expired'
+          error error
+          error details
+        when 'runner.call.error'
+          fatal "Runner crashed due to an error: #{error}"
+          fatal details
+        when 'app.stopping.error'
+          error 'Forceful Karafka server stop'
+        when 'librdkafka.error'
+          error "librdkafka internal error occurred: #{error}"
+          error details
+        else
+          # This should never happen. Please contact the maintainers
+          raise Errors::UnsupportedCaseError, event
+        end
+      end
+
+      USED_LOG_LEVELS.each do |log_level|
+        define_method log_level do |*args|
+          Karafka.logger.send(log_level, *args)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/monitor.rb

@@ -1,69 +1,21 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Namespace for all the things related with Karafka instrumentation process
   module Instrumentation
-    # Monitor is used to hookup external monitoring services to monitor how Karafka works
-    # It provides a standardized API for checking incoming messages/enqueueing etc
-    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
-    # same time, which means that you might have for example file logging and NewRelic at the same
-    # time
-    # @note This class acts as a singleton because we are only permitted to have single monitor
-    #   per running process (just as logger)
-    class Monitor < Dry::Monitor::Notifications
-      # List of events that we support in the system and to which a monitor client can hook up
-      # @note The non-error once support timestamp benchmarking
-      # @note Depending on Karafka extensions and additional engines, this might not be the
-      #   complete list of all the events. Please use the #available_events on fully loaded
-      #   Karafka system to determine all of the events you can use.
-      #   Last 4 events are from WaterDrop but for convenience we use the same monitor for the
-      #   whole karafka ecosystem
-      BASE_EVENTS = %w[
-        params.params.deserialize
-        params.params.deserialize.error
-        connection.listener.before_fetch_loop
-        connection.listener.fetch_loop
-        connection.listener.fetch_loop.error
-        connection.client.fetch_loop.error
-        connection.batch_delegator.call
-        connection.message_delegator.call
-        fetcher.call.error
-        backends.inline.process
-        process.notice_signal
-        consumers.responders.respond_with
-        async_producer.call.error
-        async_producer.call.retry
-        sync_producer.call.error
-        sync_producer.call.retry
-        app.initializing
-        app.initialized
-        app.running
-        app.stopping
-        app.stopping.error
-        app.stopped
-      ].freeze
+    # Karafka instrumentation monitor that we use to publish events
+    # By default uses our internal notifications bus but can be used with
+    # `ActiveSupport::Notifications` as well
+    class Monitor < ::Karafka::Core::Monitoring::Monitor
+      attr_reader :notifications_bus
 
-      private_constant :BASE_EVENTS
-
-      # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
-      def initialize
-        super(:karafka)
-        BASE_EVENTS.each(&method(:register_event))
-      end
-
-      # Allows us to subscribe to events with a code that will be yielded upon events
-      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
-      #   or a listener if we decide to go with object listener
-      def subscribe(event_name_or_listener)
-        return super unless event_name_or_listener.is_a?(String)
-        return super if available_events.include?(event_name_or_listener)
-
-        raise Errors::UnregisteredMonitorEventError, event_name_or_listener
-      end
-
-      # @return [Array<String>] names of available events to which we can subscribe
-      def available_events
-        __bus__.events.keys
+      # @param notifications_bus [Object] either our internal notifications bus or
+      #   `ActiveSupport::Notifications`
+      # @param namespace [String, nil] namespace for events or nil if no namespace
+      def initialize(
+        notifications_bus = ::Karafka::Instrumentation::Notifications.new,
+        namespace = nil
+      )
+        super(notifications_bus, namespace)
       end
     end
   end

data/lib/karafka/instrumentation/notifications.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related with Karafka instrumentation process
+  module Instrumentation
+    # Monitor is used to hookup external monitoring services to monitor how Karafka works
+    # It provides a standardized API for checking incoming messages/enqueueing etc
+    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
+    # same time, which means that you might have for example file logging and NewRelic at the same
+    # time
+    # @note This class acts as a singleton because we are only permitted to have single monitor
+    #   per running process (just as logger)
+    class Notifications < Karafka::Core::Monitoring::Notifications
+      # List of events that we support in the system and to which a monitor client can hook up
+      # @note The non-error once support timestamp benchmarking
+      # @note Depending on Karafka extensions and additional engines, this might not be the
+      #   complete list of all the events. Please use the #available_events on fully loaded
+      #   Karafka system to determine all of the events you can use.
+      EVENTS = %w[
+        app.initialized
+        app.running
+        app.stopping
+        app.stopped
+
+        consumer.consumed
+        consumer.revoked
+        consumer.shutdown
+
+        process.notice_signal
+
+        connection.listener.before_fetch_loop
+        connection.listener.fetch_loop
+        connection.listener.fetch_loop.received
+
+        worker.process
+        worker.processed
+
+        statistics.emitted
+
+        error.occurred
+      ].freeze
+
+      private_constant :EVENTS
+
+      # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
+      def initialize
+        super
+        EVENTS.each { |event| register_event(event) }
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -5,19 +5,19 @@ module Karafka
     # Listener that sets a proc title with a nice descriptive value
     class ProctitleListener
       # Updates proc title to an initializing one
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_initializing(_event)
         setproctitle('initializing')
       end
 
       # Updates proc title to a running one
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_running(_event)
         setproctitle('running')
       end
 
       # Updates proc title to a stopping one
-      # @param _event [Dry::Events::Event] event details including payload
+      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
       def on_app_stopping(_event)
         setproctitle('stopping')
       end

data/lib/karafka/instrumentation/vendors/datadog/dashboard.json

@@ -0,0 +1 @@
+{"title":"Karafka monitoring dashboard","description":"","widgets":[{"id":7444969424381053,"definition":{"title":"Stability & errors","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":8304008422587936,"definition":{"title":"Client connects and disconnects","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Connects","formula":"query1"},{"alias":"Disconnects","formula":"query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.connection.connects{*} by {host}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.connection.disconnects{*} by {host}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":3722865443336921,"definition":{"title":"Errors encountered (any)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"in-karafka errors","formula":"query1"},{"alias":"librdkafka consume errors","formula":"query2"},{"alias":"librdkafka receive errors","formula":"query3"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.error_occurred{*} by {type}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consume.errors{*}.as_count()","data_source":"metrics","name":"query2"},{"query":"sum:karafka.receive.errors{*}.as_count()","data_source":"metrics","name":"query3"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":5477381252952760,"definition":{"title":"Processing errors","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.error_occurred{type:consumer.consume.error} by {partition,topic}.as_count()","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":2357301680769076,"definition":{"title":"Processing errors rate per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"% error rate per topic","formula":"(query1 / (query1 + query2)) * 100"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.error_occurred{type:consumer.consume.error} by {topic,partition}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batches{*} by {topic,partition}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":2,"width":4,"height":2}},{"id":3902930069982135,"definition":{"title":"Batches successful vs failures","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Successfully processed batch","formula":"query1"},{"alias":"Batch processing with error","formula":"query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.batches{*} by {partition,topic}.as_count()","data_source":"metrics","name":"query1"},{"query":"avg:karafka.error_occurred{type:consumer.consume.error} by {partition,topic}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":2,"width":4,"height":2}},{"id":718749162159145,"definition":{"title":"Consumer instances revocations and shutdowns","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Consumer instances revokations","formula":"query1"},{"alias":"Consumer instances shutdowns","formula":"query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.revoked{*}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.shutdown{*}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":2,"width":4,"height":2}}]},"layout":{"x":0,"y":0,"width":12,"height":5}},{"id":5988438511387100,"definition":{"title":"Workers poll","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":8769294644934352,"definition":{"title":"Enqueued jobs","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Enqueued jobs","formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.worker.enqueued_jobs.avg{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":2714502141463873,"definition":{"title":"Workers usage","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Busy workers (p95)","formula":"query1"},{"alias":"Total workers","formula":"query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.worker.processing.95percentile{*}","data_source":"metrics","name":"query1"},{"query":"sum:karafka.worker.total_threads{*}","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":5370086629441984,"definition":{"title":"Workers % utilization","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"% workers utilization","formula":"(query1 / query2) * 100"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.worker.processing.95percentile{*}","data_source":"metrics","name":"query1"},{"query":"sum:karafka.worker.total_threads{*}","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":0,"width":4,"height":2}}]},"layout":{"x":0,"y":5,"width":12,"height":3}},{"id":8544040083223278,"definition":{"title":"Throughput ","type":"group","show_title":true,"layout_type":"ordered","widgets":[{"id":3740207481939733,"definition":{"title":"Offset lag changes","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"derivative(query1)"}],"response_format":"timeseries","queries":[{"query":"max:karafka.consumer.offset{*} by {topic,partition}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":0,"width":4,"height":2}},{"id":6319110548544878,"definition":{"title":"Batches processed per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.batches{*} by {partition,topic}.as_count()","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":0,"width":4,"height":2}},{"id":6232784865331443,"definition":{"title":"Messages consumed per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Messages consumed","formula":"query1"},{"alias":"Average batch size","formula":"query1 / query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.messages{*} by {partition,topic}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batches{*} by {partition,topic}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":0,"width":4,"height":2}},{"id":2321394598982770,"definition":{"title":"Consumption lag (in seconds)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Consumption lag in s (max)","formula":"query2 / 1000"},{"alias":"Consumption lag in s (avg)","formula":"query3 / 1000"},{"alias":"Consumption lag in s (p95)","formula":"query1 / 1000"}],"response_format":"timeseries","queries":[{"query":"max:karafka.consumer.consumption_lag.max{*}","data_source":"metrics","name":"query2"},{"query":"max:karafka.consumer.consumption_lag.avg{*}","data_source":"metrics","name":"query3"},{"query":"max:karafka.consumer.consumption_lag.95percentile{*}","data_source":"metrics","name":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":2,"width":4,"height":2}},{"id":1062074781483741,"definition":{"title":"Processing lag (in ms)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Processing lag in ms (p95)","formula":"query1"},{"alias":"Processing lag in ms (max)","formula":"query2"},{"alias":"Processing lag in ms (avg)","formula":"query3"}],"response_format":"timeseries","queries":[{"query":"max:karafka.consumer.processing_lag.95percentile{*}","data_source":"metrics","name":"query1"},{"query":"max:karafka.consumer.processing_lag.max{*}","data_source":"metrics","name":"query2"},{"query":"max:karafka.consumer.processing_lag.avg{*}","data_source":"metrics","name":"query3"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":2,"width":4,"height":2}},{"id":7497794728674267,"definition":{"title":"Batch processing time","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"},{"formula":"query2"},{"formula":"query3"}],"response_format":"timeseries","queries":[{"query":"max:karafka.consumer.consumed.time_taken.95percentile{*} by {topic,partition}","data_source":"metrics","name":"query1"},{"query":"max:karafka.consumer.consumed.time_taken.max{*} by {topic,partition}","data_source":"metrics","name":"query2"},{"query":"max:karafka.consumer.consumed.time_taken.avg{*} by {topic,partition}","data_source":"metrics","name":"query3"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":2,"width":4,"height":2}},{"id":4192833027984161,"definition":{"title":"Batch size per topic","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Batch size p95","formula":"query1"},{"alias":"Batch size avg","formula":"query2"},{"alias":"Batch size max","formula":"query3"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.batch_size.95percentile{*} by {partition,topic}","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batch_size.avg{*} by {partition,topic}","data_source":"metrics","name":"query2"},{"query":"sum:karafka.consumer.batch_size.max{*} by {partition,topic}","data_source":"metrics","name":"query3"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":4,"width":4,"height":2}},{"id":4741598444771147,"definition":{"title":"Messages consumed overall","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"alias":"Messages consumed","formula":"query1"},{"alias":"Average batch size","formula":"query1 / query2"}],"response_format":"timeseries","queries":[{"query":"sum:karafka.consumer.messages{*}.as_count()","data_source":"metrics","name":"query1"},{"query":"sum:karafka.consumer.batches{*}.as_count()","data_source":"metrics","name":"query2"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":4,"y":4,"width":4,"height":2}},{"id":4502534794102513,"definition":{"title":"Polling times (ms)","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"time":{},"type":"timeseries","requests":[{"formulas":[{"alias":"p95 ms polling time","formula":"query1"},{"alias":"max ms polling time","formula":"query2"},{"alias":"average ms polling time","formula":"query3"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:karafka.listener.polling.time_taken.95percentile{*}"},{"name":"query2","data_source":"metrics","query":"avg:karafka.listener.polling.time_taken.max{*}"},{"name":"query3","data_source":"metrics","query":"avg:karafka.listener.polling.time_taken.avg{*}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":8,"y":4,"width":4,"height":2}}]},"layout":{"x":0,"y":0,"width":12,"height":7,"is_column_break":true}}],"template_variables":[],"layout_type":"ordered","is_read_only":false,"notify_list":[],"reflow_type":"fixed","id":"s3u-z47-i6u"}

data/lib/karafka/instrumentation/vendors/datadog/listener.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Namespace for vendor specific instrumentation
+    module Vendors
+      # Datadog specific instrumentation
+      module Datadog
+        # Listener that can be used to subscribe to Karafka to receive stats via StatsD
+        # and/or Datadog
+        #
+        # @note You need to setup the `dogstatsd-ruby` client and assign it
+        class Listener
+          include ::Karafka::Core::Configurable
+          extend Forwardable
+
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          # Namespace under which the DD metrics should be published
+          setting :namespace, default: 'karafka'
+
+          # Datadog client that we should use to publish the metrics
+          setting :client
+
+          # Default tags we want to publish (for example hostname)
+          # Format as followed (example for hostname): `["host:#{Socket.gethostname}"]`
+          setting :default_tags, default: []
+
+          # All the rdkafka metrics we want to publish
+          #
+          # By default we publish quite a lot so this can be tuned
+          # Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
+          setting :rd_kafka_metrics, default: [
+            # Client metrics
+            RdKafkaMetric.new(:count, :root, 'messages.consumed', 'rxmsgs_d'),
+            RdKafkaMetric.new(:count, :root, 'messages.consumed.bytes', 'rxmsg_bytes'),
+
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'consume.attempts', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'consume.errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive.errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.connects', 'connects_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection.disconnects', 'disconnects_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network.latency.p99', %w[rtt p99])
+          ].freeze
+
+          configure
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            configure
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `WaterDrop#setup`
+          def setup(&block)
+            configure(&block)
+          end
+
+          # Hooks up to WaterDrop instrumentation for emitted statistics
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics)
+            end
+          end
+
+          # Increases the errors count by 1
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(event)
+            extra_tags = ["type:#{event[:type]}"]
+
+            if event.payload[:caller].respond_to?(:messages)
+              metadata = event.payload[:caller].messages.metadata
+
+              extra_tags += [
+                "topic:#{metadata.topic}",
+                "partition:#{metadata.partition}"
+              ]
+            end
+
+            count('error_occurred', 1, tags: default_tags + extra_tags)
+          end
+
+          # Reports how many messages we've polled and how much time did we spend on it
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_connection_listener_fetch_loop_received(event)
+            time_taken = event[:time]
+            messages_count = event[:messages_buffer].size
+
+            histogram('listener.polling.time_taken', time_taken, tags: default_tags)
+            histogram('listener.polling.messages', messages_count, tags: default_tags)
+          end
+
+          # Here we report majority of things related to processing as we have access to the
+          # consumer
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consumed(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.messages', messages.count, tags: tags)
+            count('consumer.batches', 1, tags: tags)
+            gauge('consumer.offset', metadata.last_offset, tags: tags)
+            histogram('consumer.consumed.time_taken', event[:time], tags: tags)
+            histogram('consumer.batch_size', messages.count, tags: tags)
+            histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
+            histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
+          end
+
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoked(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.revoked', 1, tags: tags)
+          end
+
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutdown(event)
+            messages = event.payload[:caller].messages
+            metadata = messages.metadata
+
+            tags = default_tags + [
+              "topic:#{metadata.topic}",
+              "partition:#{metadata.partition}"
+            ]
+
+            count('consumer.shutdown', 1, tags: tags)
+          end
+
+          # Worker related metrics
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_worker_process(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            gauge('worker.total_threads', Karafka::App.config.concurrency, tags: default_tags)
+            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+            histogram('worker.enqueued_jobs', jq_stats[:enqueued], tags: default_tags)
+          end
+
+          # We report this metric before and after processing for higher accuracy
+          # Without this, the utilization would not be fully reflected
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_worker_processed(event)
+            jq_stats = event[:jobs_queue].statistics
+
+            histogram('worker.processing', jq_stats[:processing], tags: default_tags)
+          end
+
+          private
+
+          %i[
+            count
+            gauge
+            histogram
+            increment
+            decrement
+          ].each do |metric_type|
+            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+              def #{metric_type}(key, *args)
+                client.#{metric_type}(
+                  namespaced_metric(key),
+                  *args
+                )
+              end
+            METHODS
+          end
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}.#{metric_name}"
+          end
+
+          # Reports a given metric statistics to Datadog
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          def report_metric(metric, statistics)
+            case metric.scope
+            when :root
+              public_send(
+                metric.type,
+                metric.name,
+                statistics.fetch(*metric.key_location),
+                tags: default_tags
+              )
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  broker_statistics.dig(*metric.key_location),
+                  tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
+                )
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  # @note Since we can only have one statistics callbacks manager and one error callbacks manager
+  # we use WaterDrops one that is already configured.
+  module Instrumentation
+    class << self
+      # Returns a manager for statistics callbacks
+      # @return [::WaterDrop::CallbacksManager]
+      def statistics_callbacks
+        ::WaterDrop::Instrumentation.statistics_callbacks
+      end
+
+      # Returns a manager for error callbacks
+      # @return [::WaterDrop::CallbacksManager]
+      def error_callbacks
+        ::WaterDrop::Instrumentation.error_callbacks
+      end
+    end
+  end
+end