honeybadger 5.18.0 → 5.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ac625c10c4a54cedc7385e468f6c1ca58b6e70462d53569b0e06e7a71e05375
-  data.tar.gz: 608e399be30c0193ec3423f9ce61d6a5a6e4c3c6409a48766d164f2c45d6ce50
+  metadata.gz: bf4c3296c72440877caccd2ada861f4060af897ed517688fb838cd57ae0a58ff
+  data.tar.gz: 25d45250d1f2d2b0fadfa37637446316f12588f3dd134695803c3a068a0513d8
 SHA512:
-  metadata.gz: e88fdc901c68e8ac52a3668eab830103ee57d1dd2180c52a21f22c4177d74d47495570682d0b7c60fa26560cbcbc083428d9274daa8fab9c8d608dea19cea59f
-  data.tar.gz: 60f864a5799746ec90aba7576eed46d2321443114b9307bea759c314dc6783e7fc2361b2883d3c4662de3506649a9c151f19328f31a198c9a8f4e10fcf09e265
+  metadata.gz: 3c66b0c6fa819e716b18a710c1e11fe8c95d25e06ca69216efb93fb2d4bffd1d65c46e7b00b658c0321ac6608817dec2c894eb68bcc0bf39da858c37c0b96302
+  data.tar.gz: 6ef30a79138840ade5fef06ef5771f96c4a94ea942b6154b322632f73d71f8c72745be940e25c6aa19187411897c528e5f3e0325c3605cce0861cc4a6aa73a9b

data/CHANGELOG.md

@@ -1,6 +1,13 @@
 # Change Log
 
 
+## [5.19.0](https://github.com/honeybadger-io/honeybadger-ruby/compare/v5.18.0...v5.19.0) (2024-10-29)
+
+
+### Features
+
+* improved karafka plugin for insights ([#625](https://github.com/honeybadger-io/honeybadger-ruby/issues/625)) ([21880fc](https://github.com/honeybadger-io/honeybadger-ruby/commit/21880fc92493abb044ba3c0d64299ad8b92ad5f3))
+
 ## [5.18.0](https://github.com/honeybadger-io/honeybadger-ruby/compare/v5.17.0...v5.18.0) (2024-10-25)
 
 

data/lib/honeybadger/config/defaults.rb

@@ -363,6 +363,21 @@ module Honeybadger
         default: 60,
         type: Integer
       },
+      :'karafka.insights.enabled' => {
+        description: 'Enable automatic data collection for Karafka.',
+        default: true,
+        type: Boolean
+      },
+      :'karafka.insights.events' => {
+        description: 'Enable automatic event capturing for Karafka.',
+        default: true,
+        type: Boolean
+      },
+      :'karafka.insights.metrics' => {
+        description: 'Enable automatic metric data collection for Karafka.',
+        default: true,
+        type: Boolean
+      },
       :'net_http.insights.enabled' => {
         description: 'Allow automatic instrumentation of Net::HTTP requests.',
         default: true,

data/lib/honeybadger/config.rb

@@ -309,6 +309,20 @@ module Honeybadger
       !!self[:"#{name}.insights.enabled"]
     end
 
+    def load_plugin_insights_events?(name)
+      return false unless insights_enabled?
+      return false unless load_plugin_insights?(name)
+      return true if self[:"#{name}.insights.events"].nil?
+      !!self[:"#{name}.insights.events"]
+    end
+
+    def load_plugin_insights_metrics?(name)
+      return false unless insights_enabled?
+      return false unless load_plugin_insights?(name)
+      return true if self[:"#{name}.insights.metrics"].nil?
+      !!self[:"#{name}.insights.metrics"]
+    end
+
     def root_regexp
       return @root_regexp if @root_regexp
       return nil if @no_root

data/lib/honeybadger/histogram.rb

@@ -9,6 +9,15 @@ module Honeybadger
       return unless value
 
       @samples += 1
+
+      @total ||= 0
+      @total = @total + value
+
+      @min = value if @min.nil? || @min > value
+      @max = value if @max.nil? || @max < value
+      @avg = @total.to_f / @samples
+      @latest = value
+
       @bin_counts ||= Hash.new(0)
       @bin_counts[find_bin(value)] += 1
     end
@@ -25,6 +34,10 @@ module Honeybadger
 
     def payloads
       [{
+        min: @min,
+        max: @max,
+        avg: @avg,
+        latest: @latest,
         bins: (bins + [INFINITY]).map { |bin| [bin.to_f, @bin_counts[bin]] }
       }]
     end

data/lib/honeybadger/instrumentation.rb

@@ -47,63 +47,91 @@ module Honeybadger
     def time(name, *args)
       attributes = extract_attributes(args)
       callable = extract_callable(args)
-      duration = attributes.delete(:duration)
+      value = nil
 
       if callable
-        duration = monotonic_timer{ callable.call }[0]
+        value = monotonic_timer{ callable.call }[0]
       elsif block_given?
-        duration = monotonic_timer{ yield }[0]
+        value = monotonic_timer{ yield }[0]
+      else
+        value = attributes.delete(:duration) || attributes.delete(:value)
       end
 
-      raise 'No duration found' if duration.nil?
+      raise 'No value found' if value.nil?
 
       Honeybadger::Timer.register(registry, name, attributes).tap do |timer|
-        timer.record(duration)
+        timer.record(value)
       end
     end
 
     def histogram(name, *args)
       attributes = extract_attributes(args)
       callable = extract_callable(args)
-      duration = attributes.delete(:duration)
+      value = nil
 
       if callable
-        duration = monotonic_timer{ callable.call }[0]
+        value = monotonic_timer{ callable.call }[0]
       elsif block_given?
-        duration = monotonic_timer{ yield }[0]
+        value = monotonic_timer{ yield }[0]
+      else
+        value = attributes.delete(:duration) || attributes.delete(:value)
       end
 
-      raise 'No duration found' if duration.nil?
+      raise 'No value found' if value.nil?
 
       Honeybadger::Histogram.register(registry, name, attributes).tap do |histogram|
-        histogram.record(duration)
+        histogram.record(value)
       end
     end
 
     def increment_counter(name, *args)
       attributes = extract_attributes(args)
-      by = extract_callable(args)&.call || attributes.delete(:by) || 1
-      by = yield if block_given?
+      callable = extract_callable(args)
+      value = nil
+
+      if callable
+        value = callable.call
+      elsif block_given?
+        value = yield
+      else
+        value = attributes.delete(:by) || attributes.delete(:value) || 1
+      end
 
       Honeybadger::Counter.register(registry, name, attributes).tap do |counter|
-        counter.count(by)
+        counter.count(value)
       end
     end
 
     def decrement_counter(name, *args)
       attributes = extract_attributes(args)
-      by = extract_callable(args)&.call || attributes.delete(:by) || 1
-      by = yield if block_given?
+      callable = extract_callable(args)
+      value = nil
+
+      if callable
+        value = callable.call
+      elsif block_given?
+        value = yield
+      else
+        value = attributes.delete(:by) || attributes.delete(:value) || 1
+      end
 
       Honeybadger::Counter.register(registry, name, attributes).tap do |counter|
-        counter.count(by * -1)
+        counter.count(value * -1)
       end
     end
 
     def gauge(name, *args)
       attributes = extract_attributes(args)
-      value = extract_callable(args)&.call || attributes.delete(:value)
-      value = yield if block_given?
+      callable = extract_callable(args)
+      value = nil
+
+      if callable
+        value = callable.call
+      elsif block_given?
+        value = yield
+      else
+        value = attributes.delete(:value)
+      end
 
       Honeybadger::Gauge.register(registry, name, attributes).tap do |gauge|
         gauge.record(value)

data/lib/honeybadger/instrumentation_helper.rb

@@ -56,7 +56,7 @@ module Honeybadger
         metric_instrumentation.time(name, attributes, ->{ callable.call })
       elsif block_given?
         metric_instrumentation.time(name, attributes, ->{ yield })
-      elsif attributes.keys.include?(:duration)
+      else
         metric_instrumentation.time(name, attributes)
       end
     end
@@ -68,38 +68,44 @@ module Honeybadger
         metric_instrumentation.histogram(name, attributes, ->{ callable.call })
       elsif block_given?
         metric_instrumentation.histogram(name, attributes, ->{ yield })
-      elsif attributes.keys.include?(:duration)
+      else
         metric_instrumentation.histogram(name, attributes)
       end
     end
 
     def increment_counter(name, *args)
       attributes = extract_attributes(args)
-      by = extract_callable(args)&.call || attributes.delete(:by) || 1
-      if block_given?
+      callable = extract_callable(args)
+      if callable
+        metric_instrumentation.increment_counter(name, attributes, ->{ callable.call })
+      elsif block_given?
         metric_instrumentation.increment_counter(name, attributes, ->{ yield })
       else
-        metric_instrumentation.increment_counter(name, attributes.merge(by: by))
+        metric_instrumentation.increment_counter(name, attributes)
       end
     end
 
     def decrement_counter(name, *args)
       attributes = extract_attributes(args)
-      by = extract_callable(args)&.call || attributes.delete(:by) || 1
-      if block_given?
+      callable = extract_callable(args)
+      if callable
+        metric_instrumentation.decrement_counter(name, attributes, ->{ callable.call })
+      elsif block_given?
         metric_instrumentation.decrement_counter(name, attributes, ->{ yield })
       else
-        metric_instrumentation.decrement_counter(name, attributes.merge(by: by))
+        metric_instrumentation.decrement_counter(name, attributes)
       end
     end
 
     def gauge(name, *args)
       attributes = extract_attributes(args)
-      value = extract_callable(args)&.call || attributes.delete(:value)
-      if block_given?
+      callable = extract_callable(args)
+      if callable
+        metric_instrumentation.gauge(name, attributes, ->{ callable.call })
+      elsif block_given?
         metric_instrumentation.gauge(name, attributes, ->{ yield })
       else
-        metric_instrumentation.gauge(name, attributes.merge(value: value))
+        metric_instrumentation.gauge(name, attributes)
       end
     end
 

data/lib/honeybadger/karafka.rb

@@ -0,0 +1,298 @@
+require 'honeybadger/instrumentation_helper'
+
+module Honeybadger
+  module Karafka
+    class ErrorsListener
+      # Sends error details to Honeybadger
+      #
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_error_occurred(event)
+        context = {
+          type: event[:type]
+        }
+        tags = ["type:#{event[:type]}"]
+
+        if (consumer = event.payload[:caller]).respond_to?(:messages)
+          messages = consumer.messages
+          metadata = messages.metadata
+          consumer_group_id = consumer.topic.consumer_group.id
+
+          context[:topic] = metadata.topic
+          context[:partition] = metadata.partition
+          context[:consumer_group] = consumer_group_id
+        end
+
+        Honeybadger.notify(event[:error], context: context)
+      end
+    end
+
+    class InsightsListener
+      include ::Honeybadger::InstrumentationHelper
+
+      # Value object for storing a single rdkafka metric publishing details
+      RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+      # All the rdkafka metrics we want to publish
+      #
+      # By default we publish quite a lot so this can be tuned
+      # Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
+      RD_KAFKA_METRICS = [
+        # Client metrics
+        RdKafkaMetric.new(:increment_counter, :root, 'messages_consumed', 'rxmsgs_d'),
+        RdKafkaMetric.new(:increment_counter, :root, 'messages_consumed_bytes', 'rxmsg_bytes'),
+
+        # Broker metrics
+        RdKafkaMetric.new(:increment_counter, :brokers, 'consume_attempts', 'txretries_d'),
+        RdKafkaMetric.new(:increment_counter, :brokers, 'consume_errors', 'txerrs_d'),
+        RdKafkaMetric.new(:increment_counter, :brokers, 'receive_errors', 'rxerrs_d'),
+        RdKafkaMetric.new(:increment_counter, :brokers, 'connection_connects', 'connects_d'),
+        RdKafkaMetric.new(:increment_counter, :brokers, 'connection_disconnects', 'disconnects_d'),
+        RdKafkaMetric.new(:gauge, :brokers, 'network_latency_avg', %w[rtt avg]),
+        RdKafkaMetric.new(:gauge, :brokers, 'network_latency_p95', %w[rtt p95]),
+        RdKafkaMetric.new(:gauge, :brokers, 'network_latency_p99', %w[rtt p99]),
+
+        # Topics metrics
+        RdKafkaMetric.new(:gauge, :topics, 'consumer_lags', 'consumer_lag_stored'),
+        RdKafkaMetric.new(:gauge, :topics, 'consumer_lags_delta', 'consumer_lag_stored_d')
+      ].freeze
+
+      # Metrics that sum values on topics levels and not on partition levels
+      AGGREGATED_RD_KAFKA_METRICS = [
+        # Topic aggregated metrics
+        RdKafkaMetric.new(:gauge, :topics, 'consumer_aggregated_lag', 'consumer_lag_stored')
+      ].freeze
+
+      def initialize
+        metric_source("karafka")
+      end
+
+      # Hooks up to Karafka instrumentation for emitted statistics
+      #
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_statistics_emitted(event)
+        return unless Honeybadger.config.load_plugin_insights_metrics?(:karafka)
+
+        statistics = event[:statistics]
+        consumer_group_id = event[:consumer_group_id]
+
+        base_tags = { consumer_group: consumer_group_id }
+
+        RD_KAFKA_METRICS.each do |metric|
+          report_metric(metric, statistics, base_tags)
+        end
+
+        report_aggregated_topics_metrics(statistics, consumer_group_id)
+      end
+
+      # Publishes aggregated topic-level metrics that are sum of per partition metrics
+      #
+      # @param statistics [Hash] hash with all the statistics emitted
+      # @param consumer_group_id [String] cg in context which we operate
+      def report_aggregated_topics_metrics(statistics, consumer_group_id)
+        AGGREGATED_RD_KAFKA_METRICS.each do |metric|
+          statistics.fetch('topics').each do |topic_name, topic_values|
+            sum = 0
+
+            topic_values['partitions'].each do |partition_name, partition_statistics|
+              next if partition_name == '-1'
+              # Skip until lag info is available
+              next if partition_statistics['consumer_lag'] == -1
+              next if partition_statistics['consumer_lag_stored'] == -1
+
+              sum += partition_statistics.dig(*metric.key_location)
+            end
+
+            public_send(
+              metric.type,
+              metric.name,
+              value: sum,
+              consumer_group: consumer_group_id,
+              topic: topic_name
+            )
+          end
+        end
+      end
+
+      # Increases the errors count by 1
+      #
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_error_occurred(event)
+        extra_tags = { type: event[:type] }
+
+        if event.payload[:caller].respond_to?(:messages)
+          extra_tags.merge!(consumer_tags(event.payload[:caller]))
+        end
+
+        if Honeybadger.config.load_plugin_insights_events?(:karafka)
+          Honeybadger.event("error.occurred.karafka", error: event[:error], **extra_tags)
+        end
+
+        if Honeybadger.config.load_plugin_insights_metrics?(:karafka)
+          increment_counter('error_occurred', value: 1, **extra_tags)
+        end
+      end
+
+      # Reports how many messages we've polled and how much time did we spend on it
+      #
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_connection_listener_fetch_loop_received(event)
+        time_taken = event[:time]
+        messages_count = event[:messages_buffer].size
+
+        consumer_group_id = event[:subscription_group].consumer_group.id
+
+        extra_tags = { consumer_group: consumer_group_id }
+
+        if Honeybadger.config.load_plugin_insights_metrics?(:karafka)
+          histogram('listener_polling_time_taken', value: time_taken, **extra_tags)
+          histogram('listener_polling_messages', value: messages_count, **extra_tags)
+        end
+      end
+
+      # Here we report majority of things related to processing as we have access to the
+      # consumer
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_consumer_consumed(event)
+        consumer = event.payload[:caller]
+        messages = consumer.messages
+        metadata = messages.metadata
+
+        tags = consumer_tags(consumer)
+
+        if Honeybadger.config.load_plugin_insights_events?(:karafka)
+          event_context = tags.merge({
+            consumer: consumer.class.name,
+            duration: event[:time],
+            processing_lag: metadata.processing_lag,
+            consumption_lag: metadata.consumption_lag,
+            processed: messages.count
+          })
+          Honeybadger.event("consumer.consumed.karafka", event_context)
+        end
+
+        if Honeybadger.config.load_plugin_insights_metrics?(:karafka)
+          increment_counter('consumer_messages', value: messages.count, **tags)
+          increment_counter('consumer_batches', value: 1, **tags)
+          gauge('consumer_offset', value: metadata.last_offset, **tags)
+          histogram('consumer_consumed_time_taken', value: event[:time], **tags)
+          histogram('consumer_batch_size', value: messages.count, **tags)
+          histogram('consumer_processing_lag', value: metadata.processing_lag, **tags)
+          histogram('consumer_consumption_lag', value: metadata.consumption_lag, **tags)
+        end
+      end
+
+      {
+        revoked: :revoked,
+        shutdown: :shutdown,
+        ticked: :tick
+      }.each do |after, name|
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # Keeps track of user code execution
+              #
+              # @param event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{after}(event)
+                tags = consumer_tags(event.payload[:caller])
+
+                increment_counter('consumer_#{name}', value: 1, **tags)
+              end
+        RUBY
+      end
+
+      # Worker related metrics
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_worker_process(event)
+        jq_stats = event[:jobs_queue].statistics
+
+        if Honeybadger.config.load_plugin_insights_metrics?(:karafka)
+          gauge('worker_total_threads', value: ::Karafka::App.config.concurrency)
+          histogram('worker_processing', value: jq_stats[:busy])
+          histogram('worker_enqueued_jobs', value: jq_stats[:enqueued])
+        end
+      end
+
+      # We report this metric before and after processing for higher accuracy
+      # Without this, the utilization would not be fully reflected
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_worker_processed(event)
+        jq_stats = event[:jobs_queue].statistics
+
+        if Honeybadger.config.load_plugin_insights_metrics?(:karafka)
+          histogram('worker_processing', value: jq_stats[:busy])
+        end
+      end
+
+      private
+
+      # Reports a given metric statistics to Honeybadger
+      # @param metric [RdKafkaMetric] metric value object
+      # @param statistics [Hash] hash with all the statistics emitted
+      # @param base_tags [Array<String>] base tags we want to start with
+      def report_metric(metric, statistics, base_tags)
+        case metric.scope
+        when :root
+          public_send(
+            metric.type,
+            metric.name,
+            value: statistics.fetch(*metric.key_location),
+            **base_tags
+          )
+        when :brokers
+          statistics.fetch('brokers').each_value do |broker_statistics|
+            # Skip bootstrap nodes
+            # Bootstrap nodes have nodeid -1, other nodes have positive
+            # node ids
+            next if broker_statistics['nodeid'] == -1
+
+            public_send(
+              metric.type,
+              metric.name,
+              value: broker_statistics.dig(*metric.key_location),
+              **base_tags.merge(broker: broker_statistics['nodename'])
+            )
+          end
+        when :topics
+          statistics.fetch('topics').each do |topic_name, topic_values|
+            topic_values['partitions'].each do |partition_name, partition_statistics|
+              next if partition_name == '-1'
+              # Skip until lag info is available
+              next if partition_statistics['consumer_lag'] == -1
+              next if partition_statistics['consumer_lag_stored'] == -1
+
+              # Skip if we do not own the fetch assignment
+              next if partition_statistics['fetch_state'] == 'stopped'
+              next if partition_statistics['fetch_state'] == 'none'
+
+              public_send(
+                metric.type,
+                metric.name,
+                value: partition_statistics.dig(*metric.key_location),
+                **base_tags.merge({
+                  topic: topic_name,
+                  partition: partition_name
+                })
+              )
+            end
+          end
+        else
+          raise ArgumentError, metric.scope
+        end
+      end
+
+      # Builds basic per consumer tags for publication
+      #
+      # @param consumer [Karafka::BaseConsumer]
+      # @return [Array<String>]
+      def consumer_tags(consumer)
+        messages = consumer.messages
+        metadata = messages.metadata
+        consumer_group_id = consumer.topic.consumer_group.id
+
+        {
+          topic: metadata.topic,
+          partition: metadata.partition,
+          consumer_group: consumer_group_id
+        }
+      end
+    end
+  end
+end

data/lib/honeybadger/plugins/karafka.rb

@@ -6,25 +6,14 @@ module Honeybadger
       requirement { defined?(::Karafka) && ::Karafka.respond_to?(:monitor) }
 
       execution do
-        ::Karafka.monitor.subscribe("error.occurred") do |event|
-          Honeybadger.notify(event[:error])
-          Honeybadger.event("error.occurred.karafka", error: event[:error]) if config.load_plugin_insights?(:karafka)
-        end
+        require 'honeybadger/karafka'
 
-        if config.load_plugin_insights?(:karafka)
-          ::Karafka.monitor.subscribe("consumer.consumed") do |event|
-            context = {
-              duration: event.payload[:time],
-              consumer: event.payload[:caller].class.to_s,
-              id: event.payload[:caller].id,
-              topic: event.payload[:caller].messages.metadata.topic,
-              messages_count: event.payload[:caller].messages.metadata.size,
-              processing_lag: event.payload[:caller].messages.metadata.processing_lag,
-              partition: event.payload[:caller].messages.metadata.partition
-            }
+        errors_listener = ::Honeybadger::Karafka::ErrorsListener.new
+        ::Karafka.monitor.subscribe(errors_listener)
+        ::Karafka.producer.monitor.subscribe(errors_listener) if ::Karafka.respond_to?(:producer)
 
-            Honeybadger.event("consumer.consumed.karafka", context)
-          end
+        if config.load_plugin_insights?(:karafka)
+          ::Karafka.monitor.subscribe(::Honeybadger::Karafka::InsightsListener.new)
         end
       end
     end

data/lib/honeybadger/version.rb

@@ -1,4 +1,4 @@
 module Honeybadger
   # The current String Honeybadger version.
-  VERSION = '5.18.0'.freeze
+  VERSION = '5.19.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: honeybadger
 version: !ruby/object:Gem::Version
-  version: 5.18.0
+  version: 5.19.0
 platform: ruby
 authors:
 - Honeybadger Industries LLC
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-10-25 00:00:00.000000000 Z
+date: 2024-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logger
@@ -81,6 +81,7 @@ files:
 - lib/honeybadger/init/sinatra.rb
 - lib/honeybadger/instrumentation.rb
 - lib/honeybadger/instrumentation_helper.rb
+- lib/honeybadger/karafka.rb
 - lib/honeybadger/logging.rb
 - lib/honeybadger/metric.rb
 - lib/honeybadger/metrics_worker.rb