promenade 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69cb441f3ff80fd4e24c3fd1760a2930d5ec57be77d1e8defc7d5c0c91d9526a
-  data.tar.gz: b29b635ad8c18ad0cd990d764c9fcda51e2b5394818788a43478692a2f9bef09
+  metadata.gz: fc0749131d57843e9243eb47058cc0615c73865b92b753772308ceb13cd6b096
+  data.tar.gz: 66c5a9376ea0cd02dbd64bd63e67d3178f440a30701646912ef459586e14a1e6
 SHA512:
-  metadata.gz: 78899076a32afdabe648eed9c717ca0656880d83343c8101f68529302d232046a7b0659e00f8aa9d0f2c0c98de5f77c75ae9d43a3b0a656816b0cbd70c5e683c
-  data.tar.gz: 233dea29570f65d4dbed4a33c3da1898d3c41918777610fa5bba95e8f9931003554a1beb2d1ba7326eb113e0e53176f72c2176e80ab236849adfef914c1076c0
+  metadata.gz: 5569ffe88cbaf2a0a738b605c15673a6866e51c345fc53a8000f01004dd77b23a8175af32f1242ed2492aa444c6a71e20d760fb1d377b2c02ead531470108f47
+  data.tar.gz: b2fe6be44e09a45f9e6cf051971b27140d9af03e436265815ccb0e527d85dfcc890dc271654e3b404ff9cfce8c8a633ff4efe1c17797781ef6751732bb7d92f3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    promenade (0.8.0)
+    promenade (0.9.0)
       actionpack
       activesupport (> 6.0, < 8.0)
       prometheus-client-mmap (~> 0.16.0)

data/lib/promenade/karafka/consumer_subscriber.rb

@@ -0,0 +1,46 @@
+require "promenade/karafka/subscriber"
+
+module Promenade
+  module Karafka
+    class ConsumerSubscriber < Subscriber
+      attach_to "consumer.karafka"
+
+      Promenade.histogram :karafka_consumer_batch_processing_duration_seconds do
+        doc "Consumer message processing latency in seconds"
+        buckets :network
+      end
+
+      Promenade.counter :karafka_consumer_messages_processed do
+        doc "Messages processed by this consumer"
+      end
+
+      def consumed(event)
+        consumer = event.payload[:caller]
+        messages = consumer.messages
+        batch_processing_duration = convert_milliseconds_to_seconds(event.payload[:time])
+
+        labels = get_labels(consumer)
+
+        Promenade.metric(:karafka_consumer_messages_processed).increment(labels, messages.size)
+        Promenade.metric(:karafka_consumer_batch_processing_duration_seconds).observe(labels, batch_processing_duration)
+      end
+
+      private
+
+        def get_labels(consumer)
+          metadata = consumer.messages.metadata
+
+          {
+            client: consumer.topic.kafka[:"client.id"],
+            group: consumer.topic.consumer_group.id,
+            topic: metadata.topic,
+            partition: metadata.partition,
+          }
+        end
+
+        def convert_milliseconds_to_seconds(time_in_milliseconds)
+          time_in_milliseconds / 1000.to_f
+        end
+    end
+  end
+end

data/lib/promenade/karafka/error_subscriber.rb

@@ -0,0 +1,27 @@
+require "promenade/karafka/subscriber"
+
+module Promenade
+  module Karafka
+    class ErrorSubscriber < Subscriber
+      attach_to "error.karafka"
+
+      Promenade.counter :karafka_errors do
+        doc "Count of Kafka connection errors"
+      end
+
+      def occurred(event)
+        labels = get_labels(event)
+
+        Promenade.metric(:karafka_errors).increment(labels)
+      end
+
+      private
+
+        def get_labels(event)
+          {
+            error_type: event.payload[:type],
+          }
+        end
+    end
+  end
+end

data/lib/promenade/karafka/statistics_subscriber.rb

@@ -0,0 +1,80 @@
+require "promenade/karafka/subscriber"
+require "active_support/core_ext/hash"
+
+module Promenade
+  module Karafka
+    class StatisticsSubscriber < Subscriber
+      attach_to "statistics.karafka"
+
+      Promenade.histogram :karafka_connection_latency_seconds do
+        doc "Request latency (rtt) in seconds"
+        buckets :network
+      end
+
+      Promenade.counter :karafka_connection_calls do
+        doc "Count of calls made to Kafka broker"
+      end
+
+      Promenade.gauge :karafka_consumer_offset_lag do
+        doc "Lag between message create and consume time"
+      end
+
+      def emitted(event)
+        group = event.payload[:consumer_group_id]
+        statistics = event.payload[:statistics].with_indifferent_access
+        client_id = statistics[:client_id]
+
+        report_topic_metrics(statistics[:topics], group, client_id)
+        report_connection_metrics(statistics[:brokers], client_id)
+      end
+
+      private
+
+        def report_topic_metrics(topics, group, client_id)
+          topics.map do |topic_name, topic_values|
+            labels = {
+              client: client_id,
+              topic: topic_name,
+              group: group,
+            }
+            report_partition_metrics(topic_values, labels)
+          end
+        end
+
+        def report_partition_metrics(topic_values, labels)
+          topic_values[:partitions].map do |partition_name, partition_values|
+            next if partition_name == "-1"
+            next if partition_values[:consumer_lag_stored] == -1
+
+            labels = labels.merge(
+              partition: partition_name,
+            )
+
+            offset_lag = partition_values[:consumer_lag_stored]
+
+            Promenade.metric(:karafka_consumer_offset_lag).set(labels, offset_lag)
+          end
+        end
+
+        def report_connection_metrics(brokers, client_id)
+          labels = {
+            client: client_id,
+          }
+
+          brokers.map do |broker_name, broker_values|
+            next if broker_values[:nodeid] == -1
+
+            rtt = convert_microseconds_to_seconds(broker_values[:rtt][:avg])
+            connection_calls = broker_values[:connects]
+
+            Promenade.metric(:karafka_connection_calls).increment(labels.merge(broker: broker_name), connection_calls)
+            Promenade.metric(:karafka_connection_latency_seconds).observe(labels.merge(broker: broker_name), rtt)
+          end
+        end
+
+        def convert_microseconds_to_seconds(time_in_microseconds)
+          time_in_microseconds / 1_000_000.to_f
+        end
+    end
+  end
+end

data/lib/promenade/karafka/subscriber.rb

@@ -0,0 +1,8 @@
+require "active_support/subscriber"
+
+module Promenade
+  module Karafka
+    class Subscriber < ActiveSupport::Subscriber
+    end
+  end
+end

data/lib/promenade/karafka.rb

@@ -0,0 +1,3 @@
+require "promenade/karafka/consumer_subscriber"
+require "promenade/karafka/error_subscriber"
+require "promenade/karafka/statistics_subscriber"

data/lib/promenade/version.rb

@@ -1,3 +1,3 @@
 module Promenade
-  VERSION = "0.8.0".freeze
+  VERSION = "0.9.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: promenade
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Ed Robinson
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-03-02 00:00:00.000000000 Z
+date: 2023-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -288,6 +288,11 @@ files:
 - lib/promenade/kafka/fetcher_subscriber.rb
 - lib/promenade/kafka/producer_subscriber.rb
 - lib/promenade/kafka/subscriber.rb
+- lib/promenade/karafka.rb
+- lib/promenade/karafka/consumer_subscriber.rb
+- lib/promenade/karafka/error_subscriber.rb
+- lib/promenade/karafka/statistics_subscriber.rb
+- lib/promenade/karafka/subscriber.rb
 - lib/promenade/prometheus.rb
 - lib/promenade/railtie.rb
 - lib/promenade/setup.rb
@@ -313,7 +318,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Promenade makes it simple to instrument Ruby apps for prometheus scraping