yabeda-karafka-2 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 96679ba8966c5530ab8c543803f8be3f338c23cf2931e54c3a68f61d232b1543
+  data.tar.gz: 6423f70840140e060920c32766c255e1aa22b1d99b13833675722f01a90e934c
+SHA512:
+  metadata.gz: c903fadb57b8cb8db23ca50dce3e6fa20e6c85ba8010590e7fda3fc2cfd37e662331a6f359f87aa0ce38921f9ce8a7956cca5f9003f8b53b3c9e74e8f120d645
+  data.tar.gz: '082cc670d4fe7ce085070af654574efb64087208942c9553ab00d574f1b248dab79e0a3036f25a7e4109a38fca3be1533d7664520bc9f064640a4bb9a4e028cc'

data/lib/yabeda/karafka/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Yabeda
+  module Karafka
+    class Base
+      class << self
+        def register_metrics
+          Yabeda.configure do
+            group :karafka do
+              counter :errors_total,
+                      tags: %i[type base_type],
+                      comment: 'A counter of the total number of errors'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yabeda/karafka/config.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'anyway'
+
+module Yabeda
+  module Karafka
+    class Config < ::Anyway::Config
+      config_name :yabeda_karafka
+
+      # Declare consumer metrics
+      attr_config consumer_metrics: true
+
+      # Declare producer metrics
+      attr_config producer_metrics: true
+    end
+  end
+end

data/lib/yabeda/karafka/consumer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'karafka'
+
+module Yabeda
+  module Karafka
+    class Consumer
+      BATCH_PROCESSING_TIME_BUCKETS = [
+        1, 3, 5, 10, 15, 25, 50, 75, 100, 125, 150, 175, 200, 225, 250, 275,
+        300, 350, 400, 450, 500, 550, 600, 650, 700, 800, 900, 1_000, 1_500, 2_000,
+        3_000, 4_000, 5_000, 6_000, 7_000, 8_000, 9_000, 10_000
+      ].freeze
+
+      MESSAGE_PER_BATCH_BUCKETS = [
+        1, 5, 10, 15, 20, 25, 30, 40, 50, 60, 75, 100, 125, 150, 200, 250, 300, 400, 500
+      ].freeze
+
+      class << self
+        def register_metrics # rubocop:disable Metrics/MethodLength
+          Yabeda.configure do
+            group :karafka_consumer do
+              counter :received_message_total,
+                      tags: %i[topic partition consumer],
+                      comment: 'A counter of the total number of messages received'
+
+              counter :processed_message_total,
+                      tags: %i[topic partition consumer],
+                      comment: 'A counter of the total number of messages processed'
+
+              histogram :messages_per_batch,
+                        unit: :messages,
+                        per: :batch,
+                        tags: %i[topic partition consumer],
+                        buckets: MESSAGE_PER_BATCH_BUCKETS,
+                        comment: 'Quantity of messages on a given batch'
+
+              histogram :batch_processing_time,
+                        unit: :milliseconds,
+                        per: :batch,
+                        tags: %i[topic partition consumer],
+                        buckets: BATCH_PROCESSING_TIME_BUCKETS,
+                        comment: 'Time that took to process a given batch (ms)'
+            end
+          end
+        end
+
+        def register_events
+          messages_received
+          messages_consumed
+          error
+        end
+
+        private
+
+        def register_event(event_name, &block)
+          ::Karafka.monitor.subscribe(event_name, &block)
+        end
+
+        def messages_received
+          register_event('consumer.consume') do |event|
+            consumer = event[:caller]
+            labels = { topic: consumer.topic.name, partition: consumer.partition, consumer: consumer.class.name }
+            message_count = consumer.messages.count
+            Yabeda.karafka_consumer_received_message_total.increment(labels, by: message_count)
+            Yabeda.karafka_consumer_messages_per_batch.measure(labels, message_count)
+          end
+        end
+
+        def messages_consumed
+          register_event('consumer.consumed') do |event|
+            consumer = event[:caller]
+            labels = { topic: consumer.topic.name, partition: consumer.partition, consumer: consumer.class.name }.compact
+            message_count = consumer.messages.count
+            time = event[:time]
+            Yabeda.karafka_consumer_processed_message_total.increment(labels, by: message_count)
+            Yabeda.karafka_consumer_batch_processing_time.measure(labels, time)
+          end
+        end
+
+        def error
+          register_event('error.occurred') do |event|
+            type = event[:type]
+            base_type = type.split('.').first
+            labels = { type: type, base_type: base_type }.compact
+            Yabeda.karafka_errors_total.increment(labels)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yabeda/karafka/producer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'karafka'
+
+module Yabeda
+  module Karafka
+    class Producer
+      class << self
+        def register_metrics
+          Yabeda.configure do
+            group :karafka_producer do
+              counter :sent_message_total,
+                      tags: %i[topic type],
+                      comment: 'A counter of the total number of messages sent'
+            end
+          end
+        end
+
+        def register_events
+          %w[sync async].each do |type|
+            message_sent(type)
+            message_batch_sent(type)
+          end
+        end
+
+        private
+
+        def register_event(event_name, &block)
+          ::Karafka.producer.monitor.subscribe(event_name, &block)
+        end
+
+        def message_sent(type)
+          register_event("message.produced_#{type}") do |event|
+            message = event[:message]
+            labels = { topic: message.topic.name, type: type }
+            Yabeda.karafka_producer_sent_message_total.increment(labels)
+          end
+        end
+
+        def message_batch_sent(type)
+          register_event("messages.produced_#{type}") do |event|
+            messages = event[:messages]
+            messages.each do |message|
+              labels = { topic: message.topic.name, type: type }
+              Yabeda.karafka_producer_sent_message_total.increment(labels)
+            end
+          end
+        end
+
+        def error
+          register_event('error.occurred') do |event|
+            type = event[:type]
+            base_type = type.split('.').first
+            labels = { type: type, base_type: base_type }.compact
+            Yabeda.karafka_errors_total.increment(labels)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yabeda/karafka/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Yabeda
+  module Karafka
+    VERSION = '0.0.1'
+  end
+end

data/lib/yabeda/karafka.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'karafka'
+require 'yabeda'
+
+require 'yabeda/karafka/version'
+require 'yabeda/karafka/config'
+
+require 'yabeda/karafka/base'
+require 'yabeda/karafka/consumer'
+require 'yabeda/karafka/producer'
+
+module Yabeda
+  module Karafka
+    def self.config
+      @config ||= Config.new
+    end
+
+    Base.register_metrics if config.consumer_metrics || config.producer_metrics
+    Consumer.register_metrics if config.consumer_metrics
+    Producer.register_metrics if config.producer_metrics
+
+    ::Karafka.monitor.subscribe('app.initialized') do |_event|
+      Consumer.register_events if config.consumer_metrics
+      Producer.register_events if config.producer_metrics
+    end
+  end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: yabeda-karafka-2
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Henrique Bontempo
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-08-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: anyway_config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: karafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: yabeda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Extends Yabeda to collect Karafka metrics
+email: me@hbontempo.dev
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/yabeda/karafka.rb
+- lib/yabeda/karafka/base.rb
+- lib/yabeda/karafka/config.rb
+- lib/yabeda/karafka/consumer.rb
+- lib/yabeda/karafka/producer.rb
+- lib/yabeda/karafka/version.rb
+homepage: https://github.com/hbontempo-br/yabeda-karafka
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/hbontempo-br/yabeda-karafka
+  source_code_uri: https://github.com/hbontempo-br/yabeda-karafka
+  changelog_uri: https://github.com/hbontempo-br/yabeda-karafka/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/hbontempo-br/yabeda-karafka/issues
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Monitoring of Karafka operation
+test_files: []