karafka-web 0.1.0

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Listener related to tracking errors, DLQs, and retries metrics for the Web UI
+          class Errors < Base
+            # Collects errors info and counts errors
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_error_occurred(event)
+              track do |sampler|
+                # Collect extra info if it was a consumer related error.
+                # Those come from user code
+                details = if event[:caller].is_a?(Karafka::BaseConsumer)
+                            extract_consumer_info(event[:caller])
+                          else
+                            {}
+                          end
+
+                error_class, error_message, backtrace = extract_error_info(event[:error])
+
+                sampler.errors << {
+                  type: event[:type],
+                  error_class: error_class,
+                  error_message: error_message,
+                  backtrace: backtrace,
+                  details: details,
+                  occurred_at: float_now
+                }
+
+                sampler.counters[:errors] += 1
+              end
+            end
+
+            # Count dead letter queue messages dispatches
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_dead_letter_queue_dispatched(_event)
+              track do |sampler|
+                sampler.counters[:dead] += 1
+              end
+            end
+
+            # Count retries
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_consumer_consuming_retry(_event)
+              track do |sampler|
+                sampler.counters[:retries] += 1
+              end
+            end
+
+            private
+
+            # @param consumer [::Karafka::BaseConsumer]
+            # @return [Hash] hash with consumer specific info for details of error
+            def extract_consumer_info(consumer)
+              {
+                topic: consumer.topic.name,
+                consumer_group: consumer.topic.consumer_group.id,
+                partition: consumer.messages.metadata.partition,
+                first_offset: consumer.messages.first.offset,
+                last_offset: consumer.messages.last.offset,
+                comitted_offset: consumer.coordinator.seek_offset - 1,
+                consumer: consumer.class.to_s
+              }
+            end
+
+            # Extracts the basic error info
+            #
+            # @param error [StandardError] error that occurred
+            # @return [Array<String, String, String>] array with error name, message and backtrace
+            def extract_error_info(error)
+              error_message = error.message.to_s
+              error_message.force_encoding('utf-8')
+              error_message.scrub!
+
+              backtrace = (error.backtrace || [])
+
+              app_root = "#{::Karafka.root}/"
+
+              gem_home = if ENV.key?('GEM_HOME')
+                           ENV['GEM_HOME']
+                         else
+                           File.expand_path(File.join(Karafka.gem_root.to_s, '../'))
+                         end
+
+              gem_home = "#{gem_home}/"
+
+              backtrace.map! { |line| line.gsub(app_root, '') }
+              backtrace.map! { |line| line.gsub(gem_home, '') }
+
+              [
+                error.class.name,
+                error_message,
+                backtrace.join("\n")
+              ]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/listeners/pausing.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Tracks pausing and un-pausing of topics partitions for both user requested and
+          # automatic events.
+          class Pausing < Base
+            # Indicate pause
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_client_pause(event)
+              track do |sampler|
+                sampler.pauses << pause_id(event)
+              end
+            end
+
+            # Indicate pause ended
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_client_resume(event)
+              track do |sampler|
+                sampler.pauses.delete pause_id(event)
+              end
+            end
+
+            private
+
+            # @param event [Karafka::Core::Monitoring::Event]
+            # @return [String] pause id built from consumer group and topic details
+            def pause_id(event)
+              topic = event[:topic]
+              partition = event[:partition]
+              consumer_group_id = event[:subscription_group].consumer_group.id
+
+              [consumer_group_id, topic, partition].join('-')
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/listeners/processing.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Listener that is used to collect metrics related to work processing
+          class Processing < Base
+            # Collect time metrics about worker work execution time
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_worker_processed(event)
+              track do |sampler|
+                sampler.times[:total] << event[:time]
+              end
+            end
+
+            # Counts work execution and processing states in consumer instances
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_consume(event)
+              consumer = event.payload[:caller]
+              messages_count = consumer.messages.count
+              jid = job_id(consumer, 'consume')
+              job_details = job_details(consumer, 'consume')
+
+              track do |sampler|
+                # We count batches and messages prior to the execution, so they are tracked even
+                # if error occurs, etc.
+                sampler.counters[:batches] += 1
+                sampler.counters[:messages] += messages_count
+                sampler.jobs[jid] = job_details
+              end
+            end
+
+            # Removes failed job from active jobs
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_error_occurred(event)
+              track do |sampler|
+                type = case event[:type]
+                       when 'consumer.consume.error'
+                         'consume'
+                       when 'consumer.revoked.error'
+                         'revoked'
+                       when 'consumer.shutdown.error'
+                         'shutdown'
+                       end
+
+                sampler.jobs.delete(
+                  job_id(event[:caller], type)
+                )
+              end
+            end
+
+            # Collect info about consumption event that occurred and its metrics
+            # Removes the job from running jobs
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_consumed(event)
+              consumer = event.payload[:caller]
+              topic = consumer.topic
+              consumer_group_id = topic.consumer_group.id
+              messages_count = consumer.messages.count
+              time = event[:time]
+              jid = job_id(consumer, 'consume')
+
+              track do |sampler|
+                sampler.jobs.delete(jid)
+                sampler.times[consumer_group_id] << [topic.name, time, messages_count]
+              end
+            end
+
+            # Stores this job details
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_revoke(event)
+              consumer = event.payload[:caller]
+              jid = job_id(consumer, 'revoked')
+              job_details = job_details(consumer, 'revoked')
+
+              track do |sampler|
+                sampler.jobs[jid] = job_details
+              end
+            end
+
+            # Removes the job from running jobs
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_revoked(event)
+              consumer = event.payload[:caller]
+              jid = job_id(consumer, 'revoked')
+
+              track do |sampler|
+                sampler.jobs.delete(jid)
+              end
+            end
+
+            # Stores this job details
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_shutting_down(event)
+              consumer = event.payload[:caller]
+              jid = job_id(consumer, 'shutdown')
+              job_details = job_details(consumer, 'shutdown')
+
+              track do |sampler|
+                sampler.jobs[jid] = job_details
+              end
+            end
+
+            # Removes the job from running jobs
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_shutdown(event)
+              consumer = event.payload[:caller]
+              jid = job_id(consumer, 'shutdown')
+
+              track do |sampler|
+                sampler.jobs.delete(jid)
+              end
+            end
+
+            private
+
+            # Generates a job id that we can use to track jobs in an unique way
+            #
+            # @param consumer [::Karafka::BaseConsumer] consumer instance
+            # @param type [String] job type
+            def job_id(consumer, type)
+              "#{consumer.id}-#{type}"
+            end
+
+            # Gets consumer details for job tracking
+            #
+            # @param consumer [::Karafka::BaseConsumer] consumer instance
+            # @param type [String] job type
+            def job_details(consumer, type)
+              {
+                started_at: float_now,
+                topic: consumer.topic.name,
+                partition: consumer.messages.metadata.partition,
+                first_offset: consumer.messages.first.offset,
+                last_offset: consumer.messages.last.offset,
+                comitted_offset: consumer.coordinator.seek_offset - 1,
+                consumer: consumer.class.to_s,
+                consumer_group: consumer.topic.consumer_group.id,
+                type: type
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/listeners/statistics.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Listener used to collect metrics published by librdkafka
+          class Statistics < Base
+            # Collect Kafka metrics
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_statistics_emitted(event)
+              statistics = event[:statistics]
+              topics = statistics.fetch('topics')
+              cgrp = statistics.fetch('cgrp')
+              consumer_group_id = event[:consumer_group_id]
+
+              track do |sampler|
+                cg_details = extract_consumer_group_details(consumer_group_id, cgrp)
+                sampler.consumer_groups[consumer_group_id] = cg_details
+
+                topics.each do |topic_name, topic_values|
+                  partitions = topic_values.fetch('partitions')
+
+                  partitions.each do |partition_name, partition_statistics|
+                    partition_id = partition_name.to_i
+
+                    next unless partition_reportable?(partition_id, partition_statistics)
+
+                    metrics = extract_partition_metrics(partition_statistics)
+
+                    next if metrics.empty?
+
+                    topics_details = cg_details[:topics]
+
+                    topic_details = topics_details[topic_name] ||= {
+                      name: topic_name,
+                      partitions: {}
+                    }
+
+                    topic_details[:partitions][partition_id] = metrics.merge(
+                      id: partition_id,
+                      poll_state: poll_state(consumer_group_id, topic_name, partition_id)
+                    )
+                  end
+                end
+              end
+            end
+
+            private
+
+            # Extracts basic consumer group related details
+            # @param consumer_group_id [String]
+            # @param consumer_group_statistics [Hash]
+            # @return [Hash] consumer group relevant details
+            def extract_consumer_group_details(consumer_group_id, consumer_group_statistics)
+              {
+                id: consumer_group_id,
+                state: consumer_group_statistics.slice(
+                  'state',
+                  'join_state',
+                  'stateage',
+                  'rebalance_age',
+                  'rebalance_cnt',
+                  'rebalance_reason'
+                ),
+                topics: {}
+              }
+            end
+
+            # @param partition_id [Integer]
+            # @param partition_statistics [Hash]
+            # @return [Boolean] is this partition relevant to the current process, hence should we
+            #   report about it in the context of the process.
+            def partition_reportable?(partition_id, partition_statistics)
+              return false if partition_id == -1
+
+              # Skip until lag info is available
+              return false if partition_statistics['consumer_lag'] == -1
+
+              # Collect information only about what we are subscribed to and what we fetch or
+              # work in any way. Stopped means, we no longer work with it
+              return false if partition_statistics['fetch_state'] == 'stopped'
+
+              true
+            end
+
+            # Extracts and formats partition relevant metrics
+            #
+            # @param partition_statistics [Hash]
+            # @return [Hash] extracted partition metrics
+            def extract_partition_metrics(partition_statistics)
+              metrics = partition_statistics.slice(
+                'consumer_lag_stored',
+                'consumer_lag_stored_d',
+                'committed_offset',
+                'stored_offset',
+                'fetch_state'
+              )
+
+              # Rename as we do not need `consumer_` prefix
+              metrics.transform_keys! { |key| key.gsub('consumer_', '') }
+              metrics.transform_keys!(&:to_sym)
+
+              metrics
+            end
+
+            # @param consumer_group_id [String]
+            # @param topic_name [String]
+            # @param partition_id [Integer]
+            # @return [String] poll state / is partition paused or not
+            def poll_state(consumer_group_id, topic_name, partition_id)
+              pause_id = [consumer_group_id, topic_name, partition_id].join('-')
+
+              sampler.pauses.include?(pause_id) ? 'paused' : 'active'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/listeners/status.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Listener that triggers reporting on Karafka process status changes
+          # Whenever the whole process status changes, we do not want to wait and want to report
+          # as fast as possible, hence `report!`. This improves the user experience and since
+          # status changes do not happen that often, we can handle few extra reports dispatches.
+          class Status < Base
+            # @param _event [Karafka::Core::Monitoring::Event]
+            #
+            # @note We do not use `#report!` here because this kicks in for each listener loop and
+            #   those run the same time.
+            def on_connection_listener_before_fetch_loop(_event)
+              report
+            end
+
+            # Indicate as fast as possible that we've started moving to the quiet mode
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_app_quieting(_event)
+              report!
+            end
+
+            # Indicate as fast as possible that we've reached the quiet mode
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_app_quiet(_event)
+              report!
+            end
+
+            # Instrument on the fact that we're stopping
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_app_stopping(_event)
+              # Make sure this is sent before shutdown
+              report!
+            end
+
+            # Instrument on the fact that Karafka has stopped
+            #
+            # We do this actually before the process ends but we need to do this so the UI does not
+            # have this "handling" stopping process.
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_app_stopped(_event)
+              # Make sure this is sent before shutdown
+              report!
+            end
+          end
+        end
+      end
+    end
+  end
+end