karafka-web 0.7.10 → 0.8.0.rc1

data/lib/karafka/web/tracking/consumers/listeners/processing.rb

@@ -12,10 +12,33 @@ module Karafka
             # @param event [Karafka::Core::Monitoring::Event]
             def on_worker_processed(event)
               track do |sampler|
-                sampler.times[:total] << event[:time]
+                sampler.windows.m1[:processed_total_time] << event[:time]
               end
             end
 
+            # We do not track idle jobs here because they are internal and not user-facing
+            %i[
+              consume
+              revoked
+              shutdown
+              tick
+            ].each do |action|
+              # Tracks the job that is going to be scheduled so we can also display pending jobs
+              class_eval <<~RUBY, __FILE__, __LINE__ + 1
+                # @param event [Karafka::Core::Monitoring::Event]
+                def on_consumer_before_schedule_#{action}(event)
+                  consumer = event.payload[:caller]
+                  jid = job_id(consumer, '#{action}')
+                  job_details = job_details(consumer, '#{action}')
+                  job_details[:status] = 'pending'
+
+                  track do |sampler|
+                    sampler.jobs[jid] = job_details
+                  end
+                end
+              RUBY
+            end
+
             # Counts work execution and processing states in consumer instances
             #
             # @param event [Karafka::Core::Monitoring::Event]
@@ -34,6 +57,19 @@ module Karafka
               end
             end
 
+            # Collect info about consumption event that occurred and its metrics
+            # Removes the job from running jobs
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_consumed(event)
+              consumer = event.payload[:caller]
+              jid = job_id(consumer, 'consume')
+
+              track do |sampler|
+                sampler.jobs.delete(jid)
+              end
+            end
+
             # Removes failed job from active jobs
             #
             # @param event [Karafka::Core::Monitoring::Event]
@@ -46,6 +82,8 @@ module Karafka
                          'revoked'
                        when 'consumer.shutdown.error'
                          'shutdown'
+                       when 'consumer.tick.error'
+                         'tick'
                        # This is not a user facing execution flow, but internal system one
                        # that is why it will not be reported as a separate job for the UI
                        when 'consumer.idle.error'
@@ -65,72 +103,39 @@ module Karafka
               end
             end
 
-            # Collect info about consumption event that occurred and its metrics
-            # Removes the job from running jobs
-            #
-            # @param event [Karafka::Core::Monitoring::Event]
-            def on_consumer_consumed(event)
-              consumer = event.payload[:caller]
-              topic = consumer.topic
-              consumer_group_id = topic.consumer_group.id
-              messages_count = consumer.messages.size
-              time = event[:time]
-              jid = job_id(consumer, 'consume')
-
-              track do |sampler|
-                sampler.jobs.delete(jid)
-                sampler.times[consumer_group_id] << [topic.name, time, messages_count]
-              end
-            end
-
-            # Stores this job details
-            #
-            # @param event [Karafka::Core::Monitoring::Event]
-            def on_consumer_revoke(event)
-              consumer = event.payload[:caller]
-              jid = job_id(consumer, 'revoked')
-              job_details = job_details(consumer, 'revoked')
-
-              track do |sampler|
-                sampler.jobs[jid] = job_details
-              end
-            end
-
-            # Removes the job from running jobs
-            #
-            # @param event [Karafka::Core::Monitoring::Event]
-            def on_consumer_revoked(event)
-              consumer = event.payload[:caller]
-              jid = job_id(consumer, 'revoked')
-
-              track do |sampler|
-                sampler.jobs.delete(jid)
-              end
-            end
-
-            # Stores this job details
-            #
-            # @param event [Karafka::Core::Monitoring::Event]
-            def on_consumer_shutting_down(event)
-              consumer = event.payload[:caller]
-              jid = job_id(consumer, 'shutdown')
-              job_details = job_details(consumer, 'shutdown')
-
-              track do |sampler|
-                sampler.jobs[jid] = job_details
-              end
-            end
-
-            # Removes the job from running jobs
-            #
-            # @param event [Karafka::Core::Monitoring::Event]
-            def on_consumer_shutdown(event)
-              consumer = event.payload[:caller]
-              jid = job_id(consumer, 'shutdown')
-
-              track do |sampler|
-                sampler.jobs.delete(jid)
-              end
+            # Consume has a bit different reporting flow than other jobs because it bumps certain
+            # counters that other jobs do not. This is why it is defined above separately
+            [
+              [:revoke, :revoked, 'revoked'],
+              [:shutting_down, :shutdown, 'shutdown'],
+              [:tick, :ticked, 'tick']
+            ].each do |pre, post, action|
+              class_eval <<~METHOD, __FILE__, __LINE__ + 1
+                # Stores this job details
+                #
+                # @param event [Karafka::Core::Monitoring::Event]
+                def on_consumer_#{pre}(event)
+                  consumer = event.payload[:caller]
+                  jid = job_id(consumer, '#{action}')
+                  job_details = job_details(consumer, '#{action}')
+
+                  track do |sampler|
+                    sampler.jobs[jid] = job_details
+                  end
+                end
+
+                # Removes the job from running jobs
+                #
+                # @param event [Karafka::Core::Monitoring::Event]
+                def on_consumer_#{post}(event)
+                  consumer = event.payload[:caller]
+                  jid = job_id(consumer, '#{action}')
+
+                  track do |sampler|
+                    sampler.jobs.delete(jid)
+                  end
+                end
+              METHOD
             end
 
             private
@@ -152,14 +157,16 @@ module Karafka
             #   more details.
             def job_details(consumer, type)
               {
-                started_at: float_now,
+                updated_at: float_now,
                 topic: consumer.topic.name,
                 partition: consumer.partition,
                 first_offset: consumer.messages.metadata.first_offset,
                 last_offset: consumer.messages.metadata.last_offset,
                 processing_lag: consumer.messages.metadata.processing_lag,
                 consumption_lag: consumer.messages.metadata.consumption_lag,
-                committed_offset: consumer.coordinator.seek_offset - 1,
+                # Committed offset may be -1 when there is no committed offset. This can happen in
+                # case of ticking that started before any consumption job happened
+                committed_offset: consumer.coordinator.seek_offset.to_i - 1,
                 # In theory this is redundant because we have first and last offset, but it is
                 # needed because VPs do not have linear count. For VPs first and last offset
                 # will be further away than the total messages count for a particular VP
@@ -167,7 +174,8 @@ module Karafka
                 consumer: consumer.class.to_s,
                 consumer_group: consumer.topic.consumer_group.id,
                 type: type,
-                tags: consumer.tags
+                tags: consumer.tags,
+                status: 'running'
               }
             end
           end

data/lib/karafka/web/tracking/consumers/listeners/statistics.rb

@@ -18,6 +18,8 @@ module Karafka
               sg_id = event[:subscription_group_id]
               sg_details = extract_sg_details(sg_id, cgrp)
 
+              track_transfers(statistics)
+
               # More than one subscription group from the same consumer group may be reporting
               # almost the same time. To prevent corruption of partial data, we put everything here
               # in track as we merge data from multiple subscription groups
@@ -42,25 +44,41 @@ module Karafka
                     }
 
                     topic_details[:partitions][pt_id] = metrics.merge(
-                      id: pt_id,
+                      id: pt_id
+                    ).merge(
                       # Pauses are stored on a consumer group since we do not process same topic
                       # twice in the multiple subscription groups
-                      poll_state: poll_state(cg_id, topic_name, pt_id)
+                      poll_details(sg_id, topic_name, pt_id)
                     )
                   end
                 end
 
-                sampler.consumer_groups[cg_id] ||= {
-                  id: cg_id,
-                  subscription_groups: {}
-                }
-
                 sampler.consumer_groups[cg_id][:subscription_groups][sg_id] = sg_details
               end
             end
 
             private
 
+            # Tracks network transfers from and to the client using a 1 minute rolling window
+            #
+            # @param statistics [Hash] statistics hash
+            def track_transfers(statistics)
+              brokers = statistics.fetch('brokers', {})
+
+              return if brokers.empty?
+
+              track do |sampler|
+                client_name = statistics.fetch('name')
+
+                brokers.each do |broker_name, values|
+                  scope_name = "#{client_name}-#{broker_name}"
+
+                  sampler.windows.m1["#{scope_name}-rxbytes"] << values.fetch('rxbytes', 0)
+                  sampler.windows.m1["#{scope_name}-txbytes"] << values.fetch('txbytes', 0)
+                end
+              end
+            end
+
             # Extracts basic consumer group related details
             # @param sg_id [String]
             # @param sg_stats [Hash]
@@ -75,7 +93,7 @@ module Karafka
                   'rebalance_age',
                   'rebalance_cnt',
                   'rebalance_reason'
-                ),
+                ).transform_keys(&:to_sym),
                 topics: {}
               }
             end
@@ -132,14 +150,23 @@ module Karafka
               metrics
             end
 
-            # @param cg_id [String]
+            # @param sg_id [String] subscription group id
             # @param topic_name [String]
-            # @param pt_id [Integer]
+            # @param pt_id [Integer] partition id
             # @return [String] poll state / is partition paused or not
-            def poll_state(cg_id, topic_name, pt_id)
-              pause_id = [cg_id, topic_name, pt_id].join('-')
+            def poll_details(sg_id, topic_name, pt_id)
+              pause_id = [sg_id, topic_name, pt_id].join('-')
+
+              details = { poll_state: 'active', poll_state_ch: 0 }
 
-              sampler.pauses.include?(pause_id) ? 'paused' : 'active'
+              pause_details = sampler.pauses[pause_id]
+
+              return details unless pause_details
+
+              {
+                poll_state: 'paused',
+                poll_state_ch: [(pause_details.fetch(:paused_till) - monotonic_now).round, 0].max
+              }
             end
           end
         end

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -9,18 +9,13 @@ module Karafka
         class Sampler < Tracking::Sampler
           include ::Karafka::Core::Helpers::Time
 
-          attr_reader :counters, :consumer_groups, :errors, :times, :pauses, :jobs
+          attr_reader :counters, :consumer_groups, :subscription_groups, :errors,
+                      :pauses, :jobs, :windows
 
           # Current schema version
-          # This can be used in the future for detecting incompatible changes and writing
-          # migrations
-          SCHEMA_VERSION = '1.2.3'
-
-          # 60 seconds window for time tracked window-based metrics
-          TIMES_TTL = 60
-
-          # Times ttl in ms
-          TIMES_TTL_MS = TIMES_TTL * 1_000
+          # This is used for detecting incompatible changes and not using outdated data during
+          # upgrades
+          SCHEMA_VERSION = '1.2.9'
 
           # Counters that count events occurrences during the given window
           COUNTERS_BASE = {
@@ -36,17 +31,23 @@ module Karafka
             dead: 0
           }.freeze
 
-          private_constant :TIMES_TTL, :TIMES_TTL_MS, :COUNTERS_BASE
+          private_constant :COUNTERS_BASE
 
           def initialize
             super
 
+            @windows = Helpers::Ttls::Windows.new
             @counters = COUNTERS_BASE.dup
-            @times = TtlHash.new(TIMES_TTL_MS)
-            @consumer_groups = {}
+            @consumer_groups = Hash.new do |h, cg_id|
+              h[cg_id] = {
+                id: cg_id,
+                subscription_groups: {}
+              }
+            end
+            @subscription_groups = {}
             @errors = []
             @started_at = float_now
-            @pauses = Set.new
+            @pauses = {}
             @jobs = {}
             @shell = MemoizedShell.new
             @memory_total_usage = 0
@@ -81,7 +82,9 @@ module Karafka
                 cpus: cpus,
                 threads: threads,
                 cpu_usage: @cpu_usage,
-                tags: Karafka::Process.tags
+                tags: Karafka::Process.tags,
+                bytes_received: bytes_received,
+                bytes_sent: bytes_sent
               },
 
               versions: {
@@ -98,7 +101,7 @@ module Karafka
                 utilization: utilization
               ).merge(total: @counters),
 
-              consumer_groups: @consumer_groups,
+              consumer_groups: enriched_consumer_groups,
               jobs: jobs.values
             }
           end
@@ -130,21 +133,28 @@ module Karafka
           #   utilized all the time within the given time window. 0% means, nothing is happening
           #   most if not all the time.
           def utilization
-            return 0 if times[:total].empty?
+            totals = windows.m1[:processed_total_time]
+
+            return 0 if totals.empty?
 
-            # Max times ttl
             timefactor = float_now - @started_at
-            timefactor = timefactor > TIMES_TTL ? TIMES_TTL : timefactor
+            timefactor = timefactor > 60 ? 60 : timefactor
 
             # We divide by 1_000 to convert from milliseconds
             # We multiply by 100 to have it in % scale
-            times[:total].sum / 1_000 / workers / timefactor * 100
+            totals.sum / 1_000 / workers / timefactor * 100
           end
 
-          # @return [Integer] number of listeners
+          # @return [Hash] number of active and standby listeners
           def listeners
-            # This can be zero before the server starts
-            Karafka::Server.listeners&.count.to_i
+            if Karafka::Server.listeners
+              active = Karafka::Server.listeners.count(&:active?)
+              total = Karafka::Server.listeners.count.to_i
+
+              { active: active, standby: total - active }
+            else
+              { active: 0, standby: 0 }
+            end
           end
 
           # @return [Integer] memory used by this process in kilobytes
@@ -175,9 +185,14 @@ module Karafka
           # @return [Hash] job queue statistics
           def jobs_queue_statistics
             # We return empty stats in case jobs queue is not yet initialized
+            base = Karafka::Server.jobs_queue&.statistics || { busy: 0, enqueued: 0 }
+            stats = base.slice(:busy, :enqueued, :waiting)
+            stats[:waiting] ||= 0
             # busy - represents number of jobs that are being executed currently
-            # enqueued - represents number of jobs that are enqueued to be processed
-            Karafka::Server.jobs_queue&.statistics || { busy: 0, enqueued: 0 }
+            # enqueued - jobs that are in the queue but not being picked up yet
+            # waiting - jobs that are not scheduled on the queue but will be
+            # be enqueued in case of advanced schedulers
+            stats
           end
 
           # Total memory used in the OS
@@ -265,6 +280,48 @@ module Karafka
                                    @memory_threads_ps = false
                                  end
           end
+
+          # Consumer group details need to be enriched with details about polling that comes from
+          # Karafka level. It is also time based, hence we need to materialize it only at the
+          # moment of message dispatch to have it accurate.
+          def enriched_consumer_groups
+            @consumer_groups.each_value do |cg_details|
+              cg_details.each do
+                cg_details.fetch(:subscription_groups, {}).each do |sg_id, sg_details|
+                  # This should be always available, since we subscription group polled at time
+                  # is first initialized before we start polling, there should be no case where
+                  # we have statistics about a given subscription group but we do not have the
+                  # last polling time
+                  polled_at = subscription_groups.fetch(sg_id).fetch(:polled_at)
+                  sg_details[:state][:poll_age] = monotonic_now - polled_at
+                end
+              end
+            end
+
+            @consumer_groups
+          end
+
+          # @return [Integer] number of bytes received per second out of a one minute time window
+          #   by all the consumers
+          # @note We use one minute window to compensate for cases where metrics would be reported
+          #   or recorded faster or slower. This normalizes data
+          def bytes_received
+            @windows
+              .m1
+              .stats_from { |k, _v| k.end_with?('rxbytes') }
+              .rps
+              .round
+          end
+
+          # @return [Integer] number of bytes sent per second out of a one minute time window by
+          #   all the consumers
+          def bytes_sent
+            @windows
+              .m1
+              .stats_from { |k, _v| k.end_with?('txbytes') }
+              .rps
+              .round
+          end
         end
       end
     end

data/lib/karafka/web/tracking/helpers/ttls/array.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Helpers
+        # Namespace for time sensitive related buffers and operators
+        module Ttls
+          # Array that allows us to store data points that expire over time automatically.
+          class Array
+            include ::Karafka::Core::Helpers::Time
+            include Enumerable
+
+            # @param ttl [Integer] milliseconds ttl
+            def initialize(ttl)
+              @ttl = ttl
+              @accu = []
+            end
+
+            # Iterates over only active elements
+            def each
+              clear
+
+              @accu.each do |sample|
+                yield sample[:value]
+              end
+            end
+
+            # @param value [Object] adds value to the array
+            # @return [Object] added element
+            def <<(value)
+              @accu << { value: value, added_at: monotonic_now }
+
+              clear
+
+              value
+            end
+
+            # @return [Boolean] is the array empty
+            def empty?
+              clear
+              @accu.empty?
+            end
+
+            # Samples that are within our TTL time window with the times
+            #
+            # @return [Hash]
+            def samples
+              clear
+              @accu
+            end
+
+            # @return [::Array] pure array version with only active elements
+            def to_a
+              clear
+              super
+            end
+
+            private
+
+            # Evicts outdated samples
+            def clear
+              @accu.delete_if do |sample|
+                monotonic_now - sample[:added_at] > @ttl
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/helpers/ttls/hash.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Helpers
+        module Ttls
+          # Hash that accumulates data that has an expiration date (ttl)
+          # Used to keep track of metrics in a window
+          class Hash < Hash
+            # @param ttl [Integer] milliseconds ttl
+            def initialize(ttl)
+              super() { |k, v| k[v] = Ttls::Array.new(ttl) }
+            end
+
+            # Takes a block where we provide a hash select filtering to select keys we are
+            # interested in using for aggregated stats. Once filtered, builds a Stats object out
+            # of the candidates
+            #
+            # @param block [Proc] block for selection of elements for stats
+            # @yieldparam [String] key
+            # @yieldparam [Ttls::Array] samples
+            # @return [Stats]
+            def stats_from(&block)
+              Stats.new(
+                select(&block)
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/helpers/ttls/stats.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Helpers
+        module Ttls
+          # Object that simplifies computing aggregated statistics out of ttl data
+          # For TTL based operations we may collect samples from multiple consumers/producers etc
+          # but in the end we are interested in the collective result of the whole process.
+          #
+          # For example when we talk about data received from Kafka, we want to materialize total
+          # number of bytes and not bytes per given client connection. This layer simplifies this
+          # by doing necessary aggregations and providing the final results
+          class Stats
+            # @param ttls_hash [Ttls::Hash, Hash] hash with window based samples
+            def initialize(ttls_hash)
+              @data = ttls_hash
+                      .values
+                      .map(&:samples)
+                      .map(&:to_a)
+                      .delete_if { |samples| samples.size < 2 }
+                      .map { |samples| samples.map(&:values) }
+            end
+
+            # Computes the rate out of the samples provided on a per second basis. The samples need
+            #   to come from the window aggregations
+            #
+            # @return [Float] per second rate value
+            def rps
+              sub_results = @data.map do |samples|
+                oldest = samples.first
+                newest = samples.last
+
+                value = oldest[0] - newest[0]
+                # Convert to seconds as we want to have it in a 1 sec pace
+                time = (oldest[1] - newest[1]) / 1_000
+
+                value / time.to_f
+              end
+
+              sub_results.flatten.sum
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/helpers/ttls/windows.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Helpers
+        module Ttls
+          # Object used to track process metrics in time windows. Those are shared, meaning they do
+          # not refer to particular metric type but allow us to store whatever we want.
+          #
+          # We have following time windows:
+          #   - m1 - one minute big
+          #   - m5 - five minute big
+          Windows = Struct.new(:m1, :m5) do
+            # @return [Ttls::Windows]
+            def initialize
+              super(
+                Ttls::Hash.new(60 * 1_000),
+                Ttls::Hash.new(5 * 60 * 1_000)
+              )
+            end
+
+            # Clears the TTLs windows
+            def clear
+              values.each(&:clear)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/reporter.rb

@@ -16,6 +16,7 @@ module Karafka
         def active?
           return false unless ::Karafka::Web.producer
           return false unless ::Karafka::Web.producer.status.active?
+          return false unless ::Karafka::Web.config.tracking.active
 
           true
         end