karafka-web 0.6.3 → 0.7.0

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      # Namespace for consumer sub-components
+      module Consumers
+        module Aggregators
+          # Aggregator that tracks consumers processes states, aggregates the metrics and converts
+          # data points into a materialized current state.
+          #
+          # There are two types of metrics:
+          #   - totals - metrics that represent absolute values like number of messages processed
+          #     in total. Things that need to be incremented/updated with each incoming consumer
+          #     process report. They cannot be "batch computed" because they do not represent a
+          #     a state of time but progress.
+          #   - aggregated state - a state that represents a "snapshot" of things happening right
+          #     now. Right now is the moment of time on which we operate.
+          class State < Base
+            # Current schema version
+            # This can be used in the future for detecting incompatible changes and writing
+            # migrations
+            SCHEMA_VERSION = '1.1.0'
+
+            # @param schema_manager [Karafka::Web::Processing::Consumers::SchemaManager] schema
+            #   manager that tracks the compatibility of schemas.
+            def initialize(schema_manager)
+              super()
+              @schema_manager = schema_manager
+            end
+
+            # Uses provided process state report to update the current materialized state
+            # @param report [Hash] consumer process state report
+            # @param offset [Integer] offset of the message with the state report. This offset is
+            #   needed as we need to be able to get all the consumers reports from a given offset.
+            def add(report, offset)
+              super(report)
+              increment_total_counters(report)
+              update_process_state(report, offset)
+              # We always evict after counters updates because we want to use expired (stopped)
+              # data for counters as it was valid previously. This can happen only when web consumer
+              # had a lag and is catching up.
+              evict_expired_processes
+              # current means current in the context of processing window (usually now but in case
+              # of lag, this state may be from the past)
+              refresh_current_stats
+            end
+
+            # @return [Array<Hash, Float>] aggregated current stats value and time from which this
+            #   aggregation comes from
+            #
+            # @note We return a copy, because we use the internal one to track state changes and
+            #   unless we would return a copy, other aggregators could have this mutated in an
+            #   unexpected way
+            def stats
+              state.fetch(:stats).dup
+            end
+
+            # Sets the dispatch time and returns the hash that can be shipped to the states topic
+            #
+            # @param _args [Object] extra parsing arguments (not used)
+            # @return [Hash] Hash that we can use to ship states data to Kafka
+            def to_h(*_args)
+              state[:schema_version] = SCHEMA_VERSION
+              state[:dispatched_at] = float_now
+              state[:schema_state] = @schema_manager.to_s
+
+              state
+            end
+
+            private
+
+            # @return [Hash] hash with current state from Kafka
+            def state
+              @state ||= Consumers::State.current!
+            end
+
+            # Increments the total counters based on the provided report
+            # @param report [Hash]
+            def increment_total_counters(report)
+              report[:stats][:total].each do |key, value|
+                state[:stats][key] ||= 0
+                state[:stats][key] += value
+              end
+            end
+
+            # Registers or updates the given process state based on the report
+            #
+            # @param report [Hash]
+            # @param offset [Integer]
+            def update_process_state(report, offset)
+              process_name = report[:process][:name]
+
+              state[:processes][process_name] = {
+                dispatched_at: report[:dispatched_at],
+                offset: offset
+              }
+            end
+
+            # Evicts expired processes from the current state
+            # We consider processes dead if they do not report often enough
+            # @note We do not evict based on states (stopped), because we want to report the
+            #   stopped processes for extra time within the ttl limitations. This makes tracking of
+            #   things from UX perspective nicer.
+            def evict_expired_processes
+              max_ttl = @aggregated_from - ::Karafka::Web.config.ttl / 1_000
+
+              state[:processes].delete_if do |_name, details|
+                details[:dispatched_at] < max_ttl
+              end
+
+              @active_reports.delete_if do |_name, details|
+                details[:dispatched_at] < max_ttl
+              end
+            end
+
+            # Refreshes the counters that are computed based on incoming reports and not a total sum.
+            # For this we use active reports we have in memory. It may not be accurate for the first
+            # few seconds but it is much more optimal from performance perspective than computing
+            # this fetching all data from Kafka for each view.
+            def refresh_current_stats
+              stats = state[:stats]
+
+              stats[:busy] = 0
+              stats[:enqueued] = 0
+              stats[:workers] = 0
+              stats[:processes] = 0
+              stats[:rss] = 0
+              stats[:listeners] = 0
+              stats[:lag] = 0
+              stats[:lag_stored] = 0
+              utilization = 0
+
+              @active_reports
+                .values
+                .reject { |report| report[:process][:status] == 'stopped' }
+                .each do |report|
+                  report_stats = report[:stats]
+                  report_process = report[:process]
+
+                  lags = []
+                  lags_stored = []
+
+                  iterate_partitions(report) do |partition_stats|
+                    lags << partition_stats[:lag]
+                    lags_stored << partition_stats[:lag_stored]
+                  end
+
+                  stats[:busy] += report_stats[:busy]
+                  stats[:enqueued] += report_stats[:enqueued]
+                  stats[:workers] += report_process[:workers] || 0
+                  stats[:listeners] += report_process[:listeners] || 0
+                  stats[:processes] += 1
+                  stats[:rss] += report_process[:memory_usage]
+                  stats[:lag] += lags.reject(&:negative?).sum
+                  stats[:lag_stored] += lags_stored.reject(&:negative?).sum
+                  utilization += report_stats[:utilization]
+                end
+
+              stats[:utilization] = utilization / (stats[:processes] + 0.0001)
+            end
+
+            # @param report [Hash]
+            def iterate_partitions(report)
+              report[:consumer_groups].each_value do |consumer_group|
+                consumer_group[:subscription_groups].each_value do |subscription_group|
+                  subscription_group[:topics].each_value do |topic|
+                    topic[:partitions].each_value do |partition|
+                      yield(partition)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        # Consumer tracking related contracts
+        module Contracts
+          # Contract used to validate the stats that are both present in state and metrics
+          class AggregatedStats < Web::Contracts::Base
+            configure
+
+            required(:batches) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:messages) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:retries) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:dead) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:errors) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:busy) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:enqueued) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:workers) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:processes) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:rss) { |val| val.is_a?(Numeric) && val >= 0 }
+            required(:listeners) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:utilization) { |val| val.is_a?(Numeric) && val >= 0 }
+            required(:lag_stored) { |val| val.is_a?(Integer) }
+            required(:lag) { |val| val.is_a?(Integer) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/contracts/metrics.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        module Contracts
+          # Contract that describes the schema for metric reporting
+          class Metrics < Web::Contracts::Base
+            configure
+
+            required(:dispatched_at) { |val| val.is_a?(Numeric) && val.positive? }
+            required(:schema_version) { |val| val.is_a?(String) && !val.empty? }
+
+            # Ensure, that all aggregated metrics are as expected (values)
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              stats_contract = Contracts::AggregatedStats.new
+
+              data.fetch(:aggregated).each_value do |range_sample|
+                # Older metrics should have been validated previously so we need to check only
+                # the most recently materialized one
+                stats_contract.validate!(range_sample.last.last)
+              end
+
+              nil
+            end
+
+            # Ensure that all the consumer groups topics details are as expected
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              topic_contract = Contracts::TopicStats.new
+
+              data.fetch(:consumer_groups).each_value do |range_sample|
+                consumer_group = range_sample.last.last
+
+                consumer_group.each_value do |topics|
+                  topics.each_value do |topic_stats|
+                    topic_contract.validate!(topic_stats)
+                  end
+                end
+              end
+
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/contracts/process.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        module Contracts
+          # State process details contract
+          class Process < Web::Contracts::Base
+            configure
+
+            required(:dispatched_at) { |val| val.is_a?(Numeric) && val.positive? }
+            required(:offset) { |val| val.is_a?(Integer) && val >= 0 }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/contracts/state.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        # Consumer tracking related contracts
+        module Contracts
+          # Contract used to ensure the consistency of the data generated to the consumers states
+          # topic
+          class State < Web::Contracts::Base
+            configure
+
+            # Valid schema manager states
+            VALID_SCHEMA_STATES = %w[compatible incompatible].freeze
+
+            private_constant :VALID_SCHEMA_STATES
+
+            required(:schema_version) { |val| val.is_a?(String) && !val.empty? }
+            required(:dispatched_at) { |val| val.is_a?(Numeric) && val.positive? }
+            required(:stats) { |val| val.is_a?(Hash) }
+            required(:processes) { |val| val.is_a?(Hash) }
+            required(:schema_state) { |val| VALID_SCHEMA_STATES.include?(val) }
+
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              Contracts::AggregatedStats.new.validate!(data.fetch(:stats))
+
+              nil
+            end
+
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              process_contract = Contracts::Process.new
+
+              data.fetch(:processes).each_value do |details|
+                process_contract.validate!(details)
+              end
+
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/contracts/topic_stats.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        module Contracts
+          # Topic metrics checks
+          class TopicStats < Web::Contracts::Base
+            configure
+
+            required(:lag_stored) { |val| val.is_a?(Integer) }
+            required(:lag) { |val| val.is_a?(Integer) }
+            required(:pace) { |val| val.is_a?(Integer) }
+            required(:ls_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/metrics.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        # Fetches the current consumers historical metrics data
+        class Metrics
+          class << self
+            # Fetch the current metrics data that is expected to exist
+            #
+            # @return [Hash] latest (current) aggregated metrics state
+            def current!
+              metrics_message = ::Karafka::Admin.read_topic(
+                Karafka::Web.config.topics.consumers.metrics,
+                0,
+                1
+              ).last
+
+              return metrics_message.payload if metrics_message
+
+              raise(::Karafka::Web::Errors::Processing::MissingConsumersMetricsError)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/schema_manager.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        # Schema manager is responsible for making sure, that the consumers reports messages that
+        # we consume have a compatible schema with the current process that is suppose to
+        # materialize them.
+        #
+        # In general we always support at least one major version back and we recommend upgrades
+        # to previous versions (0.5 => 0.6 => 0.7)
+        #
+        # This is needed in scenarios where a rolling deploy would get new karafka processes
+        # reporting data but consumption would still run in older.
+        class SchemaManager
+          # Current reports version for comparing
+          CURRENT_VERSION = ::Gem::Version.new(
+            ::Karafka::Web::Tracking::Consumers::Sampler::SCHEMA_VERSION
+          )
+
+          private_constant :CURRENT_VERSION
+
+          def initialize
+            @cache = {}
+            @valid = true
+          end
+
+          # @param message [Karafka::Messages::Message] consumer report
+          # @return [Boolean] true if all good or false if incompatible
+          #
+          # @note The state switch is one-direction only. If we encounter an incompatible message
+          #   we need to stop processing so further checks even with valid should not switch it
+          #   back to valid
+          def compatible?(message)
+            schema_version = message.payload[:schema_version]
+
+            # Save on memory allocation by reusing
+            # Most of the time we will deal with compatible schemas, so it is not worth creating
+            # an object with each message
+            message_version = @cache[schema_version] ||= ::Gem::Version.new(schema_version)
+
+            return true if message_version <= CURRENT_VERSION
+
+            @valid = false
+          end
+
+          # @return [String] state that we can use in the materialized state for the UI reporting
+          def to_s
+            @valid ? 'compatible' : 'incompatible'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/state.rb

@@ -6,23 +6,20 @@ module Karafka
       module Consumers
         # Fetches the current consumer processes aggregated state
         class State
-          extend ::Karafka::Core::Helpers::Time
-
           class << self
-            # Try bootstrapping from the current state from Kafka if exists and if not, just use
-            # a blank state. Blank state will not be flushed because materialization into Kafka
-            # happens only after first report is received.
+            # Fetch the current consumers state that is expected to exist
             #
-            # @return [Hash, false] last (current) aggregated processes state or false if no
-            #   state is available
-            def current
+            # @return [Hash] last (current) aggregated processes state
+            def current!
               state_message = ::Karafka::Admin.read_topic(
                 Karafka::Web.config.topics.consumers.states,
                 0,
                 1
               ).last
 
-              state_message ? state_message.payload : { processes: {}, stats: {} }
+              return state_message.payload if state_message
+
+              raise(::Karafka::Web::Errors::Processing::MissingConsumersStateError)
             end
           end
         end

data/lib/karafka/web/processing/time_series_tracker.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      # Allows us to accumulate and track time series data with given resolution
+      #
+      # We aggregate for last:
+      #   - 7 days (every day)
+      #   - 24 hours (every hour)
+      #   - 1 hour (every minute) + the most recent as an update every time (leading)
+      #
+      # @note Please note we publish always **absolute** metrics and not deltas in reference to
+      #   a given time window. This needs to be computed in the frontend as we want to have
+      #   state facts in the storage.
+      #
+      # @note Please note we evict and cleanup data only before we want to use it. This will put
+      #   more stress on memory but makes this tracker 70-90% faster. Since by default we anyhow
+      #   sample every few seconds, this trade-off makes sense.
+      class TimeSeriesTracker
+        include ::Karafka::Core::Helpers::Time
+
+        # How many samples and in what resolution should we track for given time range
+        # @note We add one more than we want to display for delta computation when ranges
+        # are full in the UI
+        TIME_RANGES = {
+          # 7 days sampling
+          days: {
+            # Sample every 8 hours so we end up with 56 samples over a week + 1 for baseline
+            resolution: 8 * 60 * 60,
+            limit: 57
+          }.freeze,
+          # 24 hours sampling
+          hours: {
+            # Every 30 minutes for 24 hours + baseline
+            resolution: 30 * 60,
+            limit: 49
+          }.freeze,
+          # 60 minutes sampling
+          minutes: {
+            # Every one minute for an hour => 60 samples
+            resolution: 60,
+            limit: 61
+          }.freeze,
+          # 5 minutes sampling
+          seconds: {
+            # Every 5 seconds with 60 samples + baseline. That is 300 seconds => 5 minutes
+            resolution: 5,
+            limit: 61
+          }.freeze
+        }.freeze
+
+        # @param existing [Hash] existing historical metrics (may be empty for the first state)
+        def initialize(existing)
+          # Builds an empty structure for potential time ranges we are interested in
+          @historicals = TIME_RANGES.keys.map { |name| [name, []] }.to_h
+
+          # Fetch the existing (if any) historical values that we already have
+          import_existing(existing)
+        end
+
+        # Adds current state into the states for tracking
+        # @param current [Hash] hash with current state
+        # @param state_time [Float] float UTC time from which the state comes
+        def add(current, state_time)
+          # Inject the time point into all the historicals
+          inject(current, state_time)
+        end
+
+        # Evicts expired and duplicated series and returns the cleaned hash
+        # @return [Hash] aggregated historicals hash
+        def to_h
+          evict
+
+          @historicals
+        end
+
+        private
+
+        # Import existing previous historical metrics as they are
+        #
+        # @param existing [Hash] existing historical metrics
+        def import_existing(existing)
+          existing.each do |range_name, values|
+            @historicals[range_name] = values
+          end
+        end
+
+        # Injects the current most recent stats sample into each of the time ranges on which we
+        # operate. This allows us on all the charts to present the most recent value before a
+        # given time window is completed
+        #
+        # @param current [Hash] current stats
+        # @param state_time [Float] time from which this state comes
+        def inject(current, state_time)
+          @historicals.each_value do |points|
+            points << [state_time.floor, current]
+          end
+        end
+
+        # Removes historical metrics that are beyond our expected range, so we maintain a stable
+        # count and not overload the states topic with extensive data.
+        def evict
+          # Evict old metrics that are beyond our aggregated range
+          # Builds a sliding window that goes backwards
+          @historicals.each do |range_name, values|
+            rules = TIME_RANGES.fetch(range_name)
+            limit = rules.fetch(:limit)
+            resolution = rules.fetch(:resolution)
+
+            grouped = values.group_by { |sample| sample.first / resolution }
+            times = grouped.values.map(&:first)
+
+            # Inject the most recent to always have it in each reporting range
+            # Otherwise for a longer time ranges we would not have the most recent state
+            # available
+            times << values.last unless values.empty?
+
+            times.uniq!(&:first)
+
+            # Squash in case there would be two events from the same time
+            times.sort_by!(&:first)
+
+            @historicals[range_name] = times.last(limit)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/contracts/consumer_group.rb

@@ -8,7 +8,7 @@ module Karafka
         module Contracts
           # Expected data for each consumer group
           # It's mostly about subscription groups details
-          class ConsumerGroup < Tracking::Contracts::Base
+          class ConsumerGroup < Web::Contracts::Base
             configure
 
             required(:id) { |val| val.is_a?(String) && !val.empty? }
@@ -19,7 +19,7 @@ module Karafka
 
               subscription_group_contract = SubscriptionGroup.new
 
-              data.fetch(:subscription_groups).each do |_subscription_group_name, details|
+              data.fetch(:subscription_groups).each_value do |details|
                 subscription_group_contract.validate!(details)
               end
 

data/lib/karafka/web/tracking/consumers/contracts/job.rb

@@ -6,7 +6,7 @@ module Karafka
       module Consumers
         module Contracts
           # Contract for the job reporting details
-          class Job < Tracking::Contracts::Base
+          class Job < Web::Contracts::Base
             configure
 
             required(:consumer) { |val| val.is_a?(String) }
@@ -17,6 +17,7 @@ module Karafka
             required(:first_offset) { |val| val.is_a?(Integer) && (val >= 0 || val == -1001) }
             required(:last_offset) { |val| val.is_a?(Integer) && (val >= 0 || val == -1001) }
             required(:committed_offset) { |val| val.is_a?(Integer) }
+            required(:messages) { |val| val.is_a?(Integer) && val >= 0 }
             required(:type) { |val| %w[consume revoked shutdown].include?(val) }
             required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
             # -1 can be here for workless flows

data/lib/karafka/web/tracking/consumers/contracts/partition.rb

@@ -6,14 +6,27 @@ module Karafka
       module Consumers
         module Contracts
           # Partition metrics required for web to operate
-          class Partition < Tracking::Contracts::Base
+          class Partition < Web::Contracts::Base
             configure
 
             required(:id) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:lag) { |val| val.is_a?(Integer) }
+            required(:lag_d) { |val| val.is_a?(Integer) }
             required(:lag_stored) { |val| val.is_a?(Integer) }
             required(:lag_stored_d) { |val| val.is_a?(Integer) }
             required(:committed_offset) { |val| val.is_a?(Integer) }
+            required(:committed_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
             required(:stored_offset) { |val| val.is_a?(Integer) }
+            required(:stored_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:fetch_state) { |val| val.is_a?(String) && !val.empty? }
+            required(:poll_state) { |val| val.is_a?(String) && !val.empty? }
+            required(:hi_offset) { |val| val.is_a?(Integer) }
+            required(:hi_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:lo_offset) { |val| val.is_a?(Integer) }
+            required(:eof_offset) { |val| val.is_a?(Integer) }
+            required(:ls_offset) { |val| val.is_a?(Integer) }
+            required(:ls_offset_d) { |val| val.is_a?(Integer) }
+            required(:ls_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
           end
         end
       end