karafka-web 0.7.9 → 0.8.0.rc1

data/lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_states.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Since we have introduced notion of pause listeners, we need to reflect this in the
+        # UI, so the scaling changes are visible
+        class SplitListenersIntoActiveAndPausedInStates < Base
+          self.versions_until = '1.2.2'
+          self.type = :consumers_state
+
+          # @param state [Hash]
+          def migrate(state)
+            listeners = if state[:stats].key?(:listeners)
+                          state[:stats][:listeners].to_i
+                        elsif state[:stats].key?(:listeners_count)
+                          state[:stats][:listeners_count].to_i
+                        else
+                          0
+                        end
+
+            state[:stats][:listeners] = {
+              active: listeners,
+              standby: 0
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrator.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    # Namespace for all cross-context management operations that are needed to make sure everything
+    # operate as expected.
+    module Management
+      # Migrator used to run migrations on the states topics
+      # There are cases during upgrades, where extra fields may be added and other data, so in
+      # order not to deal with cases of some information missing, we can just migrate the data
+      # and ensure all the fields that we require after upgrade are present
+      #
+      # Migrations are similar to the once that are present in Ruby on Rails conceptually.
+      #
+      # We take our most recent state and we can alter it "in place". The altered result will be
+      # passed to the consecutive migrations and then republished back to Kafka. This allows us
+      # to manage Web UI aggregated data easily.
+      #
+      # @note We do not migrate the consumers reports for the following reasons:
+      #   - if would be extremely hard to migrate them as they are being published and can be still
+      #     published when the migrations are running
+      #   - we would have to run migrations on each message
+      #   - we already have a mechanism in the processing consumer that skips outdated records for
+      #     rolling migrations
+      #   - those records are short-lived and the expectation is for the user not to run old and
+      #     new consumers together for an extensive period of time
+      #
+      # @note It will raise an error if we try to run migrations but the schemas we want to operate
+      #   are newer. This will prevent us from damaging the data and ensures that we only move
+      #   forward with the migrations. This can happen in case of a rolling upgrade, where old
+      #   instance that is going to be terminated would get a temporary assignment with already
+      #   migrated state.
+      class Migrator
+        # Include this so we can reference the schema versions easily
+        include Processing::Consumers::Aggregators
+
+        # Picks needed data from Kafka, alters it with migrations and puts the updated data
+        # back into Kafka. This ensures, that our Web UI topics that hold aggregated data are
+        # always aligned with the Web UI expectations
+        #
+        # @note To simplify things we always migrate and update all the topics data even if only
+        #   part was migrated. That way we always ensure that all the elements are up to date
+        def call
+          ensure_migrable!
+          # If migrating returns `false` it means no migrations happened
+          migrate && publish
+        end
+
+        private
+
+        # Raise an exception if there would be an attempt to run migrations on a newer schema for
+        # any states we manage. We can only move forward, so attempt to migrate for example from
+        # 1.0.0 to 0.9.0 should be considered and error.
+        def ensure_migrable!
+          if consumers_state[:schema_version] > State::SCHEMA_VERSION
+            raise(
+              Errors::Management::IncompatibleSchemaError,
+              'consumers state newer than supported'
+            )
+          end
+
+          if consumers_metrics[:schema_version] > Metrics::SCHEMA_VERSION
+            raise(
+              Errors::Management::IncompatibleSchemaError,
+              'consumers metrics newer than supported'
+            )
+          end
+
+          true
+        end
+
+        # Applies migrations if needed and mutates the in-memory data
+        #
+        # @return [Boolean] were there any migrations applied
+        def migrate
+          any_migrations = false
+
+          Migrations::Base.sorted_descendants.each do |migration_class|
+            data = send(migration_class.type)
+
+            next unless migration_class.applicable?(data[:schema_version])
+
+            migration_class.new.migrate(data)
+
+            any_migrations = true
+          end
+
+          any_migrations
+        end
+
+        # Publishes all the states migrated records
+        def publish
+          consumers_state[:schema_version] = State::SCHEMA_VERSION
+          consumers_metrics[:schema_version] = Metrics::SCHEMA_VERSION
+
+          # Migrator may run in the context of the processing consumer prior to any states
+          # fetching related to processing. We use sync to make sure, that the following
+          # processing related states fetched fetch the new states
+          Processing::Publisher.publish!(
+            consumers_state,
+            consumers_metrics
+          )
+        end
+
+        # @return [Hash] current consumers states most recent state
+        def consumers_state
+          @consumers_state ||= Processing::Consumers::State.current!
+        end
+
+        # @return [Hash] current consumers metrics most recent state
+        def consumers_metrics
+          @consumers_metrics ||= Processing::Consumers::Metrics.current!
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumer.rb

@@ -11,35 +11,19 @@ module Karafka
       class Consumer < Karafka::BaseConsumer
         include ::Karafka::Core::Helpers::Time
 
-        # @param args [Object] all the arguments `Karafka::BaseConsumer` accepts by default
-        def initialize(*args)
-          super
-
-          @flush_interval = ::Karafka::Web.config.processing.interval
-
-          @schema_manager = Consumers::SchemaManager.new
-          @state_aggregator = Consumers::Aggregators::State.new(@schema_manager)
-          @state_contract = Consumers::Contracts::State.new
-
-          @metrics_aggregator = Consumers::Aggregators::Metrics.new
-          @metrics_contract = Consumers::Contracts::Metrics.new
-
-          # We set this that way so we report with first batch and so we report as fast as possible
-          @flushed_at = monotonic_now - @flush_interval
-          @established = false
-        end
-
         # Aggregates consumers state into a single current state representation
         def consume
+          bootstrap!
+
           consumers_messages = messages.select { |message| message.payload[:type] == 'consumer' }
 
           # If there is even one incompatible message, we need to stop
           consumers_messages.each do |message|
-            case @schema_manager.call(message)
+            case @reports_schema_manager.call(message)
             when :current
               true
             when :newer
-              @schema_manager.invalidate!
+              @reports_schema_manager.invalidate!
 
               dispatch
 
@@ -49,6 +33,9 @@ module Karafka
             # requests without significant or any impact on data quality but without having to
             # worry about backwards compatibility. Errors are tracked independently, so it should
             # not be a problem.
+            #
+            # In case user wants to do a rolling upgrade, the user docs state that this can happen
+            # and it is something user should be aware
             when :older
               next
             else
@@ -83,6 +70,35 @@ module Karafka
 
         private
 
+        # Prepares all the initial objects and ensures all the needed states are as expected
+        # @note We do not run it in the `#initialize` anymore as `#initialize` happens before
+        #   the work starts so errors there are handled differently. We want this initial setup
+        #   to operate and fail (if needed) during messages consumption phase
+        def bootstrap!
+          return if @bootstrapped
+
+          # Run the migrator on the assignment to make sure all our data is as expected
+          # While users may run the CLI command this is a fail-safe for zero downtime deployments
+          # It costs us two extra requests to Kafka topics as we migrate prior to fetching the
+          # states to the aggregators but this is done on purpose not to mix those two contexts.
+          Management::Migrator.new.call
+
+          @flush_interval = ::Karafka::Web.config.processing.interval
+
+          @reports_schema_manager = Consumers::SchemaManager.new
+          @state_aggregator = Consumers::Aggregators::State.new(@reports_schema_manager)
+          @state_contract = Consumers::Contracts::State.new
+
+          @metrics_aggregator = Consumers::Aggregators::Metrics.new
+          @metrics_contract = Consumers::Contracts::Metrics.new
+
+          # We set this that way so we report with first batch and so we report as fast as possible
+          @flushed_at = monotonic_now - @flush_interval
+          @established = false
+
+          @bootstrapped = true
+        end
+
         # Flushes the state of the Web-UI to the DB
         def dispatch
           return unless @established
@@ -114,24 +130,9 @@ module Karafka
         def flush
           @flushed_at = monotonic_now
 
-          ::Karafka::Web.producer.produce_many_async(
-            [
-              {
-                topic: Karafka::Web.config.topics.consumers.states,
-                payload: Zlib::Deflate.deflate(@state.to_json),
-                # This will ensure that the consumer states are compacted
-                key: Karafka::Web.config.topics.consumers.states,
-                partition: 0,
-                headers: { 'zlib' => 'true' }
-              },
-              {
-                topic: Karafka::Web.config.topics.consumers.metrics,
-                payload: Zlib::Deflate.deflate(@metrics.to_json),
-                key: Karafka::Web.config.topics.consumers.metrics,
-                partition: 0,
-                headers: { 'zlib' => 'true' }
-              }
-            ]
+          Publisher.publish(
+            @state,
+            @metrics
           )
         end
       end

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -10,9 +10,8 @@ module Karafka
           # values for charts and metrics
           class Metrics < Base
             # Current schema version
-            # This can be used in the future for detecting incompatible changes and writing
-            # migrations
-            SCHEMA_VERSION = '1.0.0'
+            # This is used for detecting incompatible changes and writing migrations
+            SCHEMA_VERSION = '1.1.2'
 
             def initialize
               super
@@ -107,9 +106,18 @@ module Karafka
 
                 # Last stable offsets freeze durations - we pick the max freeze to indicate
                 # the longest open transaction that potentially may be hanging
-                ls_offsets_fd = partitions_data
-                                .map { |p_details| p_details.fetch(:ls_offset_fd, 0) }
-                                .reject(&:negative?)
+                # We select only those partitions for which LSO != HO as in any other case this
+                # just means we've reached the end of data and ls may freeze because there is no
+                # more data flowing. Such cases should not be reported as ls offset freezes because
+                # there is no more data to be processed and can grow until more data is present
+                # this does not indicate "bad" freezing that we are interested in
+                ls_offsets_fds = partitions_data.map do |p_details|
+                  next if p_details.fetch(:ls_offset, 0) == p_details.fetch(:hi_offset, 0)
+
+                  ls_offset_fd = p_details.fetch(:ls_offset_fd, 0)
+
+                  ls_offset_fd.negative? ? nil : ls_offset_fd
+                end
 
                 cgs[group_name] ||= {}
                 cgs[group_name][topic_name] = {
@@ -119,7 +127,7 @@ module Karafka
                   # Take max last stable offset duration without any change. This can
                   # indicate a hanging transaction, because the offset will not move forward
                   # and will stay with a growing freeze duration when stuck
-                  ls_offset_fd: ls_offsets_fd.max || 0
+                  ls_offset_fd: ls_offsets_fds.compact.max || 0
                 }
               end
 

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -20,7 +20,7 @@ module Karafka
             # Current schema version
             # This can be used in the future for detecting incompatible changes and writing
             # migrations
-            SCHEMA_VERSION = '1.1.0'
+            SCHEMA_VERSION = '1.2.2'
 
             # @param schema_manager [Karafka::Web::Processing::Consumers::SchemaManager] schema
             #   manager that tracks the compatibility of schemas.
@@ -126,9 +126,11 @@ module Karafka
               stats[:workers] = 0
               stats[:processes] = 0
               stats[:rss] = 0
-              stats[:listeners] = 0
+              stats[:listeners] = { active: 0, standby: 0 }
               stats[:lag] = 0
               stats[:lag_stored] = 0
+              stats[:bytes_received] = 0
+              stats[:bytes_sent] = 0
               utilization = 0
 
               @active_reports
@@ -149,7 +151,10 @@ module Karafka
                   stats[:busy] += report_stats[:busy]
                   stats[:enqueued] += report_stats[:enqueued]
                   stats[:workers] += report_process[:workers] || 0
-                  stats[:listeners] += report_process[:listeners] || 0
+                  stats[:bytes_received] += report_process[:bytes_received] || 0
+                  stats[:bytes_sent] += report_process[:bytes_sent] || 0
+                  stats[:listeners][:active] += report_process[:listeners][:active]
+                  stats[:listeners][:standby] += report_process[:listeners][:standby]
                   stats[:processes] += 1
                   stats[:rss] += report_process[:memory_usage]
                   stats[:lag] += lags.compact.reject(&:negative?).sum

data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb

@@ -20,10 +20,14 @@ module Karafka
             required(:workers) { |val| val.is_a?(Integer) && val >= 0 }
             required(:processes) { |val| val.is_a?(Integer) && val >= 0 }
             required(:rss) { |val| val.is_a?(Numeric) && val >= 0 }
-            required(:listeners) { |val| val.is_a?(Integer) && val >= 0 }
             required(:utilization) { |val| val.is_a?(Numeric) && val >= 0 }
             required(:lag_stored) { |val| val.is_a?(Integer) }
             required(:lag) { |val| val.is_a?(Integer) }
+
+            nested(:listeners) do
+              required(:active) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:standby) { |val| val.is_a?(Integer) && val >= 0 }
+            end
           end
         end
       end

data/lib/karafka/web/processing/publisher.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      # Object responsible for publishing states data back into Kafka so it can be used in the UI
+      class Publisher
+        class << self
+          # Publishes data back to Kafka in an async fashion
+          #
+          # @param consumers_state [Hash] consumers current state
+          # @param consumers_metrics [Hash] consumers current metrics
+          def publish(consumers_state, consumers_metrics)
+            ::Karafka::Web.producer.produce_many_async(
+              prepare_data(consumers_state, consumers_metrics)
+            )
+          end
+
+          # Publishes data back to Kafka in a sync fashion
+          #
+          # @param consumers_state [Hash] consumers current state
+          # @param consumers_metrics [Hash] consumers current metrics
+          def publish!(consumers_state, consumers_metrics)
+            ::Karafka::Web.producer.produce_many_sync(
+              prepare_data(consumers_state, consumers_metrics)
+            )
+          end
+
+          private
+
+          # Converts the states into format that we can dispatch to Kafka
+          #
+          # @param consumers_state [Hash] consumers current state
+          # @param consumers_metrics [Hash] consumers current metrics
+          # @return [Array<Hash>]
+          def prepare_data(consumers_state, consumers_metrics)
+            [
+              {
+                topic: Karafka::Web.config.topics.consumers.states,
+                payload: Zlib::Deflate.deflate(consumers_state.to_json),
+                # This will ensure that the consumer states are compacted
+                key: Karafka::Web.config.topics.consumers.states,
+                partition: 0,
+                headers: { 'zlib' => 'true' }
+              },
+              {
+                topic: Karafka::Web.config.topics.consumers.metrics,
+                payload: Zlib::Deflate.deflate(consumers_metrics.to_json),
+                key: Karafka::Web.config.topics.consumers.metrics,
+                partition: 0,
+                headers: { 'zlib' => 'true' }
+              }
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/contracts/job.rb

@@ -11,18 +11,19 @@ module Karafka
 
             required(:consumer) { |val| val.is_a?(String) }
             required(:consumer_group) { |val| val.is_a?(String) }
-            required(:started_at) { |val| val.is_a?(Float) && val >= 0 }
+            required(:updated_at) { |val| val.is_a?(Float) && val >= 0 }
             required(:topic) { |val| val.is_a?(String) }
             required(:partition) { |val| val.is_a?(Integer) && val >= 0 }
             required(:first_offset) { |val| val.is_a?(Integer) && (val >= 0 || val == -1001) }
             required(:last_offset) { |val| val.is_a?(Integer) && (val >= 0 || val == -1001) }
             required(:committed_offset) { |val| val.is_a?(Integer) }
             required(:messages) { |val| val.is_a?(Integer) && val >= 0 }
-            required(:type) { |val| %w[consume revoked shutdown].include?(val) }
+            required(:type) { |val| %w[consume revoked shutdown tick].include?(val) }
             required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
             # -1 can be here for workless flows
             required(:consumption_lag) { |val| val.is_a?(Integer) && (val >= 0 || val == -1) }
             required(:processing_lag) { |val| val.is_a?(Integer) && (val >= 0 || val == -1) }
+            required(:status) { |val| %w[running pending].include?(val) }
           end
         end
       end

data/lib/karafka/web/tracking/consumers/contracts/partition.rb

@@ -20,6 +20,7 @@ module Karafka
             required(:stored_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
             required(:fetch_state) { |val| val.is_a?(String) && !val.empty? }
             required(:poll_state) { |val| val.is_a?(String) && !val.empty? }
+            required(:poll_state_ch) { |val| val.is_a?(Integer) && val >= 0 }
             required(:hi_offset) { |val| val.is_a?(Integer) }
             required(:hi_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
             required(:lo_offset) { |val| val.is_a?(Integer) }

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -27,10 +27,14 @@ module Karafka
               required(:memory_size) { |val| val.is_a?(Integer) && val >= 0 }
               required(:status) { |val| ::Karafka::Status::STATES.key?(val.to_s.to_sym) }
               required(:threads) { |val| val.is_a?(Integer) && val >= 0 }
-              required(:listeners) { |val| val.is_a?(Integer) && val >= 0 }
               required(:workers) { |val| val.is_a?(Integer) && val.positive? }
               required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
 
+              nested(:listeners) do
+                required(:active) { |val| val.is_a?(Integer) && val >= 0 }
+                required(:standby) { |val| val.is_a?(Integer) && val >= 0 }
+              end
+
               required(:cpu_usage) do |val|
                 val.is_a?(Array) &&
                   val.all? { |key| key.is_a?(Numeric) } &&
@@ -52,6 +56,7 @@ module Karafka
             nested(:stats) do
               required(:busy) { |val| val.is_a?(Integer) && val >= 0 }
               required(:enqueued) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:waiting) { |val| val.is_a?(Integer) && val >= 0 }
               required(:utilization) { |val| val.is_a?(Numeric) && val >= 0 }
 
               nested(:total) do

data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb

@@ -12,7 +12,16 @@ module Karafka
 
             required(:id) { |val| val.is_a?(String) && !val.empty? }
             required(:topics) { |val| val.is_a?(Hash) }
-            required(:state) { |val| val.is_a?(Hash) }
+
+            nested(:state) do
+              required(:state) { |val| val.is_a?(String) && !val.empty? }
+              required(:join_state) { |val| val.is_a?(String) && !val.empty? }
+              required(:stateage) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:rebalance_age) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:rebalance_cnt) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:rebalance_reason) { |val| val.is_a?(String) && !val.empty? }
+              required(:poll_age) { |val| val.is_a?(Numeric) && val >= 0 }
+            end
 
             virtual do |data, errors|
               next unless errors.empty?

data/lib/karafka/web/tracking/consumers/listeners/connections.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Listener for listening on connections related events like polling, etc
+          class Connections < Base
+            # Set first poll time before we start fetching so we always have a poll time
+            # and we don't have to worry about it being always available
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_connection_listener_before_fetch_loop(event)
+              on_connection_listener_fetch_loop_received(event)
+            end
+
+            # When fetch loop is done it means this subscription group is no longer active and we
+            # should stop reporting. The listener was stopped.
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_connection_listener_after_fetch_loop(event)
+              subscription_group = event[:subscription_group]
+              sg_id = subscription_group.id
+              cg_id = subscription_group.consumer_group.id
+
+              track do |sampler|
+                sampler.consumer_groups[cg_id][:subscription_groups].delete(sg_id)
+                sampler.subscription_groups.delete(sg_id)
+              end
+            end
+
+            # Tracks the moment a poll happened on a given subscription group
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_connection_listener_fetch_loop_received(event)
+              sg_id = event[:subscription_group].id
+
+              track do |sampler|
+                sampler.subscription_groups[sg_id] = {
+                  polled_at: monotonic_now
+                }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/listeners/pausing.rb

@@ -11,9 +11,12 @@ module Karafka
             # Indicate pause
             #
             # @param event [Karafka::Core::Monitoring::Event]
-            def on_client_pause(event)
+            def on_consumer_consuming_pause(event)
               track do |sampler|
-                sampler.pauses << pause_id(event)
+                sampler.pauses[pause_id(event)] = {
+                  timeout: event[:timeout],
+                  paused_till: monotonic_now + event[:timeout]
+                }
               end
             end
 
@@ -33,9 +36,9 @@ module Karafka
             def pause_id(event)
               topic = event[:topic]
               partition = event[:partition]
-              consumer_group_id = event[:subscription_group].consumer_group.id
+              subscription_group_id = event[:subscription_group].id
 
-              [consumer_group_id, topic, partition].join('-')
+              [subscription_group_id, topic, partition].join('-')
             end
           end
         end