karafka-web 0.6.3 → 0.7.0

data/lib/karafka/web/management/create_topics.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      # Creates all the needed topics (if they don't exist).
+      # It does **not** populate data.
+      class CreateTopics < Base
+        # Runs the creation process
+        #
+        # @param replication_factor [Integer] replication factor for Web-UI topics
+        def call(replication_factor)
+          consumers_states_topic = ::Karafka::Web.config.topics.consumers.states
+          consumers_metrics_topic = ::Karafka::Web.config.topics.consumers.metrics
+          consumers_reports_topic = ::Karafka::Web.config.topics.consumers.reports
+          errors_topic = ::Karafka::Web.config.topics.errors
+
+          # Create only if needed
+          if existing_topics_names.include?(consumers_states_topic)
+            exists(consumers_states_topic)
+          else
+            creating(consumers_states_topic)
+            # This topic needs to have one partition
+            ::Karafka::Admin.create_topic(
+              consumers_states_topic,
+              1,
+              replication_factor,
+              # We care only about the most recent state, previous are irrelevant. So we can easily
+              # compact after one minute. We do not use this beyond the most recent collective
+              # state, hence it all can easily go away. We also limit the segment size to at most
+              # 100MB not to use more space ever.
+              {
+                'cleanup.policy': 'compact',
+                'retention.ms': 60 * 60 * 1_000,
+                'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
+                'segment.bytes': 104_857_600 # 100MB
+              }
+            )
+            created(consumers_states_topic)
+          end
+
+          if existing_topics_names.include?(consumers_metrics_topic)
+            exists(consumers_metrics_topic)
+          else
+            creating(consumers_metrics_topic)
+            # This topic needs to have one partition
+            # Same as states - only most recent is relevant as it is a materialized state
+            ::Karafka::Admin.create_topic(
+              consumers_metrics_topic,
+              1,
+              replication_factor,
+              {
+                'cleanup.policy': 'compact',
+                'retention.ms': 60 * 60 * 1_000, # 1h
+                'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
+                'segment.bytes': 104_857_600 # 100MB
+              }
+            )
+            created(consumers_metrics_topic)
+          end
+
+          if existing_topics_names.include?(consumers_reports_topic)
+            exists(consumers_reports_topic)
+          else
+            creating(consumers_reports_topic)
+            # This topic needs to have one partition
+            ::Karafka::Admin.create_topic(
+              consumers_reports_topic,
+              1,
+              replication_factor,
+              # We do not need to to store this data for longer than 1 day as this data is only
+              # used to materialize the end states
+              # On the other hand we do not want to have it really short-living because in case of
+              # a consumer crash, we may want to use this info to catch up and backfill the state.
+              # In case its not consumed because no processes are running, it also usually means
+              # there's no data to consume because no karafka servers report
+              {
+                'retention.ms': 24 * 60 * 60 * 1_000 # 1 day
+              }
+            )
+            created(consumers_reports_topic)
+          end
+
+          if existing_topics_names.include?(errors_topic)
+            exists(errors_topic)
+          else
+            creating(errors_topic)
+            # All the errors will be dispatched here
+            # This topic can have multiple partitions but we go with one by default. A single Ruby
+            # process should not crash that often and if there is an expectation of a higher volume
+            # of errors, this can be changed by the end user
+            ::Karafka::Admin.create_topic(
+              errors_topic,
+              1,
+              replication_factor,
+              # Remove really old errors (older than 3 months just to preserve space)
+              {
+                'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
+              }
+            )
+            created(errors_topic)
+          end
+        end
+
+        private
+
+        # @param topic_name [String] name of the topic that exists
+        # @return [String] formatted message
+        def exists(topic_name)
+          puts("Topic #{topic_name} #{already} exists.")
+        end
+
+        # @param topic_name [String] name of the topic that we are creating
+        # @return [String] formatted message
+        def creating(topic_name)
+          puts("Creating topic #{topic_name}...")
+        end
+
+        # @param topic_name [String] name of the topic that we created
+        # @return [String] formatted message
+        def created(topic_name)
+          puts("Topic #{topic_name} #{successfully} created.")
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/delete_topics.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      # Removes the Web-UI topics from Kafka
+      class DeleteTopics < Base
+        # Removes the Web-UI topics
+        def call
+          [
+            ::Karafka::Web.config.topics.consumers.states,
+            ::Karafka::Web.config.topics.consumers.reports,
+            ::Karafka::Web.config.topics.consumers.metrics,
+            ::Karafka::Web.config.topics.errors
+          ].each do |topic_name|
+            if existing_topics_names.include?(topic_name.to_s)
+              puts "Removing #{topic_name}..."
+              ::Karafka::Admin.delete_topic(topic_name)
+              puts "Topic #{topic_name} #{successfully} deleted."
+            else
+              puts "Topic #{topic_name} not found."
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/enable.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      # @note This runs on each process start that has `karafka.rb`. It needs to be executed
+      #   also in the context of other processes types and not only karafka server, because it
+      #   installs producers instrumentation and routing as well.
+      class Enable < Base
+        # Enables routing consumer group and subscribes Web-UI listeners
+        def call
+          extend_routing
+          subscribe_to_monitor
+        end
+
+        private
+
+        # Enables all the needed routes
+        def extend_routing
+          ::Karafka::App.routes.draw do
+            web_deserializer = ::Karafka::Web::Deserializer.new
+
+            consumer_group ::Karafka::Web.config.processing.consumer_group do
+              # Topic we listen on to materialize the states
+              topic ::Karafka::Web.config.topics.consumers.reports do
+                config(active: false)
+                active ::Karafka::Web.config.processing.active
+                # Since we materialize state in intervals, we can poll for half of this time without
+                # impacting the reporting responsiveness
+                max_wait_time ::Karafka::Web.config.processing.interval / 2
+                max_messages 1_000
+                consumer ::Karafka::Web::Processing::Consumer
+                # This needs to be true in order not to reload the consumer in dev. This consumer
+                # should not be affected by the end user development process
+                consumer_persistence true
+                deserializer web_deserializer
+                manual_offset_management true
+                # Start from the most recent data, do not materialize historical states
+                # This prevents us from dealing with cases, where client id would be changed and
+                # consumer group name would be renamed and we would start consuming all historical
+                initial_offset 'latest'
+              end
+
+              # We define those three here without consumption, so Web understands how to deserialize
+              # them when used / viewed
+              topic ::Karafka::Web.config.topics.consumers.states do
+                config(active: false)
+                active false
+                deserializer web_deserializer
+              end
+
+              topic ::Karafka::Web.config.topics.consumers.metrics do
+                config(active: false)
+                active false
+                deserializer web_deserializer
+              end
+
+              topic ::Karafka::Web.config.topics.errors do
+                config(active: false)
+                active false
+                deserializer web_deserializer
+              end
+            end
+          end
+        end
+
+        # Subscribes with all needed listeners
+        def subscribe_to_monitor
+          # Installs all the consumer related listeners
+          ::Karafka::Web.config.tracking.consumers.listeners.each do |listener|
+            ::Karafka.monitor.subscribe(listener)
+          end
+
+          # Installs all the producer related listeners
+          ::Karafka::Web.config.tracking.producers.listeners.each do |listener|
+            ::Karafka.producer.monitor.subscribe(listener)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/extend_boot_file.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      # Extends the boot file with Web components
+      class ExtendBootFile < Base
+        # Code that is needed in the `karafka.rb` to connect Web UI to Karafka
+        ENABLER_CODE = 'Karafka::Web.enable!'
+
+        # Template with initial Web UI configuration
+        # Session secret needs to be set per user and per env
+        SETUP_TEMPLATE = <<~CONFIG.freeze
+          Karafka::Web.setup do |config|
+            # You may want to set it per ENV. This value was randomly generated.
+            config.ui.sessions.secret = '#{SecureRandom.hex(32)}'
+          end
+
+          #{ENABLER_CODE}
+        CONFIG
+
+        # Adds needed code
+        def call
+          if File.read(Karafka.boot_file).include?(ENABLER_CODE)
+            puts "Web UI #{already} installed."
+          else
+            puts 'Updating the Karafka boot file...'
+            File.open(Karafka.boot_file, 'a') do |f|
+              f << "\n#{SETUP_TEMPLATE}\n"
+            end
+            puts "Karafka boot file #{successfully} updated."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumer.rb

@@ -15,50 +15,106 @@ module Karafka
         def initialize(*args)
           super
 
-          @flush_interval = ::Karafka::Web.config.processing.interval / 1_000
-          @consumers_aggregator = ::Karafka::Web.config.processing.consumers.aggregator
-          # We set this that way so we report with first batch and so we report in the development
-          # mode. In the development mode, there is a new instance per each invocation, thus we need
-          # to always initially report, so the web UI works well in the dev mode where consumer
-          # instances are not long-living.
+          @flush_interval = ::Karafka::Web.config.processing.interval
+
+          @schema_manager = Consumers::SchemaManager.new
+          @state_aggregator = Consumers::Aggregators::State.new(@schema_manager)
+          @state_contract = Consumers::Contracts::State.new
+
+          @metrics_aggregator = Consumers::Aggregators::Metrics.new
+          @metrics_contract = Consumers::Contracts::Metrics.new
+
+          # We set this that way so we report with first batch and so we report as fast as possible
           @flushed_at = monotonic_now - @flush_interval
         end
 
         # Aggregates consumers state into a single current state representation
         def consume
-          messages
-            .select { |message| message.payload[:type] == 'consumer' }
-            .each { |message| @consumers_aggregator.add(message.payload, message.offset) }
+          consumers_messages = messages.select { |message| message.payload[:type] == 'consumer' }
+
+          # If there is even one incompatible message, we need to stop
+          consumers_messages.each do |message|
+            unless @schema_manager.compatible?(message)
+              dispatch
+
+              raise ::Karafka::Web::Errors::Processing::IncompatibleSchemaError
+            end
+
+            # We need to run the aggregations on each message in order to compensate for
+            # potential lags.
+            @state_aggregator.add(message.payload, message.offset)
+            @metrics_aggregator.add_report(message.payload)
+            @metrics_aggregator.add_stats(@state_aggregator.stats)
+
+            # Optimize memory usage in pro
+            message.clean! if Karafka.pro?
+          end
 
           return unless periodic_flush?
 
-          flush
+          dispatch
 
           mark_as_consumed(messages.last)
         end
 
         # Flush final state on shutdown
         def shutdown
-          flush if @consumers_aggregator
+          return unless @state_aggregator
+
+          materialize
+          validate!
+          flush
         end
 
         private
 
+        # Flushes the state of the Web-UI to the DB
+        def dispatch
+          materialize
+          validate!
+          flush
+        end
+
         # @return [Boolean] is it time to persist the new current state
         def periodic_flush?
           (monotonic_now - @flushed_at) > @flush_interval
         end
 
+        # Materializes the current state and metrics for flushing
+        def materialize
+          @state = @state_aggregator.to_h
+          @metrics = @metrics_aggregator.to_h
+        end
+
+        # Ensures that the aggregated data complies with our schema expectation.
+        # If you ever get to this place, this is probably a bug and you should report it.
+        def validate!
+          @state_contract.validate!(@state)
+          @metrics_contract.validate!(@metrics)
+        end
+
         # Persists the new current state by flushing it to Kafka
         def flush
           @flushed_at = monotonic_now
 
-          producer.produce_async(
-            topic: Karafka::Web.config.topics.consumers.states,
-            payload: @consumers_aggregator.to_json,
-            # This will ensure that the consumer states are compacted
-            key: Karafka::Web.config.topics.consumers.states,
-            partition: 0
+          producer.produce_many_async(
+            [
+              {
+                topic: Karafka::Web.config.topics.consumers.states,
+                payload: Zlib::Deflate.deflate(@state.to_json),
+                # This will ensure that the consumer states are compacted
+                key: Karafka::Web.config.topics.consumers.states,
+                partition: 0,
+                headers: { 'zlib' => 'true' }
+              },
+              {
+                topic: Karafka::Web.config.topics.consumers.metrics,
+                payload: Zlib::Deflate.deflate(@metrics.to_json),
+                key: Karafka::Web.config.topics.consumers.metrics,
+                partition: 0,
+                headers: { 'zlib' => 'true' }
+              }
+            ]
           )
         end
       end

data/lib/karafka/web/processing/consumers/aggregators/base.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        # Namespace for data aggregators that track changes based on the incoming reports and
+        # aggregate metrics over time
+        module Aggregators
+          # Base for all the consumer related aggregators that operate on processes reports
+          #
+          # @note It is important to understand, that we operate here on a moment in time and this
+          #   moment may not mean "current" now. There might have been a lag and we may be catching
+          #   up on older states. This is why we use `@aggregated_from` time instead of the real
+          #   now. In case of a lag, we want to aggregate and catch up with data, without
+          #   assigning it to the time of processing but aligning it with the time from which the
+          #   given reports came. This allows us to compensate for the potential lag related to
+          #   rebalances, downtimes, failures, etc.
+          class Base
+            include ::Karafka::Core::Helpers::Time
+
+            def initialize
+              @active_reports = {}
+            end
+
+            # Adds report to the internal active reports hash and updates the aggregation time
+            # for internal time reference usage
+            # @param report [Hash] incoming process state report
+            def add(report)
+              memoize_process_report(report)
+              update_aggregated_from
+            end
+
+            private
+
+            # Updates the report for given process in memory
+            # @param report [Hash]
+            def memoize_process_report(report)
+              @active_reports[report[:process][:name]] = report
+            end
+
+            # Updates the time of the aggregation
+            #
+            # @return [Float] time of the aggregation
+            #
+            # @note Since this runs before eviction because of age, we always assume there is at
+            #   least one report from which we can take the dispatch time
+            def update_aggregated_from
+              @aggregated_from = @active_reports.values.map { |report| report[:dispatched_at] }.max
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Processing
+      module Consumers
+        module Aggregators
+          # Aggregates metrics for metrics topic. Tracks consumers data and converts it into a
+          # state that can then be used to enrich previous time based states to get a time-series
+          # values for charts and metrics
+          class Metrics < Base
+            # Current schema version
+            # This can be used in the future for detecting incompatible changes and writing
+            # migrations
+            SCHEMA_VERSION = '1.0.0'
+
+            def initialize
+              super
+              @aggregated_tracker = TimeSeriesTracker.new(metrics.fetch(:aggregated))
+              @consumer_groups_tracker = TimeSeriesTracker.new(metrics.fetch(:consumer_groups))
+            end
+
+            # Adds the current report to active reports and removes old once
+            #
+            # @param report [Hash] single process full report
+            def add_report(report)
+              add(report)
+              evict_expired_processes
+              add_consumers_groups_metrics
+            end
+
+            # Updates the aggregated stats metrics
+            #
+            # @param stats [Hash] aggregated statistics
+            def add_stats(stats)
+              metrics[:aggregated] = @aggregated_tracker.add(
+                stats,
+                @aggregated_from
+              )
+            end
+
+            # Converts our current knowledge into a report hash.
+            #
+            # @return [Hash] Statistics hash
+            #
+            # @note We materialize the consumers groups time series only here and not in real time,
+            #   because we materialize it based on the tracked active collective state. Materializing
+            #   on each update that would not be dispatched would be pointless.
+            def to_h
+              metrics[:schema_version] = SCHEMA_VERSION
+              metrics[:dispatched_at] = float_now
+              metrics[:aggregated] = @aggregated_tracker.to_h
+              metrics[:consumer_groups] = @consumer_groups_tracker.to_h
+
+              metrics
+            end
+
+            private
+
+            # @return [Hash] the initial metric taken from Kafka
+            def metrics
+              @metrics ||= Consumers::Metrics.current!
+            end
+
+            # Evicts outdated reports.
+            #
+            # @note This eviction differs from the one that we have for the states. For states we
+            #   do not evict stopped because we want to report them for a moment. Here we do not
+            #   care about what a stopped process was doing and we can also remove it from active
+            #   reports.
+            def evict_expired_processes
+              max_ttl = @aggregated_from - ::Karafka::Web.config.ttl / 1_000
+
+              @active_reports.delete_if do |_name, report|
+                report[:dispatched_at] < max_ttl || report[:process][:status] == 'stopped'
+              end
+            end
+
+            # Materialize and add consumers groups states into the tracker
+            def add_consumers_groups_metrics
+              @consumer_groups_tracker.add(
+                materialize_consumers_groups_current_state,
+                @aggregated_from
+              )
+            end
+
+            # Materializes the current state of consumers group data
+            #
+            # At the moment we report only topics lags but the format we are using supports
+            # extending this information in the future if it would be needed.
+            #
+            # @return [Hash] hash with nested consumers and their topics details structure
+            # @note We do **not** report on a per partition basis because it would significantly
+            #   increase needed storage.
+            def materialize_consumers_groups_current_state
+              cgs = {}
+
+              @active_reports.each do |_, details|
+                details.fetch(:consumer_groups).each do |group_name, group_details|
+                  group_details.fetch(:subscription_groups).each do |_sg_name, sg_details|
+                    sg_details.fetch(:topics).each do |topic_name, topic_details|
+                      partitions_data = topic_details.fetch(:partitions).values
+
+                      lags = partitions_data
+                             .map { |p_details| p_details[:lag] || 0 }
+                             .reject(&:negative?)
+
+                      lags_stored = partitions_data
+                                    .map { |p_details| p_details.fetch(:lag_stored) }
+                                    .reject(&:negative?)
+
+                      offsets_hi = partitions_data
+                                   .map { |p_details| p_details.fetch(:hi_offset) }
+                                   .reject(&:negative?)
+
+                      # Last stable offsets freeze durations - we pick the max freeze to indicate
+                      # the longest open transaction that potentially may be hanging
+                      ls_offsets_fd = partitions_data
+                                      .map { |p_details| p_details.fetch(:ls_offset_fd) }
+                                      .reject(&:negative?)
+
+                      # If there is no lag that would not be negative, it means we did not mark
+                      # any messages as consumed on this topic in any partitions, hence we cannot
+                      # compute lag easily
+                      # We do not want to initialize any data for this topic, when there is nothing
+                      # useful we could present
+                      #
+                      # In theory lag stored must mean that lag must exist but just to be sure we
+                      # check both here
+                      next if lags.empty? || lags_stored.empty?
+
+                      cgs[group_name] ||= {}
+                      cgs[group_name][topic_name] = {
+                        lag_stored: lags_stored.sum,
+                        lag: lags.sum,
+                        pace: offsets_hi.sum,
+                        # Take max last stable offset duration without any change. This can
+                        # indicate a hanging transaction, because the offset will not move forward
+                        # and will stay with a growing freeze duration when stuck
+                        ls_offset_fd: ls_offsets_fd.max
+                      }
+                    end
+                  end
+                end
+              end
+
+              cgs
+            end
+          end
+        end
+      end
+    end
+  end
+end