karafka-web 0.7.10 → 0.8.0.rc1

data/lib/karafka/web/management/actions/create_topics.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Actions
+        # Creates all the needed topics (if they don't exist).
+        # It does **not** populate data.
+        class CreateTopics < Base
+          # Runs the creation process
+          #
+          # @param replication_factor [Integer] replication factor for Web-UI topics
+          #
+          # @note The order of creation of those topics is important. In order to support the
+          #   zero-downtime bootstrap, we use the presence of the states topic and its initial
+          #   state existence as an indicator that the setup went as expected. It the consumers
+          #   states topic exists and contains needed data, it means all went as expected and that
+          #   topics created before it also exist (as no error).
+          def call(replication_factor)
+            consumers_states_topic = ::Karafka::Web.config.topics.consumers.states
+            consumers_metrics_topic = ::Karafka::Web.config.topics.consumers.metrics
+            consumers_reports_topic = ::Karafka::Web.config.topics.consumers.reports
+            errors_topic = ::Karafka::Web.config.topics.errors
+
+            if existing_topics_names.include?(errors_topic)
+              exists(errors_topic)
+            else
+              creating(errors_topic)
+              # All the errors will be dispatched here
+              # This topic can have multiple partitions but we go with one by default. A single
+              # Ruby process should not crash that often and if there is an expectation of a higher
+              # volume of errors, this can be changed by the end user
+              ::Karafka::Admin.create_topic(
+                errors_topic,
+                1,
+                replication_factor,
+                # Remove really old errors (older than 3 months just to preserve space)
+                {
+                  'cleanup.policy': 'delete',
+                  'retention.ms': 3 * 31 * 24 * 60 * 60 * 1_000 # 3 months
+                }
+              )
+              created(errors_topic)
+            end
+
+            if existing_topics_names.include?(consumers_reports_topic)
+              exists(consumers_reports_topic)
+            else
+              creating(consumers_reports_topic)
+              # This topic needs to have one partition
+              ::Karafka::Admin.create_topic(
+                consumers_reports_topic,
+                1,
+                replication_factor,
+                # We do not need to to store this data for longer than 1 day as this data is only
+                # used to materialize the end states
+                # On the other hand we do not want to have it really short-living because in case
+                # of a consumer crash, we may want to use this info to catch up and backfill the
+                # state.
+                #
+                # In case its not consumed because no processes are running, it also usually means
+                # there's no data to consume because no karafka servers report
+                {
+                  'cleanup.policy': 'delete',
+                  'retention.ms': 24 * 60 * 60 * 1_000 # 1 day
+                }
+              )
+              created(consumers_reports_topic)
+            end
+
+            if existing_topics_names.include?(consumers_metrics_topic)
+              exists(consumers_metrics_topic)
+            else
+              creating(consumers_metrics_topic)
+              # This topic needs to have one partition
+              # Same as states - only most recent is relevant as it is a materialized state
+              ::Karafka::Admin.create_topic(
+                consumers_metrics_topic,
+                1,
+                replication_factor,
+                {
+                  'cleanup.policy': 'compact',
+                  'retention.ms': 60 * 60 * 1_000, # 1h
+                  'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
+                  'segment.bytes': 104_857_600 # 100MB
+                }
+              )
+              created(consumers_metrics_topic)
+            end
+
+            # Create only if needed
+            if existing_topics_names.include?(consumers_states_topic)
+              exists(consumers_states_topic)
+            else
+              creating(consumers_states_topic)
+              # This topic needs to have one partition
+              ::Karafka::Admin.create_topic(
+                consumers_states_topic,
+                1,
+                replication_factor,
+                # We care only about the most recent state, previous are irrelevant. So we can
+                # easily compact after one minute. We do not use this beyond the most recent
+                # collective state, hence it all can easily go away. We also limit the segment
+                # size to at most 100MB not to use more space ever.
+                {
+                  'cleanup.policy': 'compact',
+                  'retention.ms': 60 * 60 * 1_000,
+                  'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
+                  'segment.bytes': 104_857_600 # 100MB
+                }
+              )
+              created(consumers_states_topic)
+            end
+          end
+
+          private
+
+          # @param topic_name [String] name of the topic that exists
+          # @return [String] formatted message
+          def exists(topic_name)
+            puts("Topic #{topic_name} #{already} exists.")
+          end
+
+          # @param topic_name [String] name of the topic that we are creating
+          # @return [String] formatted message
+          def creating(topic_name)
+            puts("Creating topic #{topic_name}...")
+          end
+
+          # @param topic_name [String] name of the topic that we created
+          # @return [String] formatted message
+          def created(topic_name)
+            puts("Topic #{topic_name} #{successfully} created.")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/actions/delete_topics.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Actions
+        # Removes the Web-UI topics from Kafka
+        class DeleteTopics < Base
+          # Removes the Web-UI topics
+          def call
+            [
+              ::Karafka::Web.config.topics.consumers.states,
+              ::Karafka::Web.config.topics.consumers.reports,
+              ::Karafka::Web.config.topics.consumers.metrics,
+              ::Karafka::Web.config.topics.errors
+            ].each do |topic_name|
+              if existing_topics_names.include?(topic_name.to_s)
+                puts "Removing #{topic_name}..."
+                ::Karafka::Admin.delete_topic(topic_name)
+                puts "Topic #{topic_name} #{successfully} deleted."
+              else
+                puts "Topic #{topic_name} not found."
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/actions/enable.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Actions
+        # @note This runs on each process start that has `karafka.rb`. It needs to be executed
+        #   also in the context of other processes types and not only karafka server, because it
+        #   installs producers instrumentation and routing as well.
+        class Enable < Base
+          # Enables routing consumer group and subscribes Web-UI listeners
+          def call
+            extend_routing
+            setup_tracking_activity
+
+            # Do not subscribe monitors or do anything else if tracking is disabled
+            return unless ::Karafka::Web.config.tracking.active
+
+            subscribe_to_monitor
+            subscribe_to_close_web_producer
+          end
+
+          private
+
+          # Enables tracking if it was not explicitly disabled by the user
+          def setup_tracking_activity
+            return unless ::Karafka::Web.config.tracking.active.nil?
+
+            ::Karafka::Web.config.tracking.active = true
+          end
+
+          # Enables all the needed routes
+          def extend_routing
+            ::Karafka::App.routes.draw do
+              web_deserializer = ::Karafka::Web::Deserializer.new
+
+              consumer_group ::Karafka::Web.config.processing.consumer_group do
+                # Topic we listen on to materialize the states
+                topic ::Karafka::Web.config.topics.consumers.reports do
+                  config(active: false)
+                  active ::Karafka::Web.config.processing.active
+                  # Since we materialize state in intervals, we can poll for half of this time
+                  # without impacting the reporting responsiveness
+                  max_wait_time ::Karafka::Web.config.processing.interval / 2
+                  max_messages 1_000
+                  consumer ::Karafka::Web::Processing::Consumer
+                  # This needs to be true in order not to reload the consumer in dev. This consumer
+                  # should not be affected by the end user development process
+                  consumer_persistence true
+                  deserializer web_deserializer
+                  manual_offset_management true
+                  # Start from the most recent data, do not materialize historical states
+                  # This prevents us from dealing with cases, where client id would be changed and
+                  # consumer group name would be renamed and we would start consuming all
+                  # historical
+                  initial_offset 'latest'
+                end
+
+                # We define those three here without consumption, so Web understands how to
+                # deserialize them when used / viewed
+                topic ::Karafka::Web.config.topics.consumers.states do
+                  config(active: false)
+                  active false
+                  deserializer web_deserializer
+                end
+
+                topic ::Karafka::Web.config.topics.consumers.metrics do
+                  config(active: false)
+                  active false
+                  deserializer web_deserializer
+                end
+
+                topic ::Karafka::Web.config.topics.errors do
+                  config(active: false)
+                  active false
+                  deserializer web_deserializer
+                end
+              end
+            end
+          end
+
+          # Subscribes with all needed listeners
+          def subscribe_to_monitor
+            # Installs all the consumer related listeners
+            ::Karafka::Web.config.tracking.consumers.listeners.each do |listener|
+              ::Karafka.monitor.subscribe(listener)
+            end
+
+            # Installs all the producer related listeners into Karafka default listener and
+            # into Karafka::Web listener in case it would be different than the Karafka one
+            ::Karafka::Web.config.tracking.producers.listeners.each do |listener|
+              ::Karafka.producer.monitor.subscribe(listener)
+
+              # Do not instrument twice in case only one default producer is used
+              next if ::Karafka.producer == ::Karafka::Web.producer
+
+              ::Karafka::Web.producer.monitor.subscribe(listener)
+            end
+          end
+
+          # In most cases we want to close the producer if possible.
+          # While we cannot do it easily in user processes and we should rely on WaterDrop
+          # finalization logic, we can do it in `karafka server` on terminate
+          #
+          # In other places, this producer anyhow should not be used.
+          def subscribe_to_close_web_producer
+            ::Karafka::App.monitor.subscribe('app.terminated') do
+              # If Web producer is the same as `Karafka.producer` it will do nothing as you can
+              # call `#close` multiple times without side effects
+              ::Karafka::Web.producer.close
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/actions/extend_boot_file.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Actions
+        # Extends the boot file with Web components
+        class ExtendBootFile < Base
+          # Code that is needed in the `karafka.rb` to connect Web UI to Karafka
+          ENABLER_CODE = 'Karafka::Web.enable!'
+
+          # Template with initial Web UI configuration
+          # Session secret needs to be set per user and per env
+          SETUP_TEMPLATE = <<~CONFIG.freeze
+            Karafka::Web.setup do |config|
+              # You may want to set it per ENV. This value was randomly generated.
+              config.ui.sessions.secret = '#{SecureRandom.hex(32)}'
+            end
+
+            #{ENABLER_CODE}
+          CONFIG
+
+          # Adds needed code
+          def call
+            if File.read(Karafka.boot_file).include?(ENABLER_CODE)
+              puts "Web UI #{already} installed."
+            else
+              puts 'Updating the Karafka boot file...'
+              File.open(Karafka.boot_file, 'a') do |f|
+                f << "\n#{SETUP_TEMPLATE}\n"
+              end
+              puts "Karafka boot file #{successfully} updated."
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/actions/migrate_states_data.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Actions
+        # Command to migrate states data
+        # Useful when we have older schema and need to move forward
+        class MigrateStatesData < Base
+          # Runs needed migrations (if any) on the states topics
+          def call
+            Migrator.new.call
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/0_base.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      # Namespace for storing migrations of our Web UI topics data
+      module Migrations
+        # Base for all our migrations
+        #
+        # Each migration **MUST** have a `#migrate` method defined
+        # Migrations are expected to modify the provided state **IN PLACE**
+        class Base
+          include Karafka::Core::Helpers::Time
+
+          class << self
+            # First version that should **NOT** be affected by this migration
+            attr_accessor :versions_until
+            # What resource does it relate it
+            # One migration should modify only one resource type
+            attr_accessor :type
+
+            # @param version [String] sem-ver version
+            # @return [Boolean] is the given migration applicable
+            def applicable?(version)
+              version < versions_until
+            end
+
+            # @param state [Hash] deserialized state to be modified
+            def migrate(state)
+              raise NotImplementedError, 'Implement in a subclass'
+            end
+
+            # @return [Integer] index for sorting. Older migrations are always applied first
+            def index
+              instance_method(:migrate)
+                .source_location
+                .first
+                .split('/')
+                .last
+                .split('_')
+                .first
+                .to_i
+            end
+
+            # @return [Array<Class>] array with migrations sorted from oldest to latest. This is
+            #   the order in which they need to be applied
+            def sorted_descendants
+              ObjectSpace
+                .each_object(Class)
+                .select { |klass| klass < self }
+                .sort_by(&:index)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/0_set_initial_consumers_metrics.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Initial migration that sets the consumers metrics initial first state.
+        # This is the basic of metrics as they were when they were introduced.
+        class SetInitialConsumersMetrics < Base
+          # Always migrate from empty up
+          self.versions_until = '0.0.1'
+          self.type = :consumers_metrics
+
+          # @param state [Hash] initial empty state
+          def migrate(state)
+            state.merge!(
+              aggregated: {
+                days: [],
+                hours: [],
+                minutes: [],
+                seconds: []
+              },
+              consumer_groups: {
+                days: [],
+                hours: [],
+                minutes: [],
+                seconds: []
+              },
+              dispatched_at: float_now
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/0_set_initial_consumers_state.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Initial migration that sets the consumers state initial first state.
+        # This is the basic of state as they were when they were introduced.
+        class SetInitialConsumersState < Base
+          # Run this only on the first setup
+          self.versions_until = '0.0.1'
+          self.type = :consumers_state
+
+          # @param state [Hash]
+          def migrate(state)
+            state.merge!(
+              processes: {},
+              stats: {
+                batches: 0,
+                messages: 0,
+                retries: 0,
+                dead: 0,
+                busy: 0,
+                enqueued: 0,
+                processing: 0,
+                workers: 0,
+                processes: 0,
+                rss: 0,
+                listeners: 0,
+                utilization: 0,
+                errors: 0,
+                lag_stored: 0,
+                lag: 0
+              },
+              schema_state: 'accepted',
+              dispatched_at: float_now
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1699543515_fill_missing_received_and_sent_bytes_in_consumers_metrics.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Adds bytes_sent and bytes_received to all the aggregated metrics samples, so we have
+        # charts that do not have to fill gaps or check anything
+        class FillMissingReceivedAndSentBytesInConsumersMetrics < Base
+          self.versions_until = '1.1.0'
+          self.type = :consumers_metrics
+
+          # @param state [Hash] metrics state
+          def migrate(state)
+            state[:aggregated].each_value do |metrics|
+              metrics.each do |metric|
+                metric.last[:bytes_sent] = 0
+                metric.last[:bytes_received] = 0
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1699543515_fill_missing_received_and_sent_bytes_in_consumers_state.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Similar to filling in consumers metrics, we initialize this with zeros so it is always
+        # present as expected
+        class FillMissingReceivedAndSentBytesInConsumersState < Base
+          # Network metrics were introduced with schema 1.2.0
+          self.versions_until = '1.2.0'
+          self.type = :consumers_state
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:stats][:bytes_sent] = 0
+            state[:stats][:bytes_received] = 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1700234522_introduce_waiting_in_consumers_metrics.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Introduce waiting in consumers metrics to complement busy and enqueued for jobs metrics
+        class IntroduceWaitingInConsumersMetrics < Base
+          self.versions_until = '1.1.1'
+          self.type = :consumers_metrics
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:aggregated].each_value do |metrics|
+              metrics.each do |metric|
+                metric.last[:waiting] = 0
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1700234522_introduce_waiting_in_consumers_state.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Introduce waiting in consumers metrics to complement busy and enqueued for jobs stats
+        class IntroduceWaitingInConsumersState < Base
+          self.versions_until = '1.2.1'
+          self.type = :consumers_state
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:stats][:waiting] = 0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1700234522_remove_processing_from_consumers_metrics.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Moves unused "processing" that was used instead of "busy" in older versions
+        class RemoveProcessingFromConsumersMetrics < Base
+          self.versions_until = '1.1.1'
+          self.type = :consumers_metrics
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:aggregated].each_value do |metrics|
+              metrics.each do |metric|
+                metric.last.delete(:processing)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1700234522_remove_processing_from_consumers_state.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Moves unused "processing" that was used instead of "busy" in older versions
+        class RemoveProcessingFromConsumersState < Base
+          self.versions_until = '1.2.1'
+          self.type = :consumers_state
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:stats].delete(:processing)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_metrics.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Since we have introduced notion of pause listeners, we need to reflect this in the
+        # UI, so the scaling changes are visible
+        class SplitListenersIntoActiveAndPausedInMetrics < Base
+          self.versions_until = '1.1.2'
+          self.type = :consumers_metrics
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:aggregated].each_value do |metrics|
+              metrics.each do |metric|
+                listeners = if metric.last.key?(:listeners)
+                              metric.last[:listeners].to_i
+                            elsif metric.last.key?(:listeners_count)
+                              metric.last[:listeners_count].to_i
+                            else
+                              0
+                            end
+
+                metric.last[:listeners] = {
+                  active: listeners,
+                  standby: 0
+                }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end