karafka-web 0.8.0 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 446dcc1a42f38d364747702c52aeefbdede83b203eb13bfa755c8e2e1f05438e
-  data.tar.gz: 20e7cc3cb0b400b382715b184db667d4791aeb0d626b4e9c6af7b314394d3309
+  metadata.gz: 457da6c2e847225be5d27f3a30fcf8f99a4a132f8109204315ac042e24d1bd35
+  data.tar.gz: c277f44c30a34ee2748ac4e9b56833814b522da87f6cc1841d21aca618af58be
 SHA512:
-  metadata.gz: f2ecb81d1dd77b5dc1c9feaa75dc1f262335da4a812785035c0ca85fdf5e1413cefa724a73b13032908d3fc02e52096140ee89eae18ca7d933193c1ed7ddc011
-  data.tar.gz: 59c4a2324ee277cb61c197ffa190da2e59bb594d8c615296074a5f144b3a9d6ef3ebd31a70e37ab687e7c1a872f9efc9624b24400eb2a44be1034e0901d74586
+  metadata.gz: 9ff1df82abd7f4f9f7002ae8a9e3e4e3fef2a71fbac820ff700d52c1926a5a81bf8e141ac6b928693d6276d8c16ef496efc11cada7574da217f4fb31efe31b5b
+  data.tar.gz: 3f39b8155794c25a39464d71c35f33ee82e6b59f92b55301f20ff69f1c58bc2cdf32af976720c548c2262893e25e0f8c8d31b8c033a248feb929bd088bc46d6c

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Karafka Web changelog
 
+## 0.8.2 (2024-02-16)
+- [Enhancement] Defer scheduler background thread creation until needed allowing for forks.
+- [Enhancement] Tag forks with fork indication + ppid reference when operating in swarm.
+- [Fix] Fix issue where Health tabs would not be visible when no data reported.
+- [Fix] Stopped processes subscriptions lacks indicator of no groups.
+- [Fix] Terminated process state not supported in the web ui.
+- [Fix] Rebalance reason can be empty on a SG when no network.
+
+## 0.8.1 (2024-02-01)
+- [Enhancement] Introduce "Lags" health view.
+- [Enhancement] Remove "Stored Lag" and "Committed Offset" from Health Overview due to Lags Tab.
+- [Enhancement] Report lag on consumers that did not yet marked offsets.
+- [Enhancement] Use more accurate lag reporting that compensates for lack of stored lag.
+- [Fix] When first message after process start is crashed without DLQ lag is not reported.
+- [Fix] Wrong order of enabled injection causes fresh install to crash.
+
 ## 0.8.0 (2024-01-26)
 - **[Feature]** Provide ability to sort table data for part of the views (note: not all attributes can be sorted due to technical limitations of sub-components fetching from Kafka).
 - **[Feature]** Track and report pause timeouts via "Changes" view in Health.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.8.0)
+    karafka-web (0.8.2)
       erubi (~> 1.4)
       karafka (>= 2.3.0, < 2.4.0)
       karafka-core (>= 2.3.0, < 2.4.0)
@@ -26,7 +26,7 @@ GEM
     byebug (11.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
-    diff-lcs (1.5.0)
+    diff-lcs (1.5.1)
     docile (1.4.0)
     drb (2.2.0)
       ruby2_keywords
@@ -42,14 +42,14 @@ GEM
       zeitwerk (~> 2.3)
     karafka-core (2.3.0)
       karafka-rdkafka (>= 0.14.8, < 0.15.0)
-    karafka-rdkafka (0.14.8)
+    karafka-rdkafka (0.14.9)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
     mini_portile2 (2.8.5)
-    minitest (5.21.2)
+    minitest (5.22.0)
     mutex_m (0.2.0)
-    rack (3.0.8)
+    rack (3.0.9)
     rack-test (2.1.0)
       rack (>= 1.3)
     rackup (0.2.3)
@@ -58,19 +58,19 @@ GEM
     rake (13.1.0)
     roda (3.76.0)
       rack
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.0)
     ruby2_keywords (0.0.5)
     simplecov (0.22.0)
       docile (~> 1.1)
@@ -81,11 +81,11 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.12)
+    waterdrop (2.6.14)
       karafka-core (>= 2.2.3, < 3.0.0)
       zeitwerk (~> 2.3)
     webrick (1.8.1)
-    zeitwerk (2.6.12)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   ruby
@@ -101,4 +101,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.4
+   2.5.6

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.3
+    image: confluentinc/cp-kafka:7.6.0
 
     ports:
       - 9092:9092

data/lib/karafka/web/config.rb

@@ -6,6 +6,10 @@ module Karafka
     class Config
       include ::Karafka::Core::Configurable
 
+      # Is the Web UI enabled and things were configured.
+      # Automatically set to true in case things got enabled.
+      setting :enabled, default: false
+
       # How long do we consider the process alive without receiving status info about it
       # For this long we also display dead processes (shutdown) in the UI
       # This is used both in the processing for eviction and in the UI
@@ -65,6 +69,7 @@ module Karafka
 
           # Listeners needed for the Web UI to track consumer related changes
           setting :listeners, default: [
+            Tracking::Consumers::Listeners::Booting.new,
             Tracking::Consumers::Listeners::Status.new,
             Tracking::Consumers::Listeners::Errors.new,
             Tracking::Consumers::Listeners::Connections.new,
@@ -84,6 +89,7 @@ module Karafka
 
           # Listeners needed for the Web UI to track producers related stuff
           setting :listeners, default: [
+            Tracking::Producers::Listeners::Booting.new,
             Tracking::Producers::Listeners::Errors.new
           ]
         end

data/lib/karafka/web/contracts/config.rb

@@ -10,6 +10,7 @@ module Karafka
         # Use the same regexp as Karafka for topics validation
         TOPIC_REGEXP = ::Karafka::Contracts::TOPIC_REGEXP
 
+        required(:enabled) { |val| [true, false, nil].include?(val) }
         required(:ttl) { |val| val.is_a?(Numeric) && val.positive? }
 
         nested(:topics) do

data/lib/karafka/web/installer.rb

@@ -12,9 +12,12 @@ module Karafka
       #
       # @param replication_factor [Integer] replication factor we want to use (1 by default)
       def install(replication_factor: 1)
+        enable!
         puts
         puts 'Installing Karafka Web UI...'
         puts
+        Management::Actions::ExtendBootFile.new.call
+        puts
         puts 'Creating necessary topics and populating state data...'
         puts
         Management::Actions::CreateTopics.new.call(replication_factor)
@@ -24,8 +27,6 @@ module Karafka
         puts 'Running data migrations...'
         Management::Actions::MigrateStatesData.new.call
         puts
-        Management::Actions::ExtendBootFile.new.call
-        puts
         puts("Installation #{green('completed')}. Have fun!")
         puts
       end
@@ -35,6 +36,7 @@ module Karafka
       #
       # @param replication_factor [Integer] replication factor we want to use (1 by default)
       def migrate(replication_factor: 1)
+        enable!
         puts
         puts 'Creating necessary topics and populating state data...'
         puts
@@ -52,6 +54,7 @@ module Karafka
       # Removes all the Karafka topics and creates them again with the same replication factor
       # @param replication_factor [Integer] replication factor we want to use (1 by default)
       def reset(replication_factor: 1)
+        enable!
         puts
         puts 'Resetting Karafka Web UI...'
         puts
@@ -70,6 +73,7 @@ module Karafka
 
       # Removes all the Karafka Web topics and cleans after itself.
       def uninstall
+        enable!
         puts
         puts 'Uninstalling Karafka Web UI...'
         puts

data/lib/karafka/web/management/actions/enable.rb

@@ -10,6 +10,11 @@ module Karafka
         class Enable < Base
           # Enables routing consumer group and subscribes Web-UI listeners
           def call
+            # Prevent double enabling
+            return if ::Karafka::Web.config.enabled
+
+            ::Karafka::Web.config.enabled = true
+
             extend_routing
             setup_tracking_activity
 

data/lib/karafka/web/management/actions/extend_boot_file.rb

@@ -22,7 +22,9 @@ module Karafka
 
           # Adds needed code
           def call
-            if File.read(Karafka.boot_file).include?(ENABLER_CODE)
+            # We detect this that way so in case our template or user has enabled as a comment
+            # it still adds the template and runs install
+            if File.readlines(Karafka.boot_file).any? { |line| line.start_with?(ENABLER_CODE) }
               puts "Web UI #{already} installed."
             else
               puts 'Updating the Karafka boot file...'

data/lib/karafka/web/management/migrations/1706607960_introduce_lag_total_in_metrics.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Moves to using lag total as a normalization for both lags
+        class IntroduceLagTotalInMetrics < Base
+          self.versions_until = '1.2.0'
+          self.type = :consumers_metrics
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:aggregated].each_value do |metrics|
+              metrics.each do |metric|
+                metric.last[:lag_total] = metric.last[:lag_stored]
+                metric.last.delete(:lag_stored)
+                metric.last.delete(:lag)
+              end
+            end
+
+            state[:consumer_groups].each_value do |metrics|
+              metrics.each do |metric_group|
+                metric_group.last.each_value do |metric|
+                  metric.each_value do |sample|
+                    sample[:lag_total] = sample[:lag_stored]
+                    sample.delete(:lag_stored)
+                    sample.delete(:lag)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1706607960_introduce_lag_total_in_states.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Moves to using lag total as a normalization for both lags
+        class IntroduceLagTotalInStates < Base
+          self.versions_until = '1.3.0'
+          self.type = :consumers_state
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:stats][:lag_total] = state[:stats][:lag_stored]
+            state[:stats].delete(:lag)
+            state[:stats].delete(:lag_stored)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1706611396_rename_lag_total_to_lag_hybrid_in_metrics.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Renames total lag to hybrid to better represent what it is
+        class RenameLagTotalToLagHybridInMetrics < Base
+          self.versions_until = '1.2.1'
+          self.type = :consumers_metrics
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:aggregated].each_value do |metrics|
+              metrics.each do |metric|
+                metric.last[:lag_hybrid] = metric.last[:lag_total] || 0
+                metric.last.delete(:lag_total)
+              end
+            end
+
+            state[:consumer_groups].each_value do |metrics|
+              metrics.each do |metric_group|
+                metric_group.last.each_value do |metric|
+                  metric.each_value do |sample|
+                    sample[:lag_hybrid] = sample[:lag_total]
+                    sample.delete(:lag_total)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1706611396_rename_lag_total_to_lag_hybrid_in_states.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Renames total lag to hybrid to better represent what it is
+        class RenameLagTotalToLagHybridInStates < Base
+          self.versions_until = '1.3.1'
+          self.type = :consumers_state
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:stats][:lag_hybrid] = state[:stats][:lag_total] || 0
+            state[:stats].delete(:lag_total)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -11,7 +11,7 @@ module Karafka
           class Metrics < Base
             # Current schema version
             # This is used for detecting incompatible changes and writing migrations
-            SCHEMA_VERSION = '1.1.2'
+            SCHEMA_VERSION = '1.2.1'
 
             def initialize
               super
@@ -92,12 +92,11 @@ module Karafka
               cgs = {}
 
               iterate_partitions_data do |group_name, topic_name, partitions_data|
-                lags = partitions_data
-                       .map { |p_details| p_details.fetch(:lag, -1) }
-                       .reject(&:negative?)
-
-                lags_stored = partitions_data
-                              .map { |p_details| p_details.fetch(:lag_stored, -1) }
+                lags_hybrid = partitions_data
+                              .map do |p_details|
+                                lag_stored = p_details.fetch(:lag_stored, -1)
+                                lag_stored.negative? ? p_details.fetch(:lag, -1) : lag_stored
+                              end
                               .reject(&:negative?)
 
                 offsets_hi = partitions_data
@@ -121,8 +120,7 @@ module Karafka
 
                 cgs[group_name] ||= {}
                 cgs[group_name][topic_name] = {
-                  lag_stored: lags_stored.sum,
-                  lag: lags.sum,
+                  lag_hybrid: lags_hybrid.sum,
                   pace: offsets_hi.sum,
                   # Take max last stable offset duration without any change. This can
                   # indicate a hanging transaction, because the offset will not move forward

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -20,7 +20,7 @@ module Karafka
             # Current schema version
             # This can be used in the future for detecting incompatible changes and writing
             # migrations
-            SCHEMA_VERSION = '1.2.2'
+            SCHEMA_VERSION = '1.3.1'
 
             # @param schema_manager [Karafka::Web::Processing::Consumers::SchemaManager] schema
             #   manager that tracks the compatibility of schemas.
@@ -127,8 +127,7 @@ module Karafka
               stats[:processes] = 0
               stats[:rss] = 0
               stats[:listeners] = { active: 0, standby: 0 }
-              stats[:lag] = 0
-              stats[:lag_stored] = 0
+              stats[:lag_hybrid] = 0
               stats[:bytes_received] = 0
               stats[:bytes_sent] = 0
               utilization = 0
@@ -140,12 +139,13 @@ module Karafka
                   report_stats = report[:stats]
                   report_process = report[:process]
 
-                  lags = []
-                  lags_stored = []
+                  lags_hybrid = []
 
                   iterate_partitions(report) do |partition_stats|
-                    lags << partition_stats[:lag]
-                    lags_stored << partition_stats[:lag_stored]
+                    lag_stored = partition_stats[:lag_stored]
+                    lag = partition_stats[:lag]
+
+                    lags_hybrid << (lag_stored.negative? ? lag : lag_stored)
                   end
 
                   stats[:busy] += report_stats[:busy]
@@ -157,8 +157,7 @@ module Karafka
                   stats[:listeners][:standby] += report_process[:listeners][:standby]
                   stats[:processes] += 1
                   stats[:rss] += report_process[:memory_usage]
-                  stats[:lag] += lags.compact.reject(&:negative?).sum
-                  stats[:lag_stored] += lags_stored.compact.reject(&:negative?).sum
+                  stats[:lag_hybrid] += lags_hybrid.compact.reject(&:negative?).sum
                   utilization += report_stats[:utilization]
                 end
 

data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb

@@ -21,8 +21,7 @@ module Karafka
             required(:processes) { |val| val.is_a?(Integer) && val >= 0 }
             required(:rss) { |val| val.is_a?(Numeric) && val >= 0 }
             required(:utilization) { |val| val.is_a?(Numeric) && val >= 0 }
-            required(:lag_stored) { |val| val.is_a?(Integer) }
-            required(:lag) { |val| val.is_a?(Integer) }
+            required(:lag_hybrid) { |val| val.is_a?(Integer) }
 
             nested(:listeners) do
               required(:active) { |val| val.is_a?(Integer) && val >= 0 }

data/lib/karafka/web/processing/consumers/contracts/topic_stats.rb

@@ -9,8 +9,7 @@ module Karafka
           class TopicStats < Web::Contracts::Base
             configure
 
-            required(:lag_stored) { |val| val.is_a?(Integer) }
-            required(:lag) { |val| val.is_a?(Integer) }
+            required(:lag_hybrid) { |val| val.is_a?(Integer) }
             required(:pace) { |val| val.is_a?(Integer) }
             required(:ls_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
           end

data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb

@@ -19,7 +19,7 @@ module Karafka
               required(:stateage) { |val| val.is_a?(Integer) && val >= 0 }
               required(:rebalance_age) { |val| val.is_a?(Integer) && val >= 0 }
               required(:rebalance_cnt) { |val| val.is_a?(Integer) && val >= 0 }
-              required(:rebalance_reason) { |val| val.is_a?(String) && !val.empty? }
+              required(:rebalance_reason) { |val| val.is_a?(String) }
               required(:poll_age) { |val| val.is_a?(Numeric) && val >= 0 }
             end
 

data/lib/karafka/web/tracking/consumers/listeners/booting.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Listener needed to start schedulers and other things that we need to collect and report
+          # data. We do not want to start this when code is loaded because it may not be fork
+          # compatible that way
+          class Booting < Base
+            # Starts (if needed) the Web UI tracking scheduler thread that periodically pings
+            # reporters to report needed data (when it is time).
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_app_running(_event)
+              ::Karafka::Web.config.tracking.scheduler.async_call
+            end
+
+            # Updates the web producer after fork if needed and adds ppid to nodes
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_swarm_node_after_fork(_event)
+              ::Karafka::Process.tags.add(:node_ppid, "ppid:#{::Process.ppid}")
+
+              return if Karafka::Web.config.producer == Karafka::App.config.producer
+
+              Web.config.producer = Karafka::App.config.producer
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -46,7 +46,6 @@ module Karafka
             end
             @subscription_groups = {}
             @errors = []
-            @started_at = float_now
             @pauses = {}
             @jobs = {}
             @shell = MemoizedShell.new
@@ -71,7 +70,7 @@ module Karafka
               dispatched_at: float_now,
 
               process: {
-                started_at: @started_at,
+                started_at: started_at,
                 name: process_name,
                 status: ::Karafka::App.config.internal.status.to_s,
                 listeners: listeners,
@@ -129,6 +128,13 @@ module Karafka
 
           private
 
+          # @return [Float] time of start of this process
+          # @note We memoize it on first run as forks should have their creation time matching the
+          #   fork time.
+          def started_at
+            @started_at ||= float_now
+          end
+
           # @return [Numeric] % utilization of all the threads. 100% means all the threads are
           #   utilized all the time within the given time window. 0% means, nothing is happening
           #   most if not all the time.

data/lib/karafka/web/tracking/producers/listeners/booting.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Producers
+        module Listeners
+          # Listener needed to start schedulers and other things that we need to collect and report
+          # data. We do not want to start this when code is loaded because it may not be fork
+          # compatible that way
+          class Booting < Base
+            # Starts (if needed) the Web UI tracking scheduler thread that periodically pings
+            # reporters to report needed data (when it is time).
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_producer_connected(_event)
+              ::Karafka::Web.config.tracking.scheduler.async_call
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/scheduler.rb

@@ -11,11 +11,6 @@ module Karafka
       class Scheduler
         include ::Karafka::Helpers::Async
 
-        # Creates the scheduler and runs its internal reporting
-        def initialize
-          async_call
-        end
-
         private
 
         # Reports the process state once in a while

data/lib/karafka/web/ui/controllers/consumers.rb

@@ -9,7 +9,7 @@ module Karafka
           self.sortable_attributes = %w[
             name
             started_at
-            lag_stored
+            lag_hybrid
           ].freeze
 
           # List page with consumers

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -41,11 +41,13 @@ module Karafka
           def status_bg(status)
             case status
             when 'initialized' then 'bg-success'
+            when 'supervising' then 'bg-success'
             when 'running' then 'bg-success'
             when 'quieting' then 'bg-warning'
             when 'quiet' then 'bg-warning text-dark'
             when 'stopping' then 'bg-warning text-dark'
             when 'stopped' then 'bg-danger'
+            when 'terminated' then 'bg-danger'
             else
               raise ::Karafka::Errors::UnsupportedCaseError, status
             end

data/lib/karafka/web/ui/models/metrics/charts/topics.rb

@@ -16,17 +16,17 @@ module Karafka
 
               # @return [String] JSON with lags of each of the topics + total lag of all the topics
               #   from all the consumer groups.
-              def lags_stored
+              def lags_hybrid
                 total = Hash.new { |h, v| h[v] = 0 }
 
                 @data.to_h.each_value do |metrics|
                   metrics.each do |metric|
                     time = metric.first
-                    lag_stored = metric.last[:lag_stored]
+                    lag_hybrid = metric.last[:lag_hybrid]
 
-                    if lag_stored
+                    if lag_hybrid
                       total[time] ||= 0
-                      total[time] += lag_stored
+                      total[time] += lag_hybrid
                     else
                       next if total.key?(time)
 
@@ -37,7 +37,7 @@ module Karafka
 
                 # Extract the lag stored only from all the data
                 per_topic = @data.to_h.map do |topic, metrics|
-                  extracted = metrics.map { |metric| [metric.first, metric.last[:lag_stored]] }
+                  extracted = metrics.map { |metric| [metric.first, metric.last[:lag_hybrid]] }
 
                   [topic, extracted]
                 end.to_h