karafka-web 0.8.1 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69148848d89de8a7c5a1cebec7d3f5c0c60e3bd85caa41eedea8b5082dd6bf85
-  data.tar.gz: c0f6acfdd3e732ebfc29a6e954cd8a1f8d2c75dff77eba28978a99d6394d4087
+  metadata.gz: 457da6c2e847225be5d27f3a30fcf8f99a4a132f8109204315ac042e24d1bd35
+  data.tar.gz: c277f44c30a34ee2748ac4e9b56833814b522da87f6cc1841d21aca618af58be
 SHA512:
-  metadata.gz: a89e22500fc5fedf8e432b666a84cf092d13c9b2747a4d08dc983d8ffba2f077781f06a131631b6315c64764bc00b7d06265a4d9fcd8c11bdd74d47eda5041f9
-  data.tar.gz: 7d29af2874b1df83af78fee69af89021bdcd1fa705686370f59ab276ee6808eef98e4a0b98dd273158888b472cea203bcca84f494e84c178f7733404b080166a
+  metadata.gz: 9ff1df82abd7f4f9f7002ae8a9e3e4e3fef2a71fbac820ff700d52c1926a5a81bf8e141ac6b928693d6276d8c16ef496efc11cada7574da217f4fb31efe31b5b
+  data.tar.gz: 3f39b8155794c25a39464d71c35f33ee82e6b59f92b55301f20ff69f1c58bc2cdf32af976720c548c2262893e25e0f8c8d31b8c033a248feb929bd088bc46d6c

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Karafka Web changelog
 
+## 0.8.2 (2024-02-16)
+- [Enhancement] Defer scheduler background thread creation until needed allowing for forks.
+- [Enhancement] Tag forks with fork indication + ppid reference when operating in swarm.
+- [Fix] Fix issue where Health tabs would not be visible when no data reported.
+- [Fix] Stopped processes subscriptions lacks indicator of no groups.
+- [Fix] Terminated process state not supported in the web ui.
+- [Fix] Rebalance reason can be empty on a SG when no network.
+
 ## 0.8.1 (2024-02-01)
 - [Enhancement] Introduce "Lags" health view.
 - [Enhancement] Remove "Stored Lag" and "Committed Offset" from Health Overview due to Lags Tab.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.8.1)
+    karafka-web (0.8.2)
       erubi (~> 1.4)
       karafka (>= 2.3.0, < 2.4.0)
       karafka-core (>= 2.3.0, < 2.4.0)
@@ -26,7 +26,7 @@ GEM
     byebug (11.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
-    diff-lcs (1.5.0)
+    diff-lcs (1.5.1)
     docile (1.4.0)
     drb (2.2.0)
       ruby2_keywords
@@ -42,14 +42,14 @@ GEM
       zeitwerk (~> 2.3)
     karafka-core (2.3.0)
       karafka-rdkafka (>= 0.14.8, < 0.15.0)
-    karafka-rdkafka (0.14.8)
+    karafka-rdkafka (0.14.9)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
     mini_portile2 (2.8.5)
-    minitest (5.21.2)
+    minitest (5.22.0)
     mutex_m (0.2.0)
-    rack (3.0.8)
+    rack (3.0.9)
     rack-test (2.1.0)
       rack (>= 1.3)
     rackup (0.2.3)
@@ -58,19 +58,19 @@ GEM
     rake (13.1.0)
     roda (3.76.0)
       rack
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.0)
     ruby2_keywords (0.0.5)
     simplecov (0.22.0)
       docile (~> 1.1)
@@ -81,11 +81,11 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.12)
+    waterdrop (2.6.14)
       karafka-core (>= 2.2.3, < 3.0.0)
       zeitwerk (~> 2.3)
     webrick (1.8.1)
-    zeitwerk (2.6.12)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   ruby
@@ -101,4 +101,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.4
+   2.5.6

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.3
+    image: confluentinc/cp-kafka:7.6.0
 
     ports:
       - 9092:9092

data/lib/karafka/web/config.rb

@@ -69,6 +69,7 @@ module Karafka
 
           # Listeners needed for the Web UI to track consumer related changes
           setting :listeners, default: [
+            Tracking::Consumers::Listeners::Booting.new,
             Tracking::Consumers::Listeners::Status.new,
             Tracking::Consumers::Listeners::Errors.new,
             Tracking::Consumers::Listeners::Connections.new,
@@ -88,6 +89,7 @@ module Karafka
 
           # Listeners needed for the Web UI to track producers related stuff
           setting :listeners, default: [
+            Tracking::Producers::Listeners::Booting.new,
             Tracking::Producers::Listeners::Errors.new
           ]
         end

data/lib/karafka/web/installer.rb

@@ -12,6 +12,7 @@ module Karafka
       #
       # @param replication_factor [Integer] replication factor we want to use (1 by default)
       def install(replication_factor: 1)
+        enable!
         puts
         puts 'Installing Karafka Web UI...'
         puts
@@ -21,7 +22,6 @@ module Karafka
         puts
         Management::Actions::CreateTopics.new.call(replication_factor)
         wait_for_topics
-        enable!
         Management::Actions::CreateInitialStates.new.call
         puts
         puts 'Running data migrations...'
@@ -36,6 +36,7 @@ module Karafka
       #
       # @param replication_factor [Integer] replication factor we want to use (1 by default)
       def migrate(replication_factor: 1)
+        enable!
         puts
         puts 'Creating necessary topics and populating state data...'
         puts
@@ -53,6 +54,7 @@ module Karafka
       # Removes all the Karafka topics and creates them again with the same replication factor
       # @param replication_factor [Integer] replication factor we want to use (1 by default)
       def reset(replication_factor: 1)
+        enable!
         puts
         puts 'Resetting Karafka Web UI...'
         puts
@@ -71,6 +73,7 @@ module Karafka
 
       # Removes all the Karafka Web topics and cleans after itself.
       def uninstall
+        enable!
         puts
         puts 'Uninstalling Karafka Web UI...'
         puts

data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb

@@ -19,7 +19,7 @@ module Karafka
               required(:stateage) { |val| val.is_a?(Integer) && val >= 0 }
               required(:rebalance_age) { |val| val.is_a?(Integer) && val >= 0 }
               required(:rebalance_cnt) { |val| val.is_a?(Integer) && val >= 0 }
-              required(:rebalance_reason) { |val| val.is_a?(String) && !val.empty? }
+              required(:rebalance_reason) { |val| val.is_a?(String) }
               required(:poll_age) { |val| val.is_a?(Numeric) && val >= 0 }
             end
 

data/lib/karafka/web/tracking/consumers/listeners/booting.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Listener needed to start schedulers and other things that we need to collect and report
+          # data. We do not want to start this when code is loaded because it may not be fork
+          # compatible that way
+          class Booting < Base
+            # Starts (if needed) the Web UI tracking scheduler thread that periodically pings
+            # reporters to report needed data (when it is time).
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_app_running(_event)
+              ::Karafka::Web.config.tracking.scheduler.async_call
+            end
+
+            # Updates the web producer after fork if needed and adds ppid to nodes
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_swarm_node_after_fork(_event)
+              ::Karafka::Process.tags.add(:node_ppid, "ppid:#{::Process.ppid}")
+
+              return if Karafka::Web.config.producer == Karafka::App.config.producer
+
+              Web.config.producer = Karafka::App.config.producer
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -46,7 +46,6 @@ module Karafka
             end
             @subscription_groups = {}
             @errors = []
-            @started_at = float_now
             @pauses = {}
             @jobs = {}
             @shell = MemoizedShell.new
@@ -71,7 +70,7 @@ module Karafka
               dispatched_at: float_now,
 
               process: {
-                started_at: @started_at,
+                started_at: started_at,
                 name: process_name,
                 status: ::Karafka::App.config.internal.status.to_s,
                 listeners: listeners,
@@ -129,6 +128,13 @@ module Karafka
 
           private
 
+          # @return [Float] time of start of this process
+          # @note We memoize it on first run as forks should have their creation time matching the
+          #   fork time.
+          def started_at
+            @started_at ||= float_now
+          end
+
           # @return [Numeric] % utilization of all the threads. 100% means all the threads are
           #   utilized all the time within the given time window. 0% means, nothing is happening
           #   most if not all the time.

data/lib/karafka/web/tracking/producers/listeners/booting.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Producers
+        module Listeners
+          # Listener needed to start schedulers and other things that we need to collect and report
+          # data. We do not want to start this when code is loaded because it may not be fork
+          # compatible that way
+          class Booting < Base
+            # Starts (if needed) the Web UI tracking scheduler thread that periodically pings
+            # reporters to report needed data (when it is time).
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_producer_connected(_event)
+              ::Karafka::Web.config.tracking.scheduler.async_call
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/scheduler.rb

@@ -11,11 +11,6 @@ module Karafka
       class Scheduler
         include ::Karafka::Helpers::Async
 
-        # Creates the scheduler and runs its internal reporting
-        def initialize
-          async_call
-        end
-
         private
 
         # Reports the process state once in a while

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -41,11 +41,13 @@ module Karafka
           def status_bg(status)
             case status
             when 'initialized' then 'bg-success'
+            when 'supervising' then 'bg-success'
             when 'running' then 'bg-success'
             when 'quieting' then 'bg-warning'
             when 'quiet' then 'bg-warning text-dark'
             when 'stopping' then 'bg-warning text-dark'
             when 'stopped' then 'bg-danger'
+            when 'terminated' then 'bg-danger'
             else
               raise ::Karafka::Errors::UnsupportedCaseError, status
             end

data/lib/karafka/web/ui/models/process.rb

@@ -52,6 +52,15 @@ module Karafka
               .sum
           end
 
+          # @return [Boolean] true if there are any active subscriptions, otherwise false.
+          def subscribed?
+            return false if consumer_groups.empty?
+
+            consumer_groups.any? do |cg|
+              !cg.subscription_groups.empty?
+            end
+          end
+
           # @return [Integer] number of partitions to which we are currently subscribed
           def subscribed_partitions_count
             consumer_groups

data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb

@@ -76,7 +76,13 @@
 
   <span class="text-end mt-3">
     <small>
-      Last rebalance reason: <%= subscription_group.rebalance_reason %>
+      Last rebalance reason:
+
+      <% if subscription_group.rebalance_reason.empty? %>
+        Unknown
+      <% else %>
+        <%= subscription_group.rebalance_reason %>
+      <% end %>
     </small>
   </span>
 </div>

data/lib/karafka/web/ui/pro/views/consumers/subscriptions.erb

@@ -8,16 +8,16 @@
 
 <%== partial 'consumers/consumer/tabs' %>
 
-<% if @process.consumer_groups.empty? %>
+<% if @process.subscribed? %>
+  <div class="container">
+    <%==
+      render_each(
+        @process.consumer_groups,
+        'consumers/consumer/_consumer_group',
+        local: :consumer_group
+      )
+    %>
+  </div>
+<% else %>
   <%== partial 'consumers/consumer/no_subscriptions' %>
 <% end %>
-
-<div class="container">
-  <%==
-    render_each(
-      @process.consumer_groups,
-      'consumers/consumer/_consumer_group',
-      local: :consumer_group
-    )
-  %>
-</div>

data/lib/karafka/web/ui/pro/views/health/changes.erb

@@ -1,9 +1,9 @@
 <%== view_title('Consumers groups changes details') %>
 
+<%== partial 'health/tabs' %>
+
 <% if @stats.empty? %>
   <%== partial 'health/no_data' %>
-<% else %>
-  <%== partial 'health/tabs' %>
 <% end %>
 
 <% @stats.each_with_index do |(cg_name, details), index| %>

data/lib/karafka/web/ui/pro/views/health/lags.erb

@@ -1,9 +1,9 @@
 <%== view_title('Consumers groups lags details') %>
 
+<%== partial 'health/tabs' %>
+
 <% if @stats.empty? %>
   <%== partial 'health/no_data' %>
-<% else %>
-  <%== partial 'health/tabs' %>
 <% end %>
 
 <% @stats.each_with_index do |(cg_name, details), index| %>

data/lib/karafka/web/ui/pro/views/health/offsets.erb

@@ -1,9 +1,9 @@
 <%== view_title('Consumers groups offsets details') %>
 
+<%== partial 'health/tabs' %>
+
 <% if @stats.empty? %>
   <%== partial 'health/no_data' %>
-<% else %>
-  <%== partial 'health/tabs' %>
 <% end %>
 
 <% @stats.each_with_index do |(cg_name, details), index| %>

data/lib/karafka/web/ui/pro/views/health/overview.erb

@@ -1,9 +1,9 @@
 <%== view_title('Consumers groups overview') %>
 
+<%== partial 'health/tabs' %>
+
 <% if @stats.empty? %>
   <%== partial 'health/no_data' %>
-<% else %>
-  <%== partial 'health/tabs' %>
 <% end %>
 
 <% @stats.each_with_index do |(cg_name, details), index| %>

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.8.1'
+    VERSION = '0.8.2'
   end
 end

data/lib/karafka/web.rb

@@ -15,8 +15,9 @@ module Karafka
     class << self
       # @return [WaterDrop::Producer, nil] waterdrop messages producer or nil if not yet fully
       #   initialized. It may not be fully initialized until the configuration is done
+      # @note Do NOT memoize producer as it may be updated after forking
       def producer
-        @producer ||= Web.config.producer
+        Web.config.producer
       end
 
       # @return [String] root path of this gem

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-02-01 00:00:00.000000000 Z
+date: 2024-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -231,6 +231,7 @@ files:
 - lib/karafka/web/tracking/consumers/contracts/subscription_group.rb
 - lib/karafka/web/tracking/consumers/contracts/topic.rb
 - lib/karafka/web/tracking/consumers/listeners/base.rb
+- lib/karafka/web/tracking/consumers/listeners/booting.rb
 - lib/karafka/web/tracking/consumers/listeners/connections.rb
 - lib/karafka/web/tracking/consumers/listeners/errors.rb
 - lib/karafka/web/tracking/consumers/listeners/pausing.rb
@@ -248,6 +249,7 @@ files:
 - lib/karafka/web/tracking/helpers/ttls/windows.rb
 - lib/karafka/web/tracking/memoized_shell.rb
 - lib/karafka/web/tracking/producers/listeners/base.rb
+- lib/karafka/web/tracking/producers/listeners/booting.rb
 - lib/karafka/web/tracking/producers/listeners/errors.rb
 - lib/karafka/web/tracking/producers/reporter.rb
 - lib/karafka/web/tracking/producers/sampler.rb