karafka-web 0.8.0 → 0.8.2

data/lib/karafka/web/ui/models/partition.rb

@@ -6,6 +6,28 @@ module Karafka
       module Models
         # Single topic partition data representation model
         class Partition < Lib::HashProxy
+          # @param args [Object] anything hash proxy accepts
+          def initialize(*args)
+            super
+
+            # We initialize those here because we want to have them stored in the internal has
+            # for sorting
+            lag_hybrid
+            lag_hybrid_d
+          end
+
+          # Because `lag_stored` is not available until first marking, we fallback to the lag
+          #   value that may be behind but is always available until stored lag is available.
+          # @return [Integer] hybrid log value
+          def lag_hybrid
+            self[:lag_hybrid] ||= lag_stored.negative? ? lag : lag_stored
+          end
+
+          # @return [Integer] hybrid log delta
+          def lag_hybrid_d
+            self[:lag_hybrid_d] ||= lag_stored.negative? ? lag_d : lag_stored_d
+          end
+
           # @return [Symbol] one of three states in which LSO can be in the correlation to given
           #   partition in the context of a consumer group.
           #

data/lib/karafka/web/ui/models/process.rb

@@ -41,26 +41,24 @@ module Karafka
               .then { |jobs| Jobs.new(jobs) }
           end
 
-          # @return [Integer] collective stored lag on this process
-          def lag_stored
+          # @return [Integer] collective hybrid lag on this process
+          def lag_hybrid
             consumer_groups
               .flat_map(&:subscription_groups)
               .flat_map(&:topics)
               .flat_map(&:partitions)
-              .map(&:lag_stored)
+              .map(&:lag_hybrid)
               .delete_if(&:negative?)
               .sum
           end
 
-          # @return [Integer] collective lag on this process
-          def lag
-            consumer_groups
-              .flat_map(&:subscription_groups)
-              .flat_map(&:topics)
-              .flat_map(&:partitions)
-              .map(&:lag)
-              .delete_if(&:negative?)
-              .sum
+          # @return [Boolean] true if there are any active subscriptions, otherwise false.
+          def subscribed?
+            return false if consumer_groups.empty?
+
+            consumer_groups.any? do |cg|
+              !cg.subscription_groups.empty?
+            end
           end
 
           # @return [Integer] number of partitions to which we are currently subscribed

data/lib/karafka/web/ui/pro/app.rb

@@ -163,6 +163,10 @@ module Karafka
             r.on 'health' do
               controller = Controllers::Health.new(params)
 
+              r.get 'lags' do
+                controller.lags
+              end
+
               r.get 'offsets' do
                 controller.offsets
               end

data/lib/karafka/web/ui/pro/controllers/consumers.rb

@@ -19,11 +19,11 @@ module Karafka
           # Controller for displaying consumers states and details about them
           class Consumers < Ui::Controllers::Base
             self.sortable_attributes = %w[
+              id
               name
               started_at
-              lag_stored
-              id
-              lag_stored_d
+              lag_hybrid
+              lag_hybrid_d
               committed_offset
               stored_offset
               fetch_state
@@ -93,7 +93,13 @@ module Karafka
 
               # We want to have sorting but on a per subscription group basis and not to sort
               # everything
-              @process.consumer_groups.each { |subscription_group| refine(subscription_group) }
+              @process.consumer_groups.each do |consumer_group|
+                # We need to initialize the whole structure so dynamic fields are also built into
+                # the underlying hashes for sorting
+                consumer_group.subscription_groups.flat_map(&:topics).flat_map(&:partitions)
+
+                refine(consumer_group)
+              end
 
               render
             end

data/lib/karafka/web/ui/pro/controllers/health.rb

@@ -20,8 +20,12 @@ module Karafka
           class Health < Ui::Controllers::Base
             self.sortable_attributes = %w[
               id
+              lag
+              lag_d
               lag_stored
               lag_stored_d
+              lag_hybrid
+              lag_hybrid_d
               committed_offset
               committed_offset_fd
               stored_offset
@@ -50,6 +54,14 @@ module Karafka
               render
             end
 
+            # Displays details about lags and their progression/statuses
+            def lags
+              # Same data as overview but presented differently
+              overview
+
+              render
+            end
+
             # Displays details about offsets and their progression/statuses
             def offsets
               # Same data as overview but presented differently

data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb

@@ -71,6 +71,6 @@
   </td>
 
   <td>
-    <%= process.lag_stored %>
+    <%= process.lag_hybrid %>
   </td>
 </tr>

data/lib/karafka/web/ui/pro/views/consumers/_counters.erb

@@ -19,9 +19,9 @@
               </li>
               <li class="col-sm">
                 <div class="count mb-1">
-                  <%= number_with_delimiter @counters.lag_stored, ' ' %>
+                  <%= number_with_delimiter @counters.lag_hybrid, ' ' %>
                 </div>
-                <div class="desc">Lag stored</div>
+                <div class="desc">Total lag</div>
               </li>
               <li class="col-sm">
                 <a href="<%= root_path('jobs/running') %>">

data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb

@@ -3,11 +3,11 @@
     <%= partition.id %>
   </td>
   <td>
-    <%== lag_with_label partition.lag_stored %>
+    <%== lag_with_label partition.lag_hybrid %>
   </td>
   <td>
-    <span class="badge <%= lag_trend_bg(partition.lag_stored_d) %>">
-      <%= partition.lag_stored_d %>
+    <span class="badge <%= lag_trend_bg(partition.lag_hybrid_d) %>">
+      <%= partition.lag_hybrid_d %>
     </span>
   </td>
   <td>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb

@@ -76,7 +76,13 @@
 
   <span class="text-end mt-3">
     <small>
-      Last rebalance reason: <%= subscription_group.rebalance_reason %>
+      Last rebalance reason:
+
+      <% if subscription_group.rebalance_reason.empty? %>
+        Unknown
+      <% else %>
+        <%= subscription_group.rebalance_reason %>
+      <% end %>
     </small>
   </span>
 </div>
@@ -104,8 +110,8 @@
             <tr class="align-middle">
             </tr>
               <th><%== sort_link('Partition', :id) %></th>
-              <th><%== sort_link(:lag_stored) %></th>
-              <th><%== sort_link('Lag stored trend', :lag_stored_d) %></th>
+              <th><%== sort_link('Lag', :lag_hybrid) %></th>
+              <th><%== sort_link('Lag trend', :lag_hybrid_d) %></th>
               <th><%== sort_link(:committed_offset) %></th>
               <th><%== sort_link(:stored_offset) %></th>
               <th><%== sort_link(:fetch_state) %></th>

data/lib/karafka/web/ui/pro/views/consumers/index.erb

@@ -17,7 +17,7 @@
               <th class="col-sm-1">Memory</th>
               <th class="col-sm-1">Performance</th>
               <th class="col-sm-1">Load</th>
-              <th class="col-sm-1"><%== sort_link(:lag_stored) %></th>
+              <th class="col-sm-1"><%== sort_link('Lag', :lag_hybrid) %></th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/pro/views/consumers/subscriptions.erb

@@ -8,16 +8,16 @@
 
 <%== partial 'consumers/consumer/tabs' %>
 
-<% if @process.consumer_groups.empty? %>
+<% if @process.subscribed? %>
+  <div class="container">
+    <%==
+      render_each(
+        @process.consumer_groups,
+        'consumers/consumer/_consumer_group',
+        local: :consumer_group
+      )
+    %>
+  </div>
+<% else %>
   <%== partial 'consumers/consumer/no_subscriptions' %>
 <% end %>
-
-<div class="container">
-  <%==
-    render_each(
-      @process.consumer_groups,
-      'consumers/consumer/_consumer_group',
-      local: :consumer_group
-    )
-  %>
-</div>

data/lib/karafka/web/ui/pro/views/dashboard/index.erb

@@ -13,7 +13,7 @@
         <ul class="nav nav-tabs" id="graphs1" role="tablist">
           <%== partial 'shared/tab_nav', locals: { title: 'Messages', id: 'messages', active: true } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Batches', id: 'batches' } %>
-          <%== partial 'shared/tab_nav', locals: { title: 'Lags stored', id: 'lags-stored' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Topics lags', id: 'topics-lags' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Topics pace', id: 'topics-pace' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Max LSO time', id: 'max-lso-time' } %>
         </ul>
@@ -29,8 +29,8 @@
             <%== partial 'shared/chart', locals: { data: data, id: 'batches' } %>
           </div>
 
-          <div class="tab-pane" id="lags-stored" role="tabpanel">
-            <%== partial 'shared/chart', locals: { data: @topics_charts.lags_stored, id: 'lags-stored' } %>
+          <div class="tab-pane" id="topics-lags" role="tabpanel">
+            <%== partial 'shared/chart', locals: { data: @topics_charts.lags_hybrid, id: 'topics-lags' } %>
           </div>
 
           <div class="tab-pane" id="topics-pace" role="tabpanel">

data/lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb

@@ -4,6 +4,14 @@
   </a>
 </li>
 
+<% if current_path.include?('/lags') %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('health', 'lags') %>">
+      Lags
+    </a>
+  </li>
+<% end %>
+
 <% if current_path.include?('/overview') %>
   <li class="breadcrumb-item">
     <a href="<%= root_path('health', 'overview') %>">

data/lib/karafka/web/ui/pro/views/health/_consumer_group_header.erb

@@ -0,0 +1,14 @@
+<div class="row mb-4">
+  <div class="col-sm-8">
+    <h4 class="mb-4"><%= cg_name %></h4>
+  </div>
+
+  <div class="col-sm-4">
+    <span class="float-end">
+      Last rebalance:
+      <span class="badge bg-secondary">
+        <%== relative_time(details[:rebalanced_at]) %>
+      </span>
+    </span>
+  </div>
+</div>

data/lib/karafka/web/ui/pro/views/health/_partition.erb

@@ -3,15 +3,7 @@
     <%= partition_id %>
   </td>
   <td>
-    <%== lag_with_label details.lag_stored %>
-  </td>
-  <td>
-    <span class="badge <%= lag_trend_bg(details.lag_stored_d) %>">
-      <%= details.lag_stored_d %>
-    </span>
-  </td>
-  <td>
-    <%== offset_with_label topic_name, partition_id, details.committed_offset %>
+    <%== lag_with_label details.lag_hybrid %>
   </td>
   <td>
     <%== offset_with_label topic_name, partition_id, details.stored_offset %>

data/lib/karafka/web/ui/pro/views/health/_partition_lags.erb

@@ -0,0 +1,24 @@
+<tr class="align-middle <%= lso_risk_state_bg(details) %> status-row-<%= details.process.status %>">
+  <td>
+    <%= partition_id %>
+  </td>
+  <td>
+    <%== lag_with_label details.lag %>
+  </td>
+  <td>
+    <span class="badge <%= lag_trend_bg(details.lag_d) %>">
+      <%= details.lag_d %>
+    </span>
+  </td>
+  <td>
+    <%== lag_with_label details.lag_stored %>
+  </td>
+  <td>
+    <span class="badge <%= lag_trend_bg(details.lag_stored_d) %>">
+      <%= details.lag_stored_d %>
+    </span>
+  </td>
+  <td>
+    <%== poll_state_with_change_time_label(details.poll_state, details.poll_state_ch) %>
+  </td>
+</tr>

data/lib/karafka/web/ui/pro/views/health/_partition_offset.erb

@@ -3,7 +3,7 @@
     <%= partition_id %>
   </td>
   <td>
-    <%== lag_with_label details.lag_stored %>
+    <%== lag_with_label details.lag_hybrid %>
   </td>
   <td>
     <%== offset_with_label topic_name, partition_id, details.committed_offset %>

data/lib/karafka/web/ui/pro/views/health/_tabs.erb

@@ -12,6 +12,15 @@
           </a>
         </li>
 
+        <li class="nav-item">
+          <a
+            class="nav-link <%= nav_class(include: 'lags') %>"
+            href="<%= root_path('health', 'lags') %>"
+          >
+            Lags
+          </a>
+        </li>
+
         <li class="nav-item">
           <a
             class="nav-link <%= nav_class(include: 'offsets') %>"

data/lib/karafka/web/ui/pro/views/health/changes.erb

@@ -1,28 +1,14 @@
 <%== view_title('Consumers groups changes details') %>
 
+<%== partial 'health/tabs' %>
+
 <% if @stats.empty? %>
   <%== partial 'health/no_data' %>
-<% else %>
-  <%== partial 'health/tabs' %>
 <% end %>
 
-
 <% @stats.each_with_index do |(cg_name, details), index| %>
   <div class="container mb-5">
-    <div class="row mb-4">
-      <div class="col-sm-8">
-        <h4 class="mb-4"><%= cg_name %></h4>
-      </div>
-
-      <div class="col-sm-4">
-        <span class="float-end">
-          Last rebalance:
-          <span class="badge bg-secondary">
-            <%== relative_time(details[:rebalanced_at]) %>
-          </span>
-        </span>
-      </div>
-    </div>
+    <%== partial 'health/consumer_group_header', locals: { cg_name: cg_name, details: details } %>
 
     <div class="row mb-3">
       <div class="col-sm-12">

data/lib/karafka/web/ui/pro/views/health/lags.erb

@@ -0,0 +1,52 @@
+<%== view_title('Consumers groups lags details') %>
+
+<%== partial 'health/tabs' %>
+
+<% if @stats.empty? %>
+  <%== partial 'health/no_data' %>
+<% end %>
+
+<% @stats.each_with_index do |(cg_name, details), index| %>
+  <div class="container mb-5">
+    <%== partial 'health/consumer_group_header', locals: { cg_name: cg_name, details: details } %>
+
+    <div class="row mb-3">
+      <div class="col-sm-12">
+        <% topics = details[:topics] %>
+        <% topics.each_with_index do |(topic_name, partitions), index| %>
+          <table class="processes bg-white table table-hover table-bordered table-striped align-middle <%= (index+1 < topics.size) ? 'mb-5' : 'mb-3' %>">
+            <thead>
+              <tr class="align-middle">
+                <th colspan="12">
+                  <h5 class="mb-0"><%= topic_name %></h4>
+                </th>
+              </tr>
+              <tr class="align-middle">
+                <th><%== sort_link('Partition', :id) %></th>
+                <th><%== sort_link('Lag', :lag) %></th>
+                <th><%== sort_link('Lag trend', :lag_d) %></th>
+                <th><%== sort_link('Lag stored', :lag_stored) %></th>
+                <th><%== sort_link('Lag stored trend', :lag_stored_d) %></th>
+                <th><%== sort_link(:poll_state) %></th>
+              </tr>
+            </thead>
+            <tbody>
+                <% partitions.each do |partition_id, details| %>
+                  <%==
+                    partial(
+                      'health/partition_lags',
+                      locals: {
+                        topic_name: topic_name,
+                        partition_id: partition_id,
+                        details: details
+                      }
+                    )
+                  %>
+                <% end %>
+            </tbody>
+          </table>
+        <% end %>
+      </div>
+    </div>
+  </div>
+<% end %>

data/lib/karafka/web/ui/pro/views/health/offsets.erb

@@ -1,27 +1,14 @@
 <%== view_title('Consumers groups offsets details') %>
 
+<%== partial 'health/tabs' %>
+
 <% if @stats.empty? %>
   <%== partial 'health/no_data' %>
-<% else %>
-  <%== partial 'health/tabs' %>
 <% end %>
 
 <% @stats.each_with_index do |(cg_name, details), index| %>
   <div class="container mb-5">
-    <div class="row mb-4">
-      <div class="col-sm-8">
-        <h4 class="mb-4"><%= cg_name %></h4>
-      </div>
-
-      <div class="col-sm-4">
-        <span class="float-end">
-          Last rebalance:
-          <span class="badge bg-secondary">
-            <%== relative_time(details[:rebalanced_at]) %>
-          </span>
-        </span>
-      </div>
-    </div>
+    <%== partial 'health/consumer_group_header', locals: { cg_name: cg_name, details: details } %>
 
     <div class="row mb-3">
       <div class="col-sm-12">
@@ -36,7 +23,7 @@
               </tr>
               <tr class="align-middle">
                 <th><%== sort_link('Partition', :id) %></th>
-                <th><%== sort_link(:lag_stored) %></th>
+                <th><%== sort_link('Lag', :lag_hybrid) %></th>
                 <th><%== sort_link(:committed_offset) %></th>
                 <th><%== sort_link('Committed offset change', :committed_offset_fd) %></th>
                 <th><%== sort_link(:stored_offset) %></th>

data/lib/karafka/web/ui/pro/views/health/overview.erb

@@ -1,27 +1,14 @@
 <%== view_title('Consumers groups overview') %>
 
+<%== partial 'health/tabs' %>
+
 <% if @stats.empty? %>
   <%== partial 'health/no_data' %>
-<% else %>
-  <%== partial 'health/tabs' %>
 <% end %>
 
 <% @stats.each_with_index do |(cg_name, details), index| %>
   <div class="container mb-5">
-    <div class="row mb-4">
-      <div class="col-sm-8">
-        <h4 class="mb-4"><%= cg_name %></h4>
-      </div>
-
-      <div class="col-sm-4">
-        <span class="float-end">
-          Last rebalance:
-          <span class="badge bg-secondary">
-            <%== relative_time(details[:rebalanced_at]) %>
-          </span>
-        </span>
-      </div>
-    </div>
+    <%== partial 'health/consumer_group_header', locals: { cg_name: cg_name, details: details } %>
 
     <div class="row mb-3">
       <div class="col-sm-12">
@@ -36,9 +23,7 @@
               </tr>
               <tr class="align-middle">
                 <th><%== sort_link('Partition', :id) %></th>
-                <th><%== sort_link(:lag_stored) %></th>
-                <th><%== sort_link('Lag stored trend', :lag_stored_d) %></th>
-                <th><%== sort_link(:committed_offset) %></th>
+                <th><%== sort_link('Lag', :lag_hybrid) %></th>
                 <th><%== sort_link(:stored_offset) %></th>
                 <th><%== sort_link(:fetch_state) %></th>
                 <th><%== sort_link(:poll_state) %></th>

data/lib/karafka/web/ui/views/consumers/_counters.erb

@@ -19,9 +19,9 @@
               </li>
               <li class="col-sm">
                 <div class="count mb-1">
-                  <%= number_with_delimiter @counters.lag_stored, ' ' %>
+                  <%= number_with_delimiter @counters.lag_hybrid, ' ' %>
                 </div>
-                <div class="desc">Lag stored</div>
+                <div class="desc">Total lag</div>
               </li>
               <li class="col-sm">
                 <a href="<%= root_path('jobs/running') %>">

data/lib/karafka/web/ui/views/consumers/index.erb

@@ -16,7 +16,7 @@
               <th class="col-sm-2"><%== sort_link('Started', :started_at, rev: true) %></th>
               <th class="col-sm-1">Memory</th>
               <th class="col-sm-1">Utilization</th>
-              <th class="col-sm-1"><%== sort_link(:lag_stored) %></th>
+              <th class="col-sm-1"><%== sort_link('Lag', :lag_hybrid) %></th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/views/dashboard/index.erb

@@ -23,7 +23,7 @@
         <ul class="nav nav-tabs" id="graphs1" role="tablist">
           <%== partial 'shared/tab_nav', locals: { title: 'Messages', id: 'messages', active: true } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Batches', id: 'batches' } %>
-          <%== partial 'shared/tab_nav', locals: { title: 'Lags stored', id: 'lags-stored' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Topics Lags', id: 'topics-lags' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Topics pace', id: 'topics-pace' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Max LSO time', id: 'max-lso-time' } %>
         </ul>
@@ -39,10 +39,10 @@
             <%== partial 'shared/chart', locals: { data: data, id: 'batches' } %>
           </div>
 
-          <div class="tab-pane" id="lags-stored" role="tabpanel">
+          <div class="tab-pane" id="topics-lags" role="tabpanel">
             <%== partial 'dashboard/feature_pro' %>
             <% data = { enqueued: set.call, busy: set.call }.to_json %>
-            <%== partial 'shared/chart', locals: { data: data, id: 'lags-stored', blurred: true } %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'topics-lags', blurred: true } %>
           </div>
 
           <div class="tab-pane" id="topics-pace" role="tabpanel">

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.8.0'
+    VERSION = '0.8.2'
   end
 end

data/lib/karafka/web.rb

@@ -15,8 +15,9 @@ module Karafka
     class << self
       # @return [WaterDrop::Producer, nil] waterdrop messages producer or nil if not yet fully
       #   initialized. It may not be fully initialized until the configuration is done
+      # @note Do NOT memoize producer as it may be updated after forking
       def producer
-        @producer ||= Web.config.producer
+        Web.config.producer
       end
 
       # @return [String] root path of this gem