karafka-web 0.8.0.rc1 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3e890ec145255109e1f29ea742e6e0ed2887b0b4ca1de6047973816307744c1
-  data.tar.gz: 55766edeb8f40c68c1200b577697728037882e2af3dd0dfb9723741be0343b7e
+  metadata.gz: 69148848d89de8a7c5a1cebec7d3f5c0c60e3bd85caa41eedea8b5082dd6bf85
+  data.tar.gz: c0f6acfdd3e732ebfc29a6e954cd8a1f8d2c75dff77eba28978a99d6394d4087
 SHA512:
-  metadata.gz: 29dccc30c89888815b33a703317ca44bcf1b0a5f37228e9db7ba26eeba2d3b5ce5a66f5246121c4b9c1bb9d248e9027f3e8992f324109fd807d62a94c70c47ff
-  data.tar.gz: 32ed3012d866810f85db46c8c82f20888693e1c533edb9bcdc54cf098c2bece4478e8fe82f8a3c62c313f219d622f8b42e25bd4f38793d82951dec311f164c5c
+  metadata.gz: a89e22500fc5fedf8e432b666a84cf092d13c9b2747a4d08dc983d8ffba2f077781f06a131631b6315c64764bc00b7d06265a4d9fcd8c11bdd74d47eda5041f9
+  data.tar.gz: 7d29af2874b1df83af78fee69af89021bdcd1fa705686370f59ab276ee6808eef98e4a0b98dd273158888b472cea203bcca84f494e84c178f7733404b080166a

data/CHANGELOG.md

@@ -1,6 +1,14 @@
 # Karafka Web changelog
 
-## 0.8.0 (Unreleased)
+## 0.8.1 (2024-02-01)
+- [Enhancement] Introduce "Lags" health view.
+- [Enhancement] Remove "Stored Lag" and "Committed Offset" from Health Overview due to Lags Tab.
+- [Enhancement] Report lag on consumers that did not yet marked offsets.
+- [Enhancement] Use more accurate lag reporting that compensates for lack of stored lag.
+- [Fix] When first message after process start is crashed without DLQ lag is not reported.
+- [Fix] Wrong order of enabled injection causes fresh install to crash.
+
+## 0.8.0 (2024-01-26)
 - **[Feature]** Provide ability to sort table data for part of the views (note: not all attributes can be sorted due to technical limitations of sub-components fetching from Kafka).
 - **[Feature]** Track and report pause timeouts via "Changes" view in Health.
 - **[Feature]** Introduce pending jobs visibility alongside of running jobs both in total and per process.

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.8.0.rc1)
+    karafka-web (0.8.1)
       erubi (~> 1.4)
-      karafka (>= 2.3.0.rc1, < 2.4.0)
-      karafka-core (>= 2.3.0.rc1, < 2.4.0)
+      karafka (>= 2.3.0, < 2.4.0)
+      karafka-core (>= 2.3.0, < 2.4.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
 
@@ -36,13 +36,13 @@ GEM
     ffi (1.16.3)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka (2.3.0.rc1)
-      karafka-core (>= 2.3.0.rc1, < 2.4.0)
+    karafka (2.3.0)
+      karafka-core (>= 2.3.0, < 2.4.0)
       waterdrop (>= 2.6.12, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.3.0.rc1)
-      karafka-rdkafka (>= 0.14.7, < 0.15.0)
-    karafka-rdkafka (0.14.7)
+    karafka-core (2.3.0)
+      karafka-rdkafka (>= 0.14.8, < 0.15.0)
+    karafka-rdkafka (0.14.8)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)

data/karafka-web.gemspec

@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0 Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.3.0.rc1', '< 2.4.0'
-  spec.add_dependency 'karafka-core', '>= 2.3.0.rc1', '< 2.4.0'
+  spec.add_dependency 'karafka', '>= 2.3.0', '< 2.4.0'
+  spec.add_dependency 'karafka-core', '>= 2.3.0', '< 2.4.0'
   spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'
 

data/lib/karafka/web/config.rb

@@ -6,6 +6,10 @@ module Karafka
     class Config
       include ::Karafka::Core::Configurable
 
+      # Is the Web UI enabled and things were configured.
+      # Automatically set to true in case things got enabled.
+      setting :enabled, default: false
+
       # How long do we consider the process alive without receiving status info about it
       # For this long we also display dead processes (shutdown) in the UI
       # This is used both in the processing for eviction and in the UI

data/lib/karafka/web/contracts/config.rb

@@ -10,6 +10,7 @@ module Karafka
         # Use the same regexp as Karafka for topics validation
         TOPIC_REGEXP = ::Karafka::Contracts::TOPIC_REGEXP
 
+        required(:enabled) { |val| [true, false, nil].include?(val) }
         required(:ttl) { |val| val.is_a?(Numeric) && val.positive? }
 
         nested(:topics) do

data/lib/karafka/web/installer.rb

@@ -15,17 +15,18 @@ module Karafka
         puts
         puts 'Installing Karafka Web UI...'
         puts
+        Management::Actions::ExtendBootFile.new.call
+        puts
         puts 'Creating necessary topics and populating state data...'
         puts
         Management::Actions::CreateTopics.new.call(replication_factor)
         wait_for_topics
+        enable!
         Management::Actions::CreateInitialStates.new.call
         puts
         puts 'Running data migrations...'
         Management::Actions::MigrateStatesData.new.call
         puts
-        Management::Actions::ExtendBootFile.new.call
-        puts
         puts("Installation #{green('completed')}. Have fun!")
         puts
       end

data/lib/karafka/web/management/actions/enable.rb

@@ -10,6 +10,11 @@ module Karafka
         class Enable < Base
           # Enables routing consumer group and subscribes Web-UI listeners
           def call
+            # Prevent double enabling
+            return if ::Karafka::Web.config.enabled
+
+            ::Karafka::Web.config.enabled = true
+
             extend_routing
             setup_tracking_activity
 

data/lib/karafka/web/management/actions/extend_boot_file.rb

@@ -22,7 +22,9 @@ module Karafka
 
           # Adds needed code
           def call
-            if File.read(Karafka.boot_file).include?(ENABLER_CODE)
+            # We detect this that way so in case our template or user has enabled as a comment
+            # it still adds the template and runs install
+            if File.readlines(Karafka.boot_file).any? { |line| line.start_with?(ENABLER_CODE) }
               puts "Web UI #{already} installed."
             else
               puts 'Updating the Karafka boot file...'

data/lib/karafka/web/management/migrations/1706607960_introduce_lag_total_in_metrics.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Moves to using lag total as a normalization for both lags
+        class IntroduceLagTotalInMetrics < Base
+          self.versions_until = '1.2.0'
+          self.type = :consumers_metrics
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:aggregated].each_value do |metrics|
+              metrics.each do |metric|
+                metric.last[:lag_total] = metric.last[:lag_stored]
+                metric.last.delete(:lag_stored)
+                metric.last.delete(:lag)
+              end
+            end
+
+            state[:consumer_groups].each_value do |metrics|
+              metrics.each do |metric_group|
+                metric_group.last.each_value do |metric|
+                  metric.each_value do |sample|
+                    sample[:lag_total] = sample[:lag_stored]
+                    sample.delete(:lag_stored)
+                    sample.delete(:lag)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1706607960_introduce_lag_total_in_states.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Moves to using lag total as a normalization for both lags
+        class IntroduceLagTotalInStates < Base
+          self.versions_until = '1.3.0'
+          self.type = :consumers_state
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:stats][:lag_total] = state[:stats][:lag_stored]
+            state[:stats].delete(:lag)
+            state[:stats].delete(:lag_stored)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1706611396_rename_lag_total_to_lag_hybrid_in_metrics.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Renames total lag to hybrid to better represent what it is
+        class RenameLagTotalToLagHybridInMetrics < Base
+          self.versions_until = '1.2.1'
+          self.type = :consumers_metrics
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:aggregated].each_value do |metrics|
+              metrics.each do |metric|
+                metric.last[:lag_hybrid] = metric.last[:lag_total] || 0
+                metric.last.delete(:lag_total)
+              end
+            end
+
+            state[:consumer_groups].each_value do |metrics|
+              metrics.each do |metric_group|
+                metric_group.last.each_value do |metric|
+                  metric.each_value do |sample|
+                    sample[:lag_hybrid] = sample[:lag_total]
+                    sample.delete(:lag_total)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/management/migrations/1706611396_rename_lag_total_to_lag_hybrid_in_states.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Management
+      module Migrations
+        # Renames total lag to hybrid to better represent what it is
+        class RenameLagTotalToLagHybridInStates < Base
+          self.versions_until = '1.3.1'
+          self.type = :consumers_state
+
+          # @param state [Hash]
+          def migrate(state)
+            state[:stats][:lag_hybrid] = state[:stats][:lag_total] || 0
+            state[:stats].delete(:lag_total)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -11,7 +11,7 @@ module Karafka
           class Metrics < Base
             # Current schema version
             # This is used for detecting incompatible changes and writing migrations
-            SCHEMA_VERSION = '1.1.2'
+            SCHEMA_VERSION = '1.2.1'
 
             def initialize
               super
@@ -92,12 +92,11 @@ module Karafka
               cgs = {}
 
               iterate_partitions_data do |group_name, topic_name, partitions_data|
-                lags = partitions_data
-                       .map { |p_details| p_details.fetch(:lag, -1) }
-                       .reject(&:negative?)
-
-                lags_stored = partitions_data
-                              .map { |p_details| p_details.fetch(:lag_stored, -1) }
+                lags_hybrid = partitions_data
+                              .map do |p_details|
+                                lag_stored = p_details.fetch(:lag_stored, -1)
+                                lag_stored.negative? ? p_details.fetch(:lag, -1) : lag_stored
+                              end
                               .reject(&:negative?)
 
                 offsets_hi = partitions_data
@@ -121,8 +120,7 @@ module Karafka
 
                 cgs[group_name] ||= {}
                 cgs[group_name][topic_name] = {
-                  lag_stored: lags_stored.sum,
-                  lag: lags.sum,
+                  lag_hybrid: lags_hybrid.sum,
                   pace: offsets_hi.sum,
                   # Take max last stable offset duration without any change. This can
                   # indicate a hanging transaction, because the offset will not move forward

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -20,7 +20,7 @@ module Karafka
             # Current schema version
             # This can be used in the future for detecting incompatible changes and writing
             # migrations
-            SCHEMA_VERSION = '1.2.2'
+            SCHEMA_VERSION = '1.3.1'
 
             # @param schema_manager [Karafka::Web::Processing::Consumers::SchemaManager] schema
             #   manager that tracks the compatibility of schemas.
@@ -127,8 +127,7 @@ module Karafka
               stats[:processes] = 0
               stats[:rss] = 0
               stats[:listeners] = { active: 0, standby: 0 }
-              stats[:lag] = 0
-              stats[:lag_stored] = 0
+              stats[:lag_hybrid] = 0
               stats[:bytes_received] = 0
               stats[:bytes_sent] = 0
               utilization = 0
@@ -140,12 +139,13 @@ module Karafka
                   report_stats = report[:stats]
                   report_process = report[:process]
 
-                  lags = []
-                  lags_stored = []
+                  lags_hybrid = []
 
                   iterate_partitions(report) do |partition_stats|
-                    lags << partition_stats[:lag]
-                    lags_stored << partition_stats[:lag_stored]
+                    lag_stored = partition_stats[:lag_stored]
+                    lag = partition_stats[:lag]
+
+                    lags_hybrid << (lag_stored.negative? ? lag : lag_stored)
                   end
 
                   stats[:busy] += report_stats[:busy]
@@ -157,8 +157,7 @@ module Karafka
                   stats[:listeners][:standby] += report_process[:listeners][:standby]
                   stats[:processes] += 1
                   stats[:rss] += report_process[:memory_usage]
-                  stats[:lag] += lags.compact.reject(&:negative?).sum
-                  stats[:lag_stored] += lags_stored.compact.reject(&:negative?).sum
+                  stats[:lag_hybrid] += lags_hybrid.compact.reject(&:negative?).sum
                   utilization += report_stats[:utilization]
                 end
 

data/lib/karafka/web/processing/consumers/contracts/aggregated_stats.rb

@@ -21,8 +21,7 @@ module Karafka
             required(:processes) { |val| val.is_a?(Integer) && val >= 0 }
             required(:rss) { |val| val.is_a?(Numeric) && val >= 0 }
             required(:utilization) { |val| val.is_a?(Numeric) && val >= 0 }
-            required(:lag_stored) { |val| val.is_a?(Integer) }
-            required(:lag) { |val| val.is_a?(Integer) }
+            required(:lag_hybrid) { |val| val.is_a?(Integer) }
 
             nested(:listeners) do
               required(:active) { |val| val.is_a?(Integer) && val >= 0 }

data/lib/karafka/web/processing/consumers/contracts/topic_stats.rb

@@ -9,8 +9,7 @@ module Karafka
           class TopicStats < Web::Contracts::Base
             configure
 
-            required(:lag_stored) { |val| val.is_a?(Integer) }
-            required(:lag) { |val| val.is_a?(Integer) }
+            required(:lag_hybrid) { |val| val.is_a?(Integer) }
             required(:pace) { |val| val.is_a?(Integer) }
             required(:ls_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
           end

data/lib/karafka/web/ui/controllers/consumers.rb

@@ -9,7 +9,7 @@ module Karafka
           self.sortable_attributes = %w[
             name
             started_at
-            lag_stored
+            lag_hybrid
           ].freeze
 
           # List page with consumers

data/lib/karafka/web/ui/models/metrics/charts/topics.rb

@@ -16,17 +16,17 @@ module Karafka
 
               # @return [String] JSON with lags of each of the topics + total lag of all the topics
               #   from all the consumer groups.
-              def lags_stored
+              def lags_hybrid
                 total = Hash.new { |h, v| h[v] = 0 }
 
                 @data.to_h.each_value do |metrics|
                   metrics.each do |metric|
                     time = metric.first
-                    lag_stored = metric.last[:lag_stored]
+                    lag_hybrid = metric.last[:lag_hybrid]
 
-                    if lag_stored
+                    if lag_hybrid
                       total[time] ||= 0
-                      total[time] += lag_stored
+                      total[time] += lag_hybrid
                     else
                       next if total.key?(time)
 
@@ -37,7 +37,7 @@ module Karafka
 
                 # Extract the lag stored only from all the data
                 per_topic = @data.to_h.map do |topic, metrics|
-                  extracted = metrics.map { |metric| [metric.first, metric.last[:lag_stored]] }
+                  extracted = metrics.map { |metric| [metric.first, metric.last[:lag_hybrid]] }
 
                   [topic, extracted]
                 end.to_h

data/lib/karafka/web/ui/models/partition.rb

@@ -6,6 +6,28 @@ module Karafka
       module Models
         # Single topic partition data representation model
         class Partition < Lib::HashProxy
+          # @param args [Object] anything hash proxy accepts
+          def initialize(*args)
+            super
+
+            # We initialize those here because we want to have them stored in the internal has
+            # for sorting
+            lag_hybrid
+            lag_hybrid_d
+          end
+
+          # Because `lag_stored` is not available until first marking, we fallback to the lag
+          #   value that may be behind but is always available until stored lag is available.
+          # @return [Integer] hybrid log value
+          def lag_hybrid
+            self[:lag_hybrid] ||= lag_stored.negative? ? lag : lag_stored
+          end
+
+          # @return [Integer] hybrid log delta
+          def lag_hybrid_d
+            self[:lag_hybrid_d] ||= lag_stored.negative? ? lag_d : lag_stored_d
+          end
+
           # @return [Symbol] one of three states in which LSO can be in the correlation to given
           #   partition in the context of a consumer group.
           #

data/lib/karafka/web/ui/models/process.rb

@@ -41,24 +41,13 @@ module Karafka
               .then { |jobs| Jobs.new(jobs) }
           end
 
-          # @return [Integer] collective stored lag on this process
-          def lag_stored
+          # @return [Integer] collective hybrid lag on this process
+          def lag_hybrid
             consumer_groups
               .flat_map(&:subscription_groups)
               .flat_map(&:topics)
               .flat_map(&:partitions)
-              .map(&:lag_stored)
-              .delete_if(&:negative?)
-              .sum
-          end
-
-          # @return [Integer] collective lag on this process
-          def lag
-            consumer_groups
-              .flat_map(&:subscription_groups)
-              .flat_map(&:topics)
-              .flat_map(&:partitions)
-              .map(&:lag)
+              .map(&:lag_hybrid)
               .delete_if(&:negative?)
               .sum
           end

data/lib/karafka/web/ui/pro/app.rb

@@ -163,6 +163,10 @@ module Karafka
             r.on 'health' do
               controller = Controllers::Health.new(params)
 
+              r.get 'lags' do
+                controller.lags
+              end
+
               r.get 'offsets' do
                 controller.offsets
               end

data/lib/karafka/web/ui/pro/controllers/consumers.rb

@@ -19,11 +19,11 @@ module Karafka
           # Controller for displaying consumers states and details about them
           class Consumers < Ui::Controllers::Base
             self.sortable_attributes = %w[
+              id
               name
               started_at
-              lag_stored
-              id
-              lag_stored_d
+              lag_hybrid
+              lag_hybrid_d
               committed_offset
               stored_offset
               fetch_state
@@ -93,7 +93,13 @@ module Karafka
 
               # We want to have sorting but on a per subscription group basis and not to sort
               # everything
-              @process.consumer_groups.each { |subscription_group| refine(subscription_group) }
+              @process.consumer_groups.each do |consumer_group|
+                # We need to initialize the whole structure so dynamic fields are also built into
+                # the underlying hashes for sorting
+                consumer_group.subscription_groups.flat_map(&:topics).flat_map(&:partitions)
+
+                refine(consumer_group)
+              end
 
               render
             end

data/lib/karafka/web/ui/pro/controllers/health.rb

@@ -20,8 +20,12 @@ module Karafka
           class Health < Ui::Controllers::Base
             self.sortable_attributes = %w[
               id
+              lag
+              lag_d
               lag_stored
               lag_stored_d
+              lag_hybrid
+              lag_hybrid_d
               committed_offset
               committed_offset_fd
               stored_offset
@@ -50,6 +54,14 @@ module Karafka
               render
             end
 
+            # Displays details about lags and their progression/statuses
+            def lags
+              # Same data as overview but presented differently
+              overview
+
+              render
+            end
+
             # Displays details about offsets and their progression/statuses
             def offsets
               # Same data as overview but presented differently

data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb

@@ -71,6 +71,6 @@
   </td>
 
   <td>
-    <%= process.lag_stored %>
+    <%= process.lag_hybrid %>
   </td>
 </tr>

data/lib/karafka/web/ui/pro/views/consumers/_counters.erb

@@ -19,9 +19,9 @@
               </li>
               <li class="col-sm">
                 <div class="count mb-1">
-                  <%= number_with_delimiter @counters.lag_stored, ' ' %>
+                  <%= number_with_delimiter @counters.lag_hybrid, ' ' %>
                 </div>
-                <div class="desc">Lag stored</div>
+                <div class="desc">Total lag</div>
               </li>
               <li class="col-sm">
                 <a href="<%= root_path('jobs/running') %>">

data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb

@@ -3,11 +3,11 @@
     <%= partition.id %>
   </td>
   <td>
-    <%== lag_with_label partition.lag_stored %>
+    <%== lag_with_label partition.lag_hybrid %>
   </td>
   <td>
-    <span class="badge <%= lag_trend_bg(partition.lag_stored_d) %>">
-      <%= partition.lag_stored_d %>
+    <span class="badge <%= lag_trend_bg(partition.lag_hybrid_d) %>">
+      <%= partition.lag_hybrid_d %>
     </span>
   </td>
   <td>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb

@@ -104,8 +104,8 @@
             <tr class="align-middle">
             </tr>
               <th><%== sort_link('Partition', :id) %></th>
-              <th><%== sort_link(:lag_stored) %></th>
-              <th><%== sort_link('Lag stored trend', :lag_stored_d) %></th>
+              <th><%== sort_link('Lag', :lag_hybrid) %></th>
+              <th><%== sort_link('Lag trend', :lag_hybrid_d) %></th>
               <th><%== sort_link(:committed_offset) %></th>
               <th><%== sort_link(:stored_offset) %></th>
               <th><%== sort_link(:fetch_state) %></th>

data/lib/karafka/web/ui/pro/views/consumers/index.erb

@@ -17,7 +17,7 @@
               <th class="col-sm-1">Memory</th>
               <th class="col-sm-1">Performance</th>
               <th class="col-sm-1">Load</th>
-              <th class="col-sm-1"><%== sort_link(:lag_stored) %></th>
+              <th class="col-sm-1"><%== sort_link('Lag', :lag_hybrid) %></th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/pro/views/dashboard/index.erb

@@ -13,7 +13,7 @@
         <ul class="nav nav-tabs" id="graphs1" role="tablist">
           <%== partial 'shared/tab_nav', locals: { title: 'Messages', id: 'messages', active: true } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Batches', id: 'batches' } %>
-          <%== partial 'shared/tab_nav', locals: { title: 'Lags stored', id: 'lags-stored' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Topics lags', id: 'topics-lags' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Topics pace', id: 'topics-pace' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Max LSO time', id: 'max-lso-time' } %>
         </ul>
@@ -29,8 +29,8 @@
             <%== partial 'shared/chart', locals: { data: data, id: 'batches' } %>
           </div>
 
-          <div class="tab-pane" id="lags-stored" role="tabpanel">
-            <%== partial 'shared/chart', locals: { data: @topics_charts.lags_stored, id: 'lags-stored' } %>
+          <div class="tab-pane" id="topics-lags" role="tabpanel">
+            <%== partial 'shared/chart', locals: { data: @topics_charts.lags_hybrid, id: 'topics-lags' } %>
           </div>
 
           <div class="tab-pane" id="topics-pace" role="tabpanel">

data/lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb

@@ -4,6 +4,14 @@
   </a>
 </li>
 
+<% if current_path.include?('/lags') %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('health', 'lags') %>">
+      Lags
+    </a>
+  </li>
+<% end %>
+
 <% if current_path.include?('/overview') %>
   <li class="breadcrumb-item">
     <a href="<%= root_path('health', 'overview') %>">

data/lib/karafka/web/ui/pro/views/health/_consumer_group_header.erb

@@ -0,0 +1,14 @@
+<div class="row mb-4">
+  <div class="col-sm-8">
+    <h4 class="mb-4"><%= cg_name %></h4>
+  </div>
+
+  <div class="col-sm-4">
+    <span class="float-end">
+      Last rebalance:
+      <span class="badge bg-secondary">
+        <%== relative_time(details[:rebalanced_at]) %>
+      </span>
+    </span>
+  </div>
+</div>

data/lib/karafka/web/ui/pro/views/health/_partition.erb

@@ -3,15 +3,7 @@
     <%= partition_id %>
   </td>
   <td>
-    <%== lag_with_label details.lag_stored %>
-  </td>
-  <td>
-    <span class="badge <%= lag_trend_bg(details.lag_stored_d) %>">
-      <%= details.lag_stored_d %>
-    </span>
-  </td>
-  <td>
-    <%== offset_with_label topic_name, partition_id, details.committed_offset %>
+    <%== lag_with_label details.lag_hybrid %>
   </td>
   <td>
     <%== offset_with_label topic_name, partition_id, details.stored_offset %>

data/lib/karafka/web/ui/pro/views/health/_partition_lags.erb

@@ -0,0 +1,24 @@
+<tr class="align-middle <%= lso_risk_state_bg(details) %> status-row-<%= details.process.status %>">
+  <td>
+    <%= partition_id %>
+  </td>
+  <td>
+    <%== lag_with_label details.lag %>
+  </td>
+  <td>
+    <span class="badge <%= lag_trend_bg(details.lag_d) %>">
+      <%= details.lag_d %>
+    </span>
+  </td>
+  <td>
+    <%== lag_with_label details.lag_stored %>
+  </td>
+  <td>
+    <span class="badge <%= lag_trend_bg(details.lag_stored_d) %>">
+      <%= details.lag_stored_d %>
+    </span>
+  </td>
+  <td>
+    <%== poll_state_with_change_time_label(details.poll_state, details.poll_state_ch) %>
+  </td>
+</tr>

data/lib/karafka/web/ui/pro/views/health/_partition_offset.erb

@@ -3,7 +3,7 @@
     <%= partition_id %>
   </td>
   <td>
-    <%== lag_with_label details.lag_stored %>
+    <%== lag_with_label details.lag_hybrid %>
   </td>
   <td>
     <%== offset_with_label topic_name, partition_id, details.committed_offset %>

data/lib/karafka/web/ui/pro/views/health/_tabs.erb

@@ -12,6 +12,15 @@
           </a>
         </li>
 
+        <li class="nav-item">
+          <a
+            class="nav-link <%= nav_class(include: 'lags') %>"
+            href="<%= root_path('health', 'lags') %>"
+          >
+            Lags
+          </a>
+        </li>
+
         <li class="nav-item">
           <a
             class="nav-link <%= nav_class(include: 'offsets') %>"

data/lib/karafka/web/ui/pro/views/health/changes.erb

@@ -6,23 +6,9 @@
   <%== partial 'health/tabs' %>
 <% end %>
 
-
 <% @stats.each_with_index do |(cg_name, details), index| %>
   <div class="container mb-5">
-    <div class="row mb-4">
-      <div class="col-sm-8">
-        <h4 class="mb-4"><%= cg_name %></h4>
-      </div>
-
-      <div class="col-sm-4">
-        <span class="float-end">
-          Last rebalance:
-          <span class="badge bg-secondary">
-            <%== relative_time(details[:rebalanced_at]) %>
-          </span>
-        </span>
-      </div>
-    </div>
+    <%== partial 'health/consumer_group_header', locals: { cg_name: cg_name, details: details } %>
 
     <div class="row mb-3">
       <div class="col-sm-12">

data/lib/karafka/web/ui/pro/views/health/lags.erb

@@ -0,0 +1,52 @@
+<%== view_title('Consumers groups lags details') %>
+
+<% if @stats.empty? %>
+  <%== partial 'health/no_data' %>
+<% else %>
+  <%== partial 'health/tabs' %>
+<% end %>
+
+<% @stats.each_with_index do |(cg_name, details), index| %>
+  <div class="container mb-5">
+    <%== partial 'health/consumer_group_header', locals: { cg_name: cg_name, details: details } %>
+
+    <div class="row mb-3">
+      <div class="col-sm-12">
+        <% topics = details[:topics] %>
+        <% topics.each_with_index do |(topic_name, partitions), index| %>
+          <table class="processes bg-white table table-hover table-bordered table-striped align-middle <%= (index+1 < topics.size) ? 'mb-5' : 'mb-3' %>">
+            <thead>
+              <tr class="align-middle">
+                <th colspan="12">
+                  <h5 class="mb-0"><%= topic_name %></h4>
+                </th>
+              </tr>
+              <tr class="align-middle">
+                <th><%== sort_link('Partition', :id) %></th>
+                <th><%== sort_link('Lag', :lag) %></th>
+                <th><%== sort_link('Lag trend', :lag_d) %></th>
+                <th><%== sort_link('Lag stored', :lag_stored) %></th>
+                <th><%== sort_link('Lag stored trend', :lag_stored_d) %></th>
+                <th><%== sort_link(:poll_state) %></th>
+              </tr>
+            </thead>
+            <tbody>
+                <% partitions.each do |partition_id, details| %>
+                  <%==
+                    partial(
+                      'health/partition_lags',
+                      locals: {
+                        topic_name: topic_name,
+                        partition_id: partition_id,
+                        details: details
+                      }
+                    )
+                  %>
+                <% end %>
+            </tbody>
+          </table>
+        <% end %>
+      </div>
+    </div>
+  </div>
+<% end %>

data/lib/karafka/web/ui/pro/views/health/offsets.erb

@@ -8,20 +8,7 @@
 
 <% @stats.each_with_index do |(cg_name, details), index| %>
   <div class="container mb-5">
-    <div class="row mb-4">
-      <div class="col-sm-8">
-        <h4 class="mb-4"><%= cg_name %></h4>
-      </div>
-
-      <div class="col-sm-4">
-        <span class="float-end">
-          Last rebalance:
-          <span class="badge bg-secondary">
-            <%== relative_time(details[:rebalanced_at]) %>
-          </span>
-        </span>
-      </div>
-    </div>
+    <%== partial 'health/consumer_group_header', locals: { cg_name: cg_name, details: details } %>
 
     <div class="row mb-3">
       <div class="col-sm-12">
@@ -36,7 +23,7 @@
               </tr>
               <tr class="align-middle">
                 <th><%== sort_link('Partition', :id) %></th>
-                <th><%== sort_link(:lag_stored) %></th>
+                <th><%== sort_link('Lag', :lag_hybrid) %></th>
                 <th><%== sort_link(:committed_offset) %></th>
                 <th><%== sort_link('Committed offset change', :committed_offset_fd) %></th>
                 <th><%== sort_link(:stored_offset) %></th>

data/lib/karafka/web/ui/pro/views/health/overview.erb

@@ -8,20 +8,7 @@
 
 <% @stats.each_with_index do |(cg_name, details), index| %>
   <div class="container mb-5">
-    <div class="row mb-4">
-      <div class="col-sm-8">
-        <h4 class="mb-4"><%= cg_name %></h4>
-      </div>
-
-      <div class="col-sm-4">
-        <span class="float-end">
-          Last rebalance:
-          <span class="badge bg-secondary">
-            <%== relative_time(details[:rebalanced_at]) %>
-          </span>
-        </span>
-      </div>
-    </div>
+    <%== partial 'health/consumer_group_header', locals: { cg_name: cg_name, details: details } %>
 
     <div class="row mb-3">
       <div class="col-sm-12">
@@ -36,9 +23,7 @@
               </tr>
               <tr class="align-middle">
                 <th><%== sort_link('Partition', :id) %></th>
-                <th><%== sort_link(:lag_stored) %></th>
-                <th><%== sort_link('Lag stored trend', :lag_stored_d) %></th>
-                <th><%== sort_link(:committed_offset) %></th>
+                <th><%== sort_link('Lag', :lag_hybrid) %></th>
                 <th><%== sort_link(:stored_offset) %></th>
                 <th><%== sort_link(:fetch_state) %></th>
                 <th><%== sort_link(:poll_state) %></th>

data/lib/karafka/web/ui/views/consumers/_counters.erb

@@ -19,9 +19,9 @@
               </li>
               <li class="col-sm">
                 <div class="count mb-1">
-                  <%= number_with_delimiter @counters.lag_stored, ' ' %>
+                  <%= number_with_delimiter @counters.lag_hybrid, ' ' %>
                 </div>
-                <div class="desc">Lag stored</div>
+                <div class="desc">Total lag</div>
               </li>
               <li class="col-sm">
                 <a href="<%= root_path('jobs/running') %>">

data/lib/karafka/web/ui/views/consumers/index.erb

@@ -16,7 +16,7 @@
               <th class="col-sm-2"><%== sort_link('Started', :started_at, rev: true) %></th>
               <th class="col-sm-1">Memory</th>
               <th class="col-sm-1">Utilization</th>
-              <th class="col-sm-1"><%== sort_link(:lag_stored) %></th>
+              <th class="col-sm-1"><%== sort_link('Lag', :lag_hybrid) %></th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/views/dashboard/index.erb

@@ -23,7 +23,7 @@
         <ul class="nav nav-tabs" id="graphs1" role="tablist">
           <%== partial 'shared/tab_nav', locals: { title: 'Messages', id: 'messages', active: true } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Batches', id: 'batches' } %>
-          <%== partial 'shared/tab_nav', locals: { title: 'Lags stored', id: 'lags-stored' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Topics Lags', id: 'topics-lags' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Topics pace', id: 'topics-pace' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Max LSO time', id: 'max-lso-time' } %>
         </ul>
@@ -39,10 +39,10 @@
             <%== partial 'shared/chart', locals: { data: data, id: 'batches' } %>
           </div>
 
-          <div class="tab-pane" id="lags-stored" role="tabpanel">
+          <div class="tab-pane" id="topics-lags" role="tabpanel">
             <%== partial 'dashboard/feature_pro' %>
             <% data = { enqueued: set.call, busy: set.call }.to_json %>
-            <%== partial 'shared/chart', locals: { data: data, id: 'lags-stored', blurred: true } %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'topics-lags', blurred: true } %>
           </div>
 
           <div class="tab-pane" id="topics-pace" role="tabpanel">

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.8.0.rc1'
+    VERSION = '0.8.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.8.0.rc1
+  version: 0.8.1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-21 00:00:00.000000000 Z
+date: 2024-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.rc1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.rc1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.rc1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.rc1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -205,6 +205,10 @@ files:
 - lib/karafka/web/management/migrations/1700234522_remove_processing_from_consumers_state.rb
 - lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_metrics.rb
 - lib/karafka/web/management/migrations/1704722380_split_listeners_into_active_and_paused_in_states.rb
+- lib/karafka/web/management/migrations/1706607960_introduce_lag_total_in_metrics.rb
+- lib/karafka/web/management/migrations/1706607960_introduce_lag_total_in_states.rb
+- lib/karafka/web/management/migrations/1706611396_rename_lag_total_to_lag_hybrid_in_metrics.rb
+- lib/karafka/web/management/migrations/1706611396_rename_lag_total_to_lag_hybrid_in_states.rb
 - lib/karafka/web/management/migrator.rb
 - lib/karafka/web/processing/consumer.rb
 - lib/karafka/web/processing/consumers/aggregators/base.rb
@@ -369,12 +373,15 @@ files:
 - lib/karafka/web/ui/pro/views/explorer/topic/_empty.erb
 - lib/karafka/web/ui/pro/views/explorer/topic/_limited.erb
 - lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb
+- lib/karafka/web/ui/pro/views/health/_consumer_group_header.erb
 - lib/karafka/web/ui/pro/views/health/_no_data.erb
 - lib/karafka/web/ui/pro/views/health/_partition.erb
+- lib/karafka/web/ui/pro/views/health/_partition_lags.erb
 - lib/karafka/web/ui/pro/views/health/_partition_offset.erb
 - lib/karafka/web/ui/pro/views/health/_partition_times.erb
 - lib/karafka/web/ui/pro/views/health/_tabs.erb
 - lib/karafka/web/ui/pro/views/health/changes.erb
+- lib/karafka/web/ui/pro/views/health/lags.erb
 - lib/karafka/web/ui/pro/views/health/offsets.erb
 - lib/karafka/web/ui/pro/views/health/overview.erb
 - lib/karafka/web/ui/pro/views/jobs/_job.erb