karafka-web 0.6.0 → 0.6.2

data/lib/karafka/web/ui/pro/controllers/explorer.rb

@@ -18,10 +18,11 @@ module Karafka
         module Controllers
           # Data explorer controller
           class Explorer < Ui::Controllers::Base
+            include Ui::Lib::Paginations
+
             # Lists all the topics we can explore
             def index
-              @topics = Karafka::Admin
-                        .cluster_info
+              @topics = Models::ClusterInfo
                         .topics
                         .reject { |topic| topic[:topic_name] == '__consumer_offsets' }
                         .sort_by { |topic| topic[:topic_name] }
@@ -29,6 +30,34 @@ module Karafka
               respond
             end
 
+            # Displays aggregated messages from (potentially) all partitions of a topic
+            #
+            # @param topic_id [String]
+            #
+            # @note This view may not be 100% accurate because we merge multiple partitions data
+            #   into a single view and this is never accurate. It can be used however to quickly
+            #   look at most recent data flowing, etc, hence it is still useful for aggregated
+            #   metrics information
+            #
+            # @note We cannot use offset references here because each of the partitions may have
+            #   completely different values
+            def topic(topic_id)
+              @topic_id = topic_id
+              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+
+              @active_partitions, materialized_page, @limited = Paginators::Partitions.call(
+                @partitions_count, @params.current_page
+              )
+
+              @messages, next_page = Models::Message.topic_page(
+                topic_id, @active_partitions, materialized_page
+              )
+
+              paginate(@params.current_page, next_page)
+
+              respond
+            end
+
             # Shows messages available in a given partition
             #
             # @param topic_id [String]
@@ -36,13 +65,16 @@ module Karafka
             def partition(topic_id, partition_id)
               @topic_id = topic_id
               @partition_id = partition_id
-
               @watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
 
-              @previous_page, @messages, @next_page, @partitions_count = Ui::Models::Message.page(
-                @topic_id,
-                @partition_id,
-                @params.current_page
+              previous_offset, @messages, next_offset = current_partition_data
+
+              paginate(
+                previous_offset,
+                @params.current_offset,
+                next_offset,
+                @messages.map(&:offset)
               )
 
               respond
@@ -74,6 +106,19 @@ module Karafka
 
               respond
             end
+
+            private
+
+            # Fetches current page data
+            # @return [Array] fetched data with pagination information for the requested partition
+            def current_partition_data
+              Ui::Models::Message.offset_page(
+                @topic_id,
+                @partition_id,
+                @params.current_offset,
+                @watermark_offsets
+              )
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb

@@ -113,7 +113,12 @@
               </span>
 
               <span class="badge bg-secondary">
-                karafka core
+                karafka-web
+                <%= @process.karafka_web %>
+              </span>
+
+              <span class="badge bg-secondary">
+                karafka-core
                 <%= @process.karafka_core %>
               </span>
 

data/lib/karafka/web/ui/pro/views/errors/_breadcrumbs.erb

@@ -1,14 +1,16 @@
 <li class="breadcrumb-item">
-  <a href="<%= root_path('errors', 0) %>">
+  <a href="<%= root_path('errors') %>">
     Errors
   </a>
 </li>
 
-<li class="breadcrumb-item">
-  <a href="<%= root_path('errors', @partition_id) %>">
-    Partition <%= @partition_id %>
-  </a>
-</li>
+<% if @partition_id %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('errors', @partition_id) %>">
+      Partition <%= @partition_id %>
+    </a>
+  </li>
+<% end %>
 
 <% if @offset %>
   <li class="breadcrumb-item">

data/lib/karafka/web/ui/pro/views/errors/_error.erb

@@ -1,4 +1,4 @@
-<% if error_msg.is_a?(Integer) %>
+<% if error_msg.is_a?(Array) %>
   <tr>
     <td colspan="5" class="text-center text-muted">
       This error has either been removed or compacted and is no longer available.

data/lib/karafka/web/ui/pro/views/errors/_partition_option.erb

@@ -3,5 +3,5 @@
 <% else %>
   <option value="<%= root_path('errors', partition) %>">
 <% end %>
-  <%= partition %>
+  <%= partition || 'All' %>
 </option>

data/lib/karafka/web/ui/pro/views/errors/_table.erb

@@ -0,0 +1,21 @@
+<table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
+  <thead>
+    <tr class="align-middle">
+      <th>Origin</th>
+      <th>Process name</th>
+      <th>Error</th>
+      <th>Occurred at</th>
+      <th></th>
+    </tr>
+  </thead>
+
+  <tbody>
+    <%==
+      each_partial(
+        @error_messages,
+        'errors/error',
+        local: :error_msg
+      )
+    %>
+  </tbody>
+</table>

data/lib/karafka/web/ui/pro/views/errors/_title_with_select.erb

@@ -0,0 +1,31 @@
+<div class="container mb-4">
+  <div class="row">
+    <div class="col">
+      <h3>
+        Errors
+      </h3>
+    </div>
+
+    <div class="col">
+        <div class="col-auto text-end">
+           <label class="col-form-label">Partition</label>
+        </div>
+    </div>
+
+    <div class="col pt-1 mb-0 pb-0">
+        <div class="col-auto">
+          <select class="form-select form-select-sm mb-0 form-control" id="current-partition">
+            <%==
+              each_partial(
+                [nil] + @partitions_count.times.to_a,
+                'errors/partition_option',
+                local: :partition
+              )
+            %>
+          </select>
+        </div>
+    </div>
+  </div>
+
+  <hr>
+</div>

data/lib/karafka/web/ui/pro/views/errors/index.erb

@@ -1,65 +1,18 @@
-<div class="container mb-4">
-  <div class="row">
-    <div class="col">
-      <h3>
-        Errors
-      </h3>
-    </div>
-
-    <div class="col">
-        <div class="col-auto text-end">
-           <label class="col-form-label">Partition</label>
-        </div>
-    </div>
-
-    <div class="col pt-1 mb-0 pb-0">
-        <div class="col-auto">
-          <select class="form-select form-select-sm mb-0 form-control" id="current-partition">
-            <%==
-              each_partial(
-                @partitions_count.times.to_a,
-                'errors/partition_option',
-                local: :partition
-              )
-            %>
-          </select>
-        </div>
-    </div>
-  </div>
-
-  <hr>
-</div>
+<%== partial('errors/title_with_select') %>
 
 <div class="container">
   <div class="row mb-5">
     <div class="col-sm-12">
-      <%== partial('explorer/partition/watermark_offsets') %>
+      <%== partial('explorer/topic/partitions') %>
+
+      <% if @limited %>
+        <%== partial('explorer/topic/limited') %>
+      <% end %>
 
-      <% if @watermark_offsets.empty? %>
-        <%== partial 'errors/no_errors' %>
-      <% elsif @watermark_offsets.cleaned? %>
-        <%== partial 'errors/cleaned' %>
+      <% if @error_messages.empty? && params.current_page == 1 %>
+        <%== partial 'explorer/topic/empty' %>
       <% else %>
-        <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
-          <thead>
-            <tr class="align-middle">
-              <th>Origin</th>
-              <th>Process name</th>
-              <th>Error</th>
-              <th>Occurred at</th>
-              <th></th>
-            </tr>
-          </thead>
-          <tbody>
-            <%==
-              each_partial(
-                @error_messages,
-                'errors/error',
-                local: :error_msg
-              )
-            %>
-          </tbody>
-        </table>
+        <%== partial 'errors/table' %>
       <% end %>
     </div>
   </div>

data/lib/karafka/web/ui/pro/views/errors/partition.erb

@@ -0,0 +1,17 @@
+<%== partial('errors/title_with_select') %>
+
+<div class="container">
+  <div class="row mb-5">
+    <div class="col-sm-12">
+      <%== partial('explorer/partition/watermark_offsets') %>
+
+      <% if @watermark_offsets.empty? %>
+        <%== partial 'errors/no_errors' %>
+      <% elsif @watermark_offsets.cleaned? %>
+        <%== partial 'errors/cleaned' %>
+      <% else %>
+        <%== partial 'errors/table' %>
+      <% end %>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/ui/pro/views/explorer/_breadcrumbs.erb

@@ -6,7 +6,7 @@
 
 <% if @topic_id %>
   <li class="breadcrumb-item">
-    <a href="<%= root_path('explorer', @topic_id, 0) %>">
+    <a href="<%= root_path('explorer', @topic_id) %>">
       <%= @topic_id %>
     </a>
   </li>

data/lib/karafka/web/ui/pro/views/explorer/_message.erb

@@ -1,7 +1,10 @@
-<% if message.is_a?(Integer) %>
+<% if message.is_a?(Array) %>
   <tr>
     <td class="text-muted">
-      <%= message %>
+      <%= message[0] %>
+    </td>
+    <td class="text-muted">
+      <%= message[1] %>
     </td>
     <td colspan="3" class="text-center text-muted">
       This message has either been removed or compacted and is no longer available.
@@ -9,6 +12,9 @@
   </tr>
 <% else %>
   <tr>
+    <td>
+      <%= message.partition %>
+    </td>
     <td>
       <%= message.offset %>
     </td>

data/lib/karafka/web/ui/pro/views/explorer/_partition_option.erb

@@ -3,5 +3,5 @@
 <% else %>
   <option value="<%= root_path('explorer', @topic_id, partition) %>">
 <% end %>
-  <%= partition %>
+  <%= partition || 'All' %>
 </option>

data/lib/karafka/web/ui/pro/views/explorer/_topic.erb

@@ -2,7 +2,7 @@
   <div class="card" >
     <div class="card-body p-2">
       <p class="card-text mb-0 p-2">
-        <a href="<%= root_path('explorer', topic[:topic_name], 0) %>">
+        <a href="<%= root_path('explorer', topic[:topic_name]) %>">
           <%= topic[:topic_name] %> /
           <%= topic[:partition_count] %>
         </a>

data/lib/karafka/web/ui/pro/views/explorer/partition/_messages.erb

@@ -1,6 +1,7 @@
 <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
   <thead>
     <tr class="align-middle">
+      <th>Partition</th>
       <th>Offset</th>
       <th>Timestamp</th>
       <th>Key</th>

data/lib/karafka/web/ui/pro/views/explorer/partition.erb

@@ -17,7 +17,7 @@
           <select class="form-select form-select-sm mb-0 form-control" id="current-partition">
             <%==
               each_partial(
-                @partitions_count.times.to_a,
+                [nil] + @partitions_count.times.to_a,
                 'explorer/partition_option',
                 local: :partition
               )

data/lib/karafka/web/ui/pro/views/explorer/topic/_empty.erb

@@ -0,0 +1,3 @@
+<div class="alert alert-info mt-4" role="alert">
+  This topic is empty and does not contain any data in any of the partitions.
+</div>

data/lib/karafka/web/ui/pro/views/explorer/topic/_limited.erb

@@ -0,0 +1,4 @@
+<div class="alert alert-info mt-4 mb-4" role="alert">
+  Due to many partitions, only part of the data is visible on each page.
+  Please navigate through the pages to view all data.
+</div>

data/lib/karafka/web/ui/pro/views/explorer/topic/_partitions.erb

@@ -0,0 +1,11 @@
+<p class="text-end mb-4">
+  Partitions:
+  <span class="badge bg-secondary mt-1 mb-1">
+    total: <%= @partitions_count %>
+  </span>
+
+  <span class="badge bg-secondary mt-1 mb-1">
+    visible:
+    <%= [@active_partitions.first, @active_partitions.last].uniq.join(' to ') %>
+  </span>
+</p>

data/lib/karafka/web/ui/pro/views/explorer/topic.erb

@@ -0,0 +1,49 @@
+<div class="container mb-4">
+  <div class="row">
+    <div class="col">
+      <h3>
+        <%= @topic_id %>
+      </h3>
+    </div>
+
+    <div class="col">
+        <div class="col-auto text-end">
+           <label class="col-form-label">Partition</label>
+        </div>
+    </div>
+
+    <div class="col pt-1 mb-0 pb-0">
+        <div class="col-auto">
+          <select class="form-select form-select-sm mb-0 form-control" id="current-partition">
+            <%==
+              each_partial(
+                [nil] + @partitions_count.times.to_a,
+                'explorer/partition_option',
+                local: :partition
+              )
+            %>
+          </select>
+        </div>
+    </div>
+  </div>
+
+  <hr>
+</div>
+
+<div class="container">
+  <div class="row mb-5">
+    <div class="col-sm-12">
+      <%== partial('explorer/topic/partitions') %>
+
+      <% if @limited %>
+        <%== partial('explorer/topic/limited') %>
+      <% end %>
+
+      <% if @messages.empty? && params.current_page == 1 %>
+        <%== partial 'explorer/topic/empty' %>
+      <% else %>
+        <%== partial('explorer/partition/messages') %>
+      <% end %>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/ui/pro/views/shared/_navigation.erb

@@ -31,7 +31,7 @@
       </a>
     </li>
     <li class="nav-item ms-3">
-      <a class="nav-link <%= nav_class(start_with: '/errors') %>" href="<%= root_path('errors/0') %>">
+      <a class="nav-link <%= nav_class(start_with: '/errors') %>" href="<%= root_path('errors') %>">
         Errors
       </a>
     </li>

data/lib/karafka/web/ui/views/cluster/_partition.erb

@@ -3,7 +3,7 @@
 <% if topic[:topic_name] != '__consumer_offsets' %>
   <tr>
     <td>
-      <a href="<%= root_path('explorer', topic[:topic_name], 0) %>">
+      <a href="<%= root_path('explorer', topic[:topic_name]) %>">
         <%= topic[:topic_name] %>
       </a>
     <td>

data/lib/karafka/web/ui/views/errors/_error.erb

@@ -1,4 +1,4 @@
-<% if error_msg.is_a?(Integer) %>
+<% if error_msg.is_a?(Array) %>
   <tr>
     <td colspan="5" class="text-center text-muted">
       This error has either been removed or compacted and is no longer available.

data/lib/karafka/web/ui/views/shared/_pagination.erb

@@ -1,28 +1,32 @@
-<% if @current_page && (@current_page > 1 || @next_page) %>
+<% if @pagination && @pagination.paginate? %>
   <div id="pagination" class="container mb-5">
     <div class="row">
       <nav>
         <ul class="pagination justify-content-center">
-          <li class="page-item <%= 'disabled' if @current_page <= 1 %>">
-            <a class="page-link" href="<%= current_path(page: 1) %>">
+          <li class="page-item <%= 'disabled' unless @pagination.first_offset? %>">
+            <a class="page-link" href="<%= current_path(@pagination.offset_key => @pagination.first_offset) %>">
               <span>&laquo;</span>
             </a>
           </li>
 
-          <li class="page-item <%= 'disabled' if @current_page <= 1 %>">
-            <a class="page-link" href="<%= current_path(page: @current_page - 1) %>">
+          <li class="page-item <%= 'disabled' unless @pagination.previous_offset? %>">
+            <a class="page-link" href="<%= current_path(@pagination.offset_key => @pagination.previous_offset) %>">
               <span>&lsaquo; Prev</span>
             </a>
           </li>
 
-          <li class="page-item active disabled">
-            <a class="page-link" href="<%= current_path(page: @current_page) %>">
-              <span><%= @current_page %></span>
-            </a>
-          </li>
+          <% if @pagination.current_offset? %>
+            <li class="page-item active disabled">
+              <a class="page-link" href="<%= current_path(@pagination.offset_key => @pagination.current_offset) %>">
+                <span>
+                  <%= @pagination.current_label %>
+                </span>
+              </a>
+            </li>
+          <% end %>
 
-          <li class="page-item <%= 'disabled' unless @next_page %>">
-            <a class="page-link" href="<%= current_path(page: @next_page) %>">
+          <li class="page-item <%= 'disabled' unless @pagination.next_offset? %>">
+            <a class="page-link" href="<%= current_path(@pagination.offset_key => @pagination.next_offset) %>">
               <span>Next &rsaquo;</span>
             </a>
           </li>

data/lib/karafka/web/ui/views/status/failures/_initial_state.erb

@@ -7,14 +7,5 @@
 </p>
 
 <p>
-  To fix this, you can either:
-</p>
-
-<ul>
-  <li>Run <code>bundle exec karafka-web install</code></li>
-  <li>Run at least one Karafka consumer process</li>
-</ul>
-
-<p class="mb-0">
-  Any of the above, when successful, will bootstrap the initial state.
+  To fix this, you need to ensure that the <code>bundle exec karafka-web install</code> runs successfully.
 </p>

data/lib/karafka/web/ui/views/status/info/_components.erb

@@ -13,7 +13,12 @@
   </span>
 
   <span class="badge bg-secondary">
-    karafka core
+    karafka-web
+    <%= @sampler.karafka_web_version %>
+  </span>
+
+  <span class="badge bg-secondary">
+    karafka-core
     <%= @sampler.karafka_core_version %>
   </span>
 

data/lib/karafka/web/ui/views/status/show.erb

@@ -8,7 +8,12 @@
           "status/#{@status.connection.to_s}",
           locals: {
             title: 'Connection to Kafka',
-            description: partial('status/failures/connection')
+            description: partial(
+              "status/#{@status.connection.partial_namespace}/connection",
+              locals: {
+                details: @status.connection.details
+              }
+            )
           }
         )
       %>

data/lib/karafka/web/ui/views/status/successes/_connection.erb

@@ -0,0 +1 @@
+<%# Expected to be empty. No content on success needed. %>

data/lib/karafka/web/ui/views/status/warnings/_connection.erb

@@ -0,0 +1,11 @@
+<p>
+  High latency detected when connecting to the Kafka cluster from the Web-UI.
+</p>
+
+<p>
+  Kafka is not designed to perform optimally under high network latencies; even when configurations seem appropriate, you may still encounter disconnects and other unexpected issues.
+</p>
+
+<p class="mb-0">
+  It took Web-UI <strong><%= details[:time].round(2) %> ms</strong> to get the cluster info.
+</p>

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.6.0'
+    VERSION = '0.6.2'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-13 00:00:00.000000000 Z
+date: 2023-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -218,7 +218,15 @@ files:
 - lib/karafka/web/ui/controllers/status.rb
 - lib/karafka/web/ui/helpers/application_helper.rb
 - lib/karafka/web/ui/lib/hash_proxy.rb
-- lib/karafka/web/ui/lib/paginate_array.rb
+- lib/karafka/web/ui/lib/paginations/base.rb
+- lib/karafka/web/ui/lib/paginations/offset_based.rb
+- lib/karafka/web/ui/lib/paginations/page_based.rb
+- lib/karafka/web/ui/lib/paginations/paginators/arrays.rb
+- lib/karafka/web/ui/lib/paginations/paginators/base.rb
+- lib/karafka/web/ui/lib/paginations/paginators/partitions.rb
+- lib/karafka/web/ui/lib/paginations/paginators/sets.rb
+- lib/karafka/web/ui/lib/ttl_cache.rb
+- lib/karafka/web/ui/models/cluster_info.rb
 - lib/karafka/web/ui/models/consumer_group.rb
 - lib/karafka/web/ui/models/counters.rb
 - lib/karafka/web/ui/models/health.rb
@@ -264,7 +272,10 @@ files:
 - lib/karafka/web/ui/pro/views/errors/_breadcrumbs.erb
 - lib/karafka/web/ui/pro/views/errors/_error.erb
 - lib/karafka/web/ui/pro/views/errors/_partition_option.erb
+- lib/karafka/web/ui/pro/views/errors/_table.erb
+- lib/karafka/web/ui/pro/views/errors/_title_with_select.erb
 - lib/karafka/web/ui/pro/views/errors/index.erb
+- lib/karafka/web/ui/pro/views/errors/partition.erb
 - lib/karafka/web/ui/pro/views/errors/show.erb
 - lib/karafka/web/ui/pro/views/explorer/_breadcrumbs.erb
 - lib/karafka/web/ui/pro/views/explorer/_detail.erb
@@ -280,6 +291,10 @@ files:
 - lib/karafka/web/ui/pro/views/explorer/partition/_messages.erb
 - lib/karafka/web/ui/pro/views/explorer/partition/_watermark_offsets.erb
 - lib/karafka/web/ui/pro/views/explorer/show.erb
+- lib/karafka/web/ui/pro/views/explorer/topic.erb
+- lib/karafka/web/ui/pro/views/explorer/topic/_empty.erb
+- lib/karafka/web/ui/pro/views/explorer/topic/_limited.erb
+- lib/karafka/web/ui/pro/views/explorer/topic/_partitions.erb
 - lib/karafka/web/ui/pro/views/health/_breadcrumbs.erb
 - lib/karafka/web/ui/pro/views/health/_partition.erb
 - lib/karafka/web/ui/pro/views/health/index.erb
@@ -349,6 +364,8 @@ files:
 - lib/karafka/web/ui/views/status/failures/_topics.erb
 - lib/karafka/web/ui/views/status/info/_components.erb
 - lib/karafka/web/ui/views/status/show.erb
+- lib/karafka/web/ui/views/status/successes/_connection.erb
+- lib/karafka/web/ui/views/status/warnings/_connection.erb
 - lib/karafka/web/ui/views/status/warnings/_pro_subscription.erb
 - lib/karafka/web/version.rb
 - renovate.json