karafka-web 0.7.10 → 0.8.0.rc1

data/lib/karafka/web/ui/views/cluster/brokers.erb

@@ -0,0 +1,27 @@
+<%== view_title('Cluster informations') %>
+
+<%== partial 'cluster/tabs' %>
+
+<div class="container mb-5">
+  <div class="row">
+    <div class="col-lg-12">
+        <table class="processes bg-white table table-hover table-bordered table-striped">
+          <thead>
+            <tr class="align-middle">
+              <th><%== sort_link('Id', :broker_id) %></th>
+              <th><%== sort_link('Name', :broker_name) %></th>
+              <th><%== sort_link('Port', :broker_port) %></th>
+            </tr>
+          </thead>
+          <tbody>
+            <%==
+              each_partial(
+                @brokers,
+                'cluster/broker'
+              )
+            %>
+          </tbody>
+        </table>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/ui/views/cluster/topics.erb

@@ -0,0 +1,35 @@
+<%== view_title('Cluster informations') %>
+
+<%== partial 'cluster/tabs' %>
+
+<div class="container mb-5">
+  <div class="row">
+    <div class="col-lg-12">
+      <% if @partitions.empty? && params.current_page <= 1 %>
+        <%== partial 'cluster/no_partitions' %>
+      <% elsif @partitions.empty? %>
+        <%== partial 'shared/no_paginated_data' %>
+      <% else %>
+        <table class="processes bg-white table table-hover table-bordered table-striped">
+          <thead>
+            <tr class="align-middle">
+              <th><%== sort_link(:topic_name) %></th>
+              <th><%== sort_link(:partition_id) %></th>
+              <th><%== sort_link(:leader) %></th>
+              <th><%== sort_link(:replica_count) %></th>
+              <th><%== sort_link('In sync brokers', :in_sync_replica_brokers) %></th>
+            </tr>
+          </thead>
+          <tbody>
+            <%==
+              each_partial(
+                @partitions,
+                'cluster/partition'
+              )
+            %>
+          </tbody>
+        </table>
+      <% end %>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/ui/views/consumers/_counters.erb

@@ -24,18 +24,20 @@
                 <div class="desc">Lag stored</div>
               </li>
               <li class="col-sm">
-                <a href="<%= root_path('jobs') %>">
+                <a href="<%= root_path('jobs/running') %>">
                   <div class="count mb-1">
                     <%= number_with_delimiter @counters.busy, ' ' %>
                   </div>
-                  <div class="desc">Busy</div>
+                  <div class="desc">Running</div>
                 </a>
               </li>
               <li class="col-sm">
-                <div class="count mb-1">
-                  <%= number_with_delimiter @counters.enqueued, ' ' %>
-                </div>
-                <div class="desc">Enqueued</div>
+                <a href="<%= root_path('jobs/pending') %>">
+                  <div class="count mb-1">
+                    <%= number_with_delimiter @counters.pending, ' ' %>
+                  </div>
+                  <div class="desc">Pending</div>
+                </a>
               </li>
               <li class="col-sm">
                 <a href="<%= root_path('errors') %>">

data/lib/karafka/web/ui/views/consumers/_summary.erb

@@ -36,10 +36,10 @@
         <div class="card-body">
           <p class="card-text">
             <div class="stat mb-1">
-              <%= @counters.listeners %>
+              <%= @counters.listeners.fetch(:active) %>
             </div>
             <div class="desc">
-              Listeners
+              Active Listeners
             </div>
           </p>
         </div>

data/lib/karafka/web/ui/views/consumers/index.erb

@@ -12,11 +12,11 @@
         <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
           <thead>
             <tr class="align-middle">
-              <th>Name</th>
-              <th class="col-sm-2">Started</th>
+              <th><%== sort_link(:name) %></th>
+              <th class="col-sm-2"><%== sort_link('Started', :started_at, rev: true) %></th>
               <th class="col-sm-1">Memory</th>
               <th class="col-sm-1">Utilization</th>
-              <th class="col-sm-1">Total lag</th>
+              <th class="col-sm-1"><%== sort_link(:lag_stored) %></th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/views/dashboard/_ranges_selector.erb

@@ -2,20 +2,36 @@
   <div class="row">
     <div class="col-sm-12 text-end">
       <div class="btn-group btn-group-sm" role="group" aria-label="Small button group">
-        <a type="button" href="<%= root_path('dashboard') %>" class="btn btn-outline-primary active">
+        <%
+          path = root_path('dashboard?range=seconds')
+          active = params.current_range == :seconds ? 'active' : false
+        %>
+        <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
           5 minutes
         </a>
 
-        <a type="button" href="#" class="btn btn-outline-primary disabled" title="Pro feature">
-          <s>1 hour</s>
+        <%
+          path = root_path('dashboard?range=minutes')
+          active = params.current_range == :minutes ? 'active' : false
+        %>
+        <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
+          1 hour
         </a>
 
-        <a type="button" href="#" class="btn btn-outline-primary disabled" title="Pro feature">
-          <s>24 hours</s>
+        <%
+          path = root_path('dashboard?range=hours')
+          active = params.current_range == :hours ? 'active' : false
+        %>
+        <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
+          24 hours
         </a>
 
-        <a type="button" href="#" class="btn btn-outline-primary disabled" title="Pro feature">
-          <s>7 days</s>
+        <%
+          path = root_path('dashboard?range=days')
+          active = params.current_range == :days ? 'active' : false
+        %>
+        <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
+          7 days
         </a>
       </div>
     </div>

data/lib/karafka/web/ui/views/dashboard/index.erb

@@ -25,6 +25,7 @@
           <%== partial 'shared/tab_nav', locals: { title: 'Batches', id: 'batches' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Lags stored', id: 'lags-stored' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Topics pace', id: 'topics-pace' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Max LSO time', id: 'max-lso-time' } %>
         </ul>
 
         <div class="tab-content">
@@ -34,9 +35,8 @@
           </div>
 
           <div class="tab-pane" id="batches" role="tabpanel">
-            <%== partial 'dashboard/feature_pro' %>
-            <% data = { batches: set.call(20), errors: set.call, dead: set.call, retries: set.call }.to_json %>
-            <%== partial 'shared/chart', locals: { data: data, id: 'batches', blurred: true } %>
+            <% data = @aggregated_charts.with(:batches, :errors, :dead, :retries) %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'batches' } %>
           </div>
 
           <div class="tab-pane" id="lags-stored" role="tabpanel">
@@ -50,6 +50,12 @@
             <% data = { topic1: set.call(20), topic2: set.call(10), topic3: set.call(100) }.to_json %>
             <%== partial 'shared/chart', locals: { data: data, id: 'topics-pace', blurred: true } %>
           </div>
+
+          <div class="tab-pane" id="max-lso-time" role="tabpanel">
+            <%== partial 'dashboard/feature_pro' %>
+            <% data = { max_lso: set.call(2, 0..5) }.to_json %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'max-lso-time', blurred: true } %>
+          </div>
         </div>
       </div>
     </div>
@@ -66,13 +72,13 @@
           <%== partial 'shared/tab_nav', locals: { title: 'Utilization', id: 'utilization', active: true } %>
           <%== partial 'shared/tab_nav', locals: { title: 'RSS', id: 'rss' } %>
           <%== partial 'shared/tab_nav', locals: { title: 'Concurrency', id: 'concurrency' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Data transfers', id: 'data-transfers' } %>
         </ul>
 
         <div class="tab-content">
           <div class="tab-pane show active" id="utilization" role="tabpanel">
-            <%== partial 'dashboard/feature_pro' %>
-            <% data = { utilization: set.call(50) }.to_json %>
-            <%== partial 'shared/chart', locals: { data: data, id: 'utilization', blurred: true } %>
+            <% data = @aggregated_charts.with(:utilization) %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'utilization', label_type_y: 'percentage' } %>
           </div>
 
           <div class="tab-pane show" id="rss" role="tabpanel">
@@ -82,9 +88,14 @@
           </div>
 
           <div class="tab-pane show" id="concurrency" role="tabpanel">
+            <% data = @aggregated_charts.with(:processes, :workers, :active_listeners, :standby_listeners) %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'concurrency' } %>
+          </div>
+
+          <div class="tab-pane show" id="data-transfers" role="tabpanel">
             <%== partial 'dashboard/feature_pro' %>
-            <% data = { processes: set.call(2, 0..0), workers: set.call(4, 0..0), listeners: set.call(3, 0..0) }.to_json %>
-            <%== partial 'shared/chart', locals: { data: data, id: 'concurrency', blurred: true } %>
+            <% data = { bytes_received: set.call(4, 10..15), bytes_sent: set.call(2, 0..5) }.to_json %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'data-transfers', blurred: true } %>
           </div>
         </div>
       </div>

data/lib/karafka/web/ui/views/errors/show.erb

@@ -36,7 +36,7 @@
     </div>
   </div>
 
-  <div class="row mb-2">
+  <div class="row mb-4">
     <div class="col-sm-12">
       <h5 class="mb-2">
         Backtrace
@@ -45,32 +45,11 @@
     </div>
   </div>
 
-  <div class="mb-4">
-    <%== partial 'shared/feature_pro' %>
-  </div>
-
   <div class="row mb-5">
     <div class="col-sm-12">
       <div class="card">
         <div class="card-body">
-          <pre class="m-0 p-0 blurred"><code class="wrapped json p-0 m-0">this is just an example backtrace
-please subscribe to our Pro offering to be able to view the real one
-gems/karafka-rdkafka/lib/rdkafka/consumer.rb:255:in `query_watermark_offsets'
-gems/karafka/lib/karafka/admin.rb:56:in `block in read_topic'
-gems/karafka/lib/karafka/admin.rb:184:in `with_consumer'
-gems/karafka/lib/karafka/admin.rb:55:in `read_topic'
-/mnt/software/Karafka/karafka-web/lib/karafka/web/processing/consumers/state.rb:19:in `current'
-/mnt/software/Karafka/karafka-web/lib/karafka/web/processing/consumers/aggregator.rb:45:in `state'
-/mnt/software/Karafka/karafka-web/lib/karafka/web/processing/consumers/aggregator.rb:38:in `to_json'
-gems/karafka/lib/karafka/processing/strategies/default.rb:136:in `block in handle_shutdown'
-gems/karafka-core/lib/karafka/core/monitoring/notifications.rb:118:in `measure_time_taken'
-gems/karafka-core/lib/karafka/core/monitoring/notifications.rb:94:in `instrument'
-gems/karafka-core/lib/karafka/core/monitoring/monitor.rb:34:in `instrument'
-gems/karafka/lib/karafka/processing/strategies/default.rb:135:in `handle_shutdown'
-gems/karafka/lib/karafka/base_consumer.rb:134:in `on_shutdown'
-gems/karafka/lib/karafka/processing/executor.rb:123:in `shutdown'
-gems/karafka/lib/karafka/processing/jobs/shutdown.rb:18:in `call'
-gems/karafka/lib/karafka/helpers/async.rb:28:in `block in async_call'</code></pre>
+          <pre class="m-0 p-0"><code class="wrapped json p-0 m-0"><%= @error_message.payload[:backtrace] %></code></pre>
         </div>
       </div>
     </div>

data/lib/karafka/web/ui/views/jobs/_breadcrumbs.erb

@@ -1,5 +1,21 @@
 <li class="breadcrumb-item">
   <a href="<%= root_path('jobs') %>">
-    Running jobs
+    Jobs
   </a>
 </li>
+
+<% if current_path.include?('/running') %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('jobs/running') %>">
+      Running
+    </a>
+  </li>
+<% end %>
+
+<% if current_path.include?('/pending') %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('jobs/pending') %>">
+      Pending
+    </a>
+  </li>
+<% end %>

data/lib/karafka/web/ui/views/jobs/_job.erb

@@ -22,6 +22,6 @@
     <code>#<%= job.type %></code>
   </td>
   <td>
-    <%== relative_time job.started_at %>
+    <%== relative_time job.updated_at %>
   </td>
 </tr>

data/lib/karafka/web/ui/views/jobs/_no_jobs.erb

@@ -2,7 +2,7 @@
   <div class="row">
     <div class="col-lg-12">
       <div class="alert alert-info" role="alert">
-        There are no running jobs at the moment.
+        There are no <%= type %> jobs at the moment.
       </div>
     </div>
   </div>

data/lib/karafka/web/ui/views/jobs/_tabs.erb

@@ -0,0 +1,27 @@
+<div class="container">
+  <div class="row mb-5">
+    <div class="col-sm-12">
+
+      <ul class="nav nav-tabs">
+        <li class="nav-item">
+          <a
+            class="nav-link <%= nav_class(include: 'running') %>"
+            href="<%= root_path('jobs', 'running') %>"
+          >
+            Running (<%= @jobs_counters.running %>)
+          </a>
+        </li>
+
+        <li class="nav-item">
+          <a
+            class="nav-link <%= nav_class(include: 'pending') %>"
+            href="<%= root_path('jobs', 'pending') %>"
+          >
+            Pending (<%= @jobs_counters.pending %>)
+          </a>
+        </li>
+      </ul>
+
+    </div>
+  </div>
+</div>

data/lib/karafka/web/ui/views/jobs/{index.erb → pending.erb}

@@ -1,7 +1,9 @@
-<%== view_title('Running jobs', hr: true) %>
+<%== view_title('Pending jobs', hr: false) %>
+
+<%== partial 'jobs/tabs' %>
 
 <% if @jobs.empty? && params.current_page <= 1 %>
-  <%== partial 'jobs/no_jobs' %>
+  <%== partial 'jobs/no_jobs', locals: { type: 'pending' } %>
 <% elsif @jobs.empty? %>
   <%== partial 'shared/no_paginated_data' %>
 <% else %>
@@ -11,11 +13,11 @@
         <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
           <thead>
             <tr class="align-middle">
-              <th>Process</th>
-              <th>Topic</th>
-              <th>Consumer</th>
-              <th>Type</th>
-              <th>Started at</th>
+              <th><%== sort_link('Process', :name) %></th>
+              <th><%== sort_link(:topic) %></th>
+              <th><%== sort_link(:consumer) %></th>
+              <th><%== sort_link(:type) %></th>
+              <th><%== sort_link('Created at', :updated_at, rev: true) %></th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/{pro/views/jobs/index.erb → views/jobs/running.erb}

@@ -1,7 +1,9 @@
-<%== view_title('Running jobs', hr: true) %>
+<%== view_title('Running jobs', hr: false) %>
+
+<%== partial 'jobs/tabs' %>
 
 <% if @jobs.empty? && params.current_page <= 1 %>
-  <%== partial 'jobs/no_jobs' %>
+  <%== partial 'jobs/no_jobs', locals: { type: 'running' } %>
 <% elsif @jobs.empty? %>
   <%== partial 'shared/no_paginated_data' %>
 <% else %>
@@ -11,15 +13,11 @@
         <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
           <thead>
             <tr class="align-middle">
-              <th>Process</th>
-              <th>Topic</th>
-              <th>Consumer</th>
-              <th>Type</th>
-              <th>Messages</th>
-              <th>First offset</th>
-              <th>Last offset</th>
-              <th>Committed offset</th>
-              <th>Started at</th>
+              <th><%== sort_link('Process', :name) %></th>
+              <th><%== sort_link(:topic) %></th>
+              <th><%== sort_link(:consumer) %></th>
+              <th><%== sort_link(:type) %></th>
+              <th><%== sort_link('Started at', :updated_at, rev: true) %></th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/views/routing/_consumer_group.erb

@@ -13,22 +13,24 @@
       <thead>
         <tr class="align-middle">
           <th>Subscription group</th>
-          <th>Topic</th>
-          <th>Active</th>
+          <th><%== sort_link('Topic', :name) %></th>
+          <th><%== sort_link('Active', :active?) %></th>
           <th></th>
         </tr>
       </thead>
       <tbody>
-        <% consumer_group.topics.each do |topic| %>
-          <%==
-            partial(
-              'routing/topic',
-              locals: {
-                subscription_group: topic.subscription_group,
-                topic: topic
-              }
-            )
-          %>
+        <% consumer_group.subscription_groups.each do |subscription_group| %>
+          <% subscription_group.topics.each do |topic| %>
+            <%==
+              partial(
+                'routing/topic',
+                locals: {
+                  subscription_group: subscription_group,
+                  topic: topic
+                }
+              )
+            %>
+          <% end %>
         <% end %>
       </tbody>
     </table>

data/lib/karafka/web/ui/views/shared/_navigation.erb

@@ -15,7 +15,7 @@
       </a>
     </li>
     <li class="nav-item ms-3">
-      <a class="nav-link <%= nav_class(start_with: '/jobs') %>" href="<%= root_path('jobs') %>">
+      <a class="nav-link <%= nav_class(start_with: '/jobs') %>" href="<%= root_path('jobs/running') %>">
         Jobs
       </a>
     </li>

data/lib/karafka/web/ui/views/shared/_pagination.erb

@@ -4,7 +4,7 @@
       <nav>
         <ul class="pagination justify-content-center">
           <li class="page-item <%= 'disabled' unless @pagination.first_offset? %>">
-            <a class="page-link" href="<%= current_path(@pagination.offset_key => @pagination.first_offset) %>">
+            <a class="page-link" href="<%= current_path(@pagination.offset_key => @pagination.first_offset || 1) %>">
               <span>&laquo;</span>
             </a>
           </li>

data/lib/karafka/web/ui/views/shared/exceptions/not_allowed.erb

@@ -0,0 +1,37 @@
+<main>
+  <div id="content">
+    <div class="d-flex align-items-center justify-content-center vh-100">
+      <div class="error-message text-center">
+        <h1 class="display-1 fw-bold">403</h1>
+
+        <p class="fs-3 md-5">
+          <span class="text-danger">
+            Oops!
+          </span>
+          Not permitted.
+        </p>
+
+        <div class="lead">
+          <p class="mb-5">
+            You don't have permission to view this page.
+          </p>
+
+          <p>
+            Please make sure, that:
+          </p>
+
+          <ul class="mb-5 text-start">
+            <li>You have visited the <a href="<%= root_path('status') %>">Status</a> page to troubleshoot any potential issues</li>
+            <li>You are authorized to access the resource you requested</li>
+            <li>Web UI configuration permits the action you requested</li>
+          </ul>
+
+          <p>
+            <a href="<%= root_path %>" class="btn btn-primary">Go Home</a>
+            <a href="<%= root_path('status') %>" class="btn btn-success">Status page</a>
+          </p>
+        </div>
+      </div>
+    </div>
+  </div>
+</main>

data/lib/karafka/web/ui/views/status/show.erb

@@ -7,7 +7,7 @@
         partial(
           "status/#{@status.enabled.to_s}",
           locals: {
-            title: 'Web-UI setup in karafka.rb',
+            title: 'Web UI setup in karafka.rb',
             description: partial(
               "status/#{@status.enabled.partial_namespace}/enabled",
               locals: {
@@ -37,7 +37,7 @@
         partial(
           "status/#{@status.topics.to_s}",
           locals: {
-            title: 'Topics presence',
+            title: 'Web UI topics presence',
             description: partial(
               'status/failures/topics',
               locals: {
@@ -165,6 +165,21 @@
         )
       %>
 
+      <%==
+        partial(
+          "status/#{@status.routing_topics_presence.to_s}",
+          locals: {
+            title: 'Routing topics presence',
+            description: partial(
+              'status/warnings/routing_topics_presence',
+              locals: {
+                details: @status.routing_topics_presence.details
+              }
+            )
+          }
+        )
+      %>
+
       <%==
         partial(
           "status/#{@status.pro_subscription.to_s}",

data/lib/karafka/web/ui/views/status/warnings/_routing_topics_presence.erb

@@ -0,0 +1,15 @@
+<p>
+  The following active topics present in your <code>karafka.rb</code> file were not located in the Kafka cluster:
+</p>
+
+<ul>
+  <% details.each do |topic_name| %>
+    <li>
+      <code><%= topic_name %></code>
+    </li>
+  <% end %>
+</ul>
+
+<p>
+  Although not an error, having topics in <code>karafka.rb</code> that still needs to be added to Kafka could lead to issues. It's better to create these topics beforehand to avoid potential errors.
+</p>

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.10'
+    VERSION = '0.8.0.rc1'
   end
 end

data/lib/karafka/web.rb

@@ -53,7 +53,10 @@ module Karafka
   end
 end
 
+require_relative 'web/inflector'
+
 loader = Zeitwerk::Loader.new
+
 # Make sure pro is not loaded unless Pro
 loader.ignore(Karafka::Web.gem_root.join('lib/karafka/web/ui/pro'))
 
@@ -62,9 +65,10 @@ Karafka::Licenser.detect do
   loader = Zeitwerk::Loader.new
 end
 
-root = File.expand_path('..', __dir__)
 loader.tag = 'karafka-web'
-loader.inflector = Zeitwerk::GemInflector.new("#{root}/karafka/web.rb")
+# Use our custom inflector to support migrations
+root = File.expand_path('..', __dir__)
+loader.inflector = Karafka::Web::Inflector.new("#{root}/karafka/web.rb")
 loader.push_dir(root)
 
 loader.setup