karafka-web 0.6.2 → 0.7.0

data/lib/karafka/web/ui/pro/controllers/explorer.rb

@@ -24,9 +24,12 @@ module Karafka
             def index
               @topics = Models::ClusterInfo
                         .topics
-                        .reject { |topic| topic[:topic_name] == '__consumer_offsets' }
                         .sort_by { |topic| topic[:topic_name] }
 
+              unless ::Karafka::Web.config.ui.show_internal_topics
+                @topics.reject! { |topic| topic[:topic_name].start_with?('__') }
+              end
+
               respond
             end
 
@@ -42,6 +45,8 @@ module Karafka
             # @note We cannot use offset references here because each of the partitions may have
             #   completely different values
             def topic(topic_id)
+              @visibility_filter = ::Karafka::Web.config.ui.visibility_filter
+
               @topic_id = topic_id
               @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
 
@@ -63,6 +68,7 @@ module Karafka
             # @param topic_id [String]
             # @param partition_id [Integer]
             def partition(topic_id, partition_id)
+              @visibility_filter = ::Karafka::Web.config.ui.visibility_filter
               @topic_id = topic_id
               @partition_id = partition_id
               @watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
@@ -74,7 +80,8 @@ module Karafka
                 previous_offset,
                 @params.current_offset,
                 next_offset,
-                @messages.map(&:offset)
+                # If message is an array, it means it's a compacted dummy offset representation
+                @messages.map { |message| message.is_a?(Array) ? message.last : message.offset }
               )
 
               respond
@@ -85,28 +92,106 @@ module Karafka
             # @param topic_id [String]
             # @param partition_id [Integer]
             # @param offset [Integer] offset of the message we want to display
-            def show(topic_id, partition_id, offset)
+            # @param paginate [Boolean] do we want to have pagination
+            def show(topic_id, partition_id, offset, paginate: true)
+              @visibility_filter = ::Karafka::Web.config.ui.visibility_filter
               @topic_id = topic_id
               @partition_id = partition_id
               @offset = offset
               @message = Ui::Models::Message.find(@topic_id, @partition_id, @offset)
               @payload_error = false
 
-              @decrypt = if ::Karafka::App.config.encryption.active
-                           ::Karafka::Web.config.ui.decrypt
-                         else
-                           true
-                         end
-
               begin
                 @pretty_payload = JSON.pretty_generate(@message.payload)
               rescue StandardError => e
                 @payload_error = e
               end
 
+              # This may be off for certain views like recent view where we are interested only
+              # in the most recent all the time. It does not make any sense to display pagination
+              # there
+              if paginate
+                # We need watermark offsets to decide if we can paginate left and right
+                watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+                paginate(offset, watermark_offsets.low, watermark_offsets.high)
+              end
+
               respond
             end
 
+            # Displays the most recent message on a topic/partition
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer, nil] partition we're interested in or nil if we are
+            #   interested in the most recent message from all the partitions
+            def recent(topic_id, partition_id)
+              if partition_id
+                active_partitions = [partition_id]
+              else
+                partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+                active_partitions, = Paginators::Partitions.call(partitions_count, 1)
+              end
+
+              # This selects first page with most recent messages
+              messages, = Models::Message.topic_page(topic_id, active_partitions, 1)
+
+              # Selects newest out of all partitions
+              recent = messages.max_by(&:timestamp)
+
+              recent || raise(::Karafka::Web::Errors::Ui::NotFoundError)
+
+              show(topic_id, recent.partition, recent.offset, paginate: false)
+            end
+
+            # Computes a page on which the given offset is in the middle of the page (if possible)
+            # Useful often when debugging to be able to quickly jump to the historical location
+            # of message and its surrounding to understand failure
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param offset [Integer] offset of the message we want to display
+            def surrounding(topic_id, partition_id, offset)
+              watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+
+              raise ::Karafka::Web::Errors::Ui::NotFoundError if offset < watermark_offsets.low
+              raise ::Karafka::Web::Errors::Ui::NotFoundError if offset >= watermark_offsets.high
+
+              # Assume we start from this offset
+              shift = 0
+              elements = 0
+
+              # Position the offset as close to the middle of offset based page as possible
+              ::Karafka::Web.config.ui.per_page.times do
+                break if elements >= ::Karafka::Web.config.ui.per_page
+
+                elements += 1 if offset + shift < watermark_offsets.high
+
+                if offset - shift > watermark_offsets.low
+                  shift += 1
+                  elements += 1
+                end
+              end
+
+              target = offset - shift
+
+              redirect("explorer/#{topic_id}/#{partition_id}?offset=#{target}")
+            end
+
+            # Finds the closest offset matching the requested time and redirects to this location
+            # Note, that it redirects to closest but always younger.
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param time [Time] time of the message
+            def closest(topic_id, partition_id, time)
+              target = ::Karafka::Admin.read_topic(topic_id, partition_id, 1, time).first
+
+              partition_path = "explorer/#{topic_id}/#{partition_id}"
+              partition_path += "?offset=#{target.offset}" if target
+
+              redirect(partition_path)
+            end
+
             private
 
             # Fetches current page data

data/lib/karafka/web/ui/pro/controllers/health.rb

@@ -19,12 +19,20 @@ module Karafka
           # Health state controller
           class Health < Ui::Controllers::Base
             # Displays the current system state
-            def index
-              current_state = Models::State.current!
+            def overview
+              current_state = Models::ConsumersState.current!
               @stats = Models::Health.current(current_state)
 
               respond
             end
+
+            # Displays details about offsets and their progression/statuses
+            def offsets
+              # Same data as overview but presented differently
+              overview
+
+              respond
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/messages.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Controller for working with messages
+          # While part of messages operations is done via explorer (exploring), this controller
+          # handles other cases not related to viewing data
+          class Messages < Ui::Controllers::Base
+            # Takes a requested message content and republishes it again
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param offset [Integer] offset of the message we want to republish
+            def republish(topic_id, partition_id, offset)
+              message = Ui::Models::Message.find(topic_id, partition_id, offset)
+
+              delivery = ::Karafka.producer.produce_sync(
+                topic: topic_id,
+                partition: partition_id,
+                payload: message.raw_payload,
+                headers: message.headers,
+                key: message.key
+              )
+
+              redirect(
+                :back,
+                success: reproduced(message, delivery)
+              )
+            end
+
+            private
+
+            # @param message [Karafka::Messages::Message]
+            # @param delivery [Rdkafka::Producer::DeliveryReport]
+            # @return [String] flash message about message reproducing
+            def reproduced(message, delivery)
+              <<~MSG
+                Message with offset #{message.offset}
+                has been sent again to #{message.topic}##{message.partition}
+                and received offset #{delivery.offset}.
+              MSG
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/routing.rb

@@ -18,6 +18,50 @@ module Karafka
         module Controllers
           # Routing details - same as in OSS
           class Routing < Ui::Controllers::Routing
+            # Routing list
+            def index
+              detect_patterns_routes
+
+              @routes = Karafka::App.routes
+
+              respond
+            end
+
+            # Given route details
+            #
+            # @param topic_id [String] topic id
+            def show(topic_id)
+              detect_patterns_routes
+
+              @topic = Karafka::Routing::Router.find_by(id: topic_id)
+
+              @topic || raise(::Karafka::Web::Errors::Ui::NotFoundError, topic_id)
+
+              respond
+            end
+
+            private
+
+            # Checks list of topics and tries to match them against the available patterns
+            # Uses the Pro detector to expand routes in the Web-UI so we include topics that are
+            # or will be matched using our regular expressions
+            def detect_patterns_routes
+              detector = ::Karafka::Pro::Routing::Features::Patterns::Detector.new
+              topics_names = Models::ClusterInfo.topics.map(&:topic_name)
+
+              Karafka::App
+                .routes
+                .flat_map(&:subscription_groups)
+                .each do |subscription_group|
+                  sg_topics = subscription_group.topics
+
+                  # Reject topics that are already part of routing for given subscription groups
+                  # and then for remaining try to apply patterns and expand routes
+                  topics_names
+                    .reject { |t_name| sg_topics.any? { |rtopic| rtopic.name == t_name } }
+                    .each { |t_name| detector.expand(sg_topics, t_name) }
+                end
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/views/consumers/_breadcrumbs.erb

@@ -17,11 +17,17 @@
         Running jobs
       </a>
     </li>
-  <% else %>
+  <% elsif current_path.include?('/subscriptions') %>
     <li class="breadcrumb-item">
       <a href="<%= root_path('consumers', @process.id, 'subscriptions') %>">
         Active subscriptions
       </a>
     </li>
+  <% else %>
+    <li class="breadcrumb-item">
+      <a href="<%= root_path('consumers', @process.id, 'details') %>">
+        Details
+      </a>
+    </li>
   <% end %>
 <% end %>

data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb

@@ -47,7 +47,7 @@
       <%= process.utilization.round(1) %>%
     </span>
     <span class="badge bg-primary badge-topic">
-      <%= process.concurrency %> /
+      <%= process.workers %> /
       <%= process.busy %>
     </span>
   </td>

data/lib/karafka/web/ui/pro/views/consumers/_counters.erb

@@ -14,19 +14,21 @@
                 <div class="desc">Batches</div>
               </li>
               <li class="col-sm">
-                <div class="count mb-1"><%= @processes.sum(&:lag_stored) %></div>
-                <div class="desc">Lag</div>
+                <div class="count mb-1"><%= @counters.lag_stored %></div>
+                <div class="desc">Lag stored</div>
               </li>
               <li class="col-sm">
-                <div class="count mb-1"><%= @counters.busy %></div>
-                <div class="desc">Busy</div>
+                <a href="<%= root_path('jobs') %>">
+                  <div class="count mb-1"><%= @counters.busy %></div>
+                  <div class="desc">Busy</div>
+                </a>
               </li>
               <li class="col-sm">
                 <div class="count mb-1"><%= @counters.enqueued %></div>
                 <div class="desc">Enqueued</div>
               </li>
               <li class="col-sm">
-                <a href="<%= root_path('errors/0') %>">
+                <a href="<%= root_path('errors') %>">
                   <div class="count mb-1"><%= @counters.errors %></div>
                   <div class="desc">Errors</div>
                 </a>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb

@@ -17,13 +17,13 @@
     <code>#<%= job.type %></code>
   </td>
   <td>
-    <%== offset_with_label job.first_offset %>
+    <%== offset_with_label job.topic, job.partition, job.first_offset, explore: true %>
   </td>
   <td>
-    <%== offset_with_label job.last_offset %>
+    <%== offset_with_label job.topic, job.partition, job.last_offset, explore: true %>
   </td>
   <td>
-    <%== offset_with_label job.committed_offset %>
+    <%== offset_with_label job.topic, job.partition, job.committed_offset, explore: true %>
   </td>
   <td>
     <%== relative_time job.started_at %>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb

@@ -19,6 +19,7 @@
                   <%== relative_time @process.started_at %>
                 </span>
               </li>
+
               <li class="align-items-center d-flex justify-content-between">
                 State from:
                 <span class="badge bg-secondary">
@@ -41,13 +42,13 @@
             <p class="card-text">
             <ul style="list-style: square !important;">
               <li class="align-items-center d-flex justify-content-between">
-                Threads:
+                Workers:
                 <span class="badge bg-primary">
-                  <%= @process.concurrency %>
+                  <%= @process.workers %>
                 </span>
               </li>
               <li class="align-items-center d-flex justify-content-between">
-                Threads utilization:
+                Utilization:
                 <span class="badge bg-primary">
                   <%= @process.utilization.round(2) %>%
                 </span>
@@ -55,7 +56,7 @@
               <li class="align-items-center d-flex justify-content-between">
                 CPUs:
                 <span class="badge bg-primary">
-                  <%= @process.cpu_count %>
+                  <%= @process.cpus %>
                 </span>
               </li>
               <li class="align-items-center d-flex justify-content-between">

data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb

@@ -1,4 +1,4 @@
-<tr>
+<tr class="align-middle <%= lso_risk_state_bg(partition) %>">
   <td>
     <%= topic.name %>
   </td>
@@ -6,7 +6,7 @@
     <%= partition.id %>
   </td>
   <td>
-    <%== offset_with_label partition.lag_stored.to_i %>
+    <%== lag_with_label partition.lag_stored %>
   </td>
   <td>
     <span class="badge <%= lag_trend_bg(partition.lag_stored_d) %>">
@@ -14,10 +14,14 @@
     </span>
   </td>
   <td>
-    <%== offset_with_label partition.committed_offset.to_i %>
+    <% if partition.stored_offset.negative? %>
+      <%== offset_with_label topic.name, partition.id, partition.committed_offset - 1 %>
+    <% else %>
+      <%== offset_with_label topic.name, partition.id, partition.committed_offset %>
+    <% end %>
   </td>
   <td>
-    <%== offset_with_label partition.stored_offset.to_i %>
+    <%== offset_with_label topic.name, partition.id, partition.stored_offset %>
   </td>
   <td>
     <span class="badge <%= kafka_state_bg(partition.fetch_state) %> mt-1 mb-1">
@@ -29,4 +33,9 @@
       <%= partition.poll_state %>
     </span>
   </td>
+  <td>
+    <span class="badge bg-success <%= lso_risk_state_bg(partition) %> bg-opacity-100">
+      <%= partition.lso_risk_state %>
+    </span>
+  </td>
 </tr>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb

@@ -65,7 +65,7 @@
 </div>
 
 <% if subscription_group.topics.empty? %>
-  <div class="row">
+  <div class="row mb-4">
     <div class="col-lg-12">
       <div class="alert alert-info" role="alert">
         This process does not consume any messages from any topics of this consumer group.
@@ -81,11 +81,12 @@
             <th>Topic</th>
             <th>Partition</th>
             <th>Lag stored</th>
-            <th>Lag trend</th>
+            <th>Lag stored trend</th>
             <th>Committed offset</th>
             <th>Stored offset</th>
             <th>Fetch state</th>
             <th>Poll state</th>
+            <th>LSO state</th>
           </tr>
         </thead>
         <tbody>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_tabs.erb

@@ -9,12 +9,19 @@
             (<%= @process.subscribed_partitions_count %>)
           </a>
         </li>
+
         <li class="nav-item">
           <a class="nav-link <%= nav_class(include: 'jobs') %>" href="<%= root_path('consumers', @process.id, 'jobs') %>">
             Running jobs
             (<%= @process.jobs.count %>)
           </a>
         </li>
+
+        <li class="nav-item">
+          <a class="nav-link <%= nav_class(include: 'details') %>" href="<%= root_path('consumers', @process.id, 'details') %>">
+            Details
+          </a>
+        </li>
       </ul>
 
     </div>

data/lib/karafka/web/ui/pro/views/consumers/details.erb

@@ -0,0 +1,21 @@
+<%== view_title(@process.name) %>
+
+<% if @process.status == 'stopped' %>
+  <%== partial 'consumers/consumer/stopped' %>
+<% end %>
+
+<%== partial 'consumers/consumer/metrics' %>
+
+<%== partial 'consumers/consumer/tabs' %>
+
+<div class="container">
+  <div class="row">
+    <div class="col-sm-12">
+      <div class="card">
+        <div class="card-body">
+          <pre class="m-0 p-0"><code class="wrapped json p-0 m-0"><%= JSON.pretty_generate(@process.to_h) %></code></pre>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/ui/pro/views/consumers/index.erb

@@ -4,8 +4,10 @@
 <div class="container">
   <div class="row">
     <div class="col-sm-12">
-      <% if @processes.empty? %>
+      <% if @processes.empty? && params.current_page <= 1 %>
         <%== partial 'consumers/no_consumers' %>
+      <% elsif @processes.empty? %>
+        <%== partial 'shared/no_paginated_data' %>
       <% else %>
         <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
           <thead>
@@ -15,7 +17,7 @@
               <th class="col-sm-1">Memory</th>
               <th class="col-sm-1">Performance</th>
               <th class="col-sm-1">Load</th>
-              <th class="col-sm-1">Total lag</th>
+              <th class="col-sm-1">Lag stored</th>
             </tr>
           </thead>
           <tbody>

data/lib/karafka/web/ui/pro/views/dashboard/_ranges_selector.erb

@@ -0,0 +1,39 @@
+<div class="container">
+  <div class="row">
+    <div class="col-sm-12 text-end">
+      <div class="btn-group btn-group-sm" role="group" aria-label="Small button group">
+        <%
+          path = root_path('dashboard?range=seconds')
+          active = params.current_range == :seconds ? 'active' : false
+        %>
+        <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
+          5 minutes
+        </a>
+
+        <%
+          path = root_path('dashboard?range=minutes')
+          active = params.current_range == :minutes ? 'active' : false
+        %>
+        <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
+          1 hour
+        </a>
+
+        <%
+          path = root_path('dashboard?range=hours')
+          active = params.current_range == :hours ? 'active' : false
+        %>
+        <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
+          24 hours
+        </a>
+
+        <%
+          path = root_path('dashboard?range=days')
+          active = params.current_range == :days ? 'active' : false
+        %>
+        <a type="button" href="<%= path %>" class="btn btn-outline-primary <%= active %>">
+          7 days
+        </a>
+      </div>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/ui/pro/views/dashboard/index.erb

@@ -0,0 +1,82 @@
+<%== partial 'consumers/counters' %>
+
+<% if @aggregated.sufficient? %>
+  <%== partial 'dashboard/ranges_selector' %>
+
+  <div class="container">
+    <div class="row">
+      <div class="col-sm-12">
+        <h5 class="mb-3">Processing metrics</h5>
+
+        <hr class="mb-4"/>
+
+        <ul class="nav nav-tabs" id="graphs1" role="tablist">
+          <%== partial 'shared/tab_nav', locals: { title: 'Messages', id: 'messages', active: true } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Batches', id: 'batches' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Lags stored', id: 'lags-stored' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Topics pace', id: 'topics-pace' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Max LSO time', id: 'max-lso-time' } %>
+        </ul>
+
+        <div class="tab-content">
+          <div class="tab-pane show active" id="messages" role="tabpanel">
+            <% data = @aggregated_charts.with(:messages, :errors, :dead, :retries, :batch_size) %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'messages' } %>
+          </div>
+
+          <div class="tab-pane" id="batches" role="tabpanel">
+            <% data = @aggregated_charts.with(:batches, :errors, :dead, :retries) %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'batches' } %>
+          </div>
+
+          <div class="tab-pane" id="lags-stored" role="tabpanel">
+            <%== partial 'shared/chart', locals: { data: @topics_charts.lags_stored, id: 'lags-stored' } %>
+          </div>
+
+          <div class="tab-pane" id="topics-pace" role="tabpanel">
+            <%== partial 'shared/chart', locals: { data: @topics_charts.topics_pace, id: 'topics-pace' } %>
+          </div>
+
+          <div class="tab-pane" id="max-lso-time" role="tabpanel">
+            <%== partial 'shared/chart', locals: { data: @topics_charts.max_lso_time, id: 'max-lso-time' } %>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+
+  <div class="container">
+    <div class="row">
+      <div class="col-sm-12">
+        <h5 class="mb-3">Utilization metrics</h5>
+
+        <hr class="mb-4"/>
+
+        <ul class="nav nav-tabs" id="graphs2" role="tablist">
+          <%== partial 'shared/tab_nav', locals: { title: 'Utilization', id: 'utilization', active: true } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'RSS', id: 'rss' } %>
+          <%== partial 'shared/tab_nav', locals: { title: 'Concurrency', id: 'concurrency' } %>
+        </ul>
+
+        <div class="tab-content">
+          <div class="tab-pane show active" id="utilization" role="tabpanel">
+            <% data = @aggregated_charts.with(:utilization) %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'utilization', label_type_y: 'percentage' } %>
+          </div>
+
+          <div class="tab-pane show" id="rss" role="tabpanel">
+            <% data = @aggregated_charts.with(:rss, :process_rss) %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'rss', label_type_y: 'memory' } %>
+          </div>
+
+          <div class="tab-pane show" id="concurrency" role="tabpanel">
+            <% data = @aggregated_charts.with(:processes, :workers, :listeners) %>
+            <%== partial 'shared/chart', locals: { data: data, id: 'concurrency' } %>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+<% else %>
+  <%== partial 'dashboard/not_enough_data' %>
+<% end %>

data/lib/karafka/web/ui/pro/views/dlq/_topic.erb

@@ -2,7 +2,7 @@
   <div class="card" >
     <div class="card-body p-2">
       <p class="card-text mb-0 p-2">
-        <a href="<%= root_path('explorer', topic[:topic_name], 0) %>">
+        <a href="<%= explorer_path(topic[:topic_name]) %>">
           <%= topic[:topic_name] %> /
           <%= topic[:partition_count] %>
         </a>