karafka-web 0.6.3 → 0.7.0

data/lib/karafka/web/ui/pro/controllers/errors.rb

@@ -18,18 +18,45 @@ module Karafka
         module Controllers
           # Errors details controller
           class Errors < Ui::Controllers::Base
+            include Ui::Lib::Paginations
+
+            # Lists all the errors from all the partitions
+            def index
+              @topic_id = errors_topic
+              @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
+
+              @active_partitions, materialized_page, @limited = Paginators::Partitions.call(
+                @partitions_count, @params.current_page
+              )
+
+              @error_messages, next_page = Models::Message.topic_page(
+                errors_topic, @active_partitions, materialized_page
+              )
+
+              paginate(@params.current_page, next_page)
+
+              respond
+            end
+
             # @param partition_id [Integer] id of the partition of errors we are interested in
-            def index(partition_id)
-              errors_topic = ::Karafka::Web.config.topics.errors
+            def partition(partition_id)
               @partition_id = partition_id
-              @previous_page, @error_messages, @next_page, @partitions_count = \
-                Models::Message.page(
-                  errors_topic,
-                  @partition_id,
-                  @params.current_page
-                )
-
               @watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, @partition_id)
+              @partitions_count = Models::ClusterInfo.partitions_count(errors_topic)
+
+              previous_offset, @error_messages, next_offset = Models::Message.offset_page(
+                errors_topic,
+                @partition_id,
+                @params.current_offset,
+                @watermark_offsets
+              )
+
+              paginate(
+                previous_offset,
+                @params.current_offset,
+                next_offset,
+                @error_messages.map(&:offset)
+              )
 
               respond
             end
@@ -39,7 +66,6 @@ module Karafka
             # @param partition_id [Integer]
             # @param offset [Integer]
             def show(partition_id, offset)
-              errors_topic = ::Karafka::Web.config.topics.errors
               @partition_id = partition_id
               @offset = offset
               @error_message = Models::Message.find(
@@ -48,8 +74,18 @@ module Karafka
                 @offset
               )
 
+              watermark_offsets = Ui::Models::WatermarkOffsets.find(errors_topic, partition_id)
+              paginate(offset, watermark_offsets.low, watermark_offsets.high)
+
               respond
             end
+
+            private
+
+            # @return [String] errors topic
+            def errors_topic
+              ::Karafka::Web.config.topics.errors
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/explorer.rb

@@ -18,14 +18,48 @@ module Karafka
         module Controllers
           # Data explorer controller
           class Explorer < Ui::Controllers::Base
+            include Ui::Lib::Paginations
+
             # Lists all the topics we can explore
             def index
-              @topics = Karafka::Admin
-                        .cluster_info
+              @topics = Models::ClusterInfo
                         .topics
-                        .reject { |topic| topic[:topic_name] == '__consumer_offsets' }
                         .sort_by { |topic| topic[:topic_name] }
 
+              unless ::Karafka::Web.config.ui.show_internal_topics
+                @topics.reject! { |topic| topic[:topic_name].start_with?('__') }
+              end
+
+              respond
+            end
+
+            # Displays aggregated messages from (potentially) all partitions of a topic
+            #
+            # @param topic_id [String]
+            #
+            # @note This view may not be 100% accurate because we merge multiple partitions data
+            #   into a single view and this is never accurate. It can be used however to quickly
+            #   look at most recent data flowing, etc, hence it is still useful for aggregated
+            #   metrics information
+            #
+            # @note We cannot use offset references here because each of the partitions may have
+            #   completely different values
+            def topic(topic_id)
+              @visibility_filter = ::Karafka::Web.config.ui.visibility_filter
+
+              @topic_id = topic_id
+              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+
+              @active_partitions, materialized_page, @limited = Paginators::Partitions.call(
+                @partitions_count, @params.current_page
+              )
+
+              @messages, next_page = Models::Message.topic_page(
+                topic_id, @active_partitions, materialized_page
+              )
+
+              paginate(@params.current_page, next_page)
+
               respond
             end
 
@@ -34,15 +68,20 @@ module Karafka
             # @param topic_id [String]
             # @param partition_id [Integer]
             def partition(topic_id, partition_id)
+              @visibility_filter = ::Karafka::Web.config.ui.visibility_filter
               @topic_id = topic_id
               @partition_id = partition_id
-
               @watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+              @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
 
-              @previous_page, @messages, @next_page, @partitions_count = Ui::Models::Message.page(
-                @topic_id,
-                @partition_id,
-                @params.current_page
+              previous_offset, @messages, next_offset = current_partition_data
+
+              paginate(
+                previous_offset,
+                @params.current_offset,
+                next_offset,
+                # If message is an array, it means it's a compacted dummy offset representation
+                @messages.map { |message| message.is_a?(Array) ? message.last : message.offset }
               )
 
               respond
@@ -53,27 +92,118 @@ module Karafka
             # @param topic_id [String]
             # @param partition_id [Integer]
             # @param offset [Integer] offset of the message we want to display
-            def show(topic_id, partition_id, offset)
+            # @param paginate [Boolean] do we want to have pagination
+            def show(topic_id, partition_id, offset, paginate: true)
+              @visibility_filter = ::Karafka::Web.config.ui.visibility_filter
               @topic_id = topic_id
               @partition_id = partition_id
               @offset = offset
               @message = Ui::Models::Message.find(@topic_id, @partition_id, @offset)
               @payload_error = false
 
-              @decrypt = if ::Karafka::App.config.encryption.active
-                           ::Karafka::Web.config.ui.decrypt
-                         else
-                           true
-                         end
-
               begin
                 @pretty_payload = JSON.pretty_generate(@message.payload)
               rescue StandardError => e
                 @payload_error = e
               end
 
+              # This may be off for certain views like recent view where we are interested only
+              # in the most recent all the time. It does not make any sense to display pagination
+              # there
+              if paginate
+                # We need watermark offsets to decide if we can paginate left and right
+                watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+                paginate(offset, watermark_offsets.low, watermark_offsets.high)
+              end
+
               respond
             end
+
+            # Displays the most recent message on a topic/partition
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer, nil] partition we're interested in or nil if we are
+            #   interested in the most recent message from all the partitions
+            def recent(topic_id, partition_id)
+              if partition_id
+                active_partitions = [partition_id]
+              else
+                partitions_count = Models::ClusterInfo.partitions_count(topic_id)
+                active_partitions, = Paginators::Partitions.call(partitions_count, 1)
+              end
+
+              # This selects first page with most recent messages
+              messages, = Models::Message.topic_page(topic_id, active_partitions, 1)
+
+              # Selects newest out of all partitions
+              recent = messages.max_by(&:timestamp)
+
+              recent || raise(::Karafka::Web::Errors::Ui::NotFoundError)
+
+              show(topic_id, recent.partition, recent.offset, paginate: false)
+            end
+
+            # Computes a page on which the given offset is in the middle of the page (if possible)
+            # Useful often when debugging to be able to quickly jump to the historical location
+            # of message and its surrounding to understand failure
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param offset [Integer] offset of the message we want to display
+            def surrounding(topic_id, partition_id, offset)
+              watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+
+              raise ::Karafka::Web::Errors::Ui::NotFoundError if offset < watermark_offsets.low
+              raise ::Karafka::Web::Errors::Ui::NotFoundError if offset >= watermark_offsets.high
+
+              # Assume we start from this offset
+              shift = 0
+              elements = 0
+
+              # Position the offset as close to the middle of offset based page as possible
+              ::Karafka::Web.config.ui.per_page.times do
+                break if elements >= ::Karafka::Web.config.ui.per_page
+
+                elements += 1 if offset + shift < watermark_offsets.high
+
+                if offset - shift > watermark_offsets.low
+                  shift += 1
+                  elements += 1
+                end
+              end
+
+              target = offset - shift
+
+              redirect("explorer/#{topic_id}/#{partition_id}?offset=#{target}")
+            end
+
+            # Finds the closest offset matching the requested time and redirects to this location
+            # Note, that it redirects to closest but always younger.
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param time [Time] time of the message
+            def closest(topic_id, partition_id, time)
+              target = ::Karafka::Admin.read_topic(topic_id, partition_id, 1, time).first
+
+              partition_path = "explorer/#{topic_id}/#{partition_id}"
+              partition_path += "?offset=#{target.offset}" if target
+
+              redirect(partition_path)
+            end
+
+            private
+
+            # Fetches current page data
+            # @return [Array] fetched data with pagination information for the requested partition
+            def current_partition_data
+              Ui::Models::Message.offset_page(
+                @topic_id,
+                @partition_id,
+                @params.current_offset,
+                @watermark_offsets
+              )
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/health.rb

@@ -19,12 +19,20 @@ module Karafka
           # Health state controller
           class Health < Ui::Controllers::Base
             # Displays the current system state
-            def index
-              current_state = Models::State.current!
+            def overview
+              current_state = Models::ConsumersState.current!
               @stats = Models::Health.current(current_state)
 
               respond
             end
+
+            # Displays details about offsets and their progression/statuses
+            def offsets
+              # Same data as overview but presented differently
+              overview
+
+              respond
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/controllers/messages.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Ui
+      module Pro
+        module Controllers
+          # Controller for working with messages
+          # While part of messages operations is done via explorer (exploring), this controller
+          # handles other cases not related to viewing data
+          class Messages < Ui::Controllers::Base
+            # Takes a requested message content and republishes it again
+            #
+            # @param topic_id [String]
+            # @param partition_id [Integer]
+            # @param offset [Integer] offset of the message we want to republish
+            def republish(topic_id, partition_id, offset)
+              message = Ui::Models::Message.find(topic_id, partition_id, offset)
+
+              delivery = ::Karafka.producer.produce_sync(
+                topic: topic_id,
+                partition: partition_id,
+                payload: message.raw_payload,
+                headers: message.headers,
+                key: message.key
+              )
+
+              redirect(
+                :back,
+                success: reproduced(message, delivery)
+              )
+            end
+
+            private
+
+            # @param message [Karafka::Messages::Message]
+            # @param delivery [Rdkafka::Producer::DeliveryReport]
+            # @return [String] flash message about message reproducing
+            def reproduced(message, delivery)
+              <<~MSG
+                Message with offset #{message.offset}
+                has been sent again to #{message.topic}##{message.partition}
+                and received offset #{delivery.offset}.
+              MSG
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/pro/controllers/routing.rb

@@ -18,6 +18,50 @@ module Karafka
         module Controllers
           # Routing details - same as in OSS
           class Routing < Ui::Controllers::Routing
+            # Routing list
+            def index
+              detect_patterns_routes
+
+              @routes = Karafka::App.routes
+
+              respond
+            end
+
+            # Given route details
+            #
+            # @param topic_id [String] topic id
+            def show(topic_id)
+              detect_patterns_routes
+
+              @topic = Karafka::Routing::Router.find_by(id: topic_id)
+
+              @topic || raise(::Karafka::Web::Errors::Ui::NotFoundError, topic_id)
+
+              respond
+            end
+
+            private
+
+            # Checks list of topics and tries to match them against the available patterns
+            # Uses the Pro detector to expand routes in the Web-UI so we include topics that are
+            # or will be matched using our regular expressions
+            def detect_patterns_routes
+              detector = ::Karafka::Pro::Routing::Features::Patterns::Detector.new
+              topics_names = Models::ClusterInfo.topics.map(&:topic_name)
+
+              Karafka::App
+                .routes
+                .flat_map(&:subscription_groups)
+                .each do |subscription_group|
+                  sg_topics = subscription_group.topics
+
+                  # Reject topics that are already part of routing for given subscription groups
+                  # and then for remaining try to apply patterns and expand routes
+                  topics_names
+                    .reject { |t_name| sg_topics.any? { |rtopic| rtopic.name == t_name } }
+                    .each { |t_name| detector.expand(sg_topics, t_name) }
+                end
+            end
           end
         end
       end

data/lib/karafka/web/ui/pro/views/consumers/_breadcrumbs.erb

@@ -17,11 +17,17 @@
         Running jobs
       </a>
     </li>
-  <% else %>
+  <% elsif current_path.include?('/subscriptions') %>
     <li class="breadcrumb-item">
       <a href="<%= root_path('consumers', @process.id, 'subscriptions') %>">
         Active subscriptions
       </a>
     </li>
+  <% else %>
+    <li class="breadcrumb-item">
+      <a href="<%= root_path('consumers', @process.id, 'details') %>">
+        Details
+      </a>
+    </li>
   <% end %>
 <% end %>

data/lib/karafka/web/ui/pro/views/consumers/_consumer.erb

@@ -47,7 +47,7 @@
       <%= process.utilization.round(1) %>%
     </span>
     <span class="badge bg-primary badge-topic">
-      <%= process.concurrency %> /
+      <%= process.workers %> /
       <%= process.busy %>
     </span>
   </td>

data/lib/karafka/web/ui/pro/views/consumers/_counters.erb

@@ -14,19 +14,21 @@
                 <div class="desc">Batches</div>
               </li>
               <li class="col-sm">
-                <div class="count mb-1"><%= @processes.sum(&:lag_stored) %></div>
-                <div class="desc">Lag</div>
+                <div class="count mb-1"><%= @counters.lag_stored %></div>
+                <div class="desc">Lag stored</div>
               </li>
               <li class="col-sm">
-                <div class="count mb-1"><%= @counters.busy %></div>
-                <div class="desc">Busy</div>
+                <a href="<%= root_path('jobs') %>">
+                  <div class="count mb-1"><%= @counters.busy %></div>
+                  <div class="desc">Busy</div>
+                </a>
               </li>
               <li class="col-sm">
                 <div class="count mb-1"><%= @counters.enqueued %></div>
                 <div class="desc">Enqueued</div>
               </li>
               <li class="col-sm">
-                <a href="<%= root_path('errors/0') %>">
+                <a href="<%= root_path('errors') %>">
                   <div class="count mb-1"><%= @counters.errors %></div>
                   <div class="desc">Errors</div>
                 </a>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb

@@ -17,13 +17,13 @@
     <code>#<%= job.type %></code>
   </td>
   <td>
-    <%== offset_with_label job.first_offset %>
+    <%== offset_with_label job.topic, job.partition, job.first_offset, explore: true %>
   </td>
   <td>
-    <%== offset_with_label job.last_offset %>
+    <%== offset_with_label job.topic, job.partition, job.last_offset, explore: true %>
   </td>
   <td>
-    <%== offset_with_label job.committed_offset %>
+    <%== offset_with_label job.topic, job.partition, job.committed_offset, explore: true %>
   </td>
   <td>
     <%== relative_time job.started_at %>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_metrics.erb

@@ -19,6 +19,7 @@
                   <%== relative_time @process.started_at %>
                 </span>
               </li>
+
               <li class="align-items-center d-flex justify-content-between">
                 State from:
                 <span class="badge bg-secondary">
@@ -41,13 +42,13 @@
             <p class="card-text">
             <ul style="list-style: square !important;">
               <li class="align-items-center d-flex justify-content-between">
-                Threads:
+                Workers:
                 <span class="badge bg-primary">
-                  <%= @process.concurrency %>
+                  <%= @process.workers %>
                 </span>
               </li>
               <li class="align-items-center d-flex justify-content-between">
-                Threads utilization:
+                Utilization:
                 <span class="badge bg-primary">
                   <%= @process.utilization.round(2) %>%
                 </span>
@@ -55,7 +56,7 @@
               <li class="align-items-center d-flex justify-content-between">
                 CPUs:
                 <span class="badge bg-primary">
-                  <%= @process.cpu_count %>
+                  <%= @process.cpus %>
                 </span>
               </li>
               <li class="align-items-center d-flex justify-content-between">

data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb

@@ -1,4 +1,4 @@
-<tr>
+<tr class="align-middle <%= lso_risk_state_bg(partition) %>">
   <td>
     <%= topic.name %>
   </td>
@@ -6,7 +6,7 @@
     <%= partition.id %>
   </td>
   <td>
-    <%== offset_with_label partition.lag_stored.to_i %>
+    <%== lag_with_label partition.lag_stored %>
   </td>
   <td>
     <span class="badge <%= lag_trend_bg(partition.lag_stored_d) %>">
@@ -14,10 +14,14 @@
     </span>
   </td>
   <td>
-    <%== offset_with_label partition.committed_offset.to_i %>
+    <% if partition.stored_offset.negative? %>
+      <%== offset_with_label topic.name, partition.id, partition.committed_offset - 1 %>
+    <% else %>
+      <%== offset_with_label topic.name, partition.id, partition.committed_offset %>
+    <% end %>
   </td>
   <td>
-    <%== offset_with_label partition.stored_offset.to_i %>
+    <%== offset_with_label topic.name, partition.id, partition.stored_offset %>
   </td>
   <td>
     <span class="badge <%= kafka_state_bg(partition.fetch_state) %> mt-1 mb-1">
@@ -29,4 +33,9 @@
       <%= partition.poll_state %>
     </span>
   </td>
+  <td>
+    <span class="badge bg-success <%= lso_risk_state_bg(partition) %> bg-opacity-100">
+      <%= partition.lso_risk_state %>
+    </span>
+  </td>
 </tr>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_subscription_group.erb

@@ -65,7 +65,7 @@
 </div>
 
 <% if subscription_group.topics.empty? %>
-  <div class="row">
+  <div class="row mb-4">
     <div class="col-lg-12">
       <div class="alert alert-info" role="alert">
         This process does not consume any messages from any topics of this consumer group.
@@ -81,11 +81,12 @@
             <th>Topic</th>
             <th>Partition</th>
             <th>Lag stored</th>
-            <th>Lag trend</th>
+            <th>Lag stored trend</th>
             <th>Committed offset</th>
             <th>Stored offset</th>
             <th>Fetch state</th>
             <th>Poll state</th>
+            <th>LSO state</th>
           </tr>
         </thead>
         <tbody>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_tabs.erb

@@ -9,12 +9,19 @@
             (<%= @process.subscribed_partitions_count %>)
           </a>
         </li>
+
         <li class="nav-item">
           <a class="nav-link <%= nav_class(include: 'jobs') %>" href="<%= root_path('consumers', @process.id, 'jobs') %>">
             Running jobs
             (<%= @process.jobs.count %>)
           </a>
         </li>
+
+        <li class="nav-item">
+          <a class="nav-link <%= nav_class(include: 'details') %>" href="<%= root_path('consumers', @process.id, 'details') %>">
+            Details
+          </a>
+        </li>
       </ul>
 
     </div>

data/lib/karafka/web/ui/pro/views/consumers/details.erb

@@ -0,0 +1,21 @@
+<%== view_title(@process.name) %>
+
+<% if @process.status == 'stopped' %>
+  <%== partial 'consumers/consumer/stopped' %>
+<% end %>
+
+<%== partial 'consumers/consumer/metrics' %>
+
+<%== partial 'consumers/consumer/tabs' %>
+
+<div class="container">
+  <div class="row">
+    <div class="col-sm-12">
+      <div class="card">
+        <div class="card-body">
+          <pre class="m-0 p-0"><code class="wrapped json p-0 m-0"><%= JSON.pretty_generate(@process.to_h) %></code></pre>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/ui/pro/views/consumers/index.erb

@@ -4,8 +4,10 @@
 <div class="container">
   <div class="row">
     <div class="col-sm-12">
-      <% if @processes.empty? %>
+      <% if @processes.empty? && params.current_page <= 1 %>
         <%== partial 'consumers/no_consumers' %>
+      <% elsif @processes.empty? %>
+        <%== partial 'shared/no_paginated_data' %>
       <% else %>
         <table class="processes bg-white table table-hover table-bordered table-striped mb-0 align-middle">
           <thead>
@@ -15,7 +17,7 @@
               <th class="col-sm-1">Memory</th>
               <th class="col-sm-1">Performance</th>
               <th class="col-sm-1">Load</th>
-              <th class="col-sm-1">Total lag</th>
+              <th class="col-sm-1">Lag stored</th>
             </tr>
           </thead>
           <tbody>