karafka-web 0.8.2 → 0.9.0.rc1

data/lib/karafka/web/{ui/pro/controllers/explorer.rb → pro/ui/controllers/explorer_controller.rb}

@@ -13,13 +13,11 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         module Controllers
           # Data explorer controller
-          class Explorer < Ui::Controllers::Base
-            include Ui::Lib::Paginations
-
+          class ExplorerController < BaseController
             # Lists all the topics we can explore
             def index
               @topics = Models::ClusterInfo
@@ -71,7 +69,7 @@ module Karafka
               @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
               @topic_id = topic_id
               @partition_id = partition_id
-              @watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+              @watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
               @partitions_count = Models::ClusterInfo.partitions_count(topic_id)
 
               previous_offset, @messages, next_offset = current_partition_data
@@ -94,17 +92,24 @@ module Karafka
             # @param offset [Integer] offset of the message we want to display
             # @param paginate [Boolean] do we want to have pagination
             def show(topic_id, partition_id, offset, paginate: true)
+              Lib::PatternsDetector.new.call
+
               @visibility_filter = ::Karafka::Web.config.ui.visibility.filter
               @topic_id = topic_id
               @partition_id = partition_id
               @offset = offset
-              @message = Ui::Models::Message.find(@topic_id, @partition_id, @offset)
-              @payload_error = false
+              @message = Models::Message.find(@topic_id, @partition_id, @offset)
+
+              @safe_payload = Web::Ui::Lib::SafeRunner.new do
+                JSON.pretty_generate(@message.payload)
+              end
+
+              @safe_key = Web::Ui::Lib::SafeRunner.new do
+                @message.key
+              end
 
-              begin
-                @pretty_payload = JSON.pretty_generate(@message.payload)
-              rescue StandardError => e
-                @payload_error = e
+              @safe_headers = Web::Ui::Lib::SafeRunner.new do
+                @message.headers
               end
 
               # This may be off for certain views like recent view where we are interested only
@@ -112,7 +117,7 @@ module Karafka
               # there
               if paginate
                 # We need watermark offsets to decide if we can paginate left and right
-                watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+                watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
                 paginate(offset, watermark_offsets.low, watermark_offsets.high)
               end
 
@@ -149,7 +154,7 @@ module Karafka
                 break if recent
               end
 
-              recent || raise(::Karafka::Web::Errors::Ui::NotFoundError)
+              recent || not_found!
 
               show(topic_id, recent.partition, recent.offset, paginate: false)
             end
@@ -162,10 +167,10 @@ module Karafka
             # @param partition_id [Integer]
             # @param offset [Integer] offset of the message we want to display
             def surrounding(topic_id, partition_id, offset)
-              watermark_offsets = Ui::Models::WatermarkOffsets.find(topic_id, partition_id)
+              watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
 
-              raise ::Karafka::Web::Errors::Ui::NotFoundError if offset < watermark_offsets.low
-              raise ::Karafka::Web::Errors::Ui::NotFoundError if offset >= watermark_offsets.high
+              not_found! if offset < watermark_offsets.low
+              not_found! if offset >= watermark_offsets.high
 
               # Assume we start from this offset
               shift = 0
@@ -195,7 +200,7 @@ module Karafka
             # @param partition_id [Integer]
             # @param time [Time] time of the message
             def closest(topic_id, partition_id, time)
-              target = Lib::Admin.read_topic(topic_id, partition_id, 1, time).first
+              target = Web::Ui::Lib::Admin.read_topic(topic_id, partition_id, 1, time).first
 
               partition_path = "explorer/#{topic_id}/#{partition_id}"
               partition_path += "?offset=#{target.offset}" if target
@@ -208,7 +213,7 @@ module Karafka
             # Fetches current page data
             # @return [Array] fetched data with pagination information for the requested partition
             def current_partition_data
-              Ui::Models::Message.offset_page(
+              Models::Message.offset_page(
                 @topic_id,
                 @partition_id,
                 @params.current_offset,

data/lib/karafka/web/{ui/pro/controllers/health.rb → pro/ui/controllers/health_controller.rb}

@@ -13,11 +13,11 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         module Controllers
           # Health state controller
-          class Health < Ui::Controllers::Base
+          class HealthController < BaseController
             self.sortable_attributes = %w[
               id
               lag
@@ -62,6 +62,19 @@ module Karafka
               render
             end
 
+            # Displays lags for routing defined consumer groups taken from the cluster and not
+            # the metrics reported. This is useful when we don't have any consumers running but
+            # still want to check lags because it shows what Kafka sees
+            def cluster_lags
+              @stats = Models::Health.cluster_lags_with_offsets
+
+              @stats.each_value do |cg_details|
+                cg_details.each_value { |topic_details| refine(topic_details) }
+              end
+
+              render
+            end
+
             # Displays details about offsets and their progression/statuses
             def offsets
               # Same data as overview but presented differently

data/lib/karafka/web/{ui/pro/controllers/jobs.rb → pro/ui/controllers/jobs_controller.rb}

@@ -13,13 +13,13 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         module Controllers
           # Displays list of active jobs
-          class Jobs < Ui::Controllers::Jobs
+          class JobsController < Web::Ui::Controllers::JobsController
             self.sortable_attributes = %w[
-              name
+              id
               topic
               consumer
               type

data/lib/karafka/web/{ui/pro/controllers/messages.rb → pro/ui/controllers/messages_controller.rb}

@@ -13,20 +13,20 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         module Controllers
           # Controller for working with messages
           # While part of messages operations is done via explorer (exploring), this controller
           # handles other cases not related to viewing data
-          class Messages < Ui::Controllers::Base
+          class MessagesController < BaseController
             # Takes a requested message content and republishes it again
             #
             # @param topic_id [String]
             # @param partition_id [Integer]
             # @param offset [Integer] offset of the message we want to republish
             def republish(topic_id, partition_id, offset)
-              message = Ui::Models::Message.find(topic_id, partition_id, offset)
+              message = Models::Message.find(topic_id, partition_id, offset)
 
               delivery = ::Karafka::Web.producer.produce_sync(
                 topic: topic_id,
@@ -48,7 +48,7 @@ module Karafka
             # @param partition_id [Integer]
             # @param offset [Integer] offset of the message we want to download
             def download(topic_id, partition_id, offset)
-              message = Ui::Models::Message.find(topic_id, partition_id, offset)
+              message = Models::Message.find(topic_id, partition_id, offset)
 
               # Check if downloads are allowed
               return deny unless visibility_filter.download?(message)
@@ -68,7 +68,9 @@ module Karafka
             # @param partition_id [Integer]
             # @param offset [Integer] offset of the message we want to export
             def export(topic_id, partition_id, offset)
-              message = Ui::Models::Message.find(topic_id, partition_id, offset)
+              Lib::PatternsDetector.new.call
+
+              message = Models::Message.find(topic_id, partition_id, offset)
 
               # Check if exports are allowed
               return deny unless visibility_filter.export?(message)

data/lib/karafka/web/{ui/pro/controllers/routing.rb → pro/ui/controllers/routing_controller.rb}

@@ -13,11 +13,11 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         module Controllers
           # Routing details - same as in OSS
-          class Routing < Ui::Controllers::Routing
+          class RoutingController < BaseController
             self.sortable_attributes = %w[
               name
               active?
@@ -44,32 +44,16 @@ module Karafka
 
               @topic = Karafka::Routing::Router.find_by(id: topic_id)
 
-              @topic || raise(::Karafka::Web::Errors::Ui::NotFoundError, topic_id)
+              @topic || not_found!(topic_id)
 
               render
             end
 
             private
 
-            # Checks list of topics and tries to match them against the available patterns
-            # Uses the Pro detector to expand routes in the Web-UI so we include topics that are
-            # or will be matched using our regular expressions
+            # Detect routes defined as patterns
             def detect_patterns_routes
-              detector = ::Karafka::Pro::Routing::Features::Patterns::Detector.new
-              topics_names = Models::ClusterInfo.topics.map(&:topic_name)
-
-              Karafka::App
-                .routes
-                .flat_map(&:subscription_groups)
-                .each do |subscription_group|
-                  sg_topics = subscription_group.topics
-
-                  # Reject topics that are already part of routing for given subscription groups
-                  # and then for remaining try to apply patterns and expand routes
-                  topics_names
-                    .reject { |t_name| sg_topics.any? { |rtopic| rtopic.name == t_name } }
-                    .each { |t_name| detector.expand(sg_topics, t_name) }
-                end
+              Lib::PatternsDetector.new.call
             end
           end
         end

data/lib/karafka/web/{ui/pro/controllers/status.rb → pro/ui/controllers/status_controller.rb}

@@ -13,11 +13,11 @@
 
 module Karafka
   module Web
-    module Ui
-      module Pro
+    module Pro
+      module Ui
         module Controllers
           # Status details - same as in OSS
-          class Status < Ui::Controllers::Status
+          class StatusController < Web::Ui::Controllers::StatusController
           end
         end
       end

data/lib/karafka/web/pro/ui/controllers/topics_controller.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Topics management controller
+          # Allows for exploration of settings and replication details
+          class TopicsController < BaseController
+            self.sortable_attributes = %w[
+              name
+              value
+              default?
+              read_only?
+              synonym?
+              sensitive?
+              partition_id
+              leader
+              replica_count
+              in_sync_replica_brokers
+              count
+              share
+              diff
+            ].freeze
+
+            # Lists available topics in the cluster
+            def index
+              @topics = Models::Topic.all.sort_by(&:topic_name)
+
+              unless ::Karafka::Web.config.ui.visibility.internal_topics
+                @topics.delete_if { |topic| topic[:topic_name].start_with?('__') }
+              end
+
+              render
+            end
+
+            # Displays requested topic config details
+            #
+            # @param topic_name [String] topic we're interested in
+            def config(topic_name)
+              @topic = Models::Topic.find(topic_name)
+
+              @configs = refine(@topic.configs)
+
+              render
+            end
+
+            # Displays requested topic replication details
+            #
+            # @param topic_name [String] topic we're interested in
+            def replication(topic_name)
+              @topic = Models::Topic.find(topic_name)
+
+              @partitions = refine(@topic[:partitions])
+
+              render
+            end
+
+            # Displays the messages distribution across various partitions
+            #
+            # @param topic_name [String] topic we're interested in
+            #
+            # @note Because computing distribution is fairly expensive, we paginate this. While
+            #   because of that results may not be exact, this allows us to support topics with
+            #   many partitions.
+            def distribution(topic_name)
+              @topic = Models::Topic.find(topic_name)
+
+              @active_partitions, _materialized_page, @limited = Paginators::Partitions.call(
+                @topic.partition_count, @params.current_page
+              )
+
+              @aggregated, distribution = @topic.distribution(@active_partitions)
+
+              @distribution = refine(distribution)
+
+              next_page = @active_partitions.last < @topic.partition_count - 1
+              paginate(@params.current_page, next_page)
+
+              render
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/patterns_detector.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        # Namespace for extra libs used by the Pro UI components
+        module Lib
+          # Checks list of topics and tries to match them against the available patterns
+          # Uses the Pro detector to expand routes in the Web-UI so we include topics that are
+          # or will be matched using our regular expressions
+          #
+          # This code **needs** to run when using deserializers of messages from patterns.
+          # Otherwise default deserializer will be used instead.
+          class PatternsDetector
+            # Run the detection
+            def call
+              detector = ::Karafka::Pro::Routing::Features::Patterns::Detector.new
+              topics_names = Web::Ui::Models::ClusterInfo.topics.map(&:topic_name)
+
+              Karafka::App
+                .routes
+                .flat_map(&:subscription_groups)
+                .each do |subscription_group|
+                  sg_topics = subscription_group.topics
+
+                  # Reject topics that are already part of routing for given subscription groups
+                  # and then for remaining try to apply patterns and expand routes
+                  topics_names
+                    .reject { |t_name| sg_topics.any? { |rtopic| rtopic.name == t_name } }
+                    .each { |t_name| detector.expand(sg_topics, t_name) }
+                end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/views/cluster/_breadcrumbs.erb

@@ -0,0 +1,29 @@
+<li class="breadcrumb-item">
+  <a href="<%= root_path('cluster') %>">
+    Cluster informations
+  </a>
+</li>
+
+<% unless @partitions %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('cluster') %>">
+      Brokers
+    </a>
+  </li>
+<% end %>
+
+<% if @broker %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('cluster', @broker.id) %>">
+      <%= @broker.full_name %>
+    </a>
+  </li>
+<% end %>
+
+<% if @partitions %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('cluster', 'replication') %>">
+      Replication
+    </a>
+  </li>
+<% end %>

data/lib/karafka/web/pro/ui/views/cluster/_broker.erb

@@ -0,0 +1,13 @@
+<tr>
+  <td>
+    <a href="<%= root_path('cluster', broker.id) %>">
+      <%= broker.id %>
+    </a>
+  </td>
+  <td>
+    <a href="<%= root_path('cluster', broker.id) %>">
+      <%= broker.name %>
+    </a>
+  </td>
+  <td><%= broker.port %></td>
+</tr>

data/lib/karafka/web/pro/ui/views/cluster/_config.erb

@@ -0,0 +1,13 @@
+<tr>
+  <td class="wrap-text">
+    <% all_names = ([config.name] + config.synonyms.map(&:name)).uniq %>
+    <% all_names.each do |name| %>
+      <%= name %>
+      <br/>
+    <% end %>
+  </td>
+  <td class="wrap-text"><%= config.value %></td>
+  <td><%= config.default? %></td>
+  <td><%= config.sensitive? %></td>
+  <td><%= config.read_only? %></td>
+</tr>

data/lib/karafka/web/pro/ui/views/cluster/_tabs.erb

@@ -0,0 +1,27 @@
+<div class="container">
+  <div class="row mb-4">
+    <div class="col-sm-12">
+
+      <ul class="nav nav-tabs">
+        <li class="nav-item">
+          <a
+            class="nav-link <%= nav_class(end_with: 'cluster') %>"
+            href="<%= root_path('cluster') %>"
+          >
+            Brokers
+          </a>
+        </li>
+
+        <li class="nav-item">
+          <a
+            class="nav-link <%= nav_class(include: 'replication') %>"
+            href="<%= root_path('cluster', 'replication') %>"
+          >
+            Replication
+          </a>
+        </li>
+      </ul>
+
+    </div>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/cluster/brokers.erb

@@ -0,0 +1,27 @@
+<%== view_title('Cluster Informations') %>
+
+<%== partial 'cluster/tabs' %>
+
+<div class="container mb-5">
+  <div class="row">
+    <div class="col-lg-12 table-responsive">
+      <table class="processes bg-white table table-hover table-bordered table-striped">
+        <thead>
+          <tr class="align-middle">
+            <th><%== sort_link('Id', :broker_id) %></th>
+            <th><%== sort_link('Name', :broker_name) %></th>
+            <th><%== sort_link('Port', :broker_port) %></th>
+          </tr>
+        </thead>
+        <tbody>
+          <%==
+            each_partial(
+              @brokers,
+              'cluster/broker'
+            )
+          %>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/cluster/index.erb

@@ -0,0 +1,27 @@
+<%== view_title('Cluster Informations', hr: false) %>
+
+<%== partial 'cluster/tabs' %>
+
+<div class="container mb-5">
+  <div class="row">
+    <div class="col-lg-12 table-responsive">
+      <table class="processes bg-white table table-hover table-bordered table-striped">
+        <thead>
+          <tr class="align-middle">
+            <th><%== sort_link(:id) %></th>
+            <th><%== sort_link(:name) %></th>
+            <th><%== sort_link(:port) %></th>
+          </tr>
+        </thead>
+        <tbody>
+          <%==
+            each_partial(
+              @brokers,
+              'cluster/broker'
+            )
+          %>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/cluster/show.erb

@@ -0,0 +1,27 @@
+<%== view_title(@broker.full_name, hr: true) %>
+
+<div class="container mb-5">
+  <div class="row">
+    <div class="col-lg-12 table-responsive">
+      <table class="processes bg-white table table-hover table-bordered table-striped">
+        <thead>
+          <tr class="align-middle">
+            <th><%== sort_link(:name) %></th>
+            <th><%== sort_link(:value) %></th>
+            <th><%== sort_link(:default?) %></th>
+            <th><%== sort_link(:sensitive?) %></th>
+            <th><%== sort_link(:read_only?) %></th>
+          </tr>
+        </thead>
+        <tbody>
+          <%==
+            each_partial(
+              @configs,
+              'cluster/config'
+            )
+          %>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/commands/_backtrace.erb

@@ -0,0 +1,20 @@
+<div class="container">
+  <div class="row mb-4">
+    <div class="col-sm-12">
+      <h5 class="mb-2">
+        <%= backtrace[:label] %>
+      </h5>
+      <hr/>
+    </div>
+  </div>
+
+  <div class="row mb-5">
+    <div class="col-sm-12">
+      <div class="card">
+        <div class="card-body">
+          <pre class="m-0 p-0"><code class="wrapped json p-0 m-0"><%= backtrace[:backtrace] %></code></pre>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/commands/_breadcrumbs.erb

@@ -0,0 +1,21 @@
+<li class="breadcrumb-item">
+  <a href="<%= root_path('consumers/overview') %>">
+    Consumers
+  </a>
+</li>
+
+<% if current_path.include?('/commands') %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('commands') %>">
+      Commands
+    </a>
+  </li>
+<% end %>
+
+<% if @command_message %>
+  <li class="breadcrumb-item">
+    <a href="<%= root_path('commands', @command_message.offset) %>">
+      <%= @command_message.offset %>
+    </a>
+  </li>
+<% end %>