karafka-web 0.8.1 → 0.9.0.rc1

data/lib/karafka/web/pro/ui/views/topics/config.erb

@@ -0,0 +1,29 @@
+<%== view_title(@topic.topic_name, hr: false) %>
+
+<%== partial 'topics/tabs' %>
+
+<div class="container mb-5">
+  <div class="row">
+    <div class="col-lg-12 table-responsive">
+      <table class="processes bg-white table table-hover table-bordered table-striped">
+        <thead>
+          <tr class="align-middle">
+            <th><%== sort_link(:name) %></th>
+            <th><%== sort_link(:value) %></th>
+            <th><%== sort_link(:default?) %></th>
+            <th><%== sort_link(:sensitive?) %></th>
+            <th><%== sort_link(:read_only?) %></th>
+          </tr>
+        </thead>
+        <tbody>
+          <%==
+            each_partial(
+              @configs,
+              'cluster/config'
+            )
+          %>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/distribution/_badges.erb

@@ -0,0 +1,7 @@
+<p class="text-end">
+  <span class="badge bg-primary">Total messages: <%= number_with_delimiter @aggregated.sum.round, ' ' %></span>
+  <span class="badge bg-primary">Std dev: <%= @aggregated.std_dev %></span>
+  <span class="badge bg-primary">Std dev relative: <%= @aggregated.std_dev_rel %>%</span>
+  <% partitions_range = [@active_partitions.first, @active_partitions.last].uniq.join(' to ') %>
+  <span class="badge bg-secondary">Partitions: <%= partitions_range %></span>
+</p>

data/lib/karafka/web/pro/ui/views/topics/distribution/_chart.erb

@@ -0,0 +1,2 @@
+<% data = { 'Estimated count': @distribution.map { [_1.partition_id, _1.count] } }.to_json %>
+<%== partial 'shared/charts/bar', locals: { data: data, id: 'distribution' } %>

data/lib/karafka/web/pro/ui/views/topics/distribution/_empty_partitions.erb

@@ -0,0 +1 @@
+<%== alert_info('Those partitions are empty and do not contain any data.') %>

data/lib/karafka/web/pro/ui/views/topics/distribution/_limited.erb

@@ -0,0 +1,10 @@
+<div class="mb-4">
+  <%==
+    alert_info(
+      '
+        Due to the number of partitions, distribution results are computed based only on
+        data from partitions visible on this page.
+      '
+    )
+  %>
+</div>

data/lib/karafka/web/pro/ui/views/topics/distribution/_partition.erb

@@ -0,0 +1,10 @@
+<tr>
+  <td>
+    <a href="<%= explorer_path(@topic.topic_name, partition.partition_id) %>">
+      <%= partition.partition_id %>
+    </a>
+  </td>
+  <td><%= partition.count %></td>
+  <td><%= partition.share %>%</td>
+  <td><%= partition.diff.round(2) %>%</td>
+</tr>

data/lib/karafka/web/pro/ui/views/topics/distribution.erb

@@ -0,0 +1,47 @@
+<%== view_title(@topic.topic_name, hr: false) %>
+
+<%== partial 'topics/tabs' %>
+
+<div class="container">
+  <div class="row">
+    <div class="col-lg-12">
+      <% if @limited %>
+        <%== partial 'topics/distribution/limited' %>
+      <% end %>
+
+      <% if @aggregated.sum.zero? %>
+        <%== partial 'topics/distribution/empty_partitions' %>
+      <% else %>
+        <% if @active_partitions.size >= 2 %>
+          <%== partial 'topics/distribution/chart' %>
+        <% end %>
+
+        <div id="refreshable" class="mt-4">
+          <%== partial 'topics/distribution/badges' %>
+
+          <div class="table-responsive">
+            <table class="bg-white table table-hover table-bordered table-striped mt-4 mb-4">
+              <thead>
+                <tr class="align-middle">
+                  <th><%== sort_link(:partition_id) %></th>
+                  <th><%== sort_link('Estimated count', :count) %></th>
+                  <th><%== sort_link('Total share', :share) %></th>
+                  <th><%== sort_link(:diff) %></th>
+                </tr>
+              </thead>
+              <tbody>
+                <%==
+                  each_partial(
+                    @distribution,
+                    'topics/distribution/partition'
+                  )
+                %>
+              </tbody>
+            </table>
+          </div>
+        </div>
+      <% end %>
+
+    </div>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/index.erb

@@ -0,0 +1,16 @@
+<%== view_title('Topics', hr: true) %>
+
+<div class="container">
+  <% if @topics.empty? %>
+    <%== partial 'explorer/no_topics' %>
+  <% else %>
+    <div class="row mb-5 row-cols-1 row-cols-md-4 g-4">
+      <%==
+        each_partial(
+          @topics,
+          'topics/topic'
+        )
+      %>
+    </div>
+  <% end %>
+</div>

data/lib/karafka/web/pro/ui/views/topics/replication.erb

@@ -0,0 +1,28 @@
+<%== view_title(@topic.topic_name, hr: false) %>
+
+<%== partial 'topics/tabs' %>
+
+<div class="container mb-5">
+  <div class="row">
+    <div class="col-lg-12 table-responsive">
+      <table class="processes bg-white table table-hover table-bordered table-striped">
+        <thead>
+          <tr class="align-middle">
+            <th><%== sort_link(:partition_id) %></th>
+            <th><%== sort_link(:leader) %></th>
+            <th><%== sort_link(:replica_count) %></th>
+            <th><%== sort_link('In sync brokers', :in_sync_replica_brokers) %></th>
+          </tr>
+        </thead>
+        <tbody>
+          <%==
+            each_partial(
+              @partitions,
+              'topics/partition'
+            )
+          %>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</div>

data/lib/karafka/web/processing/consumers/aggregators/base.rb

@@ -36,7 +36,7 @@ module Karafka
             # Updates the report for given process in memory
             # @param report [Hash]
             def memoize_process_report(report)
-              @active_reports[report[:process][:name]] = report
+              @active_reports[report[:process][:id]] = report
             end
 
             # Updates the time of the aggregation

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -70,7 +70,7 @@ module Karafka
             def evict_expired_processes
               max_ttl = @aggregated_from - ::Karafka::Web.config.ttl / 1_000
 
-              @active_reports.delete_if do |_name, report|
+              @active_reports.delete_if do |_id, report|
                 report[:dispatched_at] < max_ttl || report[:process][:status] == 'stopped'
               end
             end

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -89,9 +89,9 @@ module Karafka
             # @param report [Hash]
             # @param offset [Integer]
             def update_process_state(report, offset)
-              process_name = report[:process][:name]
+              process_id = report[:process][:id]
 
-              state[:processes][process_name] = {
+              state[:processes][process_id] = {
                 dispatched_at: report[:dispatched_at],
                 offset: offset
               }
@@ -105,11 +105,11 @@ module Karafka
             def evict_expired_processes
               max_ttl = @aggregated_from - ::Karafka::Web.config.ttl / 1_000
 
-              state[:processes].delete_if do |_name, details|
+              state[:processes].delete_if do |_id, details|
                 details[:dispatched_at] < max_ttl
               end
 
-              @active_reports.delete_if do |_name, details|
+              @active_reports.delete_if do |_id, details|
                 details[:dispatched_at] < max_ttl
               end
             end

data/lib/karafka/web/tracking/consumers/contracts/report.rb

@@ -20,7 +20,7 @@ module Karafka
 
             nested(:process) do
               required(:started_at) { |val| val.is_a?(Numeric) && val.positive? }
-              required(:name) { |val| val.is_a?(String) && val.count(':') >= 2 }
+              required(:id) { |val| val.is_a?(String) && val.count(':') >= 2 }
               required(:cpus) { |val| val.is_a?(Integer) && val >= 1 }
               required(:memory_usage) { |val| val.is_a?(Integer) && val >= 0 }
               required(:memory_total_usage) { |val| val.is_a?(Integer) && val >= 0 }

data/lib/karafka/web/tracking/consumers/contracts/subscription_group.rb

@@ -19,7 +19,7 @@ module Karafka
               required(:stateage) { |val| val.is_a?(Integer) && val >= 0 }
               required(:rebalance_age) { |val| val.is_a?(Integer) && val >= 0 }
               required(:rebalance_cnt) { |val| val.is_a?(Integer) && val >= 0 }
-              required(:rebalance_reason) { |val| val.is_a?(String) && !val.empty? }
+              required(:rebalance_reason) { |val| val.is_a?(String) }
               required(:poll_age) { |val| val.is_a?(Numeric) && val >= 0 }
             end
 

data/lib/karafka/web/tracking/consumers/listeners/booting.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Listener needed to start schedulers and other things that we need to collect and report
+          # data. We do not want to start this when code is loaded because it may not be fork
+          # compatible that way
+          class Booting < Base
+            # Starts (if needed) the Web UI tracking scheduler thread that periodically pings
+            # reporters to report needed data (when it is time).
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_app_running(_event)
+              ::Karafka::Web.config.tracking.scheduler.async_call(
+                'karafka.web.tracking.scheduler'
+              )
+            end
+
+            # Updates the web producer after fork if needed and adds ppid to nodes
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_swarm_node_after_fork(_event)
+              ::Karafka::Process.tags.add(:node_ppid, "ppid:#{::Process.ppid}")
+
+              return if Karafka::Web.config.producer == Karafka::App.config.producer
+
+              Web.config.producer = Karafka::App.config.producer
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -10,7 +10,7 @@ module Karafka
             include Tracking::Helpers::ErrorInfo
 
             # Schema used by consumers error reporting
-            SCHEMA_VERSION = '1.0.0'
+            SCHEMA_VERSION = '1.1.0'
 
             private_constant :SCHEMA_VERSION
 
@@ -37,7 +37,7 @@ module Karafka
                   backtrace: backtrace,
                   details: details,
                   occurred_at: float_now,
-                  process: sampler.to_report[:process].slice(:name, :tags)
+                  process: sampler.to_report[:process].slice(:id, :tags)
                 }
 
                 sampler.counters[:errors] += 1

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -65,14 +65,14 @@ module Karafka
 
               @report_contract.validate!(report)
 
-              process_name = report[:process][:name]
+              process_id = report[:process][:id]
 
               # Report consumers statuses
               messages = [
                 {
                   topic: ::Karafka::Web.config.topics.consumers.reports,
                   payload: Zlib::Deflate.deflate(report.to_json),
-                  key: process_name,
+                  key: process_id,
                   partition: 0,
                   headers: { 'zlib' => 'true' }
                 }
@@ -86,7 +86,7 @@ module Karafka
                   topic: Karafka::Web.config.topics.errors,
                   payload: Zlib::Deflate.deflate(error.to_json),
                   # Always dispatch errors from the same process to the same partition
-                  key: process_name,
+                  key: process_id,
                   headers: { 'zlib' => 'true' }
                 }
               end

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -15,7 +15,7 @@ module Karafka
           # Current schema version
           # This is used for detecting incompatible changes and not using outdated data during
           # upgrades
-          SCHEMA_VERSION = '1.2.9'
+          SCHEMA_VERSION = '1.3.0'
 
           # Counters that count events occurrences during the given window
           COUNTERS_BASE = {
@@ -46,7 +46,6 @@ module Karafka
             end
             @subscription_groups = {}
             @errors = []
-            @started_at = float_now
             @pauses = {}
             @jobs = {}
             @shell = MemoizedShell.new
@@ -71,8 +70,8 @@ module Karafka
               dispatched_at: float_now,
 
               process: {
-                started_at: @started_at,
-                name: process_name,
+                id: process_id,
+                started_at: started_at,
                 status: ::Karafka::App.config.internal.status.to_s,
                 listeners: listeners,
                 workers: workers,
@@ -129,6 +128,13 @@ module Karafka
 
           private
 
+          # @return [Float] time of start of this process
+          # @note We memoize it on first run as forks should have their creation time matching the
+          #   fork time.
+          def started_at
+            @started_at ||= float_now
+          end
+
           # @return [Numeric] % utilization of all the threads. 100% means all the threads are
           #   utilized all the time within the given time window. 0% means, nothing is happening
           #   most if not all the time.

data/lib/karafka/web/tracking/contracts/error.rb

@@ -20,7 +20,7 @@ module Karafka
           required(:occurred_at) { |val| val.is_a?(Float) }
 
           nested(:process) do
-            required(:name) { |val| val.is_a?(String) && !val.empty? }
+            required(:id) { |val| val.is_a?(String) && !val.empty? }
             # Tags may not be present for producers because they may operate from outside of
             # karafka taggable process
             optional(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }

data/lib/karafka/web/tracking/producers/listeners/booting.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Producers
+        module Listeners
+          # Listener needed to start schedulers and other things that we need to collect and report
+          # data. We do not want to start this when code is loaded because it may not be fork
+          # compatible that way
+          class Booting < Base
+            # Starts (if needed) the Web UI tracking scheduler thread that periodically pings
+            # reporters to report needed data (when it is time).
+            #
+            # @param _event [Karafka::Core::Monitoring::Event]
+            def on_producer_connected(_event)
+              ::Karafka::Web.config.tracking.scheduler.async_call(
+                'karafka.web.tracking.scheduler'
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/producers/listeners/errors.rb

@@ -10,7 +10,7 @@ module Karafka
             include Tracking::Helpers::ErrorInfo
 
             # Schema used by producers error reporting
-            SCHEMA_VERSION = '1.0.0'
+            SCHEMA_VERSION = '1.1.0'
 
             private_constant :SCHEMA_VERSION
 
@@ -42,7 +42,7 @@ module Karafka
                 details: build_details(type, event.payload),
                 occurred_at: float_now,
                 process: {
-                  name: sampler.process_name
+                  id: sampler.process_id
                 }
               }
             end

data/lib/karafka/web/tracking/producers/reporter.rb

@@ -43,7 +43,7 @@ module Karafka
                   topic: Karafka::Web.config.topics.errors,
                   payload: error.to_json,
                   # Always dispatch errors from the same process to the same partition
-                  key: error[:process][:name]
+                  key: error[:process][:id]
                 }
               end
 

data/lib/karafka/web/tracking/producers/sampler.rb

@@ -14,7 +14,7 @@ module Karafka
           # Current schema version
           # This can be used in the future for detecting incompatible changes and writing
           # migrations
-          SCHEMA_VERSION = '1.0.0'
+          SCHEMA_VERSION = '1.1.0'
 
           def initialize
             super

data/lib/karafka/web/tracking/sampler.rb

@@ -7,9 +7,9 @@ module Karafka
       # This sampler should store **only** collectors that can be used for producers, consumers and
       # the Web-UI itself. All specific to a given aspect of operations should be moved out.
       class Sampler
-        # @return [String] Unique process name
-        def process_name
-          @process_name ||= "#{Socket.gethostname}:#{::Process.pid}:#{SecureRandom.hex(6)}"
+        # @return [String] Unique process identifier
+        def process_id
+          @process_id ||= "#{Socket.gethostname}:#{::Process.pid}:#{SecureRandom.hex(6)}"
         end
 
         # @return [String] currently used ruby version with details

data/lib/karafka/web/tracking/scheduler.rb

@@ -11,11 +11,6 @@ module Karafka
       class Scheduler
         include ::Karafka::Helpers::Async
 
-        # Creates the scheduler and runs its internal reporting
-        def initialize
-          async_call
-        end
-
         private
 
         # Reports the process state once in a while

data/lib/karafka/web/ui/app.rb

@@ -22,7 +22,7 @@ module Karafka
 
           r.get 'dashboard' do
             @breadcrumbs = false
-            controller = Controllers::Dashboard.new(params)
+            controller = Controllers::DashboardController.new(params)
             controller.index
           end
 
@@ -33,7 +33,7 @@ module Karafka
 
             r.get do
               @breadcrumbs = false
-              controller = Controllers::Consumers.new(params)
+              controller = Controllers::ConsumersController.new(params)
               controller.index
             end
           end
@@ -49,7 +49,7 @@ module Karafka
           end
 
           r.on 'jobs' do
-            controller = Controllers::Jobs.new(params)
+            controller = Controllers::JobsController.new(params)
 
             r.get 'running' do
               controller.running
@@ -63,7 +63,7 @@ module Karafka
           end
 
           r.on 'routing' do
-            controller = Controllers::Routing.new(params)
+            controller = Controllers::RoutingController.new(params)
 
             r.get String do |topic_id|
               controller.show(topic_id)
@@ -75,21 +75,25 @@ module Karafka
           end
 
           r.on 'cluster' do
-            controller = Controllers::Cluster.new(params)
+            controller = Controllers::ClusterController.new(params)
 
             r.get 'brokers' do
               controller.brokers
             end
 
-            r.get 'topics' do
-              controller.topics
+            r.get 'replication' do
+              controller.replication
             end
 
             r.redirect root_path('cluster/brokers')
           end
 
+          r.on 'topics' do
+            raise Errors::Ui::ProOnlyError
+          end
+
           r.on 'errors' do
-            controller = Controllers::Errors.new(params)
+            controller = Controllers::ErrorsController.new(params)
 
             r.get Integer do |offset|
               controller.show(offset)
@@ -101,7 +105,7 @@ module Karafka
           end
 
           r.get 'status' do
-            controller = Controllers::Status.new(params)
+            controller = Controllers::StatusController.new(params)
             controller.show
           end
         end

data/lib/karafka/web/ui/base.rb

@@ -7,6 +7,7 @@ module Karafka
       class Base < Roda
         include Helpers::PathsHelper
         include Helpers::ApplicationHelper
+        include Helpers::AlertsHelper
 
         # Details that need to be evaluated in the context of OSS or Pro web UI.
         # If those would be evaluated in the base, they would not be initialized as expected

data/lib/karafka/web/ui/controllers/{base.rb → base_controller.rb}

@@ -6,7 +6,12 @@ module Karafka
       # Namespace for controller related components in the Web UI app.
       module Controllers
         # Base controller from which all the controllers should inherit.
-        class Base
+        class BaseController
+          include Web::Ui::Lib::Paginations
+
+          # Alias for easier referencing
+          Models = Web::Ui::Models
+
           class << self
             # Attributes on which we can sort in a given controller
             attr_accessor :sortable_attributes
@@ -27,7 +32,7 @@ module Karafka
           def render
             attributes = {}
 
-            scope = self.class.to_s.split('::').last.gsub(/(.)([A-Z])/, '\1_\2').downcase
+            scope = self.class.to_s.split('::').last.gsub(/(.)([A-Z])/, '\1_\2').downcase.gsub('_controller', '')
             action = caller_locations(1, 1)[0].label
 
             instance_variables.each do |iv|
@@ -91,6 +96,14 @@ module Karafka
 
             @pagination = engine.new(*args)
           end
+
+          # Raises the not found error
+          #
+          # @param resource_id [String] resource id that was not found
+          # @raise [::Karafka::Web::Errors::Ui::NotFoundError]
+          def not_found!(resource_id = '')
+            raise(::Karafka::Web::Errors::Ui::NotFoundError, resource_id)
+          end
         end
       end
     end

data/lib/karafka/web/ui/controllers/{become_pro.rb → become_pro_controller.rb}

@@ -5,7 +5,7 @@ module Karafka
     module Ui
       module Controllers
         # Pro message reporting info controller
-        class BecomePro < Base
+        class BecomeProController < BaseController
           # Display a message, that a given feature is available only in Pro
           def show
             render

data/lib/karafka/web/ui/controllers/{cluster.rb → cluster_controller.rb}

@@ -5,7 +5,7 @@ module Karafka
     module Ui
       module Controllers
         # Selects cluster info and topics basic info
-        class Cluster < Base
+        class ClusterController < BaseController
           self.sortable_attributes = %w[
             broker_id
             broker_name
@@ -24,8 +24,8 @@ module Karafka
             render
           end
 
-          # List topics and partitions with details
-          def topics
+          # List partitions replication details
+          def replication
             partitions_total = []
 
             displayable_topics(cluster_info).each do |topic|
@@ -38,7 +38,7 @@ module Karafka
               end
             end
 
-            @partitions, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
+            @partitions, last_page = Paginators::Arrays.call(
               refine(partitions_total),
               @params.current_page
             )

data/lib/karafka/web/ui/controllers/{consumers.rb → consumers_controller.rb}

@@ -5,9 +5,9 @@ module Karafka
     module Ui
       module Controllers
         # Consumers (consuming processes - `karafka server`) processes display consumer
-        class Consumers < Base
+        class ConsumersController < BaseController
           self.sortable_attributes = %w[
-            name
+            id
             started_at
             lag_hybrid
           ].freeze
@@ -17,7 +17,7 @@ module Karafka
           def index
             @current_state = Models::ConsumersState.current!
             @counters = Models::Counters.new(@current_state)
-            @processes, last_page = Ui::Lib::Paginations::Paginators::Arrays.call(
+            @processes, last_page = Paginators::Arrays.call(
               refine(Models::Processes.active(@current_state)),
               @params.current_page
             )

data/lib/karafka/web/ui/controllers/{dashboard.rb → dashboard_controller.rb}

@@ -5,7 +5,7 @@ module Karafka
     module Ui
       module Controllers
         # Main Karafka Pro Web-Ui dashboard controller
-        class Dashboard < Ui::Controllers::Base
+        class DashboardController < Ui::Controllers::BaseController
           # View with statistics dashboard details
           def index
             @current_state = Models::ConsumersState.current!