karafka-web 0.10.4 → 0.11.0

data/lib/karafka/web/ui/models/health.rb

@@ -50,15 +50,17 @@ module Karafka
             # @param state [Hash]
             # @param stats [Hash] hash where we will store all the aggregated data
             def fetch_topics_data(state, stats)
-              iterate_partitions(state) do |process, consumer_group, topic, partition|
-                cg_name = consumer_group.id
+              iterate_partitions(state) do |process, consumer_group, subscription_group, topic, partition|
+                cg_id = consumer_group.id
+                sg_id = subscription_group.id
                 t_name = topic.name
                 pt_id = partition.id
 
-                stats[cg_name] ||= { topics: {} }
-                stats[cg_name][:topics][t_name] ||= {}
-                stats[cg_name][:topics][t_name][pt_id] = partition
-                stats[cg_name][:topics][t_name][pt_id][:process] = process
+                stats[cg_id] ||= { topics: {} }
+                stats[cg_id][:topics][t_name] ||= {}
+                stats[cg_id][:topics][t_name][pt_id] = partition
+                stats[cg_id][:topics][t_name][pt_id][:process] = process
+                stats[cg_id][:topics][t_name][pt_id][:subscription_group_id] = sg_id
               end
             end
 
@@ -99,7 +101,7 @@ module Karafka
                   consumer_group.subscription_groups.each do |subscription_group|
                     subscription_group.topics.each do |topic|
                       topic.partitions.each do |partition|
-                        yield(process, consumer_group, topic, partition)
+                        yield(process, consumer_group, subscription_group, topic, partition)
                       end
                     end
                   end

data/lib/karafka/web/ui/models/message.rb

@@ -10,13 +10,21 @@ module Karafka
           extend Lib::Paginations::Paginators
 
           class << self
-            # Looks for a message from a given topic partition
+            # Looks for a message from a given topic partition. When no offsets provided, will
+            # raise if there is no data under the given offset. If watermarks were provided, it
+            # will check if this is a system entry and in such cases will return nil.
+            # Will always raise if request is out of range.
             #
             # @param topic_id [String]
             # @param partition_id [Integer]
             # @param offset [Integer]
+            # @param watermark_offsets [WatermarkOffsets, false] watermark offsets for this topic
+            #   partition or false if not provided
+            # @return [Karafka::Messages::Message, nil] found message or nil in case watermark
+            #   offsets were provided and we encountered a message matching watermarks
             # @raise [::Karafka::Web::Errors::Ui::NotFoundError] when not found
-            def find(topic_id, partition_id, offset)
+            # @note If no watermark offsets provided will always raise if no message with data
+            def find(topic_id, partition_id, offset, watermark_offsets: false)
               message = Lib::Admin.read_topic(
                 topic_id,
                 partition_id,
@@ -26,6 +34,16 @@ module Karafka
 
               return message if message
 
+              # Not found can also occur for system entries and compacted messages.
+              # Since we want to know about this in some cases we handle this case and check if the
+              # requested offset is within the range and if so, it means it has been cleaned or
+              # is a system entry. In such cases we do display user an info message.
+              return nil if watermark_offsets &&
+                            offset >= watermark_offsets.low &&
+                            offset < watermark_offsets.high
+
+              # If beyond the watermark offsets, we raise 404 as user should not reach such
+              # non-existent messages as we cannot reason about them
               raise(
                 ::Karafka::Web::Errors::Ui::NotFoundError,
                 [topic_id, partition_id, offset].join(', ')

data/lib/karafka/web/ui/models/process.rb

@@ -12,6 +12,8 @@ module Karafka
             # @param process_id [String] id of the process we are looking for
             # @return [Process] selected process or error raised
             # @raise [::Karafka::Web::Errors::Ui::NotFoundError] raised if process not found
+            # @note Keep in mind, that this search is looking only within processes with a
+            #   compatible schema, as it uses `#active` under the hood.
             def find(state, process_id)
               found_process = Processes.active(state).find { |process| process.id == process_id }
               found_process || raise(::Karafka::Web::Errors::Ui::NotFoundError, process_id)
@@ -74,6 +76,20 @@ module Karafka
               .flat_map(&:partitions)
               .count
           end
+
+          # @return [Boolean] consumer process data is only compatible if the version of its schema
+          #   matches the Web UI version of the schema. Some users have gradual deployment where
+          #   they slowly rollout new versions of consumers and for this time, we want to indicate
+          #   which instances have incompatible schema. This will allow us to show those processes
+          #   to indicate they exist, so users are not confused with them missing, but will also
+          #   block all capabilities until the full update of all components.
+          #
+          # @note We do not differentiate between reporting older or newer against Web UI Puma
+          #   instance. Any incompatibility will cause reporting of incompatible. That's for the
+          #   sake of simplicity as the long term goal for user is anyhow to align those.
+          def schema_compatible?
+            self[:schema_version] == ::Karafka::Web::Tracking::Consumers::Sampler::SCHEMA_VERSION
+          end
         end
       end
     end

data/lib/karafka/web/ui/models/processes.rb

@@ -11,22 +11,30 @@ module Karafka
           class << self
             include ::Karafka::Core::Helpers::Time
 
-            # Returns the active processes in an array and alongside of that the current state of
-            # the system. We use those together in the UI and it would be expensive to pick it up
-            # while we've already had it.
-            #
+            # Returns processes that are running or recently shutdown. It may also return processes
+            # with incompatible schema.
             # @param state [State] current system state from which we can get processes metadata
             # @return [Array<Process>]
-            def active(state)
+            def all(state)
               messages = fetch_reports(state)
               messages = squash_processes_data(messages)
               processes = messages.map(&:payload)
               evict_expired_processes(processes)
               processes = sort_processes(processes)
-
               processes.map { |process_hash| Process.new(process_hash) }
             end
 
+            # Returns the active processes in an array and alongside of that the current state of
+            # the system. We use those together in the UI and it would be expensive to pick it up
+            # while we've already had it. Active means it is running (or recently shutdown) and
+            # it has current schema. Basically any process about which we can reason
+            #
+            # @param state [State] current system state from which we can get processes metadata
+            # @return [Array<Process>]
+            def active(state)
+              all(state).delete_if { |process| !process.schema_compatible? }
+            end
+
             private
 
             # Fetches the relevant processes reports from the reports topic
@@ -45,7 +53,7 @@ module Karafka
                         .sort
 
               Lib::Admin.read_topic(
-                ::Karafka::Web.config.topics.consumers.reports,
+                ::Karafka::Web.config.topics.consumers.reports.name,
                 0,
                 # We set 10k here because we start from the latest offset of the reports, hence
                 # we will never get this much. Do do not know however exactly how many reports
@@ -79,11 +87,24 @@ module Karafka
               end
             end
 
+            # Removes processes that have schema different than the one supported by the Web UI
+            # We support incompatible schema processes reporting in the status page so users know
+            # what and how to update. For other processes we do not display them or their data
+            # as it would be too complex to support
+            #
+            # @param processes [Array<Hash>]
+            # @return [Array<Hash>] only data about processes running current schema
+            def evict_incompatible_processes(processes)
+              processes.delete_if do |details|
+                details[:schema_version] != Tracking::Consumers::Sampler::SCHEMA_VERSION
+              end
+            end
+
             # Ensures that we always return processes sorted by their id
             # @param processes [Array<Hash>]
             # @return [Array<Hash>] sorted processes data
             def sort_processes(processes)
-              processes.sort_by { |consumer| consumer[:process][:id] }
+              processes.sort_by { |consumer| consumer[:process].fetch(:id) }
             end
           end
         end

data/lib/karafka/web/ui/models/recurring_tasks/schedule.rb

@@ -22,7 +22,7 @@ module Karafka
               #   get it because requested topic/partition does not exist or nothing was present
               def current
                 messages = Karafka::Admin.read_topic(
-                  config.topics.schedules,
+                  config.topics.schedules.name,
                   0,
                   # We work here with the assumption that users won't click so fast to load
                   # more than 20 commands prior to a state flush. If that happens, this will

data/lib/karafka/web/ui/models/status.rb

@@ -16,7 +16,7 @@ module Karafka
           Step = Struct.new(:status, :details) do
             # @return [Boolean] is the given step successfully configured and working
             def success?
-              status == :success || status == :warning
+              %i[success warning].include?(status)
             end
 
             # @return [String] local namespace for partial of a given type
@@ -183,7 +183,7 @@ module Karafka
           # @return [Status::Step] could we read and operate on the current processes data (if any)
           def consumers_reports
             if initial_consumers_metrics.success?
-              @processes ||= Models::Processes.active(@current_state)
+              @processes ||= Models::Processes.all(@current_state)
               status = :success
             else
               status = :halted
@@ -209,6 +209,27 @@ module Karafka
             )
           end
 
+          # @return [Status::Step] are there any consumer processes with incompatible schema that
+          #   does not match exactly the one used by the Web UI.
+          #
+          # @note It issues a warning, not an error.
+          def consumers_schemas
+            details = { incompatible: [] }
+
+            status = if consumers_reports.success?
+                       incompatible = @processes.reject(&:schema_compatible?)
+                       details[:incompatible] = incompatible
+                       incompatible.empty? ? :success : :warning
+                     else
+                       :halted
+                     end
+
+            Step.new(
+              status,
+              details
+            )
+          end
+
           # @return [Status::Step] Is there a significant lag in the reporting of aggregated data
           #   back to the Kafka. If yes, it means that the results in the Web UI will be delayed
           #   against the reality. Often it means, that there is over-saturation on the consumer
@@ -307,22 +328,22 @@ module Karafka
 
           # @return [String] consumers states topic name
           def topics_consumers_states
-            ::Karafka::Web.config.topics.consumers.states.to_s
+            ::Karafka::Web.config.topics.consumers.states.name.to_s
           end
 
           # @return [String] consumers reports topic name
           def topics_consumers_reports
-            ::Karafka::Web.config.topics.consumers.reports.to_s
+            ::Karafka::Web.config.topics.consumers.reports.name.to_s
           end
 
           # @return [String] consumers metrics topic name
           def topics_consumers_metrics
-            ::Karafka::Web.config.topics.consumers.metrics.to_s
+            ::Karafka::Web.config.topics.consumers.metrics.name.to_s
           end
 
           # @return [String] errors topic name
           def topics_errors
-            ::Karafka::Web.config.topics.errors
+            ::Karafka::Web.config.topics.errors.name
           end
 
           # @return [Hash] hash with topics with which we work details (even if don't exist)
@@ -355,9 +376,7 @@ module Karafka
           # @note If fails, `connection_time` will be 1_000_000
           def connect
             started = Time.now.to_f
-            # For status we always need uncached data, otherwise status could cache outdated
-            # info
-            @cluster_info = Models::ClusterInfo.fetch(cached: false)
+            @cluster_info = Models::ClusterInfo.fetch
             @connection_time = (Time.now.to_f - started) * 1_000
           rescue ::Rdkafka::RdkafkaError
             @connection_time = 1_000_000

data/lib/karafka/web/ui/models/topic.rb

@@ -20,7 +20,7 @@ module Karafka
             # @return [Topic]
             # @raise [::Karafka::Web::Errors::Ui::NotFoundError]
             def find(topic_name)
-              found = all.find { |topic| topic.topic_name == topic_name }
+              found = ClusterInfo.topic(topic_name)
 
               return found if found
 
@@ -56,7 +56,6 @@ module Karafka
           #   aggregated statistics and then the second value is an array with per partition data
           def distribution(partitions)
             sum = 0.0
-            avg = 0.0
 
             counts = partitions.map do |partition_id|
               offsets = Admin.read_watermark_offsets(topic_name, partition_id)

data/lib/karafka/web/ui/public/javascripts/application.js

@@ -1,121 +1,31 @@
-// Nicer display of time distance from a given event
-function updateTimeAgo() {
-  var selection = document.querySelectorAll('time');
-
-  if (selection.length != 0) {
-    timeago.render(selection);
-    timeago.cancel()
-  }
-
-  var selection = document.getElementsByClassName('time-title')
-  var title = null
-
-  for (var i = 0; i < selection.length; i++) {
-    let element = selection[i]
-
-    title = element.getAttribute('title')
-    element.setAttribute('title', timeago.format(title))
-  }
-}
-
-// Cheap way to do breadcrumbs
-function refreshTitle() {
-  const breadcrumbs = document.querySelectorAll('.breadcrumbs a');
-  let breadcrumbTexts = Array.from(breadcrumbs).slice(1).map(crumb => crumb.textContent.trim());
-
-  if (breadcrumbTexts.length > 0) {
-    document.title = breadcrumbTexts.join(' > ') + ' - Karafka Web UI';
-  } else {
-    document.title = 'Karafka Web UI';
-  }
-}
-
-// When using explorer, we can select the desired partition. This code redirects without having
-// to press a button after a select
-function redirectToPartition() {
-  var selector = document.getElementById('current-partition');
-
-  if (selector == null) { return }
-
-  selector.addEventListener('change', function(){
-    Turbo.visit(this.value);
-  });
-}
-
-// Binds to links and forms to make sure action is what user wants
-function bindActionsConfirmations() {
-  var elements = document.getElementsByClassName('confirm-action')
-  var confirmation = 'Are you sure?'
-
-  for (var i = 0; i < elements.length; i++) {
-    let element = elements[i]
-    let action = 'click'
-
-    if (element.nodeName === 'FORM') {
-      action = 'submit'
-    }
-
-    element.addEventListener(action, function(event) {
-      if (!window.confirm(confirmation)) {
-        event.preventDefault();
-      }
-    })
-  }
-}
-
-function bindLockableButtons() {
-  document.querySelectorAll('.btn-lockable').forEach(function(button) {
-    button.addEventListener('click', function(event) {
-      // Find the form element the button is within
-      const form = button.closest('form');
-
-      // If the button is part of a form, add a submit event listener to the form
-      if (form) {
-        form.addEventListener('submit', function() {
-          button.disabled = true;
-          button.textContent += '...';
-          // also lock any modal that is open as the form is submitted
-          var modals = document.querySelectorAll('.modal');
-
-          // Add 'modal-locked' class to each modal
-          modals.forEach(function (modal) {
-            modal.classList.add('modal-locked');
-          });
-        }, { once: true });
-      } else {
-        // If the button is not part of a form, disable it immediately
-        button.disabled = true;
-        button.textContent += '...';
-      }
-    });
-  });
-}
-
 function addListeners() {
   initLivePolling();
   bindPollingButtonClick();
-  bindLockableButtons();
   setLivePollButton();
   setPollingListener();
 
   hljs.highlightAll();
-  updateTimeAgo();
-  redirectToPartition();
+  new TimeAgoManager();
+  new PartitionRedirectManager();
 
   const tabsManager = new TabsManager();
   tabsManager.manageTabs();
 
   manageCharts();
-  bindActionsConfirmations();
   loadOffsetLookupDatePicker();
 
   new BtnToggleManager();
   new BtnToggleManager('.btn-toggle-nav-collapsed', 'collapsed');
   new ThemeManager();
 
-  refreshTitle()
+  new ButtonLockManager();
+  new ActionConfirmationManager();
+  new PageTitleTracker();
   new SearchMetadataVisibilityManager();
   new SearchModalManager();
+  new TimestampSelector();
+  new AlertsManager();
+  new MessageRepublishManager();
 }
 
 document.addEventListener('turbo:load', addListeners);