karafka-web 0.10.4 → 0.11.0.beta1

data/lib/karafka/web/ui/lib/cache.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # Thread-safe in-memory cache with metadata tracking.
+        #
+        # This cache supports storing computed values, tracking the last update time,
+        # and computing a hash of the contents for change detection.
+        # It's designed for ephemeral, per-instance caching in Karafka Web controllers or libs.
+        #
+        # The cache ensures safe concurrent access via a mutex and provides utilities
+        # for cache invalidation based on external session state (timestamp + hash).
+        #
+        # @note All cache operations are mutex-synchronized for thread safety.
+        #
+        # @note We do not have granular level caching because our Web UI cache is fairly simple
+        #   and we do not want to overcomplicate things.
+        class Cache
+          # Initializes an empty cache instance
+          # @param ttl_ms [Integer] time to live of the whole cache. After this time cache will be
+          #   cleaned whether or not it is expired.
+          def initialize(ttl_ms)
+            @ttl_ms = ttl_ms
+            @values = {}
+            @timestamp = nil
+            @hash = nil
+            @mutex = Mutex.new
+          end
+
+          # Fetches or computes and stores a value under the given key.
+          #
+          # If the key already exists, returns the cached value.
+          # Otherwise, computes it via the provided block, stores it,
+          # and updates metadata (timestamp + hash).
+          #
+          # @param key [Object] key to retrieve
+          # @yield block to compute the value if key is not present
+          # @return [Object] cached or computed value
+          def fetch(key)
+            @mutex.synchronize do
+              return @values[key] if @values.key?(key)
+
+              @values[key] = yield
+              @hash = Digest::SHA256.hexdigest(
+                @values.sort.to_h.to_json
+              )
+              @timestamp = Time.now.to_f
+              @values[key]
+            end
+          end
+
+          # Clears the cache and resets metadata (timestamp and hash).
+          #
+          # If the mutex is already owned by the current thread, clears immediately.
+          # Otherwise, synchronizes first.
+          #
+          # @return [void]
+          def clear
+            cleaning = lambda do
+              @values.clear
+              @timestamp = nil
+              @hash = nil
+            end
+
+            return cleaning.call if @mutex.owned?
+
+            @mutex.synchronize do
+              cleaning.call
+            end
+          end
+
+          # Checks whether any values have been cached yet
+          #
+          # @return [Boolean] true if the cache has been written to
+          def exist?
+            !timestamp.nil?
+          end
+
+          # Returns the last update timestamp of the cache
+          #
+          # @return [Integer, nil] Unix timestamp or nil if never set
+          def timestamp
+            @mutex.synchronize { @timestamp }
+          end
+
+          # Returns the hash representing the current cached data state
+          #
+          # @return [String, nil] SHA256 hex digest or nil if never set
+          def hash
+            @mutex.synchronize { @hash }
+          end
+
+          # Clears the cache if the provided session hash and timestamp differ
+          #
+          # This is used to invalidate the cache if the external session data indicates
+          # a newer or inconsistent state.
+          #
+          # @param session_hash [String, nil] hash from the session or remote side
+          # @param session_timestamp [Integer, nil] timestamp from the session
+          # @return [Boolean] true if the cache was cleared, false otherwise
+          def clear_if_needed(session_hash, session_timestamp)
+            @mutex.synchronize do
+              return unless should_refresh?(session_hash, session_timestamp)
+
+              clear
+            end
+          end
+
+          private
+
+          # Determines whether the cache should be refreshed based on session data
+          #
+          # @param session_hash [String, nil]
+          # @param session_timestamp [Integer, nil]
+          # @return [Boolean] true if cache should be refreshed
+          def should_refresh?(session_hash, session_timestamp)
+            return true if @hash.nil? || @timestamp.nil?
+            return true if session_hash.nil? || session_timestamp.nil?
+
+            now = (Time.now.to_f * 1_000).to_i
+
+            return true if now - (@timestamp * 1_000) > @ttl_ms
+
+            return false if @hash == session_hash
+            return false if @timestamp > session_timestamp
+
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/broker.rb

@@ -9,8 +9,7 @@ module Karafka
           class << self
             # @return [Array<Broker>] all brokers in the cluster
             def all
-              # We do not cache here because we want the most recent state of brokers possible
-              ClusterInfo.fetch(cached: false).brokers.map do |broker|
+              ClusterInfo.fetch.brokers.map do |broker|
                 new(broker)
               end
             end

data/lib/karafka/web/ui/models/cluster_info.rb

@@ -10,26 +10,18 @@ module Karafka
           class << self
             # Gets us all the cluster metadata info
             #
-            # @param cached [Boolean] should we use cached data (true by default)
             # @return [Rdkafka::Metadata] cluster metadata info
-            def fetch(cached: true)
-              cache = ::Karafka::Web.config.ui.cache
-
-              cluster_info = cache.read(:cluster_info)
-
-              if cluster_info.nil? || !cached
-                cluster_info = cache.write(:cluster_info, Lib::Admin.cluster_info)
+            def fetch
+              Karafka::Web.config.ui.cache.fetch(:cluster_info) do
+                Lib::Admin.cluster_info
               end
-
-              cluster_info
             end
 
             # Returns us all the info about available topics from the cluster
             #
-            # @param cached [Boolean] should we use cached data (true by default)
             # @return [Array<Ui::Models::Topic>] topics details
-            def topics(cached: true)
-              fetch(cached: cached)
+            def topics
+              fetch
                 .topics
                 .map { |topic| Topic.new(topic) }
             end
@@ -37,19 +29,21 @@ module Karafka
             # Fetches us details about particular topic
             #
             # @param topic_name [String] name of the topic we are looking for
-            # @param cached [Boolean] should we use cached data (true by default)
             # @return [Ui::Models::Topic] topic details
-            def topic(topic_name, cached: true)
-              topics(cached: cached)
-                .find { |topic_data| topic_data.topic_name == topic_name }
-                .tap { |topic| topic || raise(Web::Errors::Ui::NotFoundError, topic_name) }
+            def topic(topic_name)
+              Lib::Admin
+                .topic_info(topic_name)
+                .then { |topic| Topic.new(topic) }
+            rescue Rdkafka::RdkafkaError => e
+              raise e unless e.code == :unknown_topic_or_part
+
+              raise(Web::Errors::Ui::NotFoundError, topic_name)
             end
 
             # @param topic_name [String] name of the topic we are looking for
-            # @param cached [Boolean] should we use cached data (true by default)
             # @return [Integer] number of partitions in a given topic
-            def partitions_count(topic_name, cached: true)
-              topic(topic_name, cached: cached).partition_count
+            def partitions_count(topic_name)
+              topic(topic_name).partition_count
             end
           end
         end

data/lib/karafka/web/ui/models/consumers_metrics.rb

@@ -33,7 +33,7 @@ module Karafka
             # @return [::Karafka::Messages::Message, nil] most recent state or nil if none
             def fetch
               Lib::Admin.read_topic(
-                Karafka::Web.config.topics.consumers.metrics,
+                Karafka::Web.config.topics.consumers.metrics.name,
                 0,
                 # We need to take last two and not the last because in case of a transactional
                 # producer, the last one will match the transaction commit message

data/lib/karafka/web/ui/models/consumers_state.rb

@@ -42,7 +42,7 @@ module Karafka
             # @return [::Karafka::Messages::Message, nil] most recent state or nil if none
             def fetch
               Lib::Admin.read_topic(
-                Karafka::Web.config.topics.consumers.states,
+                Karafka::Web.config.topics.consumers.states.name,
                 0,
                 # We need to take last two and not the last because in case of a transactional
                 # producer, the last one will match the transaction commit message

data/lib/karafka/web/ui/models/counters.rb

@@ -32,7 +32,7 @@ module Karafka
             MAX_ERROR_PARTITIONS.times do |partition|
               begin
                 offsets = Lib::Admin.read_watermark_offsets(
-                  ::Karafka::Web.config.topics.errors,
+                  ::Karafka::Web.config.topics.errors.name,
                   partition
                 )
               # We estimate that way instead of using `#cluster_info` to get the partitions count

data/lib/karafka/web/ui/models/health.rb

@@ -50,15 +50,17 @@ module Karafka
             # @param state [Hash]
             # @param stats [Hash] hash where we will store all the aggregated data
             def fetch_topics_data(state, stats)
-              iterate_partitions(state) do |process, consumer_group, topic, partition|
-                cg_name = consumer_group.id
+              iterate_partitions(state) do |process, consumer_group, subscription_group, topic, partition|
+                cg_id = consumer_group.id
+                sg_id = subscription_group.id
                 t_name = topic.name
                 pt_id = partition.id
 
-                stats[cg_name] ||= { topics: {} }
-                stats[cg_name][:topics][t_name] ||= {}
-                stats[cg_name][:topics][t_name][pt_id] = partition
-                stats[cg_name][:topics][t_name][pt_id][:process] = process
+                stats[cg_id] ||= { topics: {} }
+                stats[cg_id][:topics][t_name] ||= {}
+                stats[cg_id][:topics][t_name][pt_id] = partition
+                stats[cg_id][:topics][t_name][pt_id][:process] = process
+                stats[cg_id][:topics][t_name][pt_id][:subscription_group_id] = sg_id
               end
             end
 
@@ -99,7 +101,7 @@ module Karafka
                   consumer_group.subscription_groups.each do |subscription_group|
                     subscription_group.topics.each do |topic|
                       topic.partitions.each do |partition|
-                        yield(process, consumer_group, topic, partition)
+                        yield(process, consumer_group, subscription_group, topic, partition)
                       end
                     end
                   end

data/lib/karafka/web/ui/models/process.rb

@@ -74,6 +74,20 @@ module Karafka
               .flat_map(&:partitions)
               .count
           end
+
+          # @return [Boolean] consumer process data is only compatible if the version of its schema
+          #   matches the Web UI version of the schema. Some users have gradual deployment where
+          #   they slowly rollout new versions of consumers and for this time, we want to indicate
+          #   which instances have incompatible schema. This will allow us to show those processes
+          #   to indicate they exist, so users are not confused with them missing, but will also
+          #   block all capabilities until the full update of all components.
+          #
+          # @note We do not differentiate between reporting older or newer against Web UI Puma
+          #   instance. Any incompatibility will cause reporting of incompatible. That's for the
+          #   sake of simplicity as the long term goal for user is anyhow to align those.
+          def schema_compatible?
+            schema_version == ::Karafka::Web::Tracking::Consumers::Sampler::SCHEMA_VERSION
+          end
         end
       end
     end

data/lib/karafka/web/ui/models/processes.rb

@@ -45,7 +45,7 @@ module Karafka
                         .sort
 
               Lib::Admin.read_topic(
-                ::Karafka::Web.config.topics.consumers.reports,
+                ::Karafka::Web.config.topics.consumers.reports.name,
                 0,
                 # We set 10k here because we start from the latest offset of the reports, hence
                 # we will never get this much. Do do not know however exactly how many reports
@@ -83,7 +83,7 @@ module Karafka
             # @param processes [Array<Hash>]
             # @return [Array<Hash>] sorted processes data
             def sort_processes(processes)
-              processes.sort_by { |consumer| consumer[:process][:id] }
+              processes.sort_by { |consumer| consumer[:process].fetch(:id) }
             end
           end
         end

data/lib/karafka/web/ui/models/recurring_tasks/schedule.rb

@@ -22,7 +22,7 @@ module Karafka
               #   get it because requested topic/partition does not exist or nothing was present
               def current
                 messages = Karafka::Admin.read_topic(
-                  config.topics.schedules,
+                  config.topics.schedules.name,
                   0,
                   # We work here with the assumption that users won't click so fast to load
                   # more than 20 commands prior to a state flush. If that happens, this will

data/lib/karafka/web/ui/models/status.rb

@@ -16,7 +16,7 @@ module Karafka
           Step = Struct.new(:status, :details) do
             # @return [Boolean] is the given step successfully configured and working
             def success?
-              status == :success || status == :warning
+              %i[success warning].include?(status)
             end
 
             # @return [String] local namespace for partial of a given type
@@ -209,6 +209,27 @@ module Karafka
             )
           end
 
+          # @return [Status::Step] are there any consumer processes with incompatible schema that
+          #   does not match exactly the one used by the Web UI.
+          #
+          # @note It issues a warning, not an error.
+          def consumers_schemas
+            details = { incompatible: [] }
+
+            status = if consumers_reports.success?
+                       incompatible = @processes.reject(&:schema_compatible?)
+                       details[:incompatible] = incompatible
+                       incompatible.empty? ? :success : :warning
+                     else
+                       :halted
+                     end
+
+            Step.new(
+              status,
+              details
+            )
+          end
+
           # @return [Status::Step] Is there a significant lag in the reporting of aggregated data
           #   back to the Kafka. If yes, it means that the results in the Web UI will be delayed
           #   against the reality. Often it means, that there is over-saturation on the consumer
@@ -307,22 +328,22 @@ module Karafka
 
           # @return [String] consumers states topic name
           def topics_consumers_states
-            ::Karafka::Web.config.topics.consumers.states.to_s
+            ::Karafka::Web.config.topics.consumers.states.name.to_s
           end
 
           # @return [String] consumers reports topic name
           def topics_consumers_reports
-            ::Karafka::Web.config.topics.consumers.reports.to_s
+            ::Karafka::Web.config.topics.consumers.reports.name.to_s
           end
 
           # @return [String] consumers metrics topic name
           def topics_consumers_metrics
-            ::Karafka::Web.config.topics.consumers.metrics.to_s
+            ::Karafka::Web.config.topics.consumers.metrics.name.to_s
           end
 
           # @return [String] errors topic name
           def topics_errors
-            ::Karafka::Web.config.topics.errors
+            ::Karafka::Web.config.topics.errors.name
           end
 
           # @return [Hash] hash with topics with which we work details (even if don't exist)
@@ -355,9 +376,7 @@ module Karafka
           # @note If fails, `connection_time` will be 1_000_000
           def connect
             started = Time.now.to_f
-            # For status we always need uncached data, otherwise status could cache outdated
-            # info
-            @cluster_info = Models::ClusterInfo.fetch(cached: false)
+            @cluster_info = Models::ClusterInfo.fetch
             @connection_time = (Time.now.to_f - started) * 1_000
           rescue ::Rdkafka::RdkafkaError
             @connection_time = 1_000_000

data/lib/karafka/web/ui/models/topic.rb

@@ -20,7 +20,7 @@ module Karafka
             # @return [Topic]
             # @raise [::Karafka::Web::Errors::Ui::NotFoundError]
             def find(topic_name)
-              found = all.find { |topic| topic.topic_name == topic_name }
+              found = ClusterInfo.topic(topic_name)
 
               return found if found
 
@@ -56,7 +56,6 @@ module Karafka
           #   aggregated statistics and then the second value is an array with per partition data
           def distribution(partitions)
             sum = 0.0
-            avg = 0.0
 
             counts = partitions.map do |partition_id|
               offsets = Admin.read_watermark_offsets(topic_name, partition_id)

data/lib/karafka/web/ui/public/javascripts/application.js

@@ -1,121 +1,31 @@
-// Nicer display of time distance from a given event
-function updateTimeAgo() {
-  var selection = document.querySelectorAll('time');
-
-  if (selection.length != 0) {
-    timeago.render(selection);
-    timeago.cancel()
-  }
-
-  var selection = document.getElementsByClassName('time-title')
-  var title = null
-
-  for (var i = 0; i < selection.length; i++) {
-    let element = selection[i]
-
-    title = element.getAttribute('title')
-    element.setAttribute('title', timeago.format(title))
-  }
-}
-
-// Cheap way to do breadcrumbs
-function refreshTitle() {
-  const breadcrumbs = document.querySelectorAll('.breadcrumbs a');
-  let breadcrumbTexts = Array.from(breadcrumbs).slice(1).map(crumb => crumb.textContent.trim());
-
-  if (breadcrumbTexts.length > 0) {
-    document.title = breadcrumbTexts.join(' > ') + ' - Karafka Web UI';
-  } else {
-    document.title = 'Karafka Web UI';
-  }
-}
-
-// When using explorer, we can select the desired partition. This code redirects without having
-// to press a button after a select
-function redirectToPartition() {
-  var selector = document.getElementById('current-partition');
-
-  if (selector == null) { return }
-
-  selector.addEventListener('change', function(){
-    Turbo.visit(this.value);
-  });
-}
-
-// Binds to links and forms to make sure action is what user wants
-function bindActionsConfirmations() {
-  var elements = document.getElementsByClassName('confirm-action')
-  var confirmation = 'Are you sure?'
-
-  for (var i = 0; i < elements.length; i++) {
-    let element = elements[i]
-    let action = 'click'
-
-    if (element.nodeName === 'FORM') {
-      action = 'submit'
-    }
-
-    element.addEventListener(action, function(event) {
-      if (!window.confirm(confirmation)) {
-        event.preventDefault();
-      }
-    })
-  }
-}
-
-function bindLockableButtons() {
-  document.querySelectorAll('.btn-lockable').forEach(function(button) {
-    button.addEventListener('click', function(event) {
-      // Find the form element the button is within
-      const form = button.closest('form');
-
-      // If the button is part of a form, add a submit event listener to the form
-      if (form) {
-        form.addEventListener('submit', function() {
-          button.disabled = true;
-          button.textContent += '...';
-          // also lock any modal that is open as the form is submitted
-          var modals = document.querySelectorAll('.modal');
-
-          // Add 'modal-locked' class to each modal
-          modals.forEach(function (modal) {
-            modal.classList.add('modal-locked');
-          });
-        }, { once: true });
-      } else {
-        // If the button is not part of a form, disable it immediately
-        button.disabled = true;
-        button.textContent += '...';
-      }
-    });
-  });
-}
-
 function addListeners() {
   initLivePolling();
   bindPollingButtonClick();
-  bindLockableButtons();
   setLivePollButton();
   setPollingListener();
 
   hljs.highlightAll();
-  updateTimeAgo();
-  redirectToPartition();
+  new TimeAgoManager();
+  new PartitionRedirectManager();
 
   const tabsManager = new TabsManager();
   tabsManager.manageTabs();
 
   manageCharts();
-  bindActionsConfirmations();
   loadOffsetLookupDatePicker();
 
   new BtnToggleManager();
   new BtnToggleManager('.btn-toggle-nav-collapsed', 'collapsed');
   new ThemeManager();
 
-  refreshTitle()
+  new ButtonLockManager();
+  new ActionConfirmationManager();
+  new PageTitleTracker();
   new SearchMetadataVisibilityManager();
   new SearchModalManager();
+  new TimestampSelector();
+  new AlertsManager();
+  new MessageRepublishManager();
 }
 
 document.addEventListener('turbo:load', addListeners);