karafka-web 0.7.6 → 0.7.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32657edc369df2240e6788c6972d0c83073263ad3e2cc416ff4d24bde1677f18
-  data.tar.gz: d013c25e74a87d1820f60912805888c42c8432a24e1ed33674276f8647328585
+  metadata.gz: 2d46244304483184c3be59a0686497a9f948e9b666dccf00961b560e60db4655
+  data.tar.gz: 329686ce548063787fe06290d67650793ebef6c014f43b753780ad120671a8b1
 SHA512:
-  metadata.gz: 148e9fbe63c0029b1181aece3569f2822e108240b94c78e960b7e24a2921553dc64cd5c3d59f95fe555df2265bfd216fac97ba8906dbae2cb8c8611a66963400
-  data.tar.gz: c015c0c8cc8d15351c2620e3a4a87ee10a986277bf1f3e50fc0c7d2e9d66f839d596bf94273b067eac7fb2f1f231453ac91119ec125cea90fb8bd6eff8943333
+  metadata.gz: 5a18a3c23f63382c6c41dc5ac7f82f893bc14a55e7ec68af9cf16da0f47a45aee77ca9626973570cb9d6abe9355e0712f117424e00619b7b0198748c3bfbeb55
+  data.tar.gz: f639bd90ca21c09fcd798fc289079cafeb9c752d261ecf88df9762e347df738bf260c658f0f15b3b08c3708daf091217c0159d85ac975380463a65a45d876d0b

data/.github/workflows/ci.yml

@@ -68,7 +68,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -89,7 +89,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Run Coditsu

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Karafka Web changelog
 
+## 0.7.8 (2023-10-24)
+- [Enhancement] Support transactional producer usage with Web UI.
+- [Fix] Fix a bug where critical errors (like `IRB::Abort`) would not abort the ongoing transaction.
+- [Fix] Prevent a scenario where an ongoing transactional producer would have stats emitted and an error that could not have been dispatched because of the transaction, creating a dead-lock.
+- [Fix] Make sure that the `recent` displays the most recent non-compacted, non-system message.
+- [Fix] Improve the `recent` message display to compensate for aborted transactions.
+- [Fix] Fix `ReferenceError: response is not defined` that occurs when Web UI returns refresh non 200.
+
+## 0.7.7 (2023-10-20)
+- [Fix] Remove `thor` as a CLI engine due to breaking changes.
+
 ## 0.7.6 (2023-10-10)
 - [Fix] Fix nested SASL/SAML data visible in the routing details (#173)
 

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.7.6)
+    karafka-web (0.7.8)
       erubi (~> 1.4)
-      karafka (>= 2.2.6, < 3.0.0)
+      karafka (>= 2.2.9, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -11,35 +11,45 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.8)
+    activesupport (7.1.1)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
+    base64 (0.1.1)
+    bigdecimal (3.1.4)
     byebug (11.1.3)
     concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
     diff-lcs (1.5.0)
     docile (1.4.0)
+    drb (2.1.1)
+      ruby2_keywords
     erubi (1.12.0)
     factory_bot (6.3.0)
       activesupport (>= 5.0.0)
-    ffi (1.15.5)
+    ffi (1.16.3)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka (2.2.6)
+    karafka (2.2.9)
       karafka-core (>= 2.2.2, < 2.3.0)
-      thor (>= 0.20)
-      waterdrop (>= 2.6.6, < 3.0.0)
+      waterdrop (>= 2.6.10, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.2.2)
+    karafka-core (2.2.3)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.1, < 0.14.0)
-    karafka-rdkafka (0.13.4)
+      karafka-rdkafka (>= 0.13.6, < 0.14.0)
+    karafka-rdkafka (0.13.6)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    mini_portile2 (2.8.4)
+    mini_portile2 (2.8.5)
     minitest (5.20.0)
+    mutex_m (0.1.2)
     rack (3.0.8)
     rack-test (2.1.0)
       rack (>= 1.3)
@@ -47,7 +57,7 @@ GEM
       rack (>= 3.0.0.beta1)
       webrick
     rake (13.0.6)
-    roda (3.71.0)
+    roda (3.73.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -62,21 +72,21 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.1)
+    ruby2_keywords (0.0.5)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    tilt (2.2.0)
+    tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.7)
-      karafka-core (>= 2.1.1, < 3.0.0)
+    waterdrop (2.6.10)
+      karafka-core (>= 2.2.3, < 3.0.0)
       zeitwerk (~> 2.3)
     webrick (1.8.1)
-    zeitwerk (2.6.11)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   x86_64-linux

data/README.md

@@ -8,6 +8,9 @@ Karafka Web UI is a user interface for the [Karafka framework](https://github.co
 
 It allows for easy access to various metrics, such as the number of messages consumed, the number of errors, and the number of consumers operating. It also provides a way to view the different Kafka topics, consumers, and groups that are being used by the application.
 
+> [!IMPORTANT]
+> All of Karafka ecosystems components documentation, including the Web UI, can be found [here](https://karafka.io/docs/#web-ui).
+
 ## Getting started
 
 Karafka Web UI documentation is part of the Karafka framework documentation and can be found [here](https://karafka.io/docs).

data/bin/karafka-web

@@ -3,6 +3,7 @@
 require 'karafka'
 require 'karafka/web'
 
-::Karafka::Cli::Base.load
+ENV['KARAFKA_CLI'] = 'true'
 
+::Karafka::Cli::Base.load
 ::Karafka::Web::Cli.start

data/docker-compose.yml

@@ -21,3 +21,5 @@ services:
       KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
       ALLOW_PLAINTEXT_LISTENER: 'yes'
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

data/karafka-web.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0 Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.2.6', '< 3.0.0'
+  spec.add_dependency 'karafka', '>= 2.2.9', '< 3.0.0'
   spec.add_dependency 'karafka-core', '>= 2.2.2', '< 3.0.0'
   spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'
@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',
-    'changelog_uri' => 'https://github.com/karafka/karafka-web/blob/master/CHANGELOG.md',
+    'changelog_uri' => 'https://karafka.io/docs/Changelog-Karafka-Web-UI',
     'bug_tracker_uri' => 'https://github.com/karafka/karafka-web/issues',
     'source_code_uri' => 'https://github.com/karafka/karafka-web',
     'documentation_uri' => 'https://karafka.io/docs',

data/lib/karafka/web/cli/base.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    class Cli
+      # Base command for all the Web Cli commands
+      class Base < Karafka::Cli::Base
+        include ::Karafka::Helpers::Colorize
+
+        class << self
+          # @return [Array<Class>] available commands
+          def commands
+            ObjectSpace
+              .each_object(Class)
+              .select { |klass| klass.superclass == Karafka::Web::Cli::Base }
+              .reject { |klass| klass.to_s.end_with?('::Base') }
+              .sort_by(&:name)
+          end
+        end
+
+        private
+
+        # Takes the CLI user provided replication factor but if not present, uses the brokers count
+        # to decide. For non-dev clusters (with one broker) we usually want to have replication of
+        # two, just to have some redundancy.
+        # @param cli_replication_factor [Integer, false] user requested replication factor or false
+        #   if we are supposed to compute the factor automatically
+        # @return [Integer] replication factor for Karafka Web UI topics
+        def compute_replication_factor(cli_replication_factor)
+          cli_replication_factor || (Ui::Models::ClusterInfo.fetch.brokers.size > 1 ? 2 : 1)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/cli/help.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    class Cli
+      # Displays help
+      class Help < Base
+        desc 'Describes available commands'
+
+        # Print available commands
+        def call
+          # Find the longest command for alignment purposes
+          max_command_length = self.class.commands.map(&:name).map(&:size).max
+
+          puts 'Karafka Web UI commands:'
+
+          # Print each command formatted with its description
+          self.class.commands.each do |command|
+            puts "  #{command.name.ljust(max_command_length)}    # #{command.desc}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/cli/install.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    class Cli
+      # Installs Web UI
+      class Install < Base
+        desc 'Installs the Web UI'
+
+        option(
+          :replication_factor,
+          'Replication factor for created topics',
+          Integer,
+          ['--replication_factor [FACTOR]']
+        )
+
+        # Installs Karafka Web. Creates all needed topics, populates the data and adds the needed
+        # code to `karafka.rb`.
+        def call
+          Karafka::Web::Installer.new.install(
+            replication_factor: compute_replication_factor(options[:replication_factor])
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/cli/migrate.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    class Cli
+      # Migrates the Web UI topics and states if needed
+      class Migrate < Base
+        desc 'Runs necessary migrations of Web UI topics and states'
+
+        option(
+          :replication_factor,
+          'Replication factor for created topics',
+          Integer,
+          ['--replication_factor [FACTOR]']
+        )
+
+        # Creates new topics (if any) and populates missing data.
+        # It does **not** remove topics and will not populate data if it is already there.
+        #
+        # Useful in two scenarios:
+        #   1. When setting up Web-UI in a new environment, so the Web-UI has the proper initial
+        #      state.
+        #   2. When upgrading Web-UI in-between versions that would require extra topics and/or
+        #      extra states populated.
+        def call
+          Karafka::Web::Installer.new.migrate(
+            replication_factor: compute_replication_factor(options[:replication_factor])
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/cli/reset.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    class Cli
+      # Resets the Web UI
+      class Reset < Base
+        desc 'Resets the Web UI by removing all the Web topics and creating them again'
+
+        option(
+          :replication_factor,
+          'Replication factor for created topics',
+          Integer,
+          ['--replication_factor [FACTOR]']
+        )
+
+        # Resets Karafka Web. Removes the topics, creates them again and populates the initial
+        # state again. This is useful in case the Web-UI metrics or anything else got corrupted.
+        def call
+          Karafka::Web::Installer.new.reset(
+            replication_factor: compute_replication_factor(options[:replication_factor])
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/cli/uninstall.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    class Cli
+      # Uninstalls the Web UI
+      class Uninstall < Base
+        desc 'Removes all the Web UI topics and the enabled code'
+
+        # Uninstalls Karafka Web
+        def call
+          Karafka::Web::Installer.new.uninstall
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/cli.rb

@@ -2,82 +2,15 @@
 
 module Karafka
   module Web
-    # Karafka itself depends on Thor, so we can use it
-    class Cli < Thor
-      include ::Karafka::Helpers::Colorize
-
-      package_name 'Karafka Web'
-
-      desc 'install', 'Installs the Web UI'
-      method_option(
-        :replication_factor,
-        desc: 'Replication factor for created topics',
-        default: false,
-        check_default_type: false,
-        type: :numeric
-      )
-      # Installs Karafka Web. Creates all needed topics, populates the data and adds the needed
-      # code to `karafka.rb`.
-      def install
-        Karafka::Web::Installer.new.install(
-          replication_factor: compute_replication_factor(options[:replication_factor])
-        )
-      end
-
-      desc 'migrate', 'Creates necessary topics if not present and populates state data'
-      method_option(
-        :replication_factor,
-        desc: 'Replication factor for created topics',
-        default: false,
-        check_default_type: false,
-        type: :numeric
-      )
-      # Creates new topics (if any) and populates missing data.
-      # It does **not** remove topics and will not populate data if it is already there.
-      #
-      # Useful in two scenarios:
-      #   1. When setting up Web-UI in a new environment, so the Web-UI has the proper initial
-      #      state.
-      #   2. When upgrading Web-UI in-between versions that would require extra topics and/or extra
-      #      states populated.
-      def migrate
-        Karafka::Web::Installer.new.migrate(
-          replication_factor: compute_replication_factor(options[:replication_factor])
-        )
-      end
-
-      desc 'reset', 'Resets the Web UI by removing all the Web topics and creating them again'
-      method_option(
-        :replication_factor,
-        desc: 'Replication factor for created topics',
-        default: false,
-        check_default_type: false,
-        type: :numeric
-      )
-      # Resets Karafka Web. Removes the topics, creates them again and populates the initial state
-      # again. This is useful in case the Web-UI metrics or anything else got corrupted.
-      def reset
-        Karafka::Web::Installer.new.reset(
-          replication_factor: compute_replication_factor(options[:replication_factor])
-        )
-      end
-
-      desc 'uninstall', 'Removes all the Web UI topics and the enabled code'
-      # Uninstalls Karafka Web
-      def uninstall
-        Karafka::Web::Installer.new.uninstall
-      end
-
-      private
-
-      # Takes the CLI user provided replication factor but if not present, uses the brokers count
-      # to decide. For non-dev clusters (with one broker) we usually want to have replication of
-      # two, just to have some redundancy.
-      # @param cli_replication_factor [Integer, false] user requested replication factor or false
-      #   if we are supposed to compute the factor automatically
-      # @return [Integer] replication factor for Karafka Web UI topics
-      def compute_replication_factor(cli_replication_factor)
-        cli_replication_factor || Ui::Models::ClusterInfo.fetch.brokers.size > 1 ? 2 : 1
+    # Web CLI
+    class Cli < Karafka::Cli
+      class << self
+        private
+
+        # @return [Array<Class>] command classes
+        def commands
+          Base.commands
+        end
       end
     end
   end

data/lib/karafka/web/config.rb

@@ -36,6 +36,10 @@ module Karafka
         # 5 seconds should be enough
         setting :interval, default: 5_000
 
+        # Main Web UI reporting scheduler that runs a background thread and reports periodically
+        # from the consumer reporter and producer reporter
+        setting :scheduler, default: Tracking::Scheduler.new
+
         setting :consumers do
           # Reports the metrics collected in the sampler
           setting :reporter, default: Tracking::Consumers::Reporter.new
@@ -58,8 +62,7 @@ module Karafka
           setting :sampler, default: Tracking::Producers::Sampler.new
 
           setting :listeners, default: [
-            Tracking::Producers::Listeners::Errors.new,
-            Tracking::Producers::Listeners::Reporter.new
+            Tracking::Producers::Listeners::Errors.new
           ]
         end
       end
@@ -114,6 +117,37 @@ module Karafka
         # In some cases you may want to limit what is being displayed due to the type of data you
         # are dealing with
         setting :visibility_filter, default: Ui::Models::VisibilityFilter.new
+
+        # Specific kafka settings that are tuned to operate within the Web UI interface.
+        #
+        # Please do not change them unless you know what you are doing as their misconfiguration
+        # may cause Web UI to misbehave
+        #
+        # The settings are inherited as follows:
+        #   1. root routing level `kafka` settings
+        #   2. admin `kafka` settings
+        #   3. web ui `kafka` settings from here
+        #
+        # Those settings impact ONLY Web UI interface and do not affect other scopes. This is done
+        # on purpose as we want to improve responsiveness of the interface by tuning some of the
+        # settings and this is not that relevant for processing itself.
+        #
+        # option [Hash] extra changes to the default admin kafka settings
+        setting :kafka, default: {
+          # optimizes the responsiveness of the Web UI in three scenarios:
+          #   - topics to which writes happen only in transactions so EOF is yield faster
+          #   - heavily compacted topics
+          #   - Web UI topics read operations when using transactional producer
+          #
+          # This can be configured to be higher if you do not use transactional WaterDrop producer.
+          # This value is used when last message (first from the high watermark offset) is the
+          # transaction commit message. In cases like this the EOF gets propagated after this time
+          # so we have to wait. Default 500ms means, that for some views, where we take our data
+          # that might have been committed via transactional producer, we would wait for 1 second
+          # to get needed data. If you are experiencing timeouts or other issues with the Web IU
+          # interface, you can increase this.
+          'fetch.wait.max.ms': 100
+        }
       end
     end
   end

data/lib/karafka/web/processing/consumers/metrics.rb

@@ -14,7 +14,10 @@ module Karafka
               metrics_message = ::Karafka::Admin.read_topic(
                 Karafka::Web.config.topics.consumers.metrics,
                 0,
-                1
+                # We need to take more in case there would be transactions running.
+                # In theory we could take two but this compensates for any involuntary
+                # revocations and cases where two producers would write to the same state
+                5
               ).last
 
               return metrics_message.payload if metrics_message

data/lib/karafka/web/processing/consumers/state.rb

@@ -14,7 +14,10 @@ module Karafka
               state_message = ::Karafka::Admin.read_topic(
                 Karafka::Web.config.topics.consumers.states,
                 0,
-                1
+                # We need to take more in case there would be transactions running.
+                # In theory we could take two but this compensates for any involuntary
+                # revocations and cases where two producers would write to the same state
+                5
               ).last
 
               return state_message.payload if state_message

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -5,10 +5,7 @@ module Karafka
     module Tracking
       module Consumers
         # Reports the collected data about the process and sends it, so we can use it in the UI
-        class Reporter
-          include ::Karafka::Core::Helpers::Time
-          include ::Karafka::Helpers::Async
-
+        class Reporter < Tracking::Reporter
           # Minimum number of messages to produce to produce them in sync mode
           # This acts as a small back-off not to overload the system in case we would have
           # extremely big number of errors happening
@@ -21,12 +18,31 @@ module Karafka
           MUTEX = Mutex.new
 
           def initialize
+            super
             # Move back so first report is dispatched fast to indicate, that the process is alive
             @tracked_at = monotonic_now - 10_000
             @report_contract = Consumers::Contracts::Report.new
             @error_contract = Tracking::Contracts::Error.new
           end
 
+          # We never report in initializing phase because things are not yet fully configured
+          # We never report in the initialized because server is not yet ready until Karafka is
+          # fully running and some of the things like listeners are not yet available
+          #
+          # This method will also be `false` in case we are not running in `karafka server` or
+          # in embedding, because in those cases Karafka does not go beyond the `initialized` phase
+          #
+          # @return [Boolean] are we able to report consumer state
+          def active?
+            # If we do not have a producer that we could use to report or it was closed, we cannot
+            # and should not report
+            return false unless super
+            return false if ::Karafka::App.initializing?
+            return false if ::Karafka::App.initialized?
+
+            true
+          end
+
           # Dispatches the current state from sampler to appropriate topics
           #
           # @param forced [Boolean] should we report bypassing the time frequency or should we
@@ -41,11 +57,6 @@ module Karafka
             sampler.sample
 
             MUTEX.synchronize do
-              # Start background thread only when needed
-              # This prevents us from starting it too early or for non-consumer processes where
-              # Karafka is being included
-              async_call unless @running
-
               return unless report?(forced)
 
               @tracked_at = monotonic_now
@@ -97,31 +108,11 @@ module Karafka
 
           private
 
-          # Reports the process state once in a while
-          def call
-            @running = true
-
-            # We won't track more often anyhow but want to try frequently not to miss a window
-            # We need to convert the sleep interval into seconds for sleep
-            sleep_time = ::Karafka::Web.config.tracking.interval.to_f / 1_000 / 10
-
-            loop do
-              report
-
-              sleep(sleep_time)
-            end
-          end
-
           # @param forced [Boolean] is this report forced. Forced means that as long as we can
           #   flush we will flush
           # @return [Boolean] Should we report or is it not yet time to do so
           def report?(forced)
-            # We never report in initializing phase because things are not yet fully configured
-            return false if ::Karafka::App.initializing?
-            # We never report in the initialized because server is not yet ready until Karafka is
-            # fully running and some of the things like listeners are not yet available
-            return false if ::Karafka::App.initialized?
-
+            return false unless active?
             return true if forced
 
             (monotonic_now - @tracked_at) >= ::Karafka::Web.config.tracking.interval

data/lib/karafka/web/tracking/producers/reporter.rb

@@ -9,9 +9,7 @@ module Karafka
         # @note Producer reported does not have to operate with the `forced` dispatch mainly
         #   because there is no expectation on immediate status updates for producers and their
         #   dispatch flow is always periodic based.
-        class Reporter
-          include ::Karafka::Core::Helpers::Time
-
+        class Reporter < Tracking::Reporter
           # Minimum number of messages to produce to produce them in sync mode
           # This acts as a small back-off not to overload the system in case we would have
           # extremely big number of errors happening
@@ -24,6 +22,7 @@ module Karafka
           MUTEX = Mutex.new
 
           def initialize
+            super
             # If there are any errors right after we started sampling, dispatch them immediately
             @tracked_at = monotonic_now - 10_000
             @error_contract = Tracking::Contracts::Error.new
@@ -62,7 +61,7 @@ module Karafka
 
           # @return [Boolean] Should we report or is it not yet time to do so
           def report?
-            return false unless ::Karafka.producer.status.active?
+            return false unless active?
 
             (monotonic_now - @tracked_at) >= ::Karafka::Web.config.tracking.interval
           end
@@ -93,6 +92,9 @@ module Karafka
           # and we can just safely ignore this
           rescue WaterDrop::Errors::ProducerClosedError
             nil
+          rescue StandardError => e
+            p '------------------------------------------------'
+            p e
           end
         end
       end

data/lib/karafka/web/tracking/producers/sampler.rb

@@ -26,6 +26,9 @@ module Karafka
           # We cannot report and track the same time, that is why we use mutex here. To make sure
           # that samples aggregations and counting does not interact with reporter flushing.
           def track
+            # Prevents deadlocks when something producer related fails in the Web UI reporter
+            return yield(self) if Reporter::MUTEX.owned?
+
             Reporter::MUTEX.synchronize do
               yield(self)
             end

data/lib/karafka/web/tracking/reporter.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Base reporter from which all the reports should inherit
+      class Reporter
+        include ::Karafka::Core::Helpers::Time
+
+        # Can this reporter report. Since some reporters may report only in part of the processes
+        # where Karafka is used (like `karafka server`) each may implement more complex rules.
+        #
+        # The basic is not to report unless we have a producer and this producer is active
+        #
+        # @return [Boolean]
+        def active?
+          return false unless ::Karafka::App.producer
+          return false unless ::Karafka::App.producer.status.active?
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/scheduler.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Triggers reporters to report in an async mode in a separate thread
+      # We report this way to prevent any potential dead-locks in cases we would be emitting
+      # statistics during transactions.
+      #
+      # We should never use the notifications thread for sensitive IO bound operations.
+      class Scheduler
+        include ::Karafka::Helpers::Async
+
+        # Creates the scheduler and runs its internal reporting
+        def initialize
+          async_call
+        end
+
+        private
+
+        # Reports the process state once in a while
+        def call
+          # We won't track more often anyhow but want to try frequently not to miss a window
+          # We need to convert the sleep interval into seconds for sleep
+          sleep_time = ::Karafka::Web.config.tracking.interval.to_f / 1_000 / 10
+
+          loop do
+            # Not every reporter may be active at a given stage or in a context of a given process
+            # We select only those that decided that they are active.
+            reporters.select(&:active?).each(&:report)
+
+            sleep(sleep_time)
+          end
+        end
+
+        # @return [Array] consumers and producers reporters
+        def reporters
+          @reporters ||= [
+            ::Karafka::Web.config.tracking.consumers.reporter,
+            ::Karafka::Web.config.tracking.producers.reporter
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/admin.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # Wrapper around Karafka Admin that alters its behaviours or injects Web UI interface
+        # specific settings that optimize the responsiveness of the UI when operating on topics
+        #
+        # @note Not all commands need those optimizations, hence we alter only those that need
+        #   that and we only expose those admin commands that are used in the Web-UI interface
+        #   component.
+        #
+        # @note We expose here only admin methods used in the Web UI interface. Processing uses the
+        #   `Karafka::Admin` with the defaults
+        class Admin
+          class << self
+            extend Forwardable
+
+            def_delegators ::Karafka::Admin, :read_watermark_offsets, :cluster_info
+
+            # Allows us to read messages from the topic
+            #
+            # @param name [String, Symbol] topic name
+            # @param partition [Integer] partition
+            # @param count [Integer] how many messages we want to get at most
+            # @param start_offset [Integer, Time] offset from which we should start. If -1 is provided
+            #   (default) we will start from the latest offset. If time is provided, the appropriate
+            #   offset will be resolved. If negative beyond -1 is provided, we move backwards more.
+            # @param settings [Hash] kafka extra settings (optional)
+            #
+            # @return [Array<Karafka::Messages::Message>] array with messages
+            def read_topic(name, partition, count, start_offset = -1, settings = {})
+              ::Karafka::Admin.read_topic(
+                name,
+                partition,
+                count,
+                start_offset,
+                # Merge our Web UI specific settings
+                config.merge(settings)
+              )
+            end
+
+            private
+
+            # @return [Hash] kafka config for Web UI interface.
+            # @note It does **not** affect tracking or processing
+            def config
+              ::Karafka::Web.config.ui.kafka
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/cluster_info.rb

@@ -4,7 +4,7 @@ module Karafka
   module Web
     module Ui
       module Models
-        # Wraps around the `Karafka::Admin#cluster_info` with caching and some additional aliases
+        # Wraps around the `Lib::Admin#cluster_info` with caching and some additional aliases
         # so we can reference relevant information easily
         class ClusterInfo
           class << self
@@ -18,7 +18,7 @@ module Karafka
               cluster_info = cache.read(:cluster_info)
 
               if cluster_info.nil? || !cached
-                cluster_info = cache.write(:cluster_info, Karafka::Admin.cluster_info)
+                cluster_info = cache.write(:cluster_info, Lib::Admin.cluster_info)
               end
 
               cluster_info

data/lib/karafka/web/ui/models/consumers_metrics.rb

@@ -32,10 +32,12 @@ module Karafka
 
             # @return [::Karafka::Messages::Message, nil] most recent state or nil if none
             def fetch
-              ::Karafka::Admin.read_topic(
+              Lib::Admin.read_topic(
                 Karafka::Web.config.topics.consumers.metrics,
                 0,
-                1
+                # We need to take last two and not the last because in case of a transactional
+                # producer, the last one will match the transaction commit message
+                2
               ).last
             end
           end

data/lib/karafka/web/ui/models/consumers_state.rb

@@ -41,10 +41,12 @@ module Karafka
 
             # @return [::Karafka::Messages::Message, nil] most recent state or nil if none
             def fetch
-              ::Karafka::Admin.read_topic(
+              Lib::Admin.read_topic(
                 Karafka::Web.config.topics.consumers.states,
                 0,
-                1
+                # We need to take last two and not the last because in case of a transactional
+                # producer, the last one will match the transaction commit message
+                2
               ).last
             end
 

data/lib/karafka/web/ui/models/counters.rb

@@ -25,7 +25,7 @@ module Karafka
 
             MAX_ERROR_PARTITIONS.times do |partition|
               begin
-                offsets = Karafka::Admin.read_watermark_offsets(
+                offsets = Lib::Admin.read_watermark_offsets(
                   ::Karafka::Web.config.topics.errors,
                   partition
                 )

data/lib/karafka/web/ui/models/message.rb

@@ -17,7 +17,7 @@ module Karafka
             # @param offset [Integer]
             # @raise [::Karafka::Web::Errors::Ui::NotFoundError] when not found
             def find(topic_id, partition_id, offset)
-              message = Karafka::Admin.read_topic(
+              message = Lib::Admin.read_topic(
                 topic_id,
                 partition_id,
                 1,
@@ -195,7 +195,7 @@ module Karafka
             # @return [Array<Karafka::Messages::Message>, false] topic partition messages or false
             #   in case we hit a non-existing offset
             def read_topic(*args)
-              ::Karafka::Admin.read_topic(*args)
+              Lib::Admin.read_topic(*args)
             rescue Rdkafka::RdkafkaError => e
               return false if e.code == :auto_offset_reset
 

data/lib/karafka/web/ui/models/processes.rb

@@ -44,7 +44,7 @@ module Karafka
                         .map { |process| process[:offset] }
                         .sort
 
-              ::Karafka::Admin.read_topic(
+              Lib::Admin.read_topic(
                 ::Karafka::Web.config.topics.consumers.reports,
                 0,
                 # We set 10k here because we start from the latest offset of the reports, hence

data/lib/karafka/web/ui/models/watermark_offsets.rb

@@ -13,7 +13,7 @@ module Karafka
             # @param partition_id [Integer]
             # @return [WatermarkOffsets]
             def find(topic_id, partition_id)
-              offsets = ::Karafka::Admin.read_watermark_offsets(topic_id, partition_id)
+              offsets = Lib::Admin.read_watermark_offsets(topic_id, partition_id)
 
               new(
                 low: offsets.first,

data/lib/karafka/web/ui/pro/controllers/explorer.rb

@@ -132,11 +132,22 @@ module Karafka
                 active_partitions, = Paginators::Partitions.call(partitions_count, 1)
               end
 
-              # This selects first page with most recent messages
-              messages, = Models::Message.topic_page(topic_id, active_partitions, 1)
+              recent = nil
 
-              # Selects newest out of all partitions
-              recent = messages.max_by(&:timestamp)
+              # This selects first pages with most recent messages and moves to next if first
+              # contains only compacted data, etc.
+              #
+              # We do it until we find a message we could refer to (if doable) within first
+              # ten pages
+              10.times do |page|
+                messages, = Models::Message.topic_page(topic_id, active_partitions, page + 1)
+
+                # Selects newest out of all partitions
+                # Reject compacted messages and transaction-related once
+                recent = messages.reject { |message| message.is_a?(Array) }.max_by(&:timestamp)
+
+                break if recent
+              end
 
               recent || raise(::Karafka::Web::Errors::Ui::NotFoundError)
 
@@ -184,7 +195,7 @@ module Karafka
             # @param partition_id [Integer]
             # @param time [Time] time of the message
             def closest(topic_id, partition_id, time)
-              target = ::Karafka::Admin.read_topic(topic_id, partition_id, 1, time).first
+              target = Lib::Admin.read_topic(topic_id, partition_id, 1, time).first
 
               partition_path = "explorer/#{topic_id}/#{partition_id}"
               partition_path += "?offset=#{target.offset}" if target

data/lib/karafka/web/ui/pro/views/errors/_error.erb

@@ -1,7 +1,7 @@
 <% if error_msg.is_a?(Array) %>
   <tr>
     <td colspan="5" class="text-center text-muted">
-      This error has either been removed or compacted and is no longer available.
+      This offset does not contain error data. The message may have been compacted or is a system entry.
     </td>
   </tr>
 <% else %>

data/lib/karafka/web/ui/pro/views/explorer/_message.erb

@@ -7,7 +7,7 @@
       <%= message[1] %>
     </td>
     <td colspan="3" class="text-center text-muted">
-      This message has either been removed or compacted and is no longer available.
+      This offset does not contain any data. The message may have been compacted or is a system entry.
     </td>
   </tr>
 <% else %>

data/lib/karafka/web/ui/public/javascripts/live_poll.js

@@ -73,7 +73,7 @@ function setLivePollButton() {
 
 function checkResponse(resp) {
   if (!resp.ok) {
-    throw response.error();
+    throw resp;
   }
   return resp
 }

data/lib/karafka/web/ui/views/errors/_error.erb

@@ -1,7 +1,7 @@
 <% if error_msg.is_a?(Array) %>
   <tr>
     <td colspan="5" class="text-center text-muted">
-      This error has either been removed or compacted and is no longer available.
+      This offset does not contain error data. The message may have been compacted or is a system entry.
     </td>
   </tr>
 <% else %>

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.6'
+    VERSION = '0.7.8'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.7.8
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-10-10 00:00:00.000000000 Z
+date: 2023-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.6
+        version: 2.2.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.6
+        version: 2.2.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -173,6 +173,12 @@ files:
 - lib/karafka/web.rb
 - lib/karafka/web/app.rb
 - lib/karafka/web/cli.rb
+- lib/karafka/web/cli/base.rb
+- lib/karafka/web/cli/help.rb
+- lib/karafka/web/cli/install.rb
+- lib/karafka/web/cli/migrate.rb
+- lib/karafka/web/cli/reset.rb
+- lib/karafka/web/cli/uninstall.rb
 - lib/karafka/web/config.rb
 - lib/karafka/web/contracts/base.rb
 - lib/karafka/web/contracts/config.rb
@@ -219,10 +225,11 @@ files:
 - lib/karafka/web/tracking/memoized_shell.rb
 - lib/karafka/web/tracking/producers/listeners/base.rb
 - lib/karafka/web/tracking/producers/listeners/errors.rb
-- lib/karafka/web/tracking/producers/listeners/reporter.rb
 - lib/karafka/web/tracking/producers/reporter.rb
 - lib/karafka/web/tracking/producers/sampler.rb
+- lib/karafka/web/tracking/reporter.rb
 - lib/karafka/web/tracking/sampler.rb
+- lib/karafka/web/tracking/scheduler.rb
 - lib/karafka/web/tracking/ttl_array.rb
 - lib/karafka/web/tracking/ttl_hash.rb
 - lib/karafka/web/ui/app.rb
@@ -241,6 +248,7 @@ files:
 - lib/karafka/web/ui/controllers/status.rb
 - lib/karafka/web/ui/helpers/application_helper.rb
 - lib/karafka/web/ui/helpers/paths_helper.rb
+- lib/karafka/web/ui/lib/admin.rb
 - lib/karafka/web/ui/lib/hash_proxy.rb
 - lib/karafka/web/ui/lib/paginations/base.rb
 - lib/karafka/web/ui/lib/paginations/offset_based.rb
@@ -447,7 +455,7 @@ licenses:
 metadata:
   funding_uri: https://karafka.io/#become-pro
   homepage_uri: https://karafka.io
-  changelog_uri: https://github.com/karafka/karafka-web/blob/master/CHANGELOG.md
+  changelog_uri: https://karafka.io/docs/Changelog-Karafka-Web-UI
   bug_tracker_uri: https://github.com/karafka/karafka-web/issues
   source_code_uri: https://github.com/karafka/karafka-web
   documentation_uri: https://karafka.io/docs

data/lib/karafka/web/tracking/producers/listeners/reporter.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Web
-    module Tracking
-      module Producers
-        module Listeners
-          # Special listener that we use to report data about producers states
-          # We don't have to have a separate thread for reporting, because producers have their
-          # own internal threads for changes polling and we can utilize this thread
-          class Reporter < Base
-            # @param _event [Karafka::Core::Monitoring::Event]
-            def on_statistics_emitted(_event)
-              reporter.report
-            end
-          end
-        end
-      end
-    end
-  end
-end