karafka-web 0.7.7 → 0.7.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e4dd8f19fdad3697e837ed96aa8fafabae6eac4efe8cbf75aaa788eccebc44c
-  data.tar.gz: 33814043050afda80109fc14082a09de0f5499041fe2f4a24a90b25f7cd501c8
+  metadata.gz: 24bc5fe488a1304886f3f7f697a8662463f4c21fae60daf52d317852f9f43669
+  data.tar.gz: eaa8e4b7d9b9d70200c8929e8fa65ed7906a1b4085465b15525bfee670ebbcc9
 SHA512:
-  metadata.gz: 8c84de9a33977904aadffa3ff51af275f4301e7375f9194343b4980d82b6b4fdf4dc57aeb0f1e0cef8c66499b048fb595ec4cf7d1bdbd3011995a0223c4a70ad
-  data.tar.gz: 4c82722b6fab042562ee3f390d10775b1de7e0c77406c623e7e17f36860a9954ef132cbc540d30449c1212a9d507203e8fb9500393c92b2be09ccbc06c281f0d
+  metadata.gz: 0ba2c668103a3b25169d86f2c0624fae04d144694040694ff79c4932bf81fe648e5ba11223be5a5f9e8d8ba0bbbac9da84f2ec5312e81d07b35873707b7b754c
+  data.tar.gz: b0ec51820b50dfde4f88e942e3b0dcc2400170d0fbe256503294f366e2f4fe0057249bb525528b789bed254ba6ec035d5502e515042bd094999663155f8a221d

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Karafka Web changelog
 
+## 0.7.9 (2023-10-25)
+- [Enhancement] Allow for `Karafka::Web.producer` reconfiguration from the default (`Karafka.producer`).
+- [Change] Rely on `karafka-core` `>=` `2.2.4` to support lazy loaded custom web producer.
+
+## 0.7.8 (2023-10-24)
+- [Enhancement] Support transactional producer usage with Web UI.
+- [Fix] Prevent a scenario where an ongoing transactional producer would have stats emitted and an error that could not have been dispatched because of the transaction, creating a dead-lock.
+- [Fix] Make sure that the `recent` displays the most recent non-compacted, non-system message.
+- [Fix] Improve the `recent` message display to compensate for aborted transactions.
+- [Fix] Fix `ReferenceError: response is not defined` that occurs when Web UI returns refresh non 200.
+
 ## 0.7.7 (2023-10-20)
 - [Fix] Remove `thor` as a CLI engine due to breaking changes.
 

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.7.7)
+    karafka-web (0.7.9)
       erubi (~> 1.4)
-      karafka (>= 2.2.8.beta1, < 3.0.0)
-      karafka-core (>= 2.2.2, < 3.0.0)
+      karafka (>= 2.2.9, < 3.0.0)
+      karafka-core (>= 2.2.4, < 3.0.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
 
@@ -36,18 +36,18 @@ GEM
     ffi (1.16.3)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka (2.2.8.beta1)
+    karafka (2.2.9)
       karafka-core (>= 2.2.2, < 2.3.0)
-      waterdrop (>= 2.6.6, < 3.0.0)
+      waterdrop (>= 2.6.10, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.2.3)
+    karafka-core (2.2.4)
       concurrent-ruby (>= 1.1)
       karafka-rdkafka (>= 0.13.6, < 0.14.0)
     karafka-rdkafka (0.13.6)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    mini_portile2 (2.8.4)
+    mini_portile2 (2.8.5)
     minitest (5.20.0)
     mutex_m (0.1.2)
     rack (3.0.8)
@@ -82,8 +82,8 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.7)
-      karafka-core (>= 2.1.1, < 3.0.0)
+    waterdrop (2.6.10)
+      karafka-core (>= 2.2.3, < 3.0.0)
       zeitwerk (~> 2.3)
     webrick (1.8.1)
     zeitwerk (2.6.12)

data/karafka-web.gemspec

@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0 Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.2.8.beta1', '< 3.0.0'
-  spec.add_dependency 'karafka-core', '>= 2.2.2', '< 3.0.0'
+  spec.add_dependency 'karafka', '>= 2.2.9', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.2.4', '< 3.0.0'
   spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'
 

data/lib/karafka/web/config.rb

@@ -11,6 +11,17 @@ module Karafka
       # This is used both in the processing for eviction and in the UI
       setting :ttl, default: 30_000
 
+      # Producer for the Web UI. By default it is a `Karafka.producer`, however it may be
+      # overwritten if we want to use a separate instance in case of heavy usage of the
+      # transactional producer as a default. In cases like this, Karafka may not be able to report
+      # data because it uses this producer and it may be locked because of the transaction in a
+      # user space.
+      setting(
+        :producer,
+        constructor: -> { ::Karafka.producer },
+        lazy: true
+      )
+
       # Topics naming - used for processing and UI
       setting :topics do
         # All the errors encountered will be dispatched to this topic for inspection
@@ -36,12 +47,18 @@ module Karafka
         # 5 seconds should be enough
         setting :interval, default: 5_000
 
+        # Main Web UI reporting scheduler that runs a background thread and reports periodically
+        # from the consumer reporter and producer reporter
+        setting :scheduler, default: Tracking::Scheduler.new
+
         setting :consumers do
-          # Reports the metrics collected in the sampler
+          # Reports the metrics collected in the consumer sampler
           setting :reporter, default: Tracking::Consumers::Reporter.new
 
+          # Samples for fetching and storing metrics samples about the consumer process
           setting :sampler, default: Tracking::Consumers::Sampler.new
 
+          # Listeners needed for the Web UI to track consumer related changes
           setting :listeners, default: [
             Tracking::Consumers::Listeners::Status.new,
             Tracking::Consumers::Listeners::Errors.new,
@@ -53,13 +70,15 @@ module Karafka
         end
 
         setting :producers do
+          # Reports the metrics collected in the producer sampler
           setting :reporter, default: Tracking::Producers::Reporter.new
 
+          # Sampler for errors from producers
           setting :sampler, default: Tracking::Producers::Sampler.new
 
+          # Listeners needed for the Web UI to track producers related stuff
           setting :listeners, default: [
-            Tracking::Producers::Listeners::Errors.new,
-            Tracking::Producers::Listeners::Reporter.new
+            Tracking::Producers::Listeners::Errors.new
           ]
         end
       end
@@ -114,6 +133,37 @@ module Karafka
         # In some cases you may want to limit what is being displayed due to the type of data you
         # are dealing with
         setting :visibility_filter, default: Ui::Models::VisibilityFilter.new
+
+        # Specific kafka settings that are tuned to operate within the Web UI interface.
+        #
+        # Please do not change them unless you know what you are doing as their misconfiguration
+        # may cause Web UI to misbehave
+        #
+        # The settings are inherited as follows:
+        #   1. root routing level `kafka` settings
+        #   2. admin `kafka` settings
+        #   3. web ui `kafka` settings from here
+        #
+        # Those settings impact ONLY Web UI interface and do not affect other scopes. This is done
+        # on purpose as we want to improve responsiveness of the interface by tuning some of the
+        # settings and this is not that relevant for processing itself.
+        #
+        # option [Hash] extra changes to the default admin kafka settings
+        setting :kafka, default: {
+          # optimizes the responsiveness of the Web UI in three scenarios:
+          #   - topics to which writes happen only in transactions so EOF is yield faster
+          #   - heavily compacted topics
+          #   - Web UI topics read operations when using transactional producer
+          #
+          # This can be configured to be higher if you do not use transactional WaterDrop producer.
+          # This value is used when last message (first from the high watermark offset) is the
+          # transaction commit message. In cases like this the EOF gets propagated after this time
+          # so we have to wait. Default 500ms means, that for some views, where we take our data
+          # that might have been committed via transactional producer, we would wait for 1 second
+          # to get needed data. If you are experiencing timeouts or other issues with the Web IU
+          # interface, you can increase this.
+          'fetch.wait.max.ms': 100
+        }
       end
     end
   end

data/lib/karafka/web/management/create_initial_states.rb

@@ -55,7 +55,7 @@ module Karafka
             exists('consumers state')
           else
             creating('consumers state')
-            ::Karafka.producer.produce_sync(
+            ::Karafka::Web.producer.produce_sync(
               topic: Karafka::Web.config.topics.consumers.states,
               key: Karafka::Web.config.topics.consumers.states,
               payload: DEFAULT_STATE.to_json
@@ -67,7 +67,7 @@ module Karafka
             exists('consumers metrics')
           else
             creating('consumers metrics')
-            ::Karafka.producer.produce_sync(
+            ::Karafka::Web.producer.produce_sync(
               topic: Karafka::Web.config.topics.consumers.metrics,
               key: Karafka::Web.config.topics.consumers.metrics,
               payload: DEFAULT_METRICS.merge(dispatched_at: Time.now.to_f).to_json

data/lib/karafka/web/management/enable.rb

@@ -11,6 +11,7 @@ module Karafka
         def call
           extend_routing
           subscribe_to_monitor
+          subscribe_to_close_web_producer
         end
 
         private
@@ -71,9 +72,28 @@ module Karafka
             ::Karafka.monitor.subscribe(listener)
           end
 
-          # Installs all the producer related listeners
+          # Installs all the producer related listeners into Karafka default listener and
+          # into Karafka::Web listener in case it would be different than the Karafka one
           ::Karafka::Web.config.tracking.producers.listeners.each do |listener|
             ::Karafka.producer.monitor.subscribe(listener)
+
+            # Do not instrument twice in case only one default producer is used
+            next if ::Karafka.producer == ::Karafka::Web.producer
+
+            ::Karafka::Web.producer.monitor.subscribe(listener)
+          end
+        end
+
+        # In most cases we want to close the producer if possible.
+        # While we cannot do it easily in user processes and we should rely on WaterDrop
+        # finalization logic, we can do it in `karafka server` on terminate
+        #
+        # In other places, this producer anyhow should not be used.
+        def subscribe_to_close_web_producer
+          ::Karafka::App.monitor.subscribe('app.terminated') do
+            # If Web producer is the same as `Karafka.producer` it will do nothing as you can
+            # call `#close` multiple times without side effects
+            ::Karafka::Web.producer.close
           end
         end
       end

data/lib/karafka/web/processing/consumer.rb

@@ -114,7 +114,7 @@ module Karafka
         def flush
           @flushed_at = monotonic_now
 
-          producer.produce_many_async(
+          ::Karafka::Web.producer.produce_many_async(
             [
               {
                 topic: Karafka::Web.config.topics.consumers.states,

data/lib/karafka/web/processing/consumers/metrics.rb

@@ -14,7 +14,10 @@ module Karafka
               metrics_message = ::Karafka::Admin.read_topic(
                 Karafka::Web.config.topics.consumers.metrics,
                 0,
-                1
+                # We need to take more in case there would be transactions running.
+                # In theory we could take two but this compensates for any involuntary
+                # revocations and cases where two producers would write to the same state
+                5
               ).last
 
               return metrics_message.payload if metrics_message

data/lib/karafka/web/processing/consumers/state.rb

@@ -14,7 +14,10 @@ module Karafka
               state_message = ::Karafka::Admin.read_topic(
                 Karafka::Web.config.topics.consumers.states,
                 0,
-                1
+                # We need to take more in case there would be transactions running.
+                # In theory we could take two but this compensates for any involuntary
+                # revocations and cases where two producers would write to the same state
+                5
               ).last
 
               return state_message.payload if state_message

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -5,10 +5,7 @@ module Karafka
     module Tracking
       module Consumers
         # Reports the collected data about the process and sends it, so we can use it in the UI
-        class Reporter
-          include ::Karafka::Core::Helpers::Time
-          include ::Karafka::Helpers::Async
-
+        class Reporter < Tracking::Reporter
           # Minimum number of messages to produce to produce them in sync mode
           # This acts as a small back-off not to overload the system in case we would have
           # extremely big number of errors happening
@@ -21,12 +18,31 @@ module Karafka
           MUTEX = Mutex.new
 
           def initialize
+            super
             # Move back so first report is dispatched fast to indicate, that the process is alive
             @tracked_at = monotonic_now - 10_000
             @report_contract = Consumers::Contracts::Report.new
             @error_contract = Tracking::Contracts::Error.new
           end
 
+          # We never report in initializing phase because things are not yet fully configured
+          # We never report in the initialized because server is not yet ready until Karafka is
+          # fully running and some of the things like listeners are not yet available
+          #
+          # This method will also be `false` in case we are not running in `karafka server` or
+          # in embedding, because in those cases Karafka does not go beyond the `initialized` phase
+          #
+          # @return [Boolean] are we able to report consumer state
+          def active?
+            # If we do not have a producer that we could use to report or it was closed, we cannot
+            # and should not report
+            return false unless super
+            return false if ::Karafka::App.initializing?
+            return false if ::Karafka::App.initialized?
+
+            true
+          end
+
           # Dispatches the current state from sampler to appropriate topics
           #
           # @param forced [Boolean] should we report bypassing the time frequency or should we
@@ -41,11 +57,6 @@ module Karafka
             sampler.sample
 
             MUTEX.synchronize do
-              # Start background thread only when needed
-              # This prevents us from starting it too early or for non-consumer processes where
-              # Karafka is being included
-              async_call unless @running
-
               return unless report?(forced)
 
               @tracked_at = monotonic_now
@@ -97,31 +108,11 @@ module Karafka
 
           private
 
-          # Reports the process state once in a while
-          def call
-            @running = true
-
-            # We won't track more often anyhow but want to try frequently not to miss a window
-            # We need to convert the sleep interval into seconds for sleep
-            sleep_time = ::Karafka::Web.config.tracking.interval.to_f / 1_000 / 10
-
-            loop do
-              report
-
-              sleep(sleep_time)
-            end
-          end
-
           # @param forced [Boolean] is this report forced. Forced means that as long as we can
           #   flush we will flush
           # @return [Boolean] Should we report or is it not yet time to do so
           def report?(forced)
-            # We never report in initializing phase because things are not yet fully configured
-            return false if ::Karafka::App.initializing?
-            # We never report in the initialized because server is not yet ready until Karafka is
-            # fully running and some of the things like listeners are not yet available
-            return false if ::Karafka::App.initialized?
-
+            return false unless active?
             return true if forced
 
             (monotonic_now - @tracked_at) >= ::Karafka::Web.config.tracking.interval
@@ -144,9 +135,9 @@ module Karafka
           #   slowdown any processing.
           def produce(messages)
             if messages.count >= PRODUCE_SYNC_THRESHOLD
-              ::Karafka.producer.produce_many_sync(messages)
+              ::Karafka::Web.producer.produce_many_sync(messages)
             else
-              ::Karafka.producer.produce_many_async(messages)
+              ::Karafka::Web.producer.produce_many_async(messages)
             end
           # Since we run this in a background thread, there may be a case upon shutdown, where the
           # producer is closed right before a potential dispatch. It is not worth dealing with this

data/lib/karafka/web/tracking/producers/reporter.rb

@@ -9,9 +9,7 @@ module Karafka
         # @note Producer reported does not have to operate with the `forced` dispatch mainly
         #   because there is no expectation on immediate status updates for producers and their
         #   dispatch flow is always periodic based.
-        class Reporter
-          include ::Karafka::Core::Helpers::Time
-
+        class Reporter < Tracking::Reporter
           # Minimum number of messages to produce to produce them in sync mode
           # This acts as a small back-off not to overload the system in case we would have
           # extremely big number of errors happening
@@ -24,6 +22,7 @@ module Karafka
           MUTEX = Mutex.new
 
           def initialize
+            super
             # If there are any errors right after we started sampling, dispatch them immediately
             @tracked_at = monotonic_now - 10_000
             @error_contract = Tracking::Contracts::Error.new
@@ -62,7 +61,7 @@ module Karafka
 
           # @return [Boolean] Should we report or is it not yet time to do so
           def report?
-            return false unless ::Karafka.producer.status.active?
+            return false unless active?
 
             (monotonic_now - @tracked_at) >= ::Karafka::Web.config.tracking.interval
           end
@@ -84,15 +83,18 @@ module Karafka
           #   slowdown any processing.
           def produce(messages)
             if messages.count >= PRODUCE_SYNC_THRESHOLD
-              ::Karafka.producer.produce_many_sync(messages)
+              ::Karafka::Web.producer.produce_many_sync(messages)
             else
-              ::Karafka.producer.produce_many_async(messages)
+              ::Karafka::Web.producer.produce_many_async(messages)
             end
           # Since we run this in a background thread, there may be a case upon shutdown, where the
           # producer is closed right before a potential dispatch. It is not worth dealing with this
           # and we can just safely ignore this
           rescue WaterDrop::Errors::ProducerClosedError
             nil
+          rescue StandardError => e
+            p '------------------------------------------------'
+            p e
           end
         end
       end

data/lib/karafka/web/tracking/producers/sampler.rb

@@ -26,6 +26,9 @@ module Karafka
           # We cannot report and track the same time, that is why we use mutex here. To make sure
           # that samples aggregations and counting does not interact with reporter flushing.
           def track
+            # Prevents deadlocks when something producer related fails in the Web UI reporter
+            return yield(self) if Reporter::MUTEX.owned?
+
             Reporter::MUTEX.synchronize do
               yield(self)
             end

data/lib/karafka/web/tracking/reporter.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Base reporter from which all the reports should inherit
+      class Reporter
+        include ::Karafka::Core::Helpers::Time
+
+        # Can this reporter report. Since some reporters may report only in part of the processes
+        # where Karafka is used (like `karafka server`) each may implement more complex rules.
+        #
+        # The basic is not to report unless we have a producer and this producer is active
+        #
+        # @return [Boolean]
+        def active?
+          return false unless ::Karafka::Web.producer
+          return false unless ::Karafka::Web.producer.status.active?
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/scheduler.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Triggers reporters to report in an async mode in a separate thread
+      # We report this way to prevent any potential dead-locks in cases we would be emitting
+      # statistics during transactions.
+      #
+      # We should never use the notifications thread for sensitive IO bound operations.
+      class Scheduler
+        include ::Karafka::Helpers::Async
+
+        # Creates the scheduler and runs its internal reporting
+        def initialize
+          async_call
+        end
+
+        private
+
+        # Reports the process state once in a while
+        def call
+          # We won't track more often anyhow but want to try frequently not to miss a window
+          # We need to convert the sleep interval into seconds for sleep
+          sleep_time = ::Karafka::Web.config.tracking.interval.to_f / 1_000 / 10
+
+          loop do
+            # Not every reporter may be active at a given stage or in a context of a given process
+            # We select only those that decided that they are active.
+            reporters.select(&:active?).each(&:report)
+
+            sleep(sleep_time)
+          end
+        end
+
+        # @return [Array] consumers and producers reporters
+        def reporters
+          @reporters ||= [
+            ::Karafka::Web.config.tracking.consumers.reporter,
+            ::Karafka::Web.config.tracking.producers.reporter
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/lib/admin.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Ui
+      module Lib
+        # Wrapper around Karafka Admin that alters its behaviours or injects Web UI interface
+        # specific settings that optimize the responsiveness of the UI when operating on topics
+        #
+        # @note Not all commands need those optimizations, hence we alter only those that need
+        #   that and we only expose those admin commands that are used in the Web-UI interface
+        #   component.
+        #
+        # @note We expose here only admin methods used in the Web UI interface. Processing uses the
+        #   `Karafka::Admin` with the defaults
+        class Admin
+          class << self
+            extend Forwardable
+
+            def_delegators ::Karafka::Admin, :read_watermark_offsets, :cluster_info
+
+            # Allows us to read messages from the topic
+            #
+            # @param name [String, Symbol] topic name
+            # @param partition [Integer] partition
+            # @param count [Integer] how many messages we want to get at most
+            # @param start_offset [Integer, Time] offset from which we should start. If -1 is provided
+            #   (default) we will start from the latest offset. If time is provided, the appropriate
+            #   offset will be resolved. If negative beyond -1 is provided, we move backwards more.
+            # @param settings [Hash] kafka extra settings (optional)
+            #
+            # @return [Array<Karafka::Messages::Message>] array with messages
+            def read_topic(name, partition, count, start_offset = -1, settings = {})
+              ::Karafka::Admin.read_topic(
+                name,
+                partition,
+                count,
+                start_offset,
+                # Merge our Web UI specific settings
+                config.merge(settings)
+              )
+            end
+
+            private
+
+            # @return [Hash] kafka config for Web UI interface.
+            # @note It does **not** affect tracking or processing
+            def config
+              ::Karafka::Web.config.ui.kafka
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/ui/models/cluster_info.rb

@@ -4,7 +4,7 @@ module Karafka
   module Web
     module Ui
       module Models
-        # Wraps around the `Karafka::Admin#cluster_info` with caching and some additional aliases
+        # Wraps around the `Lib::Admin#cluster_info` with caching and some additional aliases
         # so we can reference relevant information easily
         class ClusterInfo
           class << self
@@ -18,7 +18,7 @@ module Karafka
               cluster_info = cache.read(:cluster_info)
 
               if cluster_info.nil? || !cached
-                cluster_info = cache.write(:cluster_info, Karafka::Admin.cluster_info)
+                cluster_info = cache.write(:cluster_info, Lib::Admin.cluster_info)
               end
 
               cluster_info

data/lib/karafka/web/ui/models/consumers_metrics.rb

@@ -32,10 +32,12 @@ module Karafka
 
             # @return [::Karafka::Messages::Message, nil] most recent state or nil if none
             def fetch
-              ::Karafka::Admin.read_topic(
+              Lib::Admin.read_topic(
                 Karafka::Web.config.topics.consumers.metrics,
                 0,
-                1
+                # We need to take last two and not the last because in case of a transactional
+                # producer, the last one will match the transaction commit message
+                2
               ).last
             end
           end

data/lib/karafka/web/ui/models/consumers_state.rb

@@ -41,10 +41,12 @@ module Karafka
 
             # @return [::Karafka::Messages::Message, nil] most recent state or nil if none
             def fetch
-              ::Karafka::Admin.read_topic(
+              Lib::Admin.read_topic(
                 Karafka::Web.config.topics.consumers.states,
                 0,
-                1
+                # We need to take last two and not the last because in case of a transactional
+                # producer, the last one will match the transaction commit message
+                2
               ).last
             end
 

data/lib/karafka/web/ui/models/counters.rb

@@ -25,7 +25,7 @@ module Karafka
 
             MAX_ERROR_PARTITIONS.times do |partition|
               begin
-                offsets = Karafka::Admin.read_watermark_offsets(
+                offsets = Lib::Admin.read_watermark_offsets(
                   ::Karafka::Web.config.topics.errors,
                   partition
                 )

data/lib/karafka/web/ui/models/message.rb

@@ -17,7 +17,7 @@ module Karafka
             # @param offset [Integer]
             # @raise [::Karafka::Web::Errors::Ui::NotFoundError] when not found
             def find(topic_id, partition_id, offset)
-              message = Karafka::Admin.read_topic(
+              message = Lib::Admin.read_topic(
                 topic_id,
                 partition_id,
                 1,
@@ -195,7 +195,7 @@ module Karafka
             # @return [Array<Karafka::Messages::Message>, false] topic partition messages or false
             #   in case we hit a non-existing offset
             def read_topic(*args)
-              ::Karafka::Admin.read_topic(*args)
+              Lib::Admin.read_topic(*args)
             rescue Rdkafka::RdkafkaError => e
               return false if e.code == :auto_offset_reset
 

data/lib/karafka/web/ui/models/processes.rb

@@ -44,7 +44,7 @@ module Karafka
                         .map { |process| process[:offset] }
                         .sort
 
-              ::Karafka::Admin.read_topic(
+              Lib::Admin.read_topic(
                 ::Karafka::Web.config.topics.consumers.reports,
                 0,
                 # We set 10k here because we start from the latest offset of the reports, hence

data/lib/karafka/web/ui/models/watermark_offsets.rb

@@ -13,7 +13,7 @@ module Karafka
             # @param partition_id [Integer]
             # @return [WatermarkOffsets]
             def find(topic_id, partition_id)
-              offsets = ::Karafka::Admin.read_watermark_offsets(topic_id, partition_id)
+              offsets = Lib::Admin.read_watermark_offsets(topic_id, partition_id)
 
               new(
                 low: offsets.first,

data/lib/karafka/web/ui/pro/controllers/explorer.rb

@@ -132,11 +132,22 @@ module Karafka
                 active_partitions, = Paginators::Partitions.call(partitions_count, 1)
               end
 
-              # This selects first page with most recent messages
-              messages, = Models::Message.topic_page(topic_id, active_partitions, 1)
+              recent = nil
 
-              # Selects newest out of all partitions
-              recent = messages.max_by(&:timestamp)
+              # This selects first pages with most recent messages and moves to next if first
+              # contains only compacted data, etc.
+              #
+              # We do it until we find a message we could refer to (if doable) within first
+              # ten pages
+              10.times do |page|
+                messages, = Models::Message.topic_page(topic_id, active_partitions, page + 1)
+
+                # Selects newest out of all partitions
+                # Reject compacted messages and transaction-related once
+                recent = messages.reject { |message| message.is_a?(Array) }.max_by(&:timestamp)
+
+                break if recent
+              end
 
               recent || raise(::Karafka::Web::Errors::Ui::NotFoundError)
 
@@ -184,7 +195,7 @@ module Karafka
             # @param partition_id [Integer]
             # @param time [Time] time of the message
             def closest(topic_id, partition_id, time)
-              target = ::Karafka::Admin.read_topic(topic_id, partition_id, 1, time).first
+              target = Lib::Admin.read_topic(topic_id, partition_id, 1, time).first
 
               partition_path = "explorer/#{topic_id}/#{partition_id}"
               partition_path += "?offset=#{target.offset}" if target

data/lib/karafka/web/ui/pro/controllers/messages.rb

@@ -28,7 +28,7 @@ module Karafka
             def republish(topic_id, partition_id, offset)
               message = Ui::Models::Message.find(topic_id, partition_id, offset)
 
-              delivery = ::Karafka.producer.produce_sync(
+              delivery = ::Karafka::Web.producer.produce_sync(
                 topic: topic_id,
                 partition: partition_id,
                 payload: message.raw_payload,

data/lib/karafka/web/ui/pro/views/errors/_error.erb

@@ -1,7 +1,7 @@
 <% if error_msg.is_a?(Array) %>
   <tr>
     <td colspan="5" class="text-center text-muted">
-      This error has either been removed or compacted and is no longer available.
+      This offset does not contain error data. The message may have been compacted or is a system entry.
     </td>
   </tr>
 <% else %>

data/lib/karafka/web/ui/pro/views/explorer/_message.erb

@@ -7,7 +7,7 @@
       <%= message[1] %>
     </td>
     <td colspan="3" class="text-center text-muted">
-      This message has either been removed or compacted and is no longer available.
+      This offset does not contain any data. The message may have been compacted or is a system entry.
     </td>
   </tr>
 <% else %>

data/lib/karafka/web/ui/public/javascripts/live_poll.js

@@ -73,7 +73,7 @@ function setLivePollButton() {
 
 function checkResponse(resp) {
   if (!resp.ok) {
-    throw response.error();
+    throw resp;
   }
   return resp
 }

data/lib/karafka/web/ui/views/errors/_error.erb

@@ -1,7 +1,7 @@
 <% if error_msg.is_a?(Array) %>
   <tr>
     <td colspan="5" class="text-center text-muted">
-      This error has either been removed or compacted and is no longer available.
+      This offset does not contain error data. The message may have been compacted or is a system entry.
     </td>
   </tr>
 <% else %>

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.7'
+    VERSION = '0.7.9'
   end
 end

data/lib/karafka/web.rb

@@ -13,6 +13,12 @@ module Karafka
   # Karafka Web UI + Karafka web monitoring
   module Web
     class << self
+      # @return [WaterDrop::Producer, nil] waterdrop messages producer or nil if not yet fully
+      #   initialized. It may not be fully initialized until the configuration is done
+      def producer
+        @producer ||= Web.config.producer
+      end
+
       # @return [String] root path of this gem
       def gem_root
         Pathname.new(File.expand_path('../..', __dir__))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.7.7
+  version: 0.7.9
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-10-20 00:00:00.000000000 Z
+date: 2023-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.8.beta1
+        version: 2.2.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.8.beta1
+        version: 2.2.9
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -225,10 +225,11 @@ files:
 - lib/karafka/web/tracking/memoized_shell.rb
 - lib/karafka/web/tracking/producers/listeners/base.rb
 - lib/karafka/web/tracking/producers/listeners/errors.rb
-- lib/karafka/web/tracking/producers/listeners/reporter.rb
 - lib/karafka/web/tracking/producers/reporter.rb
 - lib/karafka/web/tracking/producers/sampler.rb
+- lib/karafka/web/tracking/reporter.rb
 - lib/karafka/web/tracking/sampler.rb
+- lib/karafka/web/tracking/scheduler.rb
 - lib/karafka/web/tracking/ttl_array.rb
 - lib/karafka/web/tracking/ttl_hash.rb
 - lib/karafka/web/ui/app.rb
@@ -247,6 +248,7 @@ files:
 - lib/karafka/web/ui/controllers/status.rb
 - lib/karafka/web/ui/helpers/application_helper.rb
 - lib/karafka/web/ui/helpers/paths_helper.rb
+- lib/karafka/web/ui/lib/admin.rb
 - lib/karafka/web/ui/lib/hash_proxy.rb
 - lib/karafka/web/ui/lib/paginations/base.rb
 - lib/karafka/web/ui/lib/paginations/offset_based.rb

data/lib/karafka/web/tracking/producers/listeners/reporter.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Web
-    module Tracking
-      module Producers
-        module Listeners
-          # Special listener that we use to report data about producers states
-          # We don't have to have a separate thread for reporting, because producers have their
-          # own internal threads for changes polling and we can utilize this thread
-          class Reporter < Base
-            # @param _event [Karafka::Core::Monitoring::Event]
-            def on_statistics_emitted(_event)
-              reporter.report
-            end
-          end
-        end
-      end
-    end
-  end
-end