karafka-web 0.7.8 → 0.7.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d46244304483184c3be59a0686497a9f948e9b666dccf00961b560e60db4655
-  data.tar.gz: 329686ce548063787fe06290d67650793ebef6c014f43b753780ad120671a8b1
+  metadata.gz: '0806cac6cd5ba083c727f4737a5bacbc6fb51a45229043745e401c3f5ccfd120'
+  data.tar.gz: f6f5743e8fe1b32eec5c126f81d8ed4bcb0501569c050b721a238388d608a7aa
 SHA512:
-  metadata.gz: 5a18a3c23f63382c6c41dc5ac7f82f893bc14a55e7ec68af9cf16da0f47a45aee77ca9626973570cb9d6abe9355e0712f117424e00619b7b0198748c3bfbeb55
-  data.tar.gz: f639bd90ca21c09fcd798fc289079cafeb9c752d261ecf88df9762e347df738bf260c658f0f15b3b08c3708daf091217c0159d85ac975380463a65a45d876d0b
+  metadata.gz: a70b5779abaa3440ffb85f8d735edf6a96723ab08c0c52a982c3ac09331d4a459053de0fcf4767bd79631d0314efe9d3a13949639d1b2ee62f88cda6e6519dbb
+  data.tar.gz: 2ab8bc6927e8e5d0cf6ef63e2be932322eeec8d0f668c7a144b61d5dfc3ceae74a758dcd58f32deac4ccf42c4cad1d799e7df4f5c0b0bbb0494b245362104262

data/.github/workflows/ci.yml

@@ -1,6 +1,8 @@
 name: ci
 
-concurrency: ci-${{ github.ref }}
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 on:
   pull_request:

data/CHANGELOG.md

@@ -1,8 +1,14 @@
 # Karafka Web changelog
 
+## 0.7.10 (2023-10-31)
+- [Fix] Max LSO chart does not work as expected (#201)
+
+## 0.7.9 (2023-10-25)
+- [Enhancement] Allow for `Karafka::Web.producer` reconfiguration from the default (`Karafka.producer`).
+- [Change] Rely on `karafka-core` `>=` `2.2.4` to support lazy loaded custom web producer.
+
 ## 0.7.8 (2023-10-24)
 - [Enhancement] Support transactional producer usage with Web UI.
-- [Fix] Fix a bug where critical errors (like `IRB::Abort`) would not abort the ongoing transaction.
 - [Fix] Prevent a scenario where an ongoing transactional producer would have stats emitted and an error that could not have been dispatched because of the transaction, creating a dead-lock.
 - [Fix] Make sure that the `recent` displays the most recent non-compacted, non-system message.
 - [Fix] Improve the `recent` message display to compensate for aborted transactions.

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.7.8)
+    karafka-web (0.7.10)
       erubi (~> 1.4)
       karafka (>= 2.2.9, < 3.0.0)
-      karafka-core (>= 2.2.2, < 3.0.0)
+      karafka-core (>= 2.2.4, < 3.0.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
 
@@ -40,7 +40,7 @@ GEM
       karafka-core (>= 2.2.2, < 2.3.0)
       waterdrop (>= 2.6.10, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.2.3)
+    karafka-core (2.2.4)
       concurrent-ruby (>= 1.1)
       karafka-rdkafka (>= 0.13.6, < 0.14.0)
     karafka-rdkafka (0.13.6)

data/karafka-web.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'erubi', '~> 1.4'
   spec.add_dependency 'karafka', '>= 2.2.9', '< 3.0.0'
-  spec.add_dependency 'karafka-core', '>= 2.2.2', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.2.4', '< 3.0.0'
   spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'
 

data/lib/karafka/web/config.rb

@@ -11,6 +11,17 @@ module Karafka
       # This is used both in the processing for eviction and in the UI
       setting :ttl, default: 30_000
 
+      # Producer for the Web UI. By default it is a `Karafka.producer`, however it may be
+      # overwritten if we want to use a separate instance in case of heavy usage of the
+      # transactional producer as a default. In cases like this, Karafka may not be able to report
+      # data because it uses this producer and it may be locked because of the transaction in a
+      # user space.
+      setting(
+        :producer,
+        constructor: -> { ::Karafka.producer },
+        lazy: true
+      )
+
       # Topics naming - used for processing and UI
       setting :topics do
         # All the errors encountered will be dispatched to this topic for inspection
@@ -41,11 +52,13 @@ module Karafka
         setting :scheduler, default: Tracking::Scheduler.new
 
         setting :consumers do
-          # Reports the metrics collected in the sampler
+          # Reports the metrics collected in the consumer sampler
           setting :reporter, default: Tracking::Consumers::Reporter.new
 
+          # Samples for fetching and storing metrics samples about the consumer process
           setting :sampler, default: Tracking::Consumers::Sampler.new
 
+          # Listeners needed for the Web UI to track consumer related changes
           setting :listeners, default: [
             Tracking::Consumers::Listeners::Status.new,
             Tracking::Consumers::Listeners::Errors.new,
@@ -57,10 +70,13 @@ module Karafka
         end
 
         setting :producers do
+          # Reports the metrics collected in the producer sampler
           setting :reporter, default: Tracking::Producers::Reporter.new
 
+          # Sampler for errors from producers
           setting :sampler, default: Tracking::Producers::Sampler.new
 
+          # Listeners needed for the Web UI to track producers related stuff
           setting :listeners, default: [
             Tracking::Producers::Listeners::Errors.new
           ]

data/lib/karafka/web/management/create_initial_states.rb

@@ -55,7 +55,7 @@ module Karafka
             exists('consumers state')
           else
             creating('consumers state')
-            ::Karafka.producer.produce_sync(
+            ::Karafka::Web.producer.produce_sync(
               topic: Karafka::Web.config.topics.consumers.states,
               key: Karafka::Web.config.topics.consumers.states,
               payload: DEFAULT_STATE.to_json
@@ -67,7 +67,7 @@ module Karafka
             exists('consumers metrics')
           else
             creating('consumers metrics')
-            ::Karafka.producer.produce_sync(
+            ::Karafka::Web.producer.produce_sync(
               topic: Karafka::Web.config.topics.consumers.metrics,
               key: Karafka::Web.config.topics.consumers.metrics,
               payload: DEFAULT_METRICS.merge(dispatched_at: Time.now.to_f).to_json

data/lib/karafka/web/management/enable.rb

@@ -11,6 +11,7 @@ module Karafka
         def call
           extend_routing
           subscribe_to_monitor
+          subscribe_to_close_web_producer
         end
 
         private
@@ -71,9 +72,28 @@ module Karafka
             ::Karafka.monitor.subscribe(listener)
           end
 
-          # Installs all the producer related listeners
+          # Installs all the producer related listeners into Karafka default listener and
+          # into Karafka::Web listener in case it would be different than the Karafka one
           ::Karafka::Web.config.tracking.producers.listeners.each do |listener|
             ::Karafka.producer.monitor.subscribe(listener)
+
+            # Do not instrument twice in case only one default producer is used
+            next if ::Karafka.producer == ::Karafka::Web.producer
+
+            ::Karafka::Web.producer.monitor.subscribe(listener)
+          end
+        end
+
+        # In most cases we want to close the producer if possible.
+        # While we cannot do it easily in user processes and we should rely on WaterDrop
+        # finalization logic, we can do it in `karafka server` on terminate
+        #
+        # In other places, this producer anyhow should not be used.
+        def subscribe_to_close_web_producer
+          ::Karafka::App.monitor.subscribe('app.terminated') do
+            # If Web producer is the same as `Karafka.producer` it will do nothing as you can
+            # call `#close` multiple times without side effects
+            ::Karafka::Web.producer.close
           end
         end
       end

data/lib/karafka/web/processing/consumer.rb

@@ -114,7 +114,7 @@ module Karafka
         def flush
           @flushed_at = monotonic_now
 
-          producer.produce_many_async(
+          ::Karafka::Web.producer.produce_many_async(
             [
               {
                 topic: Karafka::Web.config.topics.consumers.states,

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -107,9 +107,18 @@ module Karafka
 
                 # Last stable offsets freeze durations - we pick the max freeze to indicate
                 # the longest open transaction that potentially may be hanging
-                ls_offsets_fd = partitions_data
-                                .map { |p_details| p_details.fetch(:ls_offset_fd, 0) }
-                                .reject(&:negative?)
+                # We select only those partitions for which LSO != HO as in any other case this
+                # just means we've reached the end of data and ls may freeze because there is no
+                # more data flowing. Such cases should not be reported as ls offset freezes because
+                # there is no more data to be processed and can grow until more data is present
+                # this does not indicate "bad" freezing that we are interested in
+                ls_offsets_fds = partitions_data.map do |p_details|
+                  next if p_details.fetch(:ls_offset, 0) == p_details.fetch(:hi_offset, 0)
+
+                  ls_offset_fd = p_details.fetch(:ls_offset_fd, 0)
+
+                  ls_offset_fd.negative? ? nil : ls_offset_fd
+                end
 
                 cgs[group_name] ||= {}
                 cgs[group_name][topic_name] = {
@@ -119,7 +128,7 @@ module Karafka
                   # Take max last stable offset duration without any change. This can
                   # indicate a hanging transaction, because the offset will not move forward
                   # and will stay with a growing freeze duration when stuck
-                  ls_offset_fd: ls_offsets_fd.max || 0
+                  ls_offset_fd: ls_offsets_fds.compact.max || 0
                 }
               end
 

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -135,9 +135,9 @@ module Karafka
           #   slowdown any processing.
           def produce(messages)
             if messages.count >= PRODUCE_SYNC_THRESHOLD
-              ::Karafka.producer.produce_many_sync(messages)
+              ::Karafka::Web.producer.produce_many_sync(messages)
             else
-              ::Karafka.producer.produce_many_async(messages)
+              ::Karafka::Web.producer.produce_many_async(messages)
             end
           # Since we run this in a background thread, there may be a case upon shutdown, where the
           # producer is closed right before a potential dispatch. It is not worth dealing with this

data/lib/karafka/web/tracking/producers/reporter.rb

@@ -83,9 +83,9 @@ module Karafka
           #   slowdown any processing.
           def produce(messages)
             if messages.count >= PRODUCE_SYNC_THRESHOLD
-              ::Karafka.producer.produce_many_sync(messages)
+              ::Karafka::Web.producer.produce_many_sync(messages)
             else
-              ::Karafka.producer.produce_many_async(messages)
+              ::Karafka::Web.producer.produce_many_async(messages)
             end
           # Since we run this in a background thread, there may be a case upon shutdown, where the
           # producer is closed right before a potential dispatch. It is not worth dealing with this

data/lib/karafka/web/tracking/reporter.rb

@@ -14,8 +14,8 @@ module Karafka
         #
         # @return [Boolean]
         def active?
-          return false unless ::Karafka::App.producer
-          return false unless ::Karafka::App.producer.status.active?
+          return false unless ::Karafka::Web.producer
+          return false unless ::Karafka::Web.producer.status.active?
 
           true
         end

data/lib/karafka/web/ui/models/metrics/charts/topics.rb

@@ -76,20 +76,11 @@ module Karafka
                   topic_without_cg = topic.split('[').first
 
                   metrics.each do |current|
-                    ls_offset = current.last[:ls_offset] || 0
                     ls_offset_fd = current.last[:ls_offset_fd] || 0
-                    hi_offset = current.last[:hi_offset] || 0
 
                     # We convert this to seconds from milliseconds due to our Web UI precision
                     # Reporting is in ms for consistency
                     normalized_fd = (ls_offset_fd / 1_000).round
-                    # In case ls_offset and hi_offset are the same, it means we're reached eof
-                    # and we just don't have more data. In cases like this, LSO freeze duration
-                    # will grow because LSO will remain unchanged, but it does not mean it is
-                    # frozen. It means there is just no more data in the topic partition
-                    # This means we need to nullify this case, otherwise it would report, that
-                    # lso is hanging.
-                    normalized_fd = 0 if ls_offset == hi_offset
 
                     topics[topic_without_cg][current.first] << normalized_fd
                   end

data/lib/karafka/web/ui/pro/controllers/messages.rb

@@ -28,7 +28,7 @@ module Karafka
             def republish(topic_id, partition_id, offset)
               message = Ui::Models::Message.find(topic_id, partition_id, offset)
 
-              delivery = ::Karafka.producer.produce_sync(
+              delivery = ::Karafka::Web.producer.produce_sync(
                 topic: topic_id,
                 partition: partition_id,
                 payload: message.raw_payload,

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.8'
+    VERSION = '0.7.10'
   end
 end

data/lib/karafka/web.rb

@@ -13,6 +13,12 @@ module Karafka
   # Karafka Web UI + Karafka web monitoring
   module Web
     class << self
+      # @return [WaterDrop::Producer, nil] waterdrop messages producer or nil if not yet fully
+      #   initialized. It may not be fully initialized until the configuration is done
+      def producer
+        @producer ||= Web.config.producer
+      end
+
       # @return [String] root path of this gem
       def gem_root
         Pathname.new(File.expand_path('../..', __dir__))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.7.10
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-10-24 00:00:00.000000000 Z
+date: 2023-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0