karafka-web 0.7.8 → 0.7.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d46244304483184c3be59a0686497a9f948e9b666dccf00961b560e60db4655
-  data.tar.gz: 329686ce548063787fe06290d67650793ebef6c014f43b753780ad120671a8b1
+  metadata.gz: 24bc5fe488a1304886f3f7f697a8662463f4c21fae60daf52d317852f9f43669
+  data.tar.gz: eaa8e4b7d9b9d70200c8929e8fa65ed7906a1b4085465b15525bfee670ebbcc9
 SHA512:
-  metadata.gz: 5a18a3c23f63382c6c41dc5ac7f82f893bc14a55e7ec68af9cf16da0f47a45aee77ca9626973570cb9d6abe9355e0712f117424e00619b7b0198748c3bfbeb55
-  data.tar.gz: f639bd90ca21c09fcd798fc289079cafeb9c752d261ecf88df9762e347df738bf260c658f0f15b3b08c3708daf091217c0159d85ac975380463a65a45d876d0b
+  metadata.gz: 0ba2c668103a3b25169d86f2c0624fae04d144694040694ff79c4932bf81fe648e5ba11223be5a5f9e8d8ba0bbbac9da84f2ec5312e81d07b35873707b7b754c
+  data.tar.gz: b0ec51820b50dfde4f88e942e3b0dcc2400170d0fbe256503294f366e2f4fe0057249bb525528b789bed254ba6ec035d5502e515042bd094999663155f8a221d

data/CHANGELOG.md

@@ -1,8 +1,11 @@
 # Karafka Web changelog
 
+## 0.7.9 (2023-10-25)
+- [Enhancement] Allow for `Karafka::Web.producer` reconfiguration from the default (`Karafka.producer`).
+- [Change] Rely on `karafka-core` `>=` `2.2.4` to support lazy loaded custom web producer.
+
 ## 0.7.8 (2023-10-24)
 - [Enhancement] Support transactional producer usage with Web UI.
-- [Fix] Fix a bug where critical errors (like `IRB::Abort`) would not abort the ongoing transaction.
 - [Fix] Prevent a scenario where an ongoing transactional producer would have stats emitted and an error that could not have been dispatched because of the transaction, creating a dead-lock.
 - [Fix] Make sure that the `recent` displays the most recent non-compacted, non-system message.
 - [Fix] Improve the `recent` message display to compensate for aborted transactions.

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.7.8)
+    karafka-web (0.7.9)
       erubi (~> 1.4)
       karafka (>= 2.2.9, < 3.0.0)
-      karafka-core (>= 2.2.2, < 3.0.0)
+      karafka-core (>= 2.2.4, < 3.0.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
 
@@ -40,7 +40,7 @@ GEM
       karafka-core (>= 2.2.2, < 2.3.0)
       waterdrop (>= 2.6.10, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.2.3)
+    karafka-core (2.2.4)
       concurrent-ruby (>= 1.1)
       karafka-rdkafka (>= 0.13.6, < 0.14.0)
     karafka-rdkafka (0.13.6)

data/karafka-web.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'erubi', '~> 1.4'
   spec.add_dependency 'karafka', '>= 2.2.9', '< 3.0.0'
-  spec.add_dependency 'karafka-core', '>= 2.2.2', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.2.4', '< 3.0.0'
   spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'
 

data/lib/karafka/web/config.rb

@@ -11,6 +11,17 @@ module Karafka
       # This is used both in the processing for eviction and in the UI
       setting :ttl, default: 30_000
 
+      # Producer for the Web UI. By default it is a `Karafka.producer`, however it may be
+      # overwritten if we want to use a separate instance in case of heavy usage of the
+      # transactional producer as a default. In cases like this, Karafka may not be able to report
+      # data because it uses this producer and it may be locked because of the transaction in a
+      # user space.
+      setting(
+        :producer,
+        constructor: -> { ::Karafka.producer },
+        lazy: true
+      )
+
       # Topics naming - used for processing and UI
       setting :topics do
         # All the errors encountered will be dispatched to this topic for inspection
@@ -41,11 +52,13 @@ module Karafka
         setting :scheduler, default: Tracking::Scheduler.new
 
         setting :consumers do
-          # Reports the metrics collected in the sampler
+          # Reports the metrics collected in the consumer sampler
           setting :reporter, default: Tracking::Consumers::Reporter.new
 
+          # Samples for fetching and storing metrics samples about the consumer process
           setting :sampler, default: Tracking::Consumers::Sampler.new
 
+          # Listeners needed for the Web UI to track consumer related changes
           setting :listeners, default: [
             Tracking::Consumers::Listeners::Status.new,
             Tracking::Consumers::Listeners::Errors.new,
@@ -57,10 +70,13 @@ module Karafka
         end
 
         setting :producers do
+          # Reports the metrics collected in the producer sampler
           setting :reporter, default: Tracking::Producers::Reporter.new
 
+          # Sampler for errors from producers
           setting :sampler, default: Tracking::Producers::Sampler.new
 
+          # Listeners needed for the Web UI to track producers related stuff
           setting :listeners, default: [
             Tracking::Producers::Listeners::Errors.new
           ]

data/lib/karafka/web/management/create_initial_states.rb

@@ -55,7 +55,7 @@ module Karafka
             exists('consumers state')
           else
             creating('consumers state')
-            ::Karafka.producer.produce_sync(
+            ::Karafka::Web.producer.produce_sync(
               topic: Karafka::Web.config.topics.consumers.states,
               key: Karafka::Web.config.topics.consumers.states,
               payload: DEFAULT_STATE.to_json
@@ -67,7 +67,7 @@ module Karafka
             exists('consumers metrics')
           else
             creating('consumers metrics')
-            ::Karafka.producer.produce_sync(
+            ::Karafka::Web.producer.produce_sync(
               topic: Karafka::Web.config.topics.consumers.metrics,
               key: Karafka::Web.config.topics.consumers.metrics,
               payload: DEFAULT_METRICS.merge(dispatched_at: Time.now.to_f).to_json

data/lib/karafka/web/management/enable.rb

@@ -11,6 +11,7 @@ module Karafka
         def call
           extend_routing
           subscribe_to_monitor
+          subscribe_to_close_web_producer
         end
 
         private
@@ -71,9 +72,28 @@ module Karafka
             ::Karafka.monitor.subscribe(listener)
           end
 
-          # Installs all the producer related listeners
+          # Installs all the producer related listeners into Karafka default listener and
+          # into Karafka::Web listener in case it would be different than the Karafka one
           ::Karafka::Web.config.tracking.producers.listeners.each do |listener|
             ::Karafka.producer.monitor.subscribe(listener)
+
+            # Do not instrument twice in case only one default producer is used
+            next if ::Karafka.producer == ::Karafka::Web.producer
+
+            ::Karafka::Web.producer.monitor.subscribe(listener)
+          end
+        end
+
+        # In most cases we want to close the producer if possible.
+        # While we cannot do it easily in user processes and we should rely on WaterDrop
+        # finalization logic, we can do it in `karafka server` on terminate
+        #
+        # In other places, this producer anyhow should not be used.
+        def subscribe_to_close_web_producer
+          ::Karafka::App.monitor.subscribe('app.terminated') do
+            # If Web producer is the same as `Karafka.producer` it will do nothing as you can
+            # call `#close` multiple times without side effects
+            ::Karafka::Web.producer.close
           end
         end
       end

data/lib/karafka/web/processing/consumer.rb

@@ -114,7 +114,7 @@ module Karafka
         def flush
           @flushed_at = monotonic_now
 
-          producer.produce_many_async(
+          ::Karafka::Web.producer.produce_many_async(
             [
               {
                 topic: Karafka::Web.config.topics.consumers.states,

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -135,9 +135,9 @@ module Karafka
           #   slowdown any processing.
           def produce(messages)
             if messages.count >= PRODUCE_SYNC_THRESHOLD
-              ::Karafka.producer.produce_many_sync(messages)
+              ::Karafka::Web.producer.produce_many_sync(messages)
             else
-              ::Karafka.producer.produce_many_async(messages)
+              ::Karafka::Web.producer.produce_many_async(messages)
             end
           # Since we run this in a background thread, there may be a case upon shutdown, where the
           # producer is closed right before a potential dispatch. It is not worth dealing with this

data/lib/karafka/web/tracking/producers/reporter.rb

@@ -83,9 +83,9 @@ module Karafka
           #   slowdown any processing.
           def produce(messages)
             if messages.count >= PRODUCE_SYNC_THRESHOLD
-              ::Karafka.producer.produce_many_sync(messages)
+              ::Karafka::Web.producer.produce_many_sync(messages)
             else
-              ::Karafka.producer.produce_many_async(messages)
+              ::Karafka::Web.producer.produce_many_async(messages)
             end
           # Since we run this in a background thread, there may be a case upon shutdown, where the
           # producer is closed right before a potential dispatch. It is not worth dealing with this

data/lib/karafka/web/tracking/reporter.rb

@@ -14,8 +14,8 @@ module Karafka
         #
         # @return [Boolean]
         def active?
-          return false unless ::Karafka::App.producer
-          return false unless ::Karafka::App.producer.status.active?
+          return false unless ::Karafka::Web.producer
+          return false unless ::Karafka::Web.producer.status.active?
 
           true
         end

data/lib/karafka/web/ui/pro/controllers/messages.rb

@@ -28,7 +28,7 @@ module Karafka
             def republish(topic_id, partition_id, offset)
               message = Ui::Models::Message.find(topic_id, partition_id, offset)
 
-              delivery = ::Karafka.producer.produce_sync(
+              delivery = ::Karafka::Web.producer.produce_sync(
                 topic: topic_id,
                 partition: partition_id,
                 payload: message.raw_payload,

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.8'
+    VERSION = '0.7.9'
   end
 end

data/lib/karafka/web.rb

@@ -13,6 +13,12 @@ module Karafka
   # Karafka Web UI + Karafka web monitoring
   module Web
     class << self
+      # @return [WaterDrop::Producer, nil] waterdrop messages producer or nil if not yet fully
+      #   initialized. It may not be fully initialized until the configuration is done
+      def producer
+        @producer ||= Web.config.producer
+      end
+
       # @return [String] root path of this gem
       def gem_root
         Pathname.new(File.expand_path('../..', __dir__))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.7.9
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-10-24 00:00:00.000000000 Z
+date: 2023-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.2
+        version: 2.2.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0