karafka-web 0.10.1 → 0.10.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40794ee9c8b084cf75de0403b904134cbc6a762dbfa338bb9ce574b56e46ac36
-  data.tar.gz: dedefe087c54e118abb90e819b36b29aa253c4351aa18bd8a78d01e2c096ce79
+  metadata.gz: 5790e7352a06e0a780810c40aac53e9b5c878a84990c70b28bdc4d49d3ab921d
+  data.tar.gz: e5ef8e226602ad0388b138b676d705f9dbda06363a0001b2c5e483abab9f2bed
 SHA512:
-  metadata.gz: 88b9b6ed8082b0484dcd32a20b2cee2b77d4df762b65d0487325e03ef375e1e7ac621d612f81b3f4a6870aa4352e07286979863d656161ca7d8d880e5709d108
-  data.tar.gz: f9350f68d872c76d289933fbf2548b692179bae30fb2c0fccf923cdefca7131bbf3e6bf362e21a442c424fc4b43472e8334115b90c32449e80ef45a3518c3228
+  metadata.gz: a645ae9fbbe73c045562b38f744e923e3bb4ac7be47f5c49dd664a1b61513ab07706348461ef7858d3c5518a0c70c5f31addf66e146cd76e91579590e7baa268
+  data.tar.gz: 4e5a11cd6d5333b1dff7f95856818cbe14a7a7fa172dc5cbf6d4a1409883b862dc97dc9a4799cf247023e12fdaecbae749e2d44647783af770ce3ca0cd95e51c

checksums.yaml.gz.sig

@@ -1 +1,2 @@
-��[����S�t���Y�Ě�D��������1�#�];Xły}�)ҙ�*��A�	��uѭȚܟ��
+�Mwi��kT�
+�͙^�Y� 	������$��n��$�"�S����0N���@���e
�r�<���xCԟ4&�ݬ#I5����Y��0�Ɓ�

data/CHANGELOG.md

@@ -1,6 +1,20 @@
 # Karafka Web Changelog
 
-## 0.10.1 (Unreleased)
+## 0.10.3 (2024-09-17)
+- **[Feature]** Introduce ability to brand Web UI with environment (Pro).
+- [Enhancement] Provide assignment status in the routing (Pro).
+- [Enhancement] Support schedule cancellation via Web UI.
+- [Enhancement] Rename "probing" to "tracing" to better reflect what this commanding option does.
+- [Fix] Fix not working primary and secondary alert styles.
+
+## 0.10.2 (2024-09-03)
+- **[Feature]** Support Future Messages management (Pro).
+- [Enhancement] Do not live-reload when form active.
+- [Fix] Undefined method `deep_merge` for an instance of Hash.
+- [Fix] Prevent live-polling on elements wrapped in a button.
+- [Fix] Fix errors extractor failure on first message-less tick / eofed
+
+## 0.10.1 (2024-08-23)
 - **[Feature]** Support Recurring Tasks management (Pro).
 - [Enhancement] Optimize command buttons so they occupy less space.
 - [Enhancement] Improve tables headers capitalization.

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.10.1)
+    karafka-web (0.10.3)
       erubi (~> 1.4)
-      karafka (>= 2.4.9, < 2.5.0)
+      karafka (>= 2.4.10, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -11,20 +11,21 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.4)
+    activesupport (7.2.1)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
     base64 (0.2.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.3.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
@@ -32,7 +33,7 @@ GEM
     erubi (1.13.0)
     et-orbi (1.2.11)
       tzinfo
-    factory_bot (6.4.6)
+    factory_bot (6.5.0)
       activesupport (>= 5.0.0)
     ffi (1.17.0)
     fugit (1.11.1)
@@ -40,7 +41,7 @@ GEM
       raabro (~> 1.4)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    karafka (2.4.9)
+    karafka (2.4.10)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
@@ -52,9 +53,9 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
+    logger (1.6.1)
     mini_portile2 (2.8.7)
-    minitest (5.24.0)
-    mutex_m (0.2.0)
+    minitest (5.25.1)
     ostruct (0.6.0)
     raabro (1.4.0)
     rack (3.1.4)
@@ -79,6 +80,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
+    securerandom (0.3.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)

data/config/locales/pro_errors.yml

@@ -2,21 +2,27 @@ en:
   validations:
     config:
       missing: needs to be present
+      key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
 
       commanding.active_format: needs to be either true or false
       commanding.consumer_group_format: 'needs to be a string with a Kafka accepted format'
       commanding.max_wait_time_format: needs to be an integer bigger than 0
       commanding.pause_timeout_format: needs to be an integer bigger than 0
-      key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       commanding.kafka_format: needs to be a filled hash
+
       ui.search.matchers_must_have_name_and_call: 'must respond to #name and its instance to #call'
       ui.search.matchers_format: must be an array with matchers
       ui.search.matchers_name_must_be_valid: all matchers names must be non-empty strings
       ui.search.limits_format: all limits need to be integers bigger than 0
       ui.search.timeout_format: must be at least 1 ms
+
       ui.policies.messages_format: cannot be nil
       ui.policies.requests_format: cannot be nil
 
+      ui.branding.type_format: 'must be :info, :error, :warning, :success, :primary or :secondary'
+      ui.branding.label_format: must be false or non-empty string
+      ui.branding.notice_format: must be false or non-empty string
+
     search_form:
       missing: needs to be present
       timestamp_key_must_be_large_enough: 'must be a Kafka message timestamp with ms precision'

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.7.0
+    image: confluentinc/cp-kafka:7.7.1
 
     ports:
       - 9092:9092

data/karafka-web.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0-only Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.4.9', '< 2.5.0'
+  spec.add_dependency 'karafka', '>= 2.4.10', '< 2.5.0'
   spec.add_dependency 'karafka-core', '>= 2.4.0', '< 2.5.0'
   spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'
@@ -31,10 +31,18 @@ Gem::Specification.new do |spec|
   end
 
   spec.cert_chain    = %w[certs/cert.pem]
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = %w[karafka-web]
   spec.require_paths = %w[lib]
 
+  spec.files = `git ls-files -z`
+               .split("\x0")
+               .reject { |f| f.match(%r{^(spec)/}) }
+               .reject { |f| f.include?('.github')  }
+               .reject { |f| f.start_with?('.')  }
+               .reject { |f| f.end_with?('.map')  }
+
+  p spec.files.select { |a| a.include?('.map')}
+
   spec.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',

data/lib/karafka/web/pro/commanding/commands/{probe.rb → trace.rb}

@@ -19,8 +19,8 @@ module Karafka
         module Commands
           # Collects all backtraces from the available Ruby threads and publishes their details
           #   back to Kafka for debug.
-          class Probe < Base
-            # Runs probing and publishes result back to Kafka
+          class Trace < Base
+            # Runs tracing and publishes result back to Kafka
             def call
               threads = {}
 
@@ -31,7 +31,7 @@ module Karafka
                 t_d[:backtrace] = (thread.backtrace || ['<no backtrace available>']).join("\n")
               end
 
-              Dispatcher.result(threads, process_id, 'probe')
+              Dispatcher.result(threads, process_id, 'trace')
             end
           end
         end

data/lib/karafka/web/pro/commanding/listener.rb

@@ -16,7 +16,7 @@ module Karafka
     module Pro
       # Namespace for things related to consumers commanding (management)
       #
-      # This feature allows for basic of consumers. They can be stopped, moved to quiet or probed
+      # This feature allows for basic of consumers. They can be stopped, moved to quiet or traced
       # via the Web UI
       module Commanding
         # Wrapper around the Pro Iterator that yields messages with commands when needed for

data/lib/karafka/web/pro/commanding/manager.rb

@@ -79,8 +79,8 @@ module Karafka
           # @param command [String] command expected to run
           def control(command)
             case command
-            when 'probe'
-              Commands::Probe.new.call
+            when 'trace'
+              Commands::Trace.new.call
             when 'stop'
               Commands::Stop.new.call
             when 'quiet'

data/lib/karafka/web/pro/commanding.rb

@@ -16,7 +16,7 @@ module Karafka
     module Pro
       # Namespace for things related to consumers commanding (management)
       #
-      # This feature allows for basic of consumers. They can be stopped, moved to quiet or probed
+      # This feature allows for basic of consumers. They can be stopped, moved to quiet or traced
       # via the Web UI
       module Commanding
         class << self

data/lib/karafka/web/pro/loader.rb

@@ -28,8 +28,9 @@ module Karafka
 
             # Expand UI config with extra search capabilities settings
             config.ui.instance_eval do
-              setting(:search, default: Ui::Lib::Search::Config.config)
+              setting(:branding, default: Ui::Lib::Branding::Config.config)
               setting(:policies, default: Ui::Lib::Policies::Config.config)
+              setting(:search, default: Ui::Lib::Search::Config.config)
             end
           end
 
@@ -38,8 +39,9 @@ module Karafka
           # @param config [Karafka::Core::Configurable::Node]
           def post_setup_all(config)
             Commanding.post_setup(config)
-            Ui::Lib::Search.post_setup(config)
+            Ui::Lib::Branding.post_setup(config)
             Ui::Lib::Policies.post_setup(config)
+            Ui::Lib::Search.post_setup(config)
           end
         end
       end

data/lib/karafka/web/pro/ui/app.rb

@@ -126,8 +126,8 @@ module Karafka
               end
 
               r.on String do |process_id|
-                r.post 'probe' do
-                  controller.probe(process_id)
+                r.post 'trace' do
+                  controller.trace(process_id)
                 end
 
                 r.post 'quiet' do
@@ -272,6 +272,51 @@ module Karafka
               end
             end
 
+            r.on 'scheduled_messages' do
+              r.on 'schedules' do
+                controller = Controllers::ScheduledMessages::SchedulesController.new(params)
+
+                r.get String do |topic_id|
+                  controller.show(topic_id)
+                end
+
+                r.get do
+                  controller.index
+                end
+              end
+
+              r.on 'explorer' do
+                controller = Controllers::ScheduledMessages::ExplorerController.new(params)
+
+                r.get String do |topic_id|
+                  controller.topic(topic_id)
+                end
+
+                r.get String, Integer do |topic_id, partition_id|
+                  controller.partition(topic_id, partition_id)
+                end
+
+                # Jumps to offset matching the expected time
+                r.get String, Integer, Time do |topic_id, partition_id, time|
+                  controller.closest(topic_id, partition_id, time)
+                end
+              end
+
+              r.on 'messages' do
+                controller = Controllers::ScheduledMessages::MessagesController.new(params)
+
+                r.post(
+                  String, Integer, Integer, 'cancel'
+                ) do |topic_id, partition_id, message_offset|
+                  controller.cancel(topic_id, partition_id, message_offset)
+                end
+              end
+
+              r.get do
+                r.redirect root_path('scheduled_messages/schedules')
+              end
+            end
+
             r.on 'health' do
               controller = Controllers::HealthController.new(params)
 

data/lib/karafka/web/pro/ui/controllers/commanding_controller.rb

@@ -19,15 +19,15 @@ module Karafka
           # Controller responsible for handling requests that should trigger some action on the
           # consumers.
           class CommandingController < BaseController
-            # Dispatches the probe request to a given process
+            # Dispatches the trace request to a given process
             #
             # @param process_id [String]
-            def probe(process_id)
-              command(:probe, process_id)
+            def trace(process_id)
+              command(:trace, process_id)
 
               redirect(
                 :back,
-                success: dispatched_to_one(:probe, process_id)
+                success: dispatched_to_one(:trace, process_id)
               )
             end
 

data/lib/karafka/web/pro/ui/controllers/routing_controller.rb

@@ -28,11 +28,28 @@ module Karafka
               detect_patterns_routes
 
               @routes = Karafka::App.routes
-
               @routes.each do |consumer_group|
                 refine(consumer_group.topics)
               end
 
+              current_state = Models::ConsumersState.current
+              @assigned = Hash.new { |h, k| h[k] = Set.new }
+
+              # If there are active processes, we can use their data to mark certain topics as
+              # assigned. This does not cover the full scope as some partitions may be assigned
+              # and some not, but provides general overview
+              if current_state
+                Models::Processes.active(current_state).each do |process|
+                  process.consumer_groups.each do |consumer_group|
+                    consumer_group.subscription_groups.each do |subscription_group|
+                      subscription_group.topics.each do |topic|
+                        @assigned[consumer_group.id.to_s] << topic.name
+                      end
+                    end
+                  end
+                end
+              end
+
               render
             end
 

data/lib/karafka/web/pro/ui/controllers/scheduled_messages/base_controller.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Namespace for all controllers related to scheduled messages
+          module ScheduledMessages
+            # Base Pro controller for scheduled messages
+            class BaseController < Controllers::BaseController
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/scheduled_messages/explorer_controller.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module ScheduledMessages
+            # Allows for exploration of dispatch messages in a less generic form that via the
+            # explorer as different details are present
+            class ExplorerController < BaseController
+              # Displays aggregated messages from (potentially) all partitions of a topic
+              #
+              # @param topic_id [String]
+              def topic(topic_id)
+                response = Controllers::ExplorerController
+                           .new(@params)
+                           .topic(topic_id)
+
+                render(attributes: response.attributes)
+              end
+
+              # Shows messages available in a given partition
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              def partition(topic_id, partition_id)
+                response = Controllers::ExplorerController
+                           .new(@params)
+                           .partition(topic_id, partition_id)
+
+                render(attributes: response.attributes)
+              end
+
+              # Finds the closest offset matching the requested time and redirects to this location
+              # Note, that it redirects to closest but always younger.
+              #
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param time [Time] time of the message
+              def closest(topic_id, partition_id, time)
+                response = Controllers::ExplorerController
+                           .new(@params)
+                           .closest(topic_id, partition_id, time)
+
+                redirect("scheduled_messages/#{response.path}")
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/scheduled_messages/messages_controller.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module ScheduledMessages
+            # Provides management of schedules related messages
+            class MessagesController < BaseController
+              # Generates a cancel request for a given schedule message
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param message_offset [Integer]
+              def cancel(topic_id, partition_id, message_offset)
+                # Fetches the message we want to cancel to get its key
+                scheduled_message = Karafka::Admin.read_topic(
+                  topic_id,
+                  partition_id,
+                  1,
+                  message_offset
+                ).first
+
+                cancel_message = ::Karafka::Pro::ScheduledMessages.cancel(
+                  key: scheduled_message.key,
+                  envelope: {
+                    topic: topic_id,
+                    partition: partition_id
+                  }
+                )
+
+                Karafka::Web.producer.produce_sync(cancel_message)
+
+                redirect(
+                  :back,
+                  success: <<~MESSAGE
+                    A scheduled message with offset #{message_offset}
+                    from #{topic_id} partition #{partition_id}
+                    had a cancel request message successfully created.
+                  MESSAGE
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/controllers/scheduled_messages/schedules_controller.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          # Namespace for all controllers related to scheduled messages
+          module ScheduledMessages
+            # Controller to display list of schedules (groups) and details about each
+            class SchedulesController < BaseController
+              # Displays list of groups
+              def index
+                topics = Models::Topic.all
+
+                # Names of scheduled messages topics defined in the routing
+                # They may not exist (yet) so we filter them based on the existing topics in the
+                # cluster
+                candidates = Karafka::App
+                             .routes
+                             .map(&:topics)
+                             .map(&:to_a)
+                             .flatten
+                             .select(&:scheduled_messages?)
+                             .reject { |topic| topic.name.end_with?(states_postfix) }
+                             .map(&:name)
+                             .sort
+
+                @topics = topics.select { |topic| candidates.include?(topic.topic_name) }
+
+                render
+              end
+
+              # Displays all partitions statistics (if any) with number of messages to dispatch
+              # @param schedule_name [String] name of the schedules messages topic
+              def show(schedule_name)
+                @schedule_name = schedule_name
+                @stats_topic_name = "#{schedule_name}#{states_postfix}"
+                @stats_info = Karafka::Admin.topic_info(@stats_topic_name)
+
+                @states = {}
+                @stats_info[:partition_count].times { |i| @states[i] = false }
+
+                Karafka::Pro::Iterator.new({ @stats_topic_name => -1 }).each do |message|
+                  @states[message.partition] = message.payload
+                end
+
+                # Sort by partition id
+                @states = @states.sort_by { |key, _| key.to_s }.to_h
+                # Sort daily from closest date
+                @states.each_value do |details|
+                  # Skip false predefined values from sorting
+                  next unless details
+
+                  details[:daily] = details[:daily].sort_by { |key, _| key.to_s }.to_h
+                end
+
+                render
+              end
+
+              private
+
+              # @return [String] states topic postfix
+              def states_postfix
+                @states_postfix ||= Karafka::App.config.scheduled_messages.states_postfix
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/branding/config.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Branding
+            # Extra configuration for pro UI branding feature
+            class Config
+              extend ::Karafka::Core::Configurable
+
+              # Type of styling aligned with Daisy. info, error, warning, success, primary
+              setting :type, default: :info
+
+              # String label with env name. Will be displayed below the logo
+              setting :label, default: false
+
+              # Additional wide alert notice to highlight extra details. Nothing if false
+              setting :notice, default: false
+
+              configure
+            end
+          end
+        end
+      end
+    end
+  end
+end