karafka-web 0.10.2 → 0.10.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d471de1bcc756c234f8db2345c269b878863759fb5deb195ac8579e57365efde
-  data.tar.gz: 12d1348a0b83353dabeeea7a5ae164be7ff834d84088cfadb7b32c7343470f66
+  metadata.gz: 13c431e0fa6d1b8927bd419298bc0ea522f0e823f91197fe989f72a9a08479ad
+  data.tar.gz: 6424f804a013bc34d1393fa79962cfacab6e6d5028726b7e90e93ada457b7a48
 SHA512:
-  metadata.gz: 3f3eab846d445aa01714c8cea282874f9228f69af2475408d19a34b3ce43793b235f32c4484b640fad3b36ae29c90a2a8112dfd0b46685159528672355945f4c
-  data.tar.gz: f44a24702afe474270232a1cd5d785737fbd3e6d3b3e574b3eeb1cba5680147000092ec564ad5da7c663d6de124c7b59d9c3bd6c4e51ec80fc249eb90627fb5c
+  metadata.gz: e3ccb77dcf9624e1ca23473a963ea59cd8efc227feae098c7a80898b45c573eb70b564fd963abc6d3b0dd01a4b9e0577c1708bc2904afa6c5523360fc3880362
+  data.tar.gz: 60f20e2ed98fb8818164157e91ce117c5fb2d6b60f5c1371abef787f0f637269f27ed378a0ca8ffe7e95c43defddfc4749cfe1c06cbf905c8e591f8b171c2c36

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Karafka Web Changelog
 
+## 0.10.4 (2024-11-26)
+- **[Breaking]** Drop Ruby `3.0` support according to the EOL schedule.
+- [Enhancement] Extract producers tracking `sync_threshold` into an internal config.
+- [Enhancement] Support complex Pro license loading strategies (Pro).
+- [Enhancement] Change default `retention.ms` for the metrics topic to support Redpanda Cloud defaults (#450).
+- [Enhancement] Include subscription group id in the consumers error tracking metadata.
+- [Enhancement] Collect metadata details of low level client errors when error tracking.
+- [Enhancement] Collect metadata details of low level listener errors when error tracking.
+- [Fix] Toggle menu button post-turbo refresh stops working.
+
+## 0.10.3 (2024-09-17)
+- **[Feature]** Introduce ability to brand Web UI with environment (Pro).
+- [Enhancement] Provide assignment status in the routing (Pro).
+- [Enhancement] Support schedule cancellation via Web UI.
+- [Enhancement] Rename "probing" to "tracing" to better reflect what this commanding option does.
+- [Fix] Fix not working primary and secondary alert styles.
+
 ## 0.10.2 (2024-09-03)
 - **[Feature]** Support Future Messages management (Pro).
 - [Enhancement] Do not live-reload when form active.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.10.2)
+    karafka-web (0.10.4)
       erubi (~> 1.4)
       karafka (>= 2.4.10, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
@@ -11,20 +11,21 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.4)
+    activesupport (7.2.1)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
     base64 (0.2.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.3.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.0)
@@ -32,7 +33,7 @@ GEM
     erubi (1.13.0)
     et-orbi (1.2.11)
       tzinfo
-    factory_bot (6.4.6)
+    factory_bot (6.5.0)
       activesupport (>= 5.0.0)
     ffi (1.17.0)
     fugit (1.11.1)
@@ -52,10 +53,10 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
+    logger (1.6.1)
     mini_portile2 (2.8.7)
-    minitest (5.24.0)
-    mutex_m (0.2.0)
-    ostruct (0.6.0)
+    minitest (5.25.1)
+    ostruct (0.6.1)
     raabro (1.4.0)
     rack (3.1.4)
     rack-test (2.1.0)
@@ -79,6 +80,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
+    securerandom (0.3.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)

data/config/locales/pro_errors.yml

@@ -2,21 +2,27 @@ en:
   validations:
     config:
       missing: needs to be present
+      key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
 
       commanding.active_format: needs to be either true or false
       commanding.consumer_group_format: 'needs to be a string with a Kafka accepted format'
       commanding.max_wait_time_format: needs to be an integer bigger than 0
       commanding.pause_timeout_format: needs to be an integer bigger than 0
-      key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       commanding.kafka_format: needs to be a filled hash
+
       ui.search.matchers_must_have_name_and_call: 'must respond to #name and its instance to #call'
       ui.search.matchers_format: must be an array with matchers
       ui.search.matchers_name_must_be_valid: all matchers names must be non-empty strings
       ui.search.limits_format: all limits need to be integers bigger than 0
       ui.search.timeout_format: must be at least 1 ms
+
       ui.policies.messages_format: cannot be nil
       ui.policies.requests_format: cannot be nil
 
+      ui.branding.type_format: 'must be :info, :error, :warning, :success, :primary or :secondary'
+      ui.branding.label_format: must be false or non-empty string
+      ui.branding.notice_format: must be false or non-empty string
+
     search_form:
       missing: needs to be present
       timestamp_key_must_be_large_enough: 'must be a Kafka message timestamp with ms precision'

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.7.0
+    image: confluentinc/cp-kafka:7.7.1
 
     ports:
       - 9092:9092

data/karafka-web.gemspec

@@ -24,17 +24,23 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'rackup', '~> 0.2'
 
-  spec.required_ruby_version = '>= 3.0.0'
+  spec.required_ruby_version = '>= 3.1.0'
 
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end
 
   spec.cert_chain    = %w[certs/cert.pem]
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = %w[karafka-web]
   spec.require_paths = %w[lib]
 
+  spec.files = `git ls-files -z`
+               .split("\x0")
+               .reject { |f| f.match(%r{^(spec)/}) }
+               .reject { |f| f.include?('.github')  }
+               .reject { |f| f.start_with?('.')  }
+               .reject { |f| f.end_with?('.map')  }
+
   spec.metadata = {
     'funding_uri' => 'https://karafka.io/#become-pro',
     'homepage_uri' => 'https://karafka.io',

data/lib/karafka/web/config.rb

@@ -76,7 +76,7 @@ module Karafka
           # Reports the metrics collected in the consumer sampler
           setting :reporter, default: Tracking::Consumers::Reporter.new
 
-          # Minimum number of messages to produce to produce them in sync mode
+          # Minimum number of messages to produce them in sync mode
           # This acts as a small back-off not to overload the system in case we would have
           # extremely big number of errors and reports happening
           setting :sync_threshold, default: 50
@@ -98,6 +98,11 @@ module Karafka
         end
 
         setting :producers do
+          # Minimum number of messages to produce them in sync mode
+          # This acts as a small back-off not to overload the system in case we would have
+          # extremely big number of errors happening
+          setting :sync_threshold, default: 25
+
           # Reports the metrics collected in the producer sampler
           setting :reporter, default: Tracking::Producers::Reporter.new
 

data/lib/karafka/web/contracts/config.rb

@@ -41,6 +41,7 @@ module Karafka
             required(:reporter) { |val| !val.nil? }
             required(:sampler) { |val| !val.nil? }
             required(:listeners) { |val| val.is_a?(Array) }
+            required(:sync_threshold) { |val| val.is_a?(Integer) && val.positive? }
           end
         end
 

data/lib/karafka/web/management/actions/create_topics.rb

@@ -81,7 +81,7 @@ module Karafka
                 replication_factor,
                 {
                   'cleanup.policy': 'compact',
-                  'retention.ms': 60 * 60 * 1_000, # 1h
+                  'retention.ms': 24 * 60 * 60 * 1_000, # 1 day
                   'segment.ms': 24 * 60 * 60 * 1_000, # 1 day
                   'segment.bytes': 104_857_600 # 100MB
                 }

data/lib/karafka/web/pro/commanding/commands/{probe.rb → trace.rb}

@@ -19,8 +19,8 @@ module Karafka
         module Commands
           # Collects all backtraces from the available Ruby threads and publishes their details
           #   back to Kafka for debug.
-          class Probe < Base
-            # Runs probing and publishes result back to Kafka
+          class Trace < Base
+            # Runs tracing and publishes result back to Kafka
             def call
               threads = {}
 
@@ -31,7 +31,7 @@ module Karafka
                 t_d[:backtrace] = (thread.backtrace || ['<no backtrace available>']).join("\n")
               end
 
-              Dispatcher.result(threads, process_id, 'probe')
+              Dispatcher.result(threads, process_id, 'trace')
             end
           end
         end

data/lib/karafka/web/pro/commanding/listener.rb

@@ -16,7 +16,7 @@ module Karafka
     module Pro
       # Namespace for things related to consumers commanding (management)
       #
-      # This feature allows for basic of consumers. They can be stopped, moved to quiet or probed
+      # This feature allows for basic of consumers. They can be stopped, moved to quiet or traced
       # via the Web UI
       module Commanding
         # Wrapper around the Pro Iterator that yields messages with commands when needed for

data/lib/karafka/web/pro/commanding/manager.rb

@@ -79,8 +79,8 @@ module Karafka
           # @param command [String] command expected to run
           def control(command)
             case command
-            when 'probe'
-              Commands::Probe.new.call
+            when 'trace'
+              Commands::Trace.new.call
             when 'stop'
               Commands::Stop.new.call
             when 'quiet'

data/lib/karafka/web/pro/commanding.rb

@@ -16,7 +16,7 @@ module Karafka
     module Pro
       # Namespace for things related to consumers commanding (management)
       #
-      # This feature allows for basic of consumers. They can be stopped, moved to quiet or probed
+      # This feature allows for basic of consumers. They can be stopped, moved to quiet or traced
       # via the Web UI
       module Commanding
         class << self

data/lib/karafka/web/pro/loader.rb

@@ -17,6 +17,25 @@ module Karafka
       # Loader requires and loads all the pro components only when they are needed
       class Loader
         class << self
+          # This loads the pro components into memory in case someone required karafka-web prior
+          # to the license usage. This can happen for users with complex require flows, where
+          # Karafka license is not part of the standard flow
+          #
+          # In such cases Web may not notice that Karafka should operate in a Pro mode when it is
+          # being required via Zeitwerk. In such cases we load Pro components prior to the setup.
+          def load_on_late_setup
+            return if defined?(Karafka::Web::Pro::Commanding)
+
+            loader = Zeitwerk::Loader.new
+            loader.push_dir(
+              File.join(Karafka::Web.gem_root, 'lib/karafka/web/pro'),
+              namespace: Karafka::Web::Pro
+            )
+
+            loader.setup
+            loader.eager_load
+          end
+
           # Loads all the Web UI pro components and configures them wherever it is expected
           # @param config [Karafka::Core::Configurable::Node] web config that we can alter with pro
           #   components
@@ -28,8 +47,9 @@ module Karafka
 
             # Expand UI config with extra search capabilities settings
             config.ui.instance_eval do
-              setting(:search, default: Ui::Lib::Search::Config.config)
+              setting(:branding, default: Ui::Lib::Branding::Config.config)
               setting(:policies, default: Ui::Lib::Policies::Config.config)
+              setting(:search, default: Ui::Lib::Search::Config.config)
             end
           end
 
@@ -38,8 +58,9 @@ module Karafka
           # @param config [Karafka::Core::Configurable::Node]
           def post_setup_all(config)
             Commanding.post_setup(config)
-            Ui::Lib::Search.post_setup(config)
+            Ui::Lib::Branding.post_setup(config)
             Ui::Lib::Policies.post_setup(config)
+            Ui::Lib::Search.post_setup(config)
           end
         end
       end

data/lib/karafka/web/pro/ui/app.rb

@@ -126,8 +126,8 @@ module Karafka
               end
 
               r.on String do |process_id|
-                r.post 'probe' do
-                  controller.probe(process_id)
+                r.post 'trace' do
+                  controller.trace(process_id)
                 end
 
                 r.post 'quiet' do
@@ -302,6 +302,16 @@ module Karafka
                 end
               end
 
+              r.on 'messages' do
+                controller = Controllers::ScheduledMessages::MessagesController.new(params)
+
+                r.post(
+                  String, Integer, Integer, 'cancel'
+                ) do |topic_id, partition_id, message_offset|
+                  controller.cancel(topic_id, partition_id, message_offset)
+                end
+              end
+
               r.get do
                 r.redirect root_path('scheduled_messages/schedules')
               end

data/lib/karafka/web/pro/ui/controllers/commanding_controller.rb

@@ -19,15 +19,15 @@ module Karafka
           # Controller responsible for handling requests that should trigger some action on the
           # consumers.
           class CommandingController < BaseController
-            # Dispatches the probe request to a given process
+            # Dispatches the trace request to a given process
             #
             # @param process_id [String]
-            def probe(process_id)
-              command(:probe, process_id)
+            def trace(process_id)
+              command(:trace, process_id)
 
               redirect(
                 :back,
-                success: dispatched_to_one(:probe, process_id)
+                success: dispatched_to_one(:trace, process_id)
               )
             end
 

data/lib/karafka/web/pro/ui/controllers/routing_controller.rb

@@ -28,11 +28,28 @@ module Karafka
               detect_patterns_routes
 
               @routes = Karafka::App.routes
-
               @routes.each do |consumer_group|
                 refine(consumer_group.topics)
               end
 
+              current_state = Models::ConsumersState.current
+              @assigned = Hash.new { |h, k| h[k] = Set.new }
+
+              # If there are active processes, we can use their data to mark certain topics as
+              # assigned. This does not cover the full scope as some partitions may be assigned
+              # and some not, but provides general overview
+              if current_state
+                Models::Processes.active(current_state).each do |process|
+                  process.consumer_groups.each do |consumer_group|
+                    consumer_group.subscription_groups.each do |subscription_group|
+                      subscription_group.topics.each do |topic|
+                        @assigned[consumer_group.id.to_s] << topic.name
+                      end
+                    end
+                  end
+                end
+              end
+
               render
             end
 

data/lib/karafka/web/pro/ui/controllers/scheduled_messages/messages_controller.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Controllers
+          module ScheduledMessages
+            # Provides management of schedules related messages
+            class MessagesController < BaseController
+              # Generates a cancel request for a given schedule message
+              # @param topic_id [String]
+              # @param partition_id [Integer]
+              # @param message_offset [Integer]
+              def cancel(topic_id, partition_id, message_offset)
+                # Fetches the message we want to cancel to get its key
+                scheduled_message = Karafka::Admin.read_topic(
+                  topic_id,
+                  partition_id,
+                  1,
+                  message_offset
+                ).first
+
+                cancel_message = ::Karafka::Pro::ScheduledMessages.cancel(
+                  key: scheduled_message.key,
+                  envelope: {
+                    topic: topic_id,
+                    partition: partition_id
+                  }
+                )
+
+                Karafka::Web.producer.produce_sync(cancel_message)
+
+                redirect(
+                  :back,
+                  success: <<~MESSAGE
+                    A scheduled message with offset #{message_offset}
+                    from #{topic_id} partition #{partition_id}
+                    had a cancel request message successfully created.
+                  MESSAGE
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/branding/config.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Branding
+            # Extra configuration for pro UI branding feature
+            class Config
+              extend ::Karafka::Core::Configurable
+
+              # Type of styling aligned with Daisy. info, error, warning, success, primary
+              setting :type, default: :info
+
+              # String label with env name. Will be displayed below the logo
+              setting :label, default: false
+
+              # Additional wide alert notice to highlight extra details. Nothing if false
+              setting :notice, default: false
+
+              configure
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/branding/contracts/config.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          module Branding
+            # Contracts for checking branding related setup
+            module Contracts
+              # Makes sure, all the expected UI branding config is defined as it should be
+              class Config < ::Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load(
+                    File.read(
+                      File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
+                    )
+                  ).fetch('en').fetch('validations').fetch('config')
+                end
+
+                nested(:ui) do
+                  nested(:branding) do
+                    required(:type) do |val|
+                      # Type of branding style wrapping needs to align with what we support
+                      # in other places
+                      ::Karafka::Web::Ui::Helpers::TailwindHelper::TYPES.include?(val)
+                    end
+
+                    required(:label) do |val|
+                      val == false || (val.is_a?(String) && val.size.positive?)
+                    end
+
+                    required(:notice) do |val|
+                      val == false || (val.is_a?(String) && val.size.positive?)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/lib/branding.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Ui
+        module Lib
+          # Namespace for UI branding related stuff
+          # Branding allows users to set an extra label and notice per env so users won't be
+          # confused by dev vs prod etc.
+          module Branding
+            class << self
+              # Validates that the UI branding config is correct
+              #
+              # @param config [Karafka::Core::Configurable::Node] web config
+              def post_setup(config)
+                Branding::Contracts::Config.new.validate!(config.to_h)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/ui/views/commands/_command.erb

@@ -15,7 +15,7 @@
       <td>
         <%
           color = case command[:command][:name]
-                  when 'probe'
+                  when 'trace'
                     'info'
                   when 'quiet'
                     'warning'

data/lib/karafka/web/pro/ui/views/consumers/_consumer_controls.erb

@@ -45,11 +45,11 @@
 
   <td class="commands-inline-3">
     <%
-      probe_path = root_path('commanding', process.id, 'probe')
+      trace_path = root_path('commanding', process.id, 'trace')
       disabled_class = process.status != 'stopped' ? '' : 'btn-disabled'
     %>
-    <form action="<%= probe_path %>" method="post" class="inline" title="Probe">
-      <%== csrf_tag(probe_path) %>
+    <form action="<%= trace_path %>" method="post" class="inline" title="Trace">
+      <%== csrf_tag(trace_path) %>
 
       <button type="submit" class="btn btn-info btn-sm <%= disabled_class %>">
         <%== icon(:gear) %>

data/lib/karafka/web/pro/ui/views/consumers/consumer/_commands.erb

@@ -1,21 +1,21 @@
 <% if Karafka::Web.config.commanding.active && !process.tags.include?('embedded') %>
   <div class="col-span-12 flex justify-end space-x-2 mb-6">
     <%
-      probe_path = root_path('commanding', process.id, 'probe')
+      trace_path = root_path('commanding', process.id, 'trace')
       disabled_class = process.status != 'stopped' ? '' : 'btn-disabled'
     %>
     <form
-      action="<%= probe_path %>"
+      action="<%= trace_path %>"
       method="post"
       class="inline-flex items-center"
     >
-      <%== csrf_tag(probe_path) %>
+      <%== csrf_tag(trace_path) %>
       <button
         type="submit"
         class="btn btn-info btn-sm <%= disabled_class %>"
       >
         <%== icon(:gear) %>
-        Probe
+        Trace
       </button>
     </form>
 

data/lib/karafka/web/pro/ui/views/recurring_tasks/_actions.erb

@@ -30,7 +30,7 @@
   <%== csrf_tag(enable_path) %>
   <button
     type="submit"
-    class="btn btn-success btn-sm   <%= disabled_class %>"
+    class="btn btn-success btn-sm <%= disabled_class %>"
   >
     <%== icon(:check) %>
   </button>
@@ -50,7 +50,7 @@
   <%== csrf_tag(disable_path) %>
   <button
     type="submit"
-    class="btn btn-warning btn-sm  <%= disabled_class %>"
+    class="btn btn-warning btn-sm <%= disabled_class %>"
 
   >
     <%== icon(:pause) %>