karafka-web 0.8.2 → 0.9.0.rc1

data/lib/karafka/web/pro/commanding/config.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        # Extra configuration for pro commanding
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          # Management of processes is enabled by default
+          setting :active, default: true
+
+          # What consumer group to use for the commanding subscription.
+          # In general this CG is irrelevant. Required but irrelevant unless you're using ACLs
+          # as in that case you will have to grant it permissions. This CG never stores any
+          # offsets as the underlying assignment is via assign API and not subscribe one.
+          setting :consumer_group, default: 'karafka_web'
+
+          # How long should we wait on command arrival before yielding. Having it too short will
+          # cause unnecessary CPU cycles. Too long will make shutdown slower.
+          setting :max_wait_time, default: 2_000
+
+          # How long should we wait when an error occurs. Since this subscription is via the assign
+          # API, we can just back-off and not care since we can always re-create the consumer on
+          # issues. We always want to prevent a case where we would create new in a loop and
+          # fail without backoff as this could overload the process.
+          #
+          # This should not happen often so waiting that long should not pose significant risks
+          # and should not cause issues with the user-experience, since this is only commanding
+          # connection
+          setting :pause_timeout, default: 10_000
+
+          # The underlying iterator requires specific settings, do not change this unless you know
+          # what you area doing
+          setting :kafka, default: {
+            'enable.partition.eof': false,
+            'auto.offset.reset': 'latest'
+          }
+
+          configure
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/contracts/config.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        # Commanding related contracts
+        module Contracts
+          # Makes sure, all the expected commanding config is defined as it should be
+          class Config < ::Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka::Web.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('config')
+            end
+
+            nested(:commanding) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
+              required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
+              required(:kafka) { |val| val.is_a?(Hash) }
+
+              required(:consumer_group) do |val|
+                val.is_a?(String) && Karafka::Web::Contracts::Config::TOPIC_REGEXP.match?(val)
+              end
+            end
+
+            # Ensure all commanding kafka keys are symbols
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              detected_errors = []
+
+              data.fetch(:commanding).fetch(:kafka).each_key do |key|
+                next if key.is_a?(Symbol)
+
+                detected_errors << [[:commanding, :kafka, key], :key_must_be_a_symbol]
+              end
+
+              detected_errors
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/dispatcher.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        # Dispatcher for sending commanding related messages. Those can be:
+        # - command (do something)
+        # - result (you wanted me to get you some info, here it is)
+        #
+        # Dispatcher requires Web UI to have a producer
+        class Dispatcher
+          # What schema do we have in current Karafka version for commands dispatches
+          SCHEMA_VERSION = '1.0.0'
+
+          class << self
+            # Dispatches the command request
+            #
+            # @param name [String, Symbol] name of the command we want to deal with in the process
+            # @param process_id [String] id of the process. We use name instead of id only
+            #   because in the web ui we work with the full name and it is easier. Since
+            def command(name, process_id)
+              producer.produce_async(
+                topic: commands_topic,
+                key: process_id,
+                partition: 0,
+                payload: {
+                  schema_version: SCHEMA_VERSION,
+                  type: 'command',
+                  command: {
+                    name: name
+                  },
+                  dispatched_at: Time.now.to_f,
+                  process: {
+                    id: process_id
+                  }
+                }.to_json
+              )
+            end
+
+            # Dispatches the result request
+            #
+            # @param result [Object] anything that can be the result of the command execution
+            # @param process_id [String] related process id
+            # @param command_name [String, Symbol] command that triggered this result
+            def result(result, process_id, command_name)
+              producer.produce_async(
+                topic: commands_topic,
+                key: process_id,
+                partition: 0,
+                payload: {
+                  schema_version: SCHEMA_VERSION,
+                  type: 'result',
+                  command: {
+                    name: command_name
+                  },
+                  result: result,
+                  dispatched_at: Time.now.to_f,
+                  process: {
+                    id: process_id
+                  }
+                }.to_json
+              )
+            end
+
+            private
+
+            # @return [::WaterDrop::Producer] web ui producer
+            def producer
+              Karafka::Web.producer
+            end
+
+            # @return [String] consumers commands topic
+            def commands_topic
+              ::Karafka::Web.config.topics.consumers.commands
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/listener.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      # Namespace for things related to consumers commanding (management)
+      #
+      # This feature allows for basic of consumers. They can be stopped, moved to quiet or probed
+      # via the Web UI
+      module Commanding
+        # Wrapper around the Pro Iterator that yields messages with commands when needed for
+        # further processing.
+        #
+        # This iterator supports error handling, basically on errors it will be reported and
+        # ignored as long as they are not critical. Critical errors will cause back-off and
+        # reconnection.
+        class Listener
+          # Runs iterator and keeps it running until not needed.
+          #
+          # @yield [Karafka::Messages::Message] command message
+          def each
+            c_config = ::Karafka::Web.config.commanding
+            t_config = Karafka::Web.config.topics
+
+            iterator = Karafka::Pro::Iterator.new(
+              { t_config.consumers.commands => true },
+              settings: c_config.kafka.merge('group.id' => c_config.consumer_group),
+              yield_nil: true,
+              max_wait_time: c_config.max_wait_time
+            )
+
+            iterator.each do |message|
+              iterator.stop if @stop
+              next if @stop
+              next unless message
+
+              yield(message)
+            rescue StandardError => e
+              report_error(e)
+
+              sleep(c_config.pause_timeout / 1_000)
+
+              next
+            end
+          rescue StandardError => e
+            report_error(e)
+
+            return if done?
+
+            sleep(c_config.pause_timeout / 1_000)
+
+            retry
+          end
+
+          # Triggers stop of the listener. Does **not** stop the listener but requests it to stop.
+          def stop
+            @stop = true
+          end
+
+          private
+
+          # @return [Boolean] true if we should stop
+          def done?
+            @stop
+          end
+
+          # Reports any error that occurred in the manager
+          #
+          # Since we have an iterator based Kafka connection here, we do not have standard Karafka
+          # error handling and retries. This means, that we have to handle errors ourselves and
+          # report them to the instrumentation pipeline.
+          #
+          # @param error [StandardError] any error that occurred in the manager
+          def report_error(error)
+            ::Karafka.monitor.instrument(
+              'error.occurred',
+              error: error,
+              caller: self,
+              type: 'web.controlling.controller.error'
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/manager.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        # Manager responsible for receiving commands and taking appropriate actions
+        # It uses the assign API instead of subscribe and it does NOT publish or change anything
+        # Since its subscription is not user-related and does not run any work in the workers, it
+        # is not visible in the statistics.
+        #
+        # There are few critical things here worth keeping in mind:
+        #   - We do **not** use dynamic routing here and we do not inject active consumption
+        #   - We use a direct assign API to get an "invisible" (from the end user) perspective
+        #     connection to the commands topic for management. This is done that way so the end
+        #     user does not see this connection within the UI as it should not be a manageable one
+        #     anyhow. Also, on top of that, because we handle it under the hood, this is also not
+        #     prone to saturation and other issues that can arise when working under stress. Thanks
+        #     to that, probing can be handled almost immediately on command arrival.
+        #   - Messages causing errors will be ignored and won't block.
+        #   - Any errors are reported back to the Karafka monitor pipeline.
+        class Manager
+          include ::Karafka::Helpers::Async
+          include Singleton
+
+          def initialize
+            @listener = Listener.new
+            @matcher = Matcher.new
+          end
+
+          # When app starts to run, we start to listen for commands
+          #
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_running(_event)
+            async_call('karafka.web.pro.commanding.manager')
+          end
+
+          # When app stops, we stop the manager
+          #
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_stopping(_event)
+            @listener.stop
+          end
+
+          # This ensures that in case of super fast shutdown, we wait on this in case it would be
+          # slower not to end up with a semi-closed iterator.
+          #
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_stopped(_event)
+            @thread&.join
+          end
+
+          private
+
+          # Main iterator code.
+          # This iterator listens to the commands topic and when it detects messages targeting
+          # current process, performs the requested command.
+          def call
+            @listener.each do |message|
+              next unless @matcher.matches?(message)
+
+              control(message.payload[:command][:name])
+            end
+          end
+
+          # Runs the expected command
+          #
+          # @param command [String] command expected to run
+          def control(command)
+            case command
+            when 'probe'
+              Commands::Probe.new.call
+            when 'stop'
+              Commands::Stop.new.call
+            when 'quiet'
+              Commands::Quiet.new.call
+            else
+              # We raise it and will be rescued, reported and ignored. We raise it as this should
+              # not happen unless there are version conflicts
+              raise ::Karafka::Errors::UnsupportedCaseError, command
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/matcher.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        # Matcher that makes a decision whether a given command message should be applied/executed
+        # within the context of the current consumer process.
+        #
+        # Since we use the `assign`, each process listens to this topic and receives messages
+        # with commands targeting all the processes, this is why it needs to be filtered.
+        class Matcher
+          # @param message [Karafka::Messages::Message] message with command
+          # @return [Boolean] is this message dedicated to current process and is actionable
+          def matches?(message)
+            matches = true
+
+            # We want to work only with commands that target all processes or our current
+            matches = false unless message.key == '*' || message.key == process_id
+            # We operate only on commands. Result messages should be ignored
+            matches = false unless message.payload[:type] == 'command'
+            # Ignore messages that have different schema. This can happen in the middle of
+            # upgrades of the framework. We ignore this not to risk compatibility issues
+            matches = false unless message.payload[:schema_version] == Dispatcher::SCHEMA_VERSION
+
+            matches
+          end
+
+          private
+
+          # @return [String] current process id
+          def process_id
+            @process_id ||= ::Karafka::Web.config.tracking.consumers.sampler.process_id
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      # Namespace for things related to consumers commanding (management)
+      #
+      # This feature allows for basic of consumers. They can be stopped, moved to quiet or probed
+      # via the Web UI
+      module Commanding
+        class << self
+          # Subscribes with the commanding manager when commanding is enabled
+          #
+          # @param config [Karafka::Core::Configurable::Node] web config
+          def post_setup(config)
+            # We do not use manager if commanding is not suppose to work at all
+            return unless config.commanding.active
+
+            Commanding::Contracts::Config.new.validate!(config.to_h)
+
+            ::Karafka.monitor.subscribe(
+              Commanding::Manager.instance
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/loader.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Web
+    module Pro
+      # Loader requires and loads all the pro components only when they are needed
+      class Loader
+        class << self
+          # Loads all the Web UI pro components and configures them wherever it is expected
+          # @param config [Karafka::Core::Configurable::Node] web config that we can alter with pro
+          #   components
+          def pre_setup_all(config)
+            # Expand the config with commanding configuration
+            config.instance_eval do
+              setting(:commanding, default: Commanding::Config.config)
+            end
+          end
+
+          # Runs post setup features configuration operations
+          #
+          # @param config [Karafka::Core::Configurable::Node]
+          def post_setup_all(config)
+            Commanding.post_setup(config)
+          end
+        end
+      end
+    end
+  end
+end