karafka 2.0.23 → 2.0.24

data/lib/karafka/pro/encryption/messages/middleware.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related messages components
+      module Messages
+        # Middleware for WaterDrop. It automatically encrypts messages payload.
+        # It is injected only if encryption is enabled.
+        class Middleware
+          # @param message [Hash] WaterDrop message hash
+          # @return [Hash] hash with encrypted payload and encryption version indicator
+          def call(message)
+            message[:headers] ||= {}
+            message[:headers]['encryption'] = version
+            message[:payload] = cipher.encrypt(message[:payload])
+            message
+          end
+
+          private
+
+          # @return [::Karafka::Pro::Encryption::Cipher]
+          def cipher
+            @cipher ||= ::Karafka::App.config.encryption.cipher
+          end
+
+          # @return [String] encryption version
+          def version
+            @version ||= ::Karafka::App.config.encryption.version
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      module Messages
+        # Pro parser that takes into consideration encryption usage
+        # @note There may be a case where someone decides not to encrypt data and we start getting
+        #   unencrypted payloads. That is why we always rely on message headers for encryption
+        #   indication.
+        class Parser < ::Karafka::Messages::Parser
+          # @param message [::Karafka::Messages::Message]
+          # @return [Object] deserialized payload
+          def call(message)
+            if active? && message.headers.key?('encryption')
+              # Decrypt raw payload so it can be handled by the default parser logic
+              message.raw_payload = cipher.decrypt(
+                message.headers['encryption'],
+                message.raw_payload
+              )
+            end
+
+            super(message)
+          end
+
+          private
+
+          # @return [::Karafka::Pro::Encryption::Cipher]
+          def cipher
+            @cipher ||= ::Karafka::App.config.encryption.cipher
+          end
+
+          # @return [Boolean] is encryption active
+          def active?
+            return @active unless @active.nil?
+
+            @active = ::Karafka::App.config.encryption.active
+
+            @active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/setup/config.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Setup and config related encryption components
+      module Setup
+        # Config for encryption
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          # Should this feature be in use
+          setting(:active, default: false)
+
+          # Supporting versions allows us to be able to rotate private and public keys in case
+          # we would need this. We can increase the version, rotate and Karafka when decrypting
+          # will figure out proper private key based on the version
+          setting(:version, default: '1')
+
+          # We always support one public key for producing messages
+          # Public key needs to be always present even if we do not plan to produce messages from
+          # a Karafka process. This is because of the web-ui and potentially other cases like this
+          setting(:public_key, default: '')
+
+          # Private keys in pem format, where the key is the version and value is the key.
+          # This allows us to support key rotation
+          setting(:private_keys, default: {})
+
+          # Cipher used to encrypt and decrypt data
+          setting(:cipher, default: Encryption::Cipher.new)
+
+          configure
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Out of the box encryption engine for both Karafka and WaterDrop
+    # It uses asymmetric encryption via RSA. We use asymmetric so we can have producers that won't
+    # have ability (when private key not added) to decrypt messages.
+    module Encryption
+      class << self
+        # Sets up additional config scope, validations and other things
+        #
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(config)
+          # Expand the config with this feature specific stuff
+          config.instance_eval do
+            setting(:encryption, default: Setup::Config.config)
+          end
+        end
+
+        # @param config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(config)
+          Encryption::Contracts::Config.new.validate!(config.to_h)
+
+          # Don't inject extra components if encryption is not active
+          return unless config.encryption.active
+
+          # This parser is encryption aware
+          config.internal.messages.parser = Messages::Parser.new
+
+          # Encryption for WaterDrop
+          config.producer.middleware.append(Messages::Middleware.new)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -23,6 +23,11 @@ module Karafka
         processing/jobs/consume_non_blocking
         processing/strategies/base
         routing/features/base
+        encryption
+        encryption/cipher
+        encryption/setup/config
+        encryption/contracts/config
+        encryption/messages/parser
       ].freeze
 
       # Zeitwerk pro loader
@@ -44,14 +49,30 @@ module Karafka
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
-        def setup(config)
+        def pre_setup(config)
+          features.each { |feature| feature.pre_setup(config) }
+
           reconfigure(config)
 
           load_topic_features
         end
 
+        # Runs post setup features configuration operations
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        def post_setup(config)
+          features.each { |feature| feature.post_setup(config) }
+        end
+
         private
 
+        # @return [Array<Module>] extra non-routing related pro features
+        def features
+          [
+            Encryption
+          ]
+        end
+
         # Sets proper config options to use pro components
         # @param config [::Karafka::Core::Configurable::Node] root config node
         def reconfigure(config)

data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb

@@ -42,7 +42,7 @@ module Karafka
                 # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
                 coordinator.pause_tracker.reset
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               else
                 coordinator.pause_tracker.reset
                 skippable_message = find_skippable_message

data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb

@@ -51,7 +51,7 @@ module Karafka
                 # If processing failed, we need to pause
                 # For long running job this will overwrite the default never-ending pause and will
                 # cause the processing to keep going after the error backoff
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end

data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb

@@ -47,7 +47,7 @@ module Karafka
 
                 mark_as_consumed(last_group_message)
               else
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -79,7 +79,7 @@ module Karafka
 
                 mark_as_consumed(last_group_message)
               else
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -36,7 +36,7 @@ module Karafka
 
                 mark_as_consumed(messages.last)
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               # If we've reached number of retries that we could, we need to skip the first message
               # that was not marked as consumed, pause and continue, while also moving this message
               # to the dead topic

data/lib/karafka/pro/processing/strategies/dlq_lrj.rb

@@ -41,7 +41,7 @@ module Karafka
 
                 resume
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               else
                 coordinator.pause_tracker.reset
 

data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb

@@ -38,7 +38,7 @@ module Karafka
 
                 resume
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               else
                 coordinator.pause_tracker.reset
 

data/lib/karafka/pro/processing/strategies/dlq_mom.rb

@@ -35,7 +35,7 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
               elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               # If we've reached number of retries that we could, we need to skip the first message
               # that was not marked as consumed, pause and continue, while also moving this message
               # to the dead topic.

data/lib/karafka/pro/processing/strategies/lrj.rb

@@ -57,7 +57,7 @@ module Karafka
                 # If processing failed, we need to pause
                 # For long running job this will overwrite the default never-ending pause and will
                 # cause the processing to keep going after the error backoff
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end

data/lib/karafka/pro/processing/strategies/lrj_mom.rb

@@ -50,7 +50,7 @@ module Karafka
 
                 resume
               else
-                pause(coordinator.seek_offset, false)
+                retry_after_pause
               end
             end
           end

data/lib/karafka/pro/processing/strategies/mom.rb

@@ -32,7 +32,7 @@ module Karafka
               if coordinator.success?
                 coordinator.pause_tracker.reset
               else
-                pause(coordinator.seek_offset, nil, false)
+                retry_after_pause
               end
             end
           end

data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb

@@ -21,9 +21,9 @@ module Karafka
             configure do |config|
               config.error_messages = YAML.safe_load(
                 File.read(
-                  File.join(Karafka.gem_root, 'config', 'errors.yml')
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('pro_topic')
+              ).fetch('en').fetch('validations').fetch('topic')
             end
 
             # Make sure that we don't use DLQ with VP

data/lib/karafka/pro/routing/features/long_running_job/contract.rb

@@ -21,9 +21,9 @@ module Karafka
             configure do |config|
               config.error_messages = YAML.safe_load(
                 File.read(
-                  File.join(Karafka.gem_root, 'config', 'errors.yml')
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('pro_topic')
+              ).fetch('en').fetch('validations').fetch('topic')
             end
 
             nested(:long_running_job) do

data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb

@@ -21,9 +21,9 @@ module Karafka
             configure do |config|
               config.error_messages = YAML.safe_load(
                 File.read(
-                  File.join(Karafka.gem_root, 'config', 'errors.yml')
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
                 )
-              ).fetch('en').fetch('validations').fetch('pro_topic')
+              ).fetch('en').fetch('validations').fetch('topic')
             end
 
             nested(:virtual_partitions) do

data/lib/karafka/processing/executor.rb

@@ -31,7 +31,7 @@ module Karafka
       # @param client [Karafka::Connection::Client] kafka client
       # @param topic [Karafka::Routing::Topic] topic for which this executor will run
       def initialize(group_id, client, topic)
-        @id = SecureRandom.uuid
+        @id = SecureRandom.hex(6)
         @group_id = group_id
         @client = client
         @topic = topic

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -26,7 +26,7 @@ module Karafka
             # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
             coordinator.pause_tracker.reset
           elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           else
             coordinator.pause_tracker.reset
             skippable_message = find_skippable_message

data/lib/karafka/processing/strategies/default.rb

@@ -59,7 +59,7 @@ module Karafka
 
             mark_as_consumed(messages.last)
           else
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           end
         end
 

data/lib/karafka/processing/strategies/dlq.rb

@@ -26,7 +26,7 @@ module Karafka
 
             mark_as_consumed(messages.last)
           elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           # If we've reached number of retries that we could, we need to skip the first message
           # that was not marked as consumed, pause and continue, while also moving this message
           # to the dead topic

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -21,7 +21,7 @@ module Karafka
           if coordinator.success?
             coordinator.pause_tracker.reset
           elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           # If we've reached number of retries that we could, we need to skip the first message
           # that was not marked as consumed, pause and continue, while also moving this message
           # to the dead topic

data/lib/karafka/processing/strategies/mom.rb

@@ -20,7 +20,7 @@ module Karafka
           if coordinator.success?
             coordinator.pause_tracker.reset
           else
-            pause(coordinator.seek_offset, nil, false)
+            retry_after_pause
           end
         end
       end

data/lib/karafka/processing/worker.rb

@@ -25,7 +25,7 @@ module Karafka
       # @param jobs_queue [JobsQueue]
       # @return [Worker]
       def initialize(jobs_queue)
-        @id = SecureRandom.uuid
+        @id = SecureRandom.hex(6)
         @jobs_queue = jobs_queue
       end
 

data/lib/karafka/railtie.rb

@@ -79,6 +79,9 @@ if rails
         ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
           # Reload code each time there is a change in the code
           next unless Rails.application.reloaders.any?(&:updated?)
+          # If consumer persistence is enabled, no reason to reload because we will still keep
+          # old consumer instances in memory.
+          next if Karafka::App.config.consumer_persistence
 
           Rails.application.reloader.reload!
         end

data/lib/karafka/routing/builder.rb

@@ -80,7 +80,7 @@ module Karafka
       # @param subscription_group_name [String, Symbol] subscription group id. When not provided,
       #   a random uuid will be used
       # @param block [Proc] further topics definitions
-      def subscription_group(subscription_group_name = SecureRandom.uuid, &block)
+      def subscription_group(subscription_group_name = SecureRandom.hex(6), &block)
         consumer_group('app') do
           target.public_send(:subscription_group=, subscription_group_name.to_s, &block)
         end

data/lib/karafka/routing/consumer_group.rb

@@ -26,7 +26,7 @@ module Karafka
         @topics = Topics.new([])
         # Initialize the subscription group so there's always a value for it, since even if not
         # defined directly, a subscription group will be created
-        @current_subscription_group_id = SecureRandom.uuid
+        @current_subscription_group_id = SecureRandom.hex(6)
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
@@ -55,7 +55,7 @@ module Karafka
       # topic definition
       # @param name [String, Symbol] name of the current subscription group
       # @param block [Proc] block that may include topics definitions
-      def subscription_group=(name = SecureRandom.uuid, &block)
+      def subscription_group=(name = SecureRandom.hex(6), &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
         @current_subscription_group_id = name
@@ -64,7 +64,7 @@ module Karafka
 
         # We need to reset the current subscription group after it is used, so it won't leak
         # outside to other topics that would be defined without a defined subscription group
-        @current_subscription_group_id = SecureRandom.uuid
+        @current_subscription_group_id = SecureRandom.hex(6)
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on

data/lib/karafka/routing/features/active_job/contract.rb

@@ -10,7 +10,7 @@ module Karafka
           configure do |config|
             config.error_messages = YAML.safe_load(
               File.read(
-                File.join(Karafka.gem_root, 'config', 'errors.yml')
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               )
             ).fetch('en').fetch('validations').fetch('topic')
           end

data/lib/karafka/routing/features/dead_letter_queue/contract.rb

@@ -9,7 +9,7 @@ module Karafka
           configure do |config|
             config.error_messages = YAML.safe_load(
               File.read(
-                File.join(Karafka.gem_root, 'config', 'errors.yml')
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               )
             ).fetch('en').fetch('validations').fetch('topic')
           end

data/lib/karafka/routing/features/manual_offset_management/contract.rb

@@ -9,7 +9,7 @@ module Karafka
           configure do |config|
             config.error_messages = YAML.safe_load(
               File.read(
-                File.join(Karafka.gem_root, 'config', 'errors.yml')
+                File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
               )
             ).fetch('en').fetch('validations').fetch('topic')
           end

data/lib/karafka/server.rb

@@ -47,7 +47,7 @@ module Karafka
         # in a separate thread (or trap context) to indicate everything is closed
         # Since `#start` is blocking, we were get here only after the runner is done. This will
         # not add any performance degradation because of that.
-        Thread.pass until Karafka::App.stopped?
+        Thread.pass until Karafka::App.terminated?
       # Try its best to shutdown underlying components before re-raising
       # rubocop:disable Lint/RescueException
       rescue Exception => e
@@ -75,6 +75,7 @@ module Karafka
         # Initialize the stopping process only if Karafka was running
         return if Karafka::App.stopping?
         return if Karafka::App.stopped?
+        return if Karafka::App.terminated?
 
         Karafka::App.stop!
 
@@ -84,13 +85,7 @@ module Karafka
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
-          if listeners.count(&:alive?).zero? &&
-             workers.count(&:alive?).zero?
-
-            Karafka::App.producer.close
-
-            return
-          end
+          return if listeners.count(&:alive?).zero? && workers.count(&:alive?).zero?
 
           sleep SUPERVISION_SLEEP
         end
@@ -122,18 +117,23 @@ module Karafka
       ensure
         # We need to check if it wasn't an early exit to make sure that only on stop invocation
         # can change the status after everything is closed
-        Karafka::App.stopped! if timeout
+        if timeout
+          Karafka::App.stopped!
+
+          # We close producer as the last thing as it can be used in the notification pipeline
+          # to dispatch state changes, etc
+          Karafka::App.producer.close
+
+          Karafka::App.terminate!
+        end
       end
 
       # Quiets the Karafka server.
       # Karafka will stop processing but won't quiet to consumer group, so no rebalance will be
       # triggered until final shutdown.
       def quiet
-        # If we are already quieting or in the stop procedures, we should not do it again.
-        return if Karafka::App.quieting?
-        return if Karafka::App.stopping?
-        return if Karafka::App.stopped?
-
+        # We don't have to safe-guard it with check states as the state transitions work only
+        # in one direction
         Karafka::App.quiet!
       end
 

data/lib/karafka/setup/config.rb

@@ -16,7 +16,10 @@ module Karafka
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
-        'client.id': 'karafka'
+        'client.id': 'karafka',
+        # We emit the statistics by default, so all the instrumentation and web-ui work out of
+        # the box, without requiring users to take any extra actions aside from enabling.
+        'statistics.interval.ms': 5_000
       }.freeze
 
       # Contains settings that should not be used in production but make life easier in dev
@@ -130,6 +133,12 @@ module Karafka
           setting :strategy_selector, default: Processing::StrategySelector.new
         end
 
+        # Things related to operating on messages
+        setting :messages do
+          # Parser is used to convert raw payload prior to deserialization
+          setting :parser, default: Messages::Parser.new
+        end
+
         # Karafka components for ActiveJob
         setting :active_job do
           # option dispatcher [Karafka::ActiveJob::Dispatcher] default dispatcher for ActiveJob
@@ -155,7 +164,7 @@ module Karafka
           # Will configure all the pro components
           # This needs to happen before end user configuration as the end user may overwrite some
           # of the pro defaults with custom components
-          Pro::Loader.setup(config) if Karafka.pro?
+          Pro::Loader.pre_setup(config) if Karafka.pro?
 
           configure(&block)
           merge_kafka_defaults!(config)
@@ -164,6 +173,10 @@ module Karafka
 
           configure_components
 
+          # Runs things that need to be executed after config is defined and all the components
+          # are also configured
+          Pro::Loader.post_setup(config) if Karafka.pro?
+
           Karafka::App.initialized!
         end