karafka 1.4.12 → 2.2.10

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ActiveJob
+      # Contract for validating the options that can be altered with `#karafka_options` per job
+      # class that works with Pro features.
+      class JobOptionsContract < Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('job_options')
+        end
+
+        optional(:partitioner) { |val| val.respond_to?(:call) }
+        optional(:partition_key_type) { |val| %i[key partition_key].include?(val) }
+        optional(:dispatch_method) do |val|
+          %i[
+            produce_async
+            produce_sync
+          ].include?(val)
+        end
+        optional(:dispatch_many_method) do |val|
+          %i[
+            produce_many_async
+            produce_many_sync
+          ].include?(val)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cleaner/errors.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Cleaner
+      # Cleaner related errors
+      module Errors
+        # Base for all the clearer errors
+        BaseError = Class.new(::Karafka::Errors::BaseError)
+
+        # Raised when trying to deserialize a message that has already been cleaned
+        MessageCleanedError = Class.new(BaseError)
+      end
+    end
+  end
+end

data/lib/karafka/pro/cleaner/messages/message.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Cleaner
+      # Cleaner messages components related enhancements
+      module Messages
+        # Extensions to the message that allow for granular memory control on a per message basis
+        module Message
+          # @return [Object] lazy-deserialized data (deserialized upon first request)
+          def payload
+            # If message has already been cleaned, it cannot be deserialized again
+            cleaned? ? raise(Errors::MessageCleanedError) : super
+          end
+
+          # @return [Boolean] true if the message has been cleaned
+          def cleaned?
+            @raw_payload == false
+          end
+
+          # Cleans the message payload and removes the deserialized data references
+          # This is useful when working with big messages that take a lot of space.
+          #
+          # After the message content is no longer needed, it can be removed so it does not consume
+          # space anymore.
+          def clean!
+            @deserialized = false
+            @raw_payload = false
+            @payload = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cleaner/messages/messages.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Cleaner
+      module Messages
+        # Extensions to the messages batch allowing for automatic cleaning of each message after
+        # message is processed.
+        module Messages
+          # @param clean [Boolean] do we want to clean each message after we're done working with
+          #   it.
+          # @yield block we want to execute per each message
+          #
+          # @note Cleaning messages after we're done with each of them and did not fail does not
+          #   affect any other functionalities. The only thing that is crucial is to make sure,
+          #   that if DLQ is used, that we mark each message as consumed when using this API as
+          #   otherwise a cleaned message may be dispatched and that should never happen
+          def each(clean: false)
+            @messages_array.each do |message|
+              yield(message)
+
+              next unless clean
+
+              message.clean!
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cleaner.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Feature that introduces a granular memory management for each message and messages iterator
+    #
+    # It allows for better resource allocation by providing an API to clear payload and raw payload
+    # from a message after those are no longer needed but before whole messages are freed and
+    # removed by Ruby GC.
+    #
+    # This can be useful when processing bigger batches or bigger messages one after another and
+    # wanting not to have all of the data loaded into memory.
+    #
+    # Can yield significant memory savings (up to 80%).
+    module Cleaner
+      class << self
+        # @param _config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(_config)
+          ::Karafka::Messages::Message.prepend(Messages::Message)
+          ::Karafka::Messages::Messages.prepend(Messages::Messages)
+        end
+
+        # @param _config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(_config)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/contracts/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro contracts that aim to replace or complement the general framework contracts
+    module Contracts
+      # Base for all the Pro contracts
+      class Base < ::Karafka::Contracts::Base
+      end
+    end
+  end
+end

data/lib/karafka/pro/contracts/server_cli_options.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Contracts
+      # Contract for validating correctness of the server cli command options.
+      # It differs slightly from the OSS one because it is aware of the routing patterns
+      class ServerCliOptions < ::Karafka::Contracts::ServerCliOptions
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('server_cli_options')
+        end
+
+        %i[
+          include
+          exclude
+        ].each do |action|
+          optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
+          optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
+          optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
+
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            value = data.fetch(:"#{action}_consumer_groups")
+
+            # If there were no consumer_groups declared in the server cli, it means that we will
+            # run all of them and no need to validate them here at all
+            next if value.empty?
+            next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
+
+            # Found unknown consumer groups
+            [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
+          end
+
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            value = data.fetch(:"#{action}_subscription_groups")
+
+            # If there were no subscription_groups declared in the server cli, it means that we
+            # will run all of them and no need to validate them here at all
+            next if value.empty?
+
+            subscription_groups = Karafka::App
+                                  .consumer_groups
+                                  .map(&:subscription_groups)
+                                  .flatten
+                                  .map(&:name)
+
+            next if (value - subscription_groups).empty?
+
+            # Found unknown subscription groups
+            [[[:"#{action}_subscription_groups"], :subscription_groups_inclusion]]
+          end
+
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            value = data.fetch(:"#{action}_topics")
+
+            # If there were no topics declared in the server cli, it means that we will
+            # run all of them and no need to validate them here at all
+            next if value.empty?
+
+            topics = Karafka::App
+                     .consumer_groups
+                     .map(&:subscription_groups)
+                     .flatten
+                     .map(&:topics)
+                     .map { |gtopics| gtopics.map(&:name) }
+                     .flatten
+
+            next if (value - topics).empty?
+
+            # If there are any patterns defined, we cannot report on topics inclusions because
+            # topics may be added during boot or runtime. We go with simple assumption:
+            # if there are patterns defined, we do not check the inclusions at all
+            next unless Karafka::App.consumer_groups.map(&:patterns).flatten.empty?
+
+            # Found unknown topics
+            [[[:"#{action}_topics"], :topics_inclusion]]
+          end
+        end
+
+        # Makes sure we have anything to subscribe to when we start the server
+        virtual do |_, errors|
+          next unless errors.empty?
+
+          next unless Karafka::App.subscription_groups.empty?
+
+          [[%i[include_topics], :topics_missing]]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/cipher.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Cipher for encrypting and decrypting data
+      class Cipher
+        def initialize
+          @private_pems = {}
+        end
+
+        # Encrypts given string content with the public key
+        # @param content [String]
+        # @return [String]
+        def encrypt(content)
+          public_pem.public_encrypt(content)
+        end
+
+        # Decrypts provided content using `version` key
+        # @param version [String] encryption version
+        # @param content [String] encrypted content
+        # @return [String] decrypted content
+        def decrypt(version, content)
+          private_pem(version).private_decrypt(content)
+        end
+
+        private
+
+        # @return [::OpenSSL::PKey::RSA] rsa public key
+        def public_pem
+          @public_pem ||= ::OpenSSL::PKey::RSA.new(::Karafka::App.config.encryption.public_key)
+        end
+
+        # @param version [String] version for which we want to get the rsa key
+        # @return [::OpenSSL::PKey::RSA] rsa private key
+        def private_pem(version)
+          return @private_pems[version] if @private_pems.key?(version)
+
+          key_string = ::Karafka::App.config.encryption.private_keys[version]
+          key_string || raise(Errors::PrivateKeyNotFound, version)
+
+          @private_pems[version] = ::OpenSSL::PKey::RSA.new(key_string)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related contracts
+      module Contracts
+        # Makes sure, all the expected config is defined as it should be
+        class Config < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('config')
+          end
+
+          nested(:encryption) do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:version) { |val| val.is_a?(String) && !val.empty? }
+            required(:public_key) { |val| val.is_a?(String) }
+
+            required(:private_keys) do |val|
+              val.is_a?(Hash) &&
+                val.keys.all? { |key| key.is_a?(String) } &&
+                val.values.all? { |key| key.is_a?(String) }
+            end
+          end
+
+          # Public key validation
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data.fetch(:encryption).fetch(:active)
+
+            key = OpenSSL::PKey::RSA.new(data.fetch(:encryption).fetch(:public_key))
+
+            next unless key.private?
+
+            [[%i[encryption public_key], :needs_to_be_public]]
+          rescue OpenSSL::PKey::RSAError
+            [[%i[encryption public_key], :invalid]]
+          end
+
+          # Private keys validation
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data.fetch(:encryption).fetch(:active)
+
+            private_keys = data.fetch(:encryption).fetch(:private_keys)
+
+            # Keys may be empty for production only envs
+            next if private_keys.empty?
+
+            keys = private_keys.each_value.map do |key|
+              OpenSSL::PKey::RSA.new(key)
+            end
+
+            next if keys.all?(&:private?)
+
+            [[%i[encryption private_keys], :need_to_be_private]]
+          rescue OpenSSL::PKey::RSAError
+            [[%i[encryption private_keys], :invalid]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/errors.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related errors
+      module Errors
+        # Base for all the encryption errors
+        BaseError = Class.new(::Karafka::Errors::BaseError)
+
+        # Raised when we have encountered encryption key with version we do not have
+        PrivateKeyNotFound = Class.new(BaseError)
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/messages/middleware.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related messages components
+      module Messages
+        # Middleware for WaterDrop. It automatically encrypts messages payload.
+        # It is injected only if encryption is enabled.
+        class Middleware
+          # @param message [Hash] WaterDrop message hash
+          # @return [Hash] hash with encrypted payload and encryption version indicator
+          def call(message)
+            message[:headers] ||= {}
+            message[:headers]['encryption'] = version
+            message[:payload] = cipher.encrypt(message[:payload])
+            message
+          end
+
+          private
+
+          # @return [::Karafka::Pro::Encryption::Cipher]
+          def cipher
+            @cipher ||= ::Karafka::App.config.encryption.cipher
+          end
+
+          # @return [String] encryption version
+          def version
+            @version ||= ::Karafka::App.config.encryption.version
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      module Messages
+        # Pro parser that takes into consideration encryption usage
+        # @note There may be a case where someone decides not to encrypt data and we start getting
+        #   unencrypted payloads. That is why we always rely on message headers for encryption
+        #   indication.
+        class Parser < ::Karafka::Messages::Parser
+          # @param message [::Karafka::Messages::Message]
+          # @return [Object] deserialized payload
+          def call(message)
+            if active? && message.headers.key?('encryption')
+              # Decrypt raw payload so it can be handled by the default parser logic
+              message.raw_payload = cipher.decrypt(
+                message.headers['encryption'],
+                message.raw_payload
+              )
+            end
+
+            super(message)
+          end
+
+          private
+
+          # @return [::Karafka::Pro::Encryption::Cipher]
+          def cipher
+            @cipher ||= ::Karafka::App.config.encryption.cipher
+          end
+
+          # @return [Boolean] is encryption active
+          def active?
+            return @active unless @active.nil?
+
+            @active = ::Karafka::App.config.encryption.active
+
+            @active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/setup/config.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Setup and config related encryption components
+      module Setup
+        # Config for encryption
+        class Config
+          extend ::Karafka::Core::Configurable
+
+          # Should this feature be in use
+          setting(:active, default: false)
+
+          # Supporting versions allows us to be able to rotate private and public keys in case
+          # we would need this. We can increase the version, rotate and Karafka when decrypting
+          # will figure out proper private key based on the version
+          setting(:version, default: '1')
+
+          # We always support one public key for producing messages
+          # Public key needs to be always present even if we do not plan to produce messages from
+          # a Karafka process. This is because of the web-ui and potentially other cases like this
+          setting(:public_key, default: '')
+
+          # Private keys in pem format, where the key is the version and value is the key.
+          # This allows us to support key rotation
+          setting(:private_keys, default: {})
+
+          # Cipher used to encrypt and decrypt data
+          setting(:cipher, default: Encryption::Cipher.new)
+
+          configure
+        end
+      end
+    end
+  end
+end