karafka 2.0.0 → 2.1.0

data/lib/karafka/licenser.rb

@@ -8,68 +8,71 @@ module Karafka
 
     private_constant :PUBLIC_KEY_LOCATION
 
-    # Tries to prepare license and verifies it
-    #
-    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
-    def prepare_and_verify(license_config)
-      prepare(license_config)
-      verify(license_config)
-    end
-
-    private
+    class << self
+      # Tries to load the license and yields if successful
+      def detect
+        # If required, do not require again
+        require('karafka-license') unless const_defined?('::Karafka::License')
 
-    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
-    def prepare(license_config)
-      # If there is token, no action needed
-      # We support a case where someone would put the token in instead of using one from the
-      # license. That's in case there are limitations to using external package sources, etc
-      return if license_config.token
+        yield
 
-      begin
-        license_config.token || require('karafka-license')
+        true
       rescue LoadError
-        return
+        false
       end
 
-      license_config.token = Karafka::License.token
-    end
-
-    # Check license and setup license details (if needed)
-    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
-    def verify(license_config)
-      # If no license, it will just run LGPL components without anything extra
-      return unless license_config.token
+      # Tries to prepare license and verifies it
+      #
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def prepare_and_verify(license_config)
+        # If license is not loaded, nothing to do
+        return unless const_defined?('::Karafka::License')
 
-      public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
+        prepare(license_config)
+        verify(license_config)
+      end
 
-      # We gsub and strip in case someone copy-pasted it as a multi line string
-      formatted_token = license_config.token.strip.delete("\n").delete(' ')
-      decoded_token = Base64.decode64(formatted_token)
+      private
 
-      begin
-        data = public_key.public_decrypt(decoded_token)
-      rescue OpenSSL::OpenSSLError
-        data = nil
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def prepare(license_config)
+        license_config.token = Karafka::License.token
       end
 
-      details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
+      # Check license and setup license details (if needed)
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def verify(license_config)
+        public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
 
-      license_config.entity = details.fetch('entity')
-    end
+        # We gsub and strip in case someone copy-pasted it as a multi line string
+        formatted_token = license_config.token.strip.delete("\n").delete(' ')
+        decoded_token = Base64.decode64(formatted_token)
 
-    # Raises an error with info, that used token is invalid
-    # @param license_config [Karafka::Core::Configurable::Node]
-    def raise_invalid_license_token(license_config)
-      # We set it to false so `Karafka.pro?` method behaves as expected
-      license_config.token = false
-
-      raise(
-        Errors::InvalidLicenseTokenError,
-        <<~MSG.tr("\n", ' ')
-          License key you provided is invalid.
-          Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
-        MSG
-      )
+        begin
+          data = public_key.public_decrypt(decoded_token)
+        rescue OpenSSL::OpenSSLError
+          data = nil
+        end
+
+        details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
+
+        license_config.entity = details.fetch('entity')
+      end
+
+      # Raises an error with info, that used token is invalid
+      # @param license_config [Karafka::Core::Configurable::Node]
+      def raise_invalid_license_token(license_config)
+        # We set it to false so `Karafka.pro?` method behaves as expected
+        license_config.token = false
+
+        raise(
+          Errors::InvalidLicenseTokenError,
+          <<~MSG.tr("\n", ' ')
+            License key you provided is invalid.
+            Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+          MSG
+        )
+      end
     end
   end
 end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -10,27 +10,27 @@ module Karafka
           #
           # @param messages [Array<Karafka::Messages::Message>] messages array
           # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @param partition [Integer] partition of this metadata
           # @param scheduled_at [Time] moment when the batch was scheduled for processing
           # @return [Karafka::Messages::BatchMetadata] batch metadata object
           #
           # @note We do not set `processed_at` as this needs to be assigned when the batch is
           #   picked up for processing.
-          def call(messages, topic, scheduled_at)
+          def call(messages, topic, partition, scheduled_at)
             Karafka::Messages::BatchMetadata.new(
               size: messages.count,
-              first_offset: messages.first.offset,
-              last_offset: messages.last.offset,
+              first_offset: messages.first&.offset || -1001,
+              last_offset: messages.last&.offset || -1001,
               deserializer: topic.deserializer,
-              partition: messages.first.partition,
+              partition: partition,
               topic: topic.name,
               # We go with the assumption that the creation of the whole batch is the last message
               # creation time
-              created_at: messages.last.timestamp,
+              created_at: messages.last&.timestamp || nil,
               # When this batch was built and scheduled for execution
               scheduled_at: scheduled_at,
-              # We build the batch metadata when we pick up the job in the worker, thus we can use
-              # current time here
-              processed_at: Time.now
+              # This needs to be set to a correct value prior to processing starting
+              processed_at: nil
             )
           end
         end

data/lib/karafka/messages/builders/messages.rb

@@ -11,14 +11,18 @@ module Karafka
           #
           # @param messages [Array<Karafka::Messages::Message>] karafka messages array
           # @param topic [Karafka::Routing::Topic] topic for which we're received messages
+          # @param partition [Integer] partition of those messages
           # @param received_at [Time] moment in time when the messages were received
           # @return [Karafka::Messages::Messages] messages batch object
-          def call(messages, topic, received_at)
+          def call(messages, topic, partition, received_at)
+            # We cannot freeze the batch metadata because it is altered with the processed_at time
+            # prior to the consumption. It is being frozen there
             metadata = BatchMetadata.call(
               messages,
               topic,
+              partition,
               received_at
-            ).freeze
+            )
 
             Karafka::Messages::Messages.new(
               messages,

data/lib/karafka/messages/message.rb

@@ -9,7 +9,19 @@ module Karafka
     class Message
       extend Forwardable
 
-      attr_reader :raw_payload, :metadata
+      class << self
+        # @return [Object] general parser
+        # @note We cache it here for performance reasons. It is 2.5x times faster than getting it
+        #   via the config chain.
+        def parser
+          @parser ||= App.config.internal.messages.parser
+        end
+      end
+
+      attr_reader :metadata
+      # raw payload needs to be mutable as we want to have option to change it in the parser
+      # prior to the final deserialization
+      attr_accessor :raw_payload
 
       def_delegators :metadata, *Metadata.members
 
@@ -42,7 +54,7 @@ module Karafka
 
       # @return [Object] deserialized data
       def deserialize
-        metadata.deserializer.call(self)
+        self.class.parser.call(self)
       end
     end
   end

data/lib/karafka/messages/messages.rb

@@ -40,6 +40,11 @@ module Karafka
         map(&:raw_payload)
       end
 
+      # @return [Boolean] is the messages batch empty
+      def empty?
+        @messages_array.empty?
+      end
+
       # @return [Karafka::Messages::Message] first message
       def first
         @messages_array.first

data/lib/karafka/messages/parser.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Default message parser. The only thing it does, is calling the deserializer
+    class Parser
+      # @param message [::Karafka::Messages::Message]
+      # @return [Object] deserialized payload
+      def call(message)
+        message.metadata.deserializer.call(message)
+      end
+    end
+  end
+end

data/lib/karafka/patches/rdkafka/bindings.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Patches
+    module Rdkafka
+      # Binding patches that slightly change how rdkafka operates in certain places
+      module Bindings
+        include ::Rdkafka::Bindings
+
+        # Alias internally
+        RB = ::Rdkafka::Bindings
+
+        class << self
+          # Handle assignments on cooperative rebalance
+          #
+          # @param client_ptr [FFI::Pointer]
+          # @param code [Integer]
+          # @param partitions_ptr [FFI::Pointer]
+          def on_cooperative_rebalance(client_ptr, code, partitions_ptr)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              RB.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              RB.rd_kafka_commit(client_ptr, nil, false)
+              RB.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
+            else
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+            end
+          end
+
+          # Handle assignments on a eager rebalance
+          #
+          # @param client_ptr [FFI::Pointer]
+          # @param code [Integer]
+          # @param partitions_ptr [FFI::Pointer]
+          def on_eager_rebalance(client_ptr, code, partitions_ptr)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              RB.rd_kafka_assign(client_ptr, partitions_ptr)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              RB.rd_kafka_commit(client_ptr, nil, false)
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+            else
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+            end
+          end
+
+          # Trigger Karafka callbacks
+          #
+          # @param code [Integer]
+          # @param opaque [Rdkafka::Opaque]
+          # @param consumer [Rdkafka::Consumer]
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          def trigger_callbacks(code, opaque, consumer, tpl)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              opaque.call_on_partitions_assigned(consumer, tpl)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              opaque.call_on_partitions_revoked(consumer, tpl)
+            end
+          rescue StandardError => e
+            Karafka.monitor.instrument(
+              'error.occurred',
+              caller: self,
+              error: e,
+              type: 'connection.client.rebalance_callback.error'
+            )
+          end
+        end
+
+        # This patch changes few things:
+        # - it commits offsets (if any) upon partition revocation, so less jobs need to be
+        #   reprocessed if they are assigned to a different process
+        # - reports callback errors into the errors instrumentation instead of the logger
+        # - catches only StandardError instead of Exception as we fully control the directly
+        #   executed callbacks
+        #
+        # @see https://docs.confluent.io/2.0.0/clients/librdkafka/classRdKafka_1_1RebalanceCb.html
+        RebalanceCallback = FFI::Function.new(
+          :void, %i[pointer int pointer pointer]
+        ) do |client_ptr, code, partitions_ptr, opaque_ptr|
+          # Patch reference
+          pr = ::Karafka::Patches::Rdkafka::Bindings
+
+          if RB.rd_kafka_rebalance_protocol(client_ptr) == 'COOPERATIVE'
+            pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr)
+          else
+            pr.on_eager_rebalance(client_ptr, code, partitions_ptr)
+          end
+
+          opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
+          return unless opaque
+
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
+          consumer = ::Rdkafka::Consumer.new(client_ptr)
+
+          pr.trigger_callbacks(code, opaque, consumer, tpl)
+        end
+      end
+    end
+  end
+end
+
+# We need to replace the original callback with ours.
+# At the moment there is no API in rdkafka-ruby to do so
+::Rdkafka::Bindings.send(
+  :remove_const,
+  'RebalanceCallback'
+)
+
+::Rdkafka::Bindings.const_set(
+  'RebalanceCallback',
+  Karafka::Patches::Rdkafka::Bindings::RebalanceCallback
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_rebalance_protocol,
+  %i[pointer],
+  :string
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_incremental_assign,
+  %i[pointer pointer],
+  :string
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_incremental_unassign,
+  %i[pointer pointer],
+  :string
+)

data/lib/karafka/pro/active_job/consumer.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -20,23 +22,22 @@ module Karafka
       #
       # It contains slightly better revocation warranties than the regular blocking consumer as
       # it can stop processing batch of jobs in the middle after the revocation.
-      class Consumer < Karafka::Pro::BaseConsumer
+      class Consumer < ::Karafka::ActiveJob::Consumer
         # Runs ActiveJob jobs processing and handles lrj if needed
         def consume
           messages.each do |message|
             # If for any reason we've lost this partition, not worth iterating over new messages
             # as they are no longer ours
             break if revoked?
-            break if Karafka::App.stopping?
 
-            ::ActiveJob::Base.execute(
-              ::ActiveSupport::JSON.decode(message.raw_payload)
-            )
+            # We cannot early stop when running virtual partitions because the intermediate state
+            # would force us not to commit the offsets. This would cause extensive
+            # double-processing
+            break if Karafka::App.stopping? && !topic.virtual_partitions?
 
-            # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
-            # this could create random markings
-            next if topic.virtual_partitioner?
+            consume_job(message)
 
+            # We can always mark because of the virtual offset management that we have in VPs
             mark_as_consumed(message)
           end
         end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -21,6 +23,7 @@ module Karafka
         # They can be updated by using `#karafka_options` on the job
         DEFAULTS = {
           dispatch_method: :produce_async,
+          dispatch_many_method: :produce_many_async,
           # We don't create a dummy proc based partitioner as we would have to evaluate it with
           # each job.
           partitioner: nil,
@@ -31,16 +34,38 @@ module Karafka
         private_constant :DEFAULTS
 
         # @param job [ActiveJob::Base] job
-        def call(job)
+        def dispatch(job)
           ::Karafka.producer.public_send(
             fetch_option(job, :dispatch_method, DEFAULTS),
             dispatch_details(job).merge!(
               topic: job.queue_name,
-              payload: ::ActiveSupport::JSON.encode(job.serialize)
+              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
             )
           )
         end
 
+        # Bulk dispatches multiple jobs using the Rails 7.1+ API
+        # @param jobs [Array<ActiveJob::Base>] jobs we want to dispatch
+        def dispatch_many(jobs)
+          dispatches = Hash.new { |hash, key| hash[key] = [] }
+
+          jobs.each do |job|
+            d_method = fetch_option(job, :dispatch_many_method, DEFAULTS)
+
+            dispatches[d_method] << dispatch_details(job).merge!(
+              topic: job.queue_name,
+              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+            )
+          end
+
+          dispatches.each do |type, messages|
+            ::Karafka.producer.public_send(
+              type,
+              messages
+            )
+          end
+        end
+
         private
 
         # @param job [ActiveJob::Base] job instance

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -18,14 +20,25 @@ module Karafka
         configure do |config|
           config.error_messages = YAML.safe_load(
             File.read(
-              File.join(Karafka.gem_root, 'config', 'errors.yml')
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
             )
           ).fetch('en').fetch('validations').fetch('job_options')
         end
 
-        optional(:dispatch_method) { |val| %i[produce_async produce_sync].include?(val) }
         optional(:partitioner) { |val| val.respond_to?(:call) }
         optional(:partition_key_type) { |val| %i[key partition_key].include?(val) }
+        optional(:dispatch_method) do |val|
+          %i[
+            produce_async
+            produce_sync
+          ].include?(val)
+        end
+        optional(:dispatch_many_method) do |val|
+          %i[
+            produce_many_async
+            produce_many_sync
+          ].include?(val)
+        end
       end
     end
   end

data/lib/karafka/pro/encryption/cipher.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Cipher for encrypting and decrypting data
+      class Cipher
+        def initialize
+          @private_pems = {}
+        end
+
+        # Encrypts given string content with the public key
+        # @param content [String]
+        # @return [String]
+        def encrypt(content)
+          public_pem.public_encrypt(content)
+        end
+
+        # Decrypts provided content using `version` key
+        # @param version [String] encryption version
+        # @param content [String] encrypted content
+        # @return [String] decrypted content
+        def decrypt(version, content)
+          private_pem(version).private_decrypt(content)
+        end
+
+        private
+
+        # @return [::OpenSSL::PKey::RSA] rsa public key
+        def public_pem
+          @public_pem ||= ::OpenSSL::PKey::RSA.new(::Karafka::App.config.encryption.public_key)
+        end
+
+        # @param version [String] version for which we want to get the rsa key
+        # @return [::OpenSSL::PKey::RSA] rsa private key
+        def private_pem(version)
+          return @private_pems[version] if @private_pems.key?(version)
+
+          key_string = ::Karafka::App.config.encryption.private_keys[version]
+          key_string || raise(Errors::PrivateKeyNotFound, version)
+
+          @private_pems[version] = ::OpenSSL::PKey::RSA.new(key_string)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related contracts
+      module Contracts
+        # Makes sure, all the expected config is defined as it should be
+        class Config < ::Karafka::Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('config')
+          end
+
+          nested(:encryption) do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:version) { |val| val.is_a?(String) && !val.empty? }
+            required(:public_key) { |val| val.is_a?(String) }
+
+            required(:private_keys) do |val|
+              val.is_a?(Hash) &&
+                val.keys.all? { |key| key.is_a?(String) } &&
+                val.values.all? { |key| key.is_a?(String) }
+            end
+          end
+
+          # Public key validation
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data.fetch(:encryption).fetch(:active)
+
+            key = OpenSSL::PKey::RSA.new(data.fetch(:encryption).fetch(:public_key))
+
+            next unless key.private?
+
+            [[%i[encryption public_key], :needs_to_be_public]]
+          rescue OpenSSL::PKey::RSAError
+            [[%i[encryption public_key], :invalid]]
+          end
+
+          # Private keys validation
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data.fetch(:encryption).fetch(:active)
+
+            private_keys = data.fetch(:encryption).fetch(:private_keys)
+
+            # Keys may be empty for production only envs
+            next if private_keys.empty?
+
+            keys = private_keys.each_value.map do |key|
+              OpenSSL::PKey::RSA.new(key)
+            end
+
+            next if keys.all?(&:private?)
+
+            [[%i[encryption private_keys], :need_to_be_private]]
+          rescue OpenSSL::PKey::RSAError
+            [[%i[encryption private_keys], :invalid]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/errors.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Encryption
+      # Encryption related errors
+      module Errors
+        # Raised when we have encountered encryption key with version we do not have
+        PrivateKeyNotFound = Class.new(::Karafka::Errors::BaseError)
+      end
+    end
+  end
+end