karafka 1.4.12 → 2.2.10

data/lib/karafka/messages/batch_metadata.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Simple batch metadata object that stores all non-message information received from Kafka
+    # cluster while fetching the data.
+    #
+    # @note This metadata object refers to per batch metadata, not `#message.metadata`
+    BatchMetadata = Struct.new(
+      :size,
+      :first_offset,
+      :last_offset,
+      :deserializer,
+      :partition,
+      :topic,
+      :created_at,
+      :scheduled_at,
+      :processed_at,
+      keyword_init: true
+    ) do
+      # This lag describes how long did it take for a message to be consumed from the moment it was
+      # created
+      #
+      #
+      # @return [Integer] number of milliseconds
+      # @note In case of usage in workless flows, this value will be set to -1
+      def consumption_lag
+        processed_at ? time_distance_in_ms(processed_at, created_at) : -1
+      end
+
+      # This lag describes how long did a batch have to wait before it was picked up by one of the
+      # workers
+      #
+      # @return [Integer] number of milliseconds
+      # @note In case of usage in workless flows, this value will be set to -1
+      def processing_lag
+        processed_at ? time_distance_in_ms(processed_at, scheduled_at) : -1
+      end
+
+      private
+
+      # Computes time distance in between two times in ms
+      #
+      # @param time1 [Time]
+      # @param time2 [Time]
+      # @return [Integer] distance in between two times in ms
+      def time_distance_in_ms(time1, time2)
+        ((time1 - time2) * 1_000).round
+      end
+    end
+  end
+end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    module Builders
+      # Builder for creating batch metadata object based on the batch informations.
+      module BatchMetadata
+        class << self
+          # Creates metadata based on the kafka batch data.
+          #
+          # @param messages [Array<Karafka::Messages::Message>] messages array
+          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @param partition [Integer] partition of this metadata
+          # @param scheduled_at [Time] moment when the batch was scheduled for processing
+          # @return [Karafka::Messages::BatchMetadata] batch metadata object
+          #
+          # @note We do not set `processed_at` as this needs to be assigned when the batch is
+          #   picked up for processing.
+          def call(messages, topic, partition, scheduled_at)
+            Karafka::Messages::BatchMetadata.new(
+              size: messages.count,
+              first_offset: messages.first&.offset || -1001,
+              last_offset: messages.last&.offset || -1001,
+              deserializer: topic.deserializer,
+              partition: partition,
+              topic: topic.name,
+              # We go with the assumption that the creation of the whole batch is the last message
+              # creation time
+              created_at: local_created_at(messages.last),
+              # When this batch was built and scheduled for execution
+              scheduled_at: scheduled_at,
+              # This needs to be set to a correct value prior to processing starting
+              processed_at: nil
+            )
+          end
+
+          private
+
+          # Code that aligns the batch creation at into our local time. If time of current machine
+          # and the Kafka cluster drift, this helps not to allow this to leak into the framework.
+          #
+          # @param last_message [::Karafka::Messages::Message, nil] last message from the batch or
+          #   nil if no message
+          # @return [Time] batch creation time. Now if no messages (workless flow) or the last
+          #   message time as long as the message is not from the future
+          # @note Message can be from the future in case consumer machine and Kafka cluster drift
+          #   apart and the machine is behind the cluster.
+          def local_created_at(last_message)
+            now = ::Time.now
+
+            return now unless last_message
+
+            timestamp = last_message.timestamp
+            timestamp > now ? now : timestamp
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/messages/builders/message.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Builders encapsulate logic related to creating messages related objects.
+    module Builders
+      # Builder of a single message based on raw rdkafka message.
+      module Message
+        class << self
+          # @param kafka_message [Rdkafka::Consumer::Message] raw fetched message
+          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
+          # @param received_at [Time] moment when we've received the message
+          # @return [Karafka::Messages::Message] message object with payload and metadata
+          def call(kafka_message, topic, received_at)
+            metadata = Karafka::Messages::Metadata.new(
+              timestamp: kafka_message.timestamp,
+              headers: kafka_message.headers,
+              key: kafka_message.key,
+              offset: kafka_message.offset,
+              deserializer: topic.deserializer,
+              partition: kafka_message.partition,
+              topic: topic.name,
+              received_at: received_at
+            ).freeze
+
+            # Get the raw payload
+            payload = kafka_message.payload
+
+            # And nullify it in the kafka message. This can save a lot of memory when used with
+            # the Pro Cleaner API
+            kafka_message.instance_variable_set('@payload', nil)
+
+            # Karafka messages cannot be frozen because of the lazy deserialization feature
+            Karafka::Messages::Message.new(payload, metadata)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/messages/builders/messages.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    module Builders
+      # Builder for creating message batch instances.
+      module Messages
+        class << self
+          # Creates messages batch with messages inside based on the incoming messages and the
+          # topic from which it comes.
+          #
+          # @param messages [Array<Karafka::Messages::Message>] karafka messages array
+          # @param topic [Karafka::Routing::Topic] topic for which we're received messages
+          # @param partition [Integer] partition of those messages
+          # @param received_at [Time] moment in time when the messages were received
+          # @return [Karafka::Messages::Messages] messages batch object
+          def call(messages, topic, partition, received_at)
+            # We cannot freeze the batch metadata because it is altered with the processed_at time
+            # prior to the consumption. It is being frozen there
+            metadata = BatchMetadata.call(
+              messages,
+              topic,
+              partition,
+              received_at
+            )
+
+            Karafka::Messages::Messages.new(
+              messages,
+              metadata
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/{params/params.rb → messages/message.rb}

@@ -1,20 +1,32 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Params namespace encapsulating all the logic that is directly related to params handling
-  module Params
+  # Messages namespace encapsulating all the logic that is directly related to messages handling
+  module Messages
     # It provides lazy loading not only until the first usage, but also allows us to skip
     # using deserializer until we execute our logic. That way we can operate with
     # heavy-deserialization data without slowing down the whole application.
-    class Params
+    class Message
       extend Forwardable
 
-      attr_reader :raw_payload, :metadata
+      class << self
+        # @return [Object] general parser
+        # @note We cache it here for performance reasons. It is 2.5x times faster than getting it
+        #   via the config chain.
+        def parser
+          @parser ||= App.config.internal.messages.parser
+        end
+      end
+
+      attr_reader :metadata
+      # raw payload needs to be mutable as we want to have option to change it in the parser
+      # prior to the final deserialization
+      attr_accessor :raw_payload
 
       def_delegators :metadata, *Metadata.members
 
       # @param raw_payload [Object] incoming payload before deserialization
-      # @param metadata [Karafka::Params::Metadata] message metadata object
+      # @param metadata [Karafka::Messages::Metadata] message metadata object
       def initialize(raw_payload, metadata)
         @raw_payload = raw_payload
         @metadata = metadata
@@ -33,21 +45,16 @@ module Karafka
         @payload
       end
 
-      # @return [Boolean] did given params payload were deserialized already
+      # @return [Boolean] did we deserialize payload already
       def deserialized?
         @deserialized
       end
 
       private
 
-      # @return [Object] tries de-serializes data
+      # @return [Object] deserialized data
       def deserialize
-        Karafka.monitor.instrument('params.params.deserialize', caller: self) do
-          metadata.deserializer.call(self)
-        end
-      rescue ::StandardError => e
-        Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)
-        raise e
+        self.class.parser.call(self)
       end
     end
   end

data/lib/karafka/messages/messages.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Messages batch represents a set of messages received from Kafka of a single topic partition.
+    class Messages
+      include Enumerable
+
+      attr_reader :metadata
+
+      # @param messages_array [Array<Karafka::Messages::Message>] array with karafka messages
+      # @param metadata [Karafka::Messages::BatchMetadata]
+      # @return [Karafka::Messages::Messages] lazy evaluated messages batch object
+      def initialize(messages_array, metadata)
+        @messages_array = messages_array
+        @metadata = metadata
+      end
+
+      # @param block [Proc] block we want to execute per each message
+      # @note Invocation of this method will not cause loading and deserializing of messages.
+      def each(&block)
+        @messages_array.each(&block)
+      end
+
+      # Runs deserialization of all the messages and returns them
+      # @return [Array<Karafka::Messages::Message>]
+      def deserialize!
+        each(&:payload)
+      end
+
+      # @return [Array<Object>] array with deserialized payloads. This method can be useful when
+      #   we don't care about metadata and just want to extract all the data payloads from the
+      #   batch
+      def payloads
+        map(&:payload)
+      end
+
+      # @return [Array<String>] array with raw, not deserialized payloads
+      def raw_payloads
+        map(&:raw_payload)
+      end
+
+      # @return [Boolean] is the messages batch empty
+      def empty?
+        @messages_array.empty?
+      end
+
+      # @return [Karafka::Messages::Message] first message
+      def first
+        @messages_array.first
+      end
+
+      # @return [Karafka::Messages::Message] last message
+      def last
+        @messages_array.last
+      end
+
+      # @return [Integer] number of messages in the batch
+      def size
+        @messages_array.size
+      end
+
+      # @return [Array<Karafka::Messages::Message>] copy of the pure array with messages
+      def to_a
+        @messages_array.dup
+      end
+
+      alias count size
+    end
+  end
+end

data/lib/karafka/{params → messages}/metadata.rb

@@ -1,18 +1,16 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Params
-    # Single message / params metadata details that can be accessed without the need for the
-    # payload deserialization
+  module Messages
+    # Single message metadata details that can be accessed without the need of deserialization.
     Metadata = Struct.new(
-      :create_time,
+      :timestamp,
       :headers,
-      :is_control_record,
       :key,
       :offset,
       :deserializer,
       :partition,
-      :receive_time,
+      :received_at,
       :topic,
       keyword_init: true
     )

data/lib/karafka/messages/parser.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # Default message parser. The only thing it does, is calling the deserializer
+    class Parser
+      # @param message [::Karafka::Messages::Message]
+      # @return [Object] deserialized payload
+      def call(message)
+        message.metadata.deserializer.call(message)
+      end
+    end
+  end
+end

data/lib/karafka/messages/seek.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Messages
+    # "Fake" message that we use as an abstraction layer when seeking back.
+    # This allows us to encapsulate a seek with a simple abstraction
+    #
+    # @note `#offset` can be either the offset value or the time of the offset
+    # (first equal or greater)
+    Seek = Struct.new(:topic, :partition, :offset)
+  end
+end

data/lib/karafka/patches/rdkafka/bindings.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for third-party libraries patches
+  module Patches
+    # Rdkafka patches specific to Karafka
+    module Rdkafka
+      # Binding patches that slightly change how rdkafka operates in certain places
+      module Bindings
+        include ::Rdkafka::Bindings
+
+        # Alias internally
+        RB = ::Rdkafka::Bindings
+
+        class << self
+          # Handle assignments on cooperative rebalance
+          #
+          # @param client_ptr [FFI::Pointer]
+          # @param code [Integer]
+          # @param partitions_ptr [FFI::Pointer]
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          # @param opaque [Rdkafka::Opaque]
+          def on_cooperative_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              opaque&.call_on_partitions_assign(tpl)
+              RB.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
+              opaque&.call_on_partitions_assigned(tpl)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              opaque&.call_on_partitions_revoke(tpl)
+              RB.rd_kafka_commit(client_ptr, nil, false)
+              RB.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
+              opaque&.call_on_partitions_revoked(tpl)
+            else
+              opaque&.call_on_partitions_assign(tpl)
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+              opaque&.call_on_partitions_assigned(tpl)
+            end
+          end
+
+          # Handle assignments on a eager rebalance
+          #
+          # @param client_ptr [FFI::Pointer]
+          # @param code [Integer]
+          # @param partitions_ptr [FFI::Pointer]
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          # @param opaque [Rdkafka::Opaque]
+          def on_eager_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              opaque&.call_on_partitions_assign(tpl)
+              RB.rd_kafka_assign(client_ptr, partitions_ptr)
+              opaque&.call_on_partitions_assigned(tpl)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              opaque&.call_on_partitions_revoke(tpl)
+              RB.rd_kafka_commit(client_ptr, nil, false)
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+              opaque&.call_on_partitions_revoked(tpl)
+            else
+              opaque&.call_on_partitions_assign(tpl)
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+              opaque&.call_on_partitions_assigned(tpl)
+            end
+          end
+        end
+
+        # This patch changes few things:
+        # - it commits offsets (if any) upon partition revocation, so less jobs need to be
+        #   reprocessed if they are assigned to a different process
+        # - reports callback errors into the errors instrumentation instead of the logger
+        # - catches only StandardError instead of Exception as we fully control the directly
+        #   executed callbacks
+        #
+        # @see https://docs.confluent.io/2.0.0/clients/librdkafka/classRdKafka_1_1RebalanceCb.html
+        RebalanceCallback = FFI::Function.new(
+          :void, %i[pointer int pointer pointer]
+        ) do |client_ptr, code, partitions_ptr, opaque_ptr|
+          # Patch reference
+          pr = ::Karafka::Patches::Rdkafka::Bindings
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
+          opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
+
+          if RB.rd_kafka_rebalance_protocol(client_ptr) == 'COOPERATIVE'
+            pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
+          else
+            pr.on_eager_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
+          end
+        end
+      end
+    end
+  end
+end
+
+# We need to replace the original callback with ours.
+# At the moment there is no API in rdkafka-ruby to do so
+::Rdkafka::Bindings.send(
+  :remove_const,
+  'RebalanceCallback'
+)
+
+::Rdkafka::Bindings.const_set(
+  'RebalanceCallback',
+  Karafka::Patches::Rdkafka::Bindings::RebalanceCallback
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_rebalance_protocol,
+  %i[pointer],
+  :string
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_incremental_assign,
+  %i[pointer pointer],
+  :string
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_incremental_unassign,
+  %i[pointer pointer],
+  :string
+)

data/lib/karafka/patches/rdkafka/opaque.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Patches
+    module Rdkafka
+      # Patches allowing us to run events on both pre and post rebalance events.
+      # Thanks to that, we can easily connect to the whole flow despite of the moment when things
+      # are happening
+      module Opaque
+        # Handles pre-assign phase of rebalance
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def call_on_partitions_assign(tpl)
+          return unless consumer_rebalance_listener
+          return unless consumer_rebalance_listener.respond_to?(:on_partitions_assign)
+
+          consumer_rebalance_listener.on_partitions_assign(tpl)
+        end
+
+        # Handles pre-revoke phase of rebalance
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def call_on_partitions_revoke(tpl)
+          return unless consumer_rebalance_listener
+          return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoke)
+
+          consumer_rebalance_listener.on_partitions_revoke(tpl)
+        end
+      end
+    end
+  end
+end
+
+::Rdkafka::Opaque.include(
+  Karafka::Patches::Rdkafka::Opaque
+)

data/lib/karafka/pro/active_job/consumer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module ActiveJob
+      # Pro ActiveJob consumer that is suppose to handle long-running jobs as well as short
+      # running jobs
+      #
+      # When in LRJ, it will pause a given partition forever and will resume its processing only
+      # when all the jobs are done processing.
+      #
+      # It contains slightly better revocation warranties than the regular blocking consumer as
+      # it can stop processing batch of jobs in the middle after the revocation.
+      class Consumer < ::Karafka::ActiveJob::Consumer
+        # Runs ActiveJob jobs processing and handles lrj if needed
+        def consume
+          messages.each(clean: true) do |message|
+            # If for any reason we've lost this partition, not worth iterating over new messages
+            # as they are no longer ours
+            break if revoked?
+
+            # We cannot early stop when running virtual partitions because the intermediate state
+            # would force us not to commit the offsets. This would cause extensive
+            # double-processing
+            break if Karafka::App.stopping? && !topic.virtual_partitions?
+
+            consume_job(message)
+
+            # We can always mark because of the virtual offset management that we have in VPs
+            mark_as_consumed(message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Karafka Pro ActiveJob components
+    module ActiveJob
+      # Pro dispatcher that sends the ActiveJob job to a proper topic based on the queue name
+      # and that allows to inject additional options into the producer, effectively allowing for a
+      # much better and more granular control over the dispatch and consumption process.
+      class Dispatcher < ::Karafka::ActiveJob::Dispatcher
+        # Defaults for dispatching
+        # They can be updated by using `#karafka_options` on the job
+        DEFAULTS = {
+          dispatch_method: :produce_async,
+          dispatch_many_method: :produce_many_async,
+          # We don't create a dummy proc based partitioner as we would have to evaluate it with
+          # each job.
+          partitioner: nil,
+          # Allows for usage of `:key` or `:partition_key`
+          partition_key_type: :key
+        }.freeze
+
+        private_constant :DEFAULTS
+
+        # @param job [ActiveJob::Base] job
+        def dispatch(job)
+          ::Karafka.producer.public_send(
+            fetch_option(job, :dispatch_method, DEFAULTS),
+            dispatch_details(job).merge!(
+              topic: job.queue_name,
+              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+            )
+          )
+        end
+
+        # Bulk dispatches multiple jobs using the Rails 7.1+ API
+        # @param jobs [Array<ActiveJob::Base>] jobs we want to dispatch
+        def dispatch_many(jobs)
+          dispatches = Hash.new { |hash, key| hash[key] = [] }
+
+          jobs.each do |job|
+            d_method = fetch_option(job, :dispatch_many_method, DEFAULTS)
+
+            dispatches[d_method] << dispatch_details(job).merge!(
+              topic: job.queue_name,
+              payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+            )
+          end
+
+          dispatches.each do |type, messages|
+            ::Karafka.producer.public_send(
+              type,
+              messages
+            )
+          end
+        end
+
+        private
+
+        # @param job [ActiveJob::Base] job instance
+        # @return [Hash] hash with dispatch details to which we merge topic and payload
+        def dispatch_details(job)
+          partitioner = fetch_option(job, :partitioner, DEFAULTS)
+          key_type = fetch_option(job, :partition_key_type, DEFAULTS)
+
+          return {} unless partitioner
+
+          {
+            key_type => partitioner.call(job)
+          }
+        end
+      end
+    end
+  end
+end