karafka 1.3.0

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Listener that sets a proc title with a nice descriptive value
+    class ProctitleListener
+      # Updates proc title to an initializing one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_initializing(_event)
+        setproctitle('initializing')
+      end
+
+      # Updates proc title to a running one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_running(_event)
+        setproctitle('running')
+      end
+
+      # Updates proc title to a stopping one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping(_event)
+        setproctitle('stopping')
+      end
+
+      private
+
+      # Sets a proper proc title with our constant prefix
+      # @param status [String] any status we want to set
+      def setproctitle(status)
+        ::Process.setproctitle(
+          "karafka #{Karafka::App.config.client_id} (#{status})"
+        )
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default listener that hooks up to our instrumentation and uses its events for logging
+    # It can be removed/replaced or anything without any harm to the Karafka app flow
+    class StdoutListener
+      # Log levels that we use in this particular listener
+      USED_LOG_LEVELS = %i[
+        debug
+        info
+        error
+        fatal
+      ].freeze
+
+      # Logs details about incoming batches and with which consumer we will consume them
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_batch_delegator_call(event)
+        consumer = event[:consumer]
+        topic = consumer.topic.name
+        kafka_messages = event[:kafka_batch].messages
+        info(
+          <<~MSG.chomp.tr("\n", ' ')
+            #{kafka_messages.count} messages
+            on #{topic} topic
+            delegated to #{consumer.class}
+          MSG
+        )
+      end
+
+      # Logs details about incoming message and with which consumer we will consume it
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_message_delegator_call(event)
+        consumer = event[:consumer]
+        topic = consumer.topic.name
+        info "1 message on #{topic} topic delegated to #{consumer.class}"
+      end
+
+      # Logs details about each received message value deserialization
+      # @param event [Dry::Events::Event] event details including payload
+      def on_params_params_deserialize(event)
+        # Keep in mind, that a caller here is a param object not a controller,
+        # so it returns a topic as a string, not a routing topic
+        debug(
+          <<~MSG.chomp.tr("\n", ' ')
+            Params deserialization for #{event[:caller].topic} topic
+            successful in #{event[:time]} ms
+          MSG
+        )
+      end
+
+      # Logs unsuccessful deserialization attempts of incoming data
+      # @param event [Dry::Events::Event] event details including payload
+      def on_params_params_deserialize_error(event)
+        error "Params deserialization error for #{event[:caller].topic} topic: #{event[:error]}"
+      end
+
+      # Logs errors that occurred in a listener fetch loop
+      # @param event [Dry::Events::Event] event details including payload
+      # @note It's an error as we can recover from it not a fatal
+      def on_connection_listener_fetch_loop_error(event)
+        error "Listener fetch loop error: #{event[:error]}"
+      end
+
+      # Logs errors that are related to the connection itself
+      # @param event [Dry::Events::Event] event details including payload
+      # @note Karafka will attempt to reconnect, so an error not a fatal
+      def on_connection_client_fetch_loop_error(event)
+        error "Client fetch loop error: #{event[:error]}"
+      end
+
+      # Logs info about crashed fetcher
+      # @param event [Dry::Events::Event] event details including payload
+      # @note If this happens, Karafka will shutdown as it means a critical error
+      #   in one of the threads
+      def on_fetcher_call_error(event)
+        fatal "Fetcher crash due to an error: #{event[:error]}"
+      end
+
+      # Logs info about processing of a certain dataset with an inline backend
+      # @param event [Dry::Events::Event] event details including payload
+      def on_backends_inline_process(event)
+        count = event[:caller].send(:params_batch).to_a.size
+        topic = event[:caller].topic.name
+        time = event[:time]
+        info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
+      end
+
+      # Logs info about system signals that Karafka received
+      # @param event [Dry::Events::Event] event details including payload
+      def on_process_notice_signal(event)
+        info "Received #{event[:signal]} system signal"
+      end
+
+      # Logs info about responder usage withing a controller flow
+      # @param event [Dry::Events::Event] event details including payload
+      def on_consumers_responders_respond_with(event)
+        calling = event[:caller]
+        responder = calling.topic.responder
+        data = event[:data]
+        info "Responded from #{calling.class} using #{responder} with following data #{data}"
+      end
+
+      # Logs info that we're initializing Karafka app
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_initializing(_event)
+        info "Initializing Karafka server #{::Process.pid}"
+      end
+
+      # Logs info that we're running Karafka app
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_running(_event)
+        info "Running Karafka server #{::Process.pid}"
+      end
+
+      # Logs info that we're going to stop the Karafka server
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping(_event)
+        # We use a separate thread as logging can't be called from trap context
+        Thread.new { info "Stopping Karafka server #{::Process.pid}" }
+      end
+
+      # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
+      #   forced exit
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping_error(_event)
+        # We use a separate thread as logging can't be called from trap context
+        Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
+      end
+
+      USED_LOG_LEVELS.each do |log_level|
+        define_method log_level do |*args|
+          Karafka.logger.send(log_level, *args)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/metadata.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    module Builders
+      # Builder for creating metadata object based on the message or batch informations
+      # @note We have 2 ways of creating metadata based on the way ruby-kafka operates
+      module Metadata
+        class << self
+          # Creates metadata based on the kafka batch data
+          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
+          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @return [Karafka::Params::Metadata] metadata object
+          def from_kafka_batch(kafka_batch, topic)
+            Karafka::Params::Metadata
+              .new
+              .merge!(
+                'batch_size' => kafka_batch.messages.count,
+                'first_offset' => kafka_batch.first_offset,
+                'highwater_mark_offset' => kafka_batch.highwater_mark_offset,
+                'last_offset' => kafka_batch.last_offset,
+                'offset_lag' => kafka_batch.offset_lag,
+                'deserializer' => topic.deserializer,
+                'partition' => kafka_batch.partition,
+                'topic' => kafka_batch.topic,
+                'unknown_last_offset' => kafka_batch.unknown_last_offset?
+              )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/params.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Due to the fact, that we create params related objects in couple contexts / places
+    # plus backends can build up them their own way we have this namespace.
+    # It allows to isolate actual params objects from their building process that can be
+    # context dependent.
+    module Builders
+      # Builder for params
+      module Params
+        class << self
+          # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
+          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
+          # @return [Karafka::Params::Params] params object
+          def from_kafka_message(kafka_message, topic)
+            Karafka::Params::Params
+              .new
+              .merge!(
+                'create_time' => kafka_message.create_time,
+                'headers' => kafka_message.headers || {},
+                'is_control_record' => kafka_message.is_control_record,
+                'key' => kafka_message.key,
+                'offset' => kafka_message.offset,
+                'deserializer' => topic.deserializer,
+                'partition' => kafka_message.partition,
+                'receive_time' => Time.now,
+                'topic' => kafka_message.topic,
+                'payload' => kafka_message.value
+              )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/params_batch.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    module Builders
+      # Builder for creating params batch instances
+      module ParamsBatch
+        class << self
+          # Creates params batch with params inside based on the incoming messages
+          # and the topic from which it comes
+          # @param kafka_messages [Array<Kafka::FetchedMessage>] raw fetched messages
+          # @param topic [Karafka::Routing::Topic] topic for which we're received messages
+          # @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
+          def from_kafka_messages(kafka_messages, topic)
+            params_array = kafka_messages.map! do |message|
+              Karafka::Params::Builders::Params.from_kafka_message(message, topic)
+            end
+
+            Karafka::Params::ParamsBatch.new(params_array)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/metadata.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Simple metadata object that stores all non-message information received from Kafka cluster
+    # while fetching the data
+    class Metadata < Hash
+      # Attributes that should be accessible as methods as well (not only hash)
+      METHOD_ATTRIBUTES = %w[
+        batch_size
+        first_offset
+        highwater_mark_offset
+        last_offset
+        offset_lag
+        deserializer
+        partition
+        topic
+      ].freeze
+
+      private_constant :METHOD_ATTRIBUTES
+
+      METHOD_ATTRIBUTES.each do |attr|
+        # Defines a method call accessor to a particular hash field.
+        define_method(attr) do
+          self[attr]
+        end
+      end
+
+      # @return [Boolean] is the last offset known or unknown
+      def unknown_last_offset?
+        self['unknown_last_offset']
+      end
+    end
+  end
+end

data/lib/karafka/params/params.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Params namespace encapsulating all the logic that is directly related to params handling
+  module Params
+    # It provides lazy loading not only until the first usage, but also allows us to skip
+    # using deserializer until we execute our logic. That way we can operate with
+    # heavy-deserialization data without slowing down the whole application.
+    class Params < Hash
+      # Params attributes that should be available via a method call invocation for Kafka
+      # client compatibility.
+      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
+      # uses those fields via method calls, so in order to be able to pass there our params
+      # objects, have to have same api.
+      METHOD_ATTRIBUTES = %w[
+        create_time
+        headers
+        is_control_record
+        key
+        offset
+        deserializer
+        deserialized
+        partition
+        receive_time
+        topic
+        payload
+      ].freeze
+
+      private_constant :METHOD_ATTRIBUTES
+
+      METHOD_ATTRIBUTES.each do |attr|
+        # Defines a method call accessor to a particular hash field.
+        # @note Won't work for complex key names that contain spaces, etc
+        # @param key [Symbol] name of a field that we want to retrieve with a method call
+        # @example
+        #   key_attr_reader :example
+        #   params.example #=> 'my example payload'
+        define_method(attr) do
+          self[attr]
+        end
+      end
+
+      # @return [Karafka::Params::Params] This method will trigger deserializer execution. If we
+      #   decide to retrieve data, deserializer will be executed to get data. Output of that will
+      #   be merged to the current object. This object will be also marked as already deserialized,
+      #   so we won't deserialize it again.
+      def deserialize!
+        return self if self['deserialized']
+
+        self['deserialized'] = true
+        self['payload'] = deserialize
+        self
+      end
+
+      private
+
+      # @return [Object] deserialized data
+      def deserialize
+        Karafka.monitor.instrument('params.params.deserialize', caller: self) do
+          self['deserializer'].call(self)
+        end
+      rescue ::StandardError => e
+        Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)
+        raise e
+      end
+    end
+  end
+end

data/lib/karafka/params/params_batch.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Params batch represents a set of messages received from Kafka.
+    # @note Params internally are lazy loaded before first use. That way we can skip
+    #   deserialization process if we have after_fetch that rejects some incoming messages
+    #   without using params It can be also used when handling really heavy data.
+    class ParamsBatch
+      include Enumerable
+
+      # @param params_array [Array<Karafka::Params::Params>] array with karafka params
+      # @return [Karafka::Params::ParamsBatch] lazy evaluated params batch object
+      def initialize(params_array)
+        @params_array = params_array
+      end
+
+      # @yieldparam [Karafka::Params::Params] each deserialized and loaded params instance
+      # @note Invocation of this method will cause loading and deserializing each param after
+      #   another. If you want to get access without deserializing, please access params_array
+      #   directly
+      def each
+        @params_array.each { |param| yield(param.deserialize!) }
+      end
+
+      # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
+      #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
+      #   be deserialized
+      def deserialize!
+        each(&:itself)
+      end
+
+      # @return [Array<Object>] array with deserialized payloads. This method can be useful when
+      #   we don't care about metadata and just want to extract all the data payloads from the
+      #   batch
+      def payloads
+        deserialize!.map(&:payload)
+      end
+
+      # @return [Karafka::Params::Params] first element after the deserialization process
+      def first
+        @params_array.first.deserialize!
+      end
+
+      # @return [Karafka::Params::Params] last element after the deserialization process
+      def last
+        @params_array.last.deserialize!
+      end
+
+      # @return [Array<Karafka::Params::Params>] pure array with params (not deserialized)
+      def to_a
+        @params_array
+      end
+
+      # @return [Integer] number of messages in the batch
+      def size
+        @params_array.size
+      end
+    end
+  end
+end

data/lib/karafka/patches/ruby_kafka.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for various other libs patches
+  module Patches
+    # Patches for Ruby Kafka gem
+    module RubyKafka
+      # This patch allows us to inject business logic in between fetches and before the consumer
+      # stop, so we can perform stop commit or anything else that we need since
+      # ruby-kafka fetch loop does not allow that directly
+      # We don't won't to use poll ruby-kafka api as it brings many more problems that we would
+      # have to take care of. That way, nothing like that ever happens but we get the control
+      # over the stopping process that we need (since we're the once that initiate it for each
+      # thread)
+      def consumer_loop
+        super do
+          consumers = Karafka::Persistence::Consumers
+                      .current
+                      .values
+                      .flat_map(&:values)
+                      .select { |consumer| consumer.class.respond_to?(:after_fetch) }
+
+          if Karafka::App.stopping?
+            publish_event(consumers, 'before_stop')
+            Karafka::Persistence::Client.read.stop
+          else
+            publish_event(consumers, 'before_poll')
+            yield
+            publish_event(consumers, 'after_poll')
+          end
+        end
+      end
+
+      private
+
+      # Notifies consumers about particular events happening
+      # @param consumers [Array<Object>] all consumers that want to be notified about an event
+      # @param event_name [String] name of the event that happened
+      def publish_event(consumers, event_name)
+        consumers.each do |consumer|
+          key = "consumers.#{Helpers::Inflector.map(consumer.class.to_s)}.#{event_name}"
+          Karafka::App.monitor.instrument(key, context: consumer)
+        end
+      end
+    end
+  end
+end