karafka 1.0.1 → 1.4.14

data/lib/karafka/helpers/multi_delegator.rb

@@ -7,7 +7,6 @@ module Karafka
     # Multidelegator is used to delegate calls to multiple targets
     class MultiDelegator
       # @param targets to which we want to delegate methods
-      #
       def initialize(*targets)
         @targets = targets
       end

data/lib/karafka/instrumentation/logger.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default logger for Event Delegator
+    # @note It uses ::Logger features - providing basic logging
+    class Logger < ::Logger
+      # Map containing information about log level for given environment
+      ENV_MAP = {
+        'production' => ::Logger::ERROR,
+        'test' => ::Logger::ERROR,
+        'development' => ::Logger::INFO,
+        'debug' => ::Logger::DEBUG,
+        'default' => ::Logger::INFO
+      }.freeze
+
+      private_constant :ENV_MAP
+
+      # Creates a new instance of logger ensuring that it has a place to write to
+      # @param _args Any arguments that we don't care about but that are needed in order to
+      #   make this logger compatible with the default Ruby one
+      def initialize(*_args)
+        super(target)
+        self.level = ENV_MAP[Karafka.env] || ENV_MAP['default']
+      end
+
+      private
+
+      # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
+      #   to which we will be writing logs
+      # We use this approach to log stuff to file and to the $stdout at the same time
+      def target
+        Karafka::Helpers::MultiDelegator
+          .delegate(:write, :close)
+          .to(*[$stdout, file].compact)
+      end
+
+      # @return [Pathname] Path to a file to which we should log
+      def log_path
+        @log_path ||= Karafka::App.root.join("log/#{Karafka.env}.log")
+      end
+
+      # @return [File] file to which we want to write our logs
+      # @note File is being opened in append mode ('a')
+      def file
+        FileUtils.mkdir_p(File.dirname(log_path))
+
+        @file ||= File.open(log_path, 'a')
+      rescue Errno::EACCES, Errno::EROFS
+        nil
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/monitor.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related with Karafka instrumentation process
+  module Instrumentation
+    # Monitor is used to hookup external monitoring services to monitor how Karafka works
+    # It provides a standardized API for checking incoming messages/enqueueing etc
+    # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
+    # same time, which means that you might have for example file logging and NewRelic at the same
+    # time
+    # @note This class acts as a singleton because we are only permitted to have single monitor
+    #   per running process (just as logger)
+    class Monitor < Dry::Monitor::Notifications
+      # List of events that we support in the system and to which a monitor client can hook up
+      # @note The non-error once support timestamp benchmarking
+      # @note Depending on Karafka extensions and additional engines, this might not be the
+      #   complete list of all the events. Please use the #available_events on fully loaded
+      #   Karafka system to determine all of the events you can use.
+      #   Last 4 events are from WaterDrop but for convenience we use the same monitor for the
+      #   whole karafka ecosystem
+      BASE_EVENTS = %w[
+        params.params.deserialize
+        params.params.deserialize.error
+        connection.listener.before_fetch_loop
+        connection.listener.fetch_loop
+        connection.listener.fetch_loop.error
+        connection.client.fetch_loop.error
+        connection.batch_delegator.call
+        connection.message_delegator.call
+        fetcher.call.error
+        backends.inline.process
+        process.notice_signal
+        consumers.responders.respond_with
+        async_producer.call.error
+        async_producer.call.retry
+        sync_producer.call.error
+        sync_producer.call.retry
+        app.initializing
+        app.initialized
+        app.running
+        app.stopping
+        app.stopping.error
+        app.stopped
+      ].freeze
+
+      private_constant :BASE_EVENTS
+
+      # @return [Karafka::Instrumentation::Monitor] monitor instance for system instrumentation
+      def initialize
+        super(:karafka)
+        BASE_EVENTS.each(&method(:register_event))
+      end
+
+      # Allows us to subscribe to events with a code that will be yielded upon events
+      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
+      #   or a listener if we decide to go with object listener
+      def subscribe(event_name_or_listener)
+        return super unless event_name_or_listener.is_a?(String)
+        return super if available_events.include?(event_name_or_listener)
+
+        raise Errors::UnregisteredMonitorEventError, event_name_or_listener
+      end
+
+      # @return [Array<String>] names of available events to which we can subscribe
+      def available_events
+        __bus__.events.keys
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Listener that sets a proc title with a nice descriptive value
+    class ProctitleListener
+      # Updates proc title to an initializing one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_initializing(_event)
+        setproctitle('initializing')
+      end
+
+      # Updates proc title to a running one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_running(_event)
+        setproctitle('running')
+      end
+
+      # Updates proc title to a stopping one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping(_event)
+        setproctitle('stopping')
+      end
+
+      private
+
+      # Sets a proper proc title with our constant prefix
+      # @param status [String] any status we want to set
+      def setproctitle(status)
+        ::Process.setproctitle(
+          "karafka #{Karafka::App.config.client_id} (#{status})"
+        )
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default listener that hooks up to our instrumentation and uses its events for logging
+    # It can be removed/replaced or anything without any harm to the Karafka app flow
+    class StdoutListener
+      # Log levels that we use in this particular listener
+      USED_LOG_LEVELS = %i[
+        debug
+        info
+        error
+        fatal
+      ].freeze
+
+      # Logs details about incoming batches and with which consumer we will consume them
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_batch_delegator_call(event)
+        consumer = event[:consumer]
+        topic = consumer.topic.name
+        kafka_messages = event[:kafka_batch].messages
+        info(
+          <<~MSG.chomp.tr("\n", ' ')
+            #{kafka_messages.count} messages
+            on #{topic} topic
+            delegated to #{consumer.class}
+          MSG
+        )
+      end
+
+      # Logs details about incoming message and with which consumer we will consume it
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_message_delegator_call(event)
+        consumer = event[:consumer]
+        topic = consumer.topic.name
+        info "1 message on #{topic} topic delegated to #{consumer.class}"
+      end
+
+      # Logs details about each received message value deserialization
+      # @param event [Dry::Events::Event] event details including payload
+      def on_params_params_deserialize(event)
+        # Keep in mind, that a caller here is a param object not a controller,
+        # so it returns a topic as a string, not a routing topic
+        debug(
+          <<~MSG.chomp.tr("\n", ' ')
+            Params deserialization for #{event[:caller].metadata.topic} topic
+            successful in #{event[:time]} ms
+          MSG
+        )
+      end
+
+      # Logs unsuccessful deserialization attempts of incoming data
+      # @param event [Dry::Events::Event] event details including payload
+      def on_params_params_deserialize_error(event)
+        topic = event[:caller].metadata.topic
+        error = event[:error]
+        error "Params deserialization error for #{topic} topic: #{error}"
+      end
+
+      # Logs errors that occurred in a listener fetch loop
+      # @param event [Dry::Events::Event] event details including payload
+      # @note It's an error as we can recover from it not a fatal
+      def on_connection_listener_fetch_loop_error(event)
+        error "Listener fetch loop error: #{event[:error]}"
+      end
+
+      # Logs errors that are related to the connection itself
+      # @param event [Dry::Events::Event] event details including payload
+      # @note Karafka will attempt to reconnect, so an error not a fatal
+      def on_connection_client_fetch_loop_error(event)
+        error "Client fetch loop error: #{event[:error]}"
+      end
+
+      # Logs info about crashed fetcher
+      # @param event [Dry::Events::Event] event details including payload
+      # @note If this happens, Karafka will shutdown as it means a critical error
+      #   in one of the threads
+      def on_fetcher_call_error(event)
+        fatal "Fetcher crash due to an error: #{event[:error]}"
+      end
+
+      # Logs info about processing of a certain dataset with an inline backend
+      # @param event [Dry::Events::Event] event details including payload
+      def on_backends_inline_process(event)
+        count = event[:caller].send(:params_batch).to_a.size
+        topic = event[:caller].topic.name
+        time = event[:time]
+        info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
+      end
+
+      # Logs info about system signals that Karafka received
+      # @param event [Dry::Events::Event] event details including payload
+      def on_process_notice_signal(event)
+        info "Received #{event[:signal]} system signal"
+      end
+
+      # Logs info about responder usage withing a controller flow
+      # @param event [Dry::Events::Event] event details including payload
+      def on_consumers_responders_respond_with(event)
+        calling = event[:caller]
+        responder = calling.topic.responder
+        data = event[:data]
+        info "Responded from #{calling.class} using #{responder} with following data #{data}"
+      end
+
+      # Logs info that we're initializing Karafka framework components
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_initializing(_event)
+        info "Initializing Karafka framework #{::Process.pid}"
+      end
+
+      # Logs info that we're running Karafka app
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_running(_event)
+        info "Running Karafka server #{::Process.pid}"
+      end
+
+      # Logs info that we're going to stop the Karafka server
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping(_event)
+        # We use a separate thread as logging can't be called from trap context
+        Thread.new { info "Stopping Karafka server #{::Process.pid}" }
+      end
+
+      # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
+      #   forced exit
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping_error(_event)
+        # We use a separate thread as logging can't be called from trap context
+        Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
+      end
+
+      USED_LOG_LEVELS.each do |log_level|
+        define_method log_level do |*args|
+          Karafka.logger.send(log_level, *args)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/batch_metadata.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Simple batch metadata object that stores all non-message information received from Kafka
+    # cluster while fetching the data
+    # @note This metadata object refers to per batch metadata, not `#params.metadata`
+    BatchMetadata = Struct.new(
+      :batch_size,
+      :first_offset,
+      :highwater_mark_offset,
+      :unknown_last_offset,
+      :last_offset,
+      :offset_lag,
+      :deserializer,
+      :partition,
+      :topic,
+      keyword_init: true
+    ) do
+      # @return [Boolean] is the last offset known or unknown
+      def unknown_last_offset?
+        unknown_last_offset
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/batch_metadata.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    module Builders
+      # Builder for creating batch metadata object based on the batch informations
+      module BatchMetadata
+        class << self
+          # Creates metadata based on the kafka batch data
+          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
+          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @return [Karafka::Params::BatchMetadata] batch metadata object
+          def from_kafka_batch(kafka_batch, topic)
+            Karafka::Params::BatchMetadata.new(
+              batch_size: kafka_batch.messages.count,
+              first_offset: kafka_batch.first_offset,
+              highwater_mark_offset: kafka_batch.highwater_mark_offset,
+              unknown_last_offset: kafka_batch.unknown_last_offset?,
+              last_offset: kafka_batch.last_offset,
+              offset_lag: kafka_batch.offset_lag,
+              deserializer: topic.deserializer,
+              partition: kafka_batch.partition,
+              topic: topic.name
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/params.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Due to the fact, that we create params related objects in couple contexts / places
+    # plus backends can build up them their own way we have this namespace.
+    # It allows to isolate actual params objects from their building process that can be
+    # context dependent.
+    module Builders
+      # Builder for params
+      module Params
+        class << self
+          # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
+          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
+          # @return [Karafka::Params::Params] params object with payload and message metadata
+          def from_kafka_message(kafka_message, topic)
+            metadata = Karafka::Params::Metadata.new(
+              create_time: kafka_message.create_time,
+              headers: kafka_message.headers || {},
+              is_control_record: kafka_message.is_control_record,
+              key: kafka_message.key,
+              offset: kafka_message.offset,
+              deserializer: topic.deserializer,
+              partition: kafka_message.partition,
+              receive_time: Time.now,
+              topic: topic.name
+            ).freeze
+
+            Karafka::Params::Params.new(
+              kafka_message.value,
+              metadata
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/params_batch.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    module Builders
+      # Builder for creating params batch instances
+      module ParamsBatch
+        class << self
+          # Creates params batch with params inside based on the incoming messages
+          # and the topic from which it comes
+          # @param kafka_messages [Array<Kafka::FetchedMessage>] raw fetched messages
+          # @param topic [Karafka::Routing::Topic] topic for which we're received messages
+          # @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
+          def from_kafka_messages(kafka_messages, topic)
+            params_array = kafka_messages.map do |message|
+              Karafka::Params::Builders::Params.from_kafka_message(message, topic)
+            end
+
+            Karafka::Params::ParamsBatch.new(params_array).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/metadata.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Single message / params metadata details that can be accessed without the need for the
+    # payload deserialization
+    Metadata = Struct.new(
+      :create_time,
+      :headers,
+      :is_control_record,
+      :key,
+      :offset,
+      :deserializer,
+      :partition,
+      :receive_time,
+      :topic,
+      keyword_init: true
+    )
+  end
+end

data/lib/karafka/params/params.rb

@@ -3,123 +3,51 @@
 module Karafka
   # Params namespace encapsulating all the logic that is directly related to params handling
   module Params
-    # Class-wrapper for hash with indifferent access with additional lazy loading feature
     # It provides lazy loading not only until the first usage, but also allows us to skip
-    # using parser until we execute our logic. That way we can operate with
-    # heavy-parsing data without slowing down the whole application.
-    class Params < HashWithIndifferentAccess
-      # Kafka::FetchedMessage attributes that we want to use inside of params
-      KAFKA_MESSAGE_ATTRIBUTES = %i[
-        value
-        partition
-        offset
-        key
-      ].freeze
-
-      # Params attributes that should be available via a method call invocation for Kafka
-      # client compatibility.
-      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
-      # uses those fields via method calls, so in order to be able to pass there our params
-      # objects, have to have same api.
-      PARAMS_METHOD_ATTRIBUTES = %i[
-        topic
-        partition
-        offset
-        key
-      ].freeze
-
-      class << self
-        # We allow building instances only via the #build method
-
-        # @param message [Kafka::FetchedMessage, Hash] message that we get out of Kafka
-        #   in case of building params inside main Karafka process in
-        #   Karafka::Connection::Consumer, or a hash when we retrieve data that is already parsed
-        # @param parser [Class] parser class that we will use to unparse data
-        # @return [Karafka::Params::Params] Karafka params object not yet used parser for
-        #   retrieving data that we've got from Kafka
-        # @example Build params instance from a hash
-        #   Karafka::Params::Params.build({ key: 'value' }) #=> params object
-        # @example Build params instance from a Kafka::FetchedMessage object
-        #   Karafka::Params::Params.build(message) #=> params object
-        def build(message, parser)
-          # Hash case happens inside backends that interchange data
-          if message.is_a?(Hash)
-            new(parser: parser).send(:merge!, message)
-          else
-            # This happens inside Kafka::FetchedMessagesProcessor
-            new(
-              parser: parser,
-              parsed: false,
-              received_at: Time.now
-            ).tap do |instance|
-              KAFKA_MESSAGE_ATTRIBUTES.each do |attribute|
-                instance[attribute] = message.send(attribute)
-              end
-
-              # When we get raw messages, they might have a topic, that was modified by a
-              # topic mapper. We need to "reverse" this change and map back to the non-modified
-              # format, so our internal flow is not corrupted with the mapping
-              instance[:topic] = Karafka::App.config.topic_mapper.incoming(message.topic)
-            end
-          end
-        end
-
-        # Defines a method call accessor to a particular hash field.
-        # @note Won't work for complex key names that contain spaces, etc
-        # @param key [Symbol] name of a field that we want to retrieve with a method call
-        # @example
-        #   key_attr_reader :example
-        #   params.example #=> 'my example value'
-        def key_attr_reader(key)
-          define_method key do
-            self[key]
-          end
-        end
+    # using deserializer until we execute our logic. That way we can operate with
+    # heavy-deserialization data without slowing down the whole application.
+    class Params
+      extend Forwardable
+
+      attr_reader :raw_payload, :metadata
+
+      def_delegators :metadata, *Metadata.members
+
+      # @param raw_payload [Object] incoming payload before deserialization
+      # @param metadata [Karafka::Params::Metadata] message metadata object
+      def initialize(raw_payload, metadata)
+        @raw_payload = raw_payload
+        @metadata = metadata
+        @deserialized = false
+        @payload = nil
       end
 
-      # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
-      #   retrieve data, parser will be executed to parse data. Output of parsing will be merged
-      #   to the current object. This object will be also marked as already parsed, so we won't
-      #   parse it again.
-      def retrieve!
-        return self if self[:parsed]
+      # @return [Object] lazy-deserialized data (deserialized upon first request)
+      def payload
+        return @payload if deserialized?
 
-        merge!(parse(delete(:value)))
+        @payload = deserialize
+        # We mark deserialization as successful after deserialization, as in case of an error
+        # this won't be falsely set to true
+        @deserialized = true
+        @payload
       end
 
-      PARAMS_METHOD_ATTRIBUTES.each(&method(:key_attr_reader))
+      # @return [Boolean] did given params payload were deserialized already
+      def deserialized?
+        @deserialized
+      end
 
       private
 
-      # Overwritten merge! method - it behaves differently for keys that are the same in our hash
-      #  and in a other_hash - it will not replace keys that are the same in our hash
-      #  and in the other one
-      # @param other_hash [Hash, HashWithIndifferentAccess] hash that we want to merge into current
-      # @return [Karafka::Params::Params] our parameters hash with merged values
-      # @example Merge with hash without same keys
-      #   new(a: 1, b: 2).merge!(c: 3) #=> { a: 1, b: 2, c: 3 }
-      # @example Merge with hash with same keys (symbol based)
-      #   new(a: 1).merge!(a: 2) #=> { a: 1 }
-      # @example Merge with hash with same keys (string based)
-      #   new(a: 1).merge!('a' => 2) #=> { a: 1 }
-      # @example Merge with hash with same keys (current string based)
-      #   new('a' => 1).merge!(a: 2) #=> { a: 1 }
-      def merge!(other_hash)
-        super(other_hash) { |_key, base_value, _new_value| base_value }
-      end
-
-      # @param value [String] Raw data that we want to parse using controller's parser
-      # @note If something goes wrong, it will return raw data in a hash with a message key
-      # @return [Hash] parsed data or a hash with message key containing raw data if something
-      #   went wrong during parsing
-      def parse(value)
-        self[:parser].parse(value)
-        # We catch both of them, because for default JSON - we use JSON parser directly
-      rescue ::Karafka::Errors::ParserError => e
-        Karafka.monitor.notice_error(self.class, e)
+      # @return [Object] tries de-serializes data
+      def deserialize
+        Karafka.monitor.instrument('params.params.deserialize', caller: self) do
+          metadata.deserializer.call(self)
+        end
+      rescue ::StandardError => e
+        Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)
         raise e
-      ensure
-        self[:parsed] = true
       end
     end
   end