karafka 1.2.2 → 1.4.0.rc1

data/lib/karafka/helpers/class_matcher.rb

@@ -8,7 +8,9 @@ module Karafka
     class ClassMatcher
       # Regexp used to remove any non classy like characters that might be in the consumer
       # class name (if defined dynamically, etc)
-      CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}
+      CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}.freeze
+
+      private_constant :CONSTANT_REGEXP
 
       # @param klass [Class] class to which we want to find a corresponding class
       # @param from [String] what type of object is it (based on postfix name part)
@@ -30,6 +32,7 @@ module Karafka
       def match
         return nil if name.empty?
         return nil unless scope.const_defined?(name)
+
         matching = scope.const_get(name)
         same_scope?(matching) ? matching : nil
       end
@@ -41,7 +44,13 @@ module Karafka
       # @example From Namespaced::Super2Consumer matching responder
       #   matcher.name #=> Super2Responder
       def name
-        inflected = @klass.to_s.split('::').last.to_s
+        inflected = +@klass.to_s.split('::').last.to_s
+        # We inject the from into the name just in case it is missing as in a situation like
+        # that it would just sanitize the name without adding the "to" postfix.
+        # It could create cases when we want to build for example a responder to a consumer
+        # that does not have the "Consumer" postfix and would do nothing returning the same name.
+        # That would be bad as the matching classes shouldn't be matched to themselves.
+        inflected << @from unless inflected.include?(@from)
         inflected.gsub!(@from, @to)
         inflected.gsub!(CONSTANT_REGEXP, '')
         inflected
@@ -65,6 +74,7 @@ module Karafka
       def scope_of(klass)
         enclosing = klass.to_s.split('::')[0...-1]
         return ::Object if enclosing.empty?
+
         ::Object.const_get(enclosing.join('::'))
       end
 

data/lib/karafka/helpers/config_retriever.rb

@@ -5,7 +5,7 @@ module Karafka
     # A helper method that allows us to build methods that try to get a given
     # attribute from its instance value and if it fails, will fallback to
     # the default config or config.kafka value for a given attribute.
-    # It is used to simplify the checkings.
+    # It is used to simplify the checks.
     # @note Worth noticing, that the value might be equal to false, so even
     #   then we need to return it. That's why we check for nil?
     # @example Define config retried attribute for start_from_beginning

data/lib/karafka/helpers/inflector.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Inflector provides inflection for the whole Karafka framework with additional inflection
+    # caching (due to the fact, that Dry::Inflector is slow)
+    module Inflector
+      # What inflection engine do we want to use
+      ENGINE = Dry::Inflector.new
+
+      @map = Concurrent::Hash.new
+
+      private_constant :ENGINE
+
+      class << self
+        # @param string [String] string that we want to convert to our underscore format
+        # @return [String] inflected string
+        # @example
+        #   Karafka::Helpers::Inflector.map('Module/ControllerName') #=> 'module_controller_name'
+        def map(string)
+          @map[string] ||= ENGINE.underscore(string).tr('/', '_')
+        end
+      end
+    end
+  end
+end

data/lib/karafka/helpers/multi_delegator.rb

@@ -7,7 +7,6 @@ module Karafka
     # Multidelegator is used to delegate calls to multiple targets
     class MultiDelegator
       # @param targets to which we want to delegate methods
-      #
       def initialize(*targets)
         @targets = targets
       end

data/lib/karafka/instrumentation/logger.rb

@@ -5,8 +5,6 @@ module Karafka
     # Default logger for Event Delegator
     # @note It uses ::Logger features - providing basic logging
     class Logger < ::Logger
-      include Singleton
-
       # Map containing information about log level for given environment
       ENV_MAP = {
         'production' => ::Logger::ERROR,
@@ -16,7 +14,11 @@ module Karafka
         'default' => ::Logger::INFO
       }.freeze
 
+      private_constant :ENV_MAP
+
       # Creates a new instance of logger ensuring that it has a place to write to
+      # @param _args Any arguments that we don't care about but that are needed in order to
+      #   make this logger compatible with the default Ruby one
       def initialize(*_args)
         ensure_dir_exists
         super(target)
@@ -26,7 +28,7 @@ module Karafka
       private
 
       # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
-      #   to which we will be writtng logs
+      #   to which we will be writing logs
       # We use this approach to log stuff to file and to the STDOUT at the same time
       def target
         Karafka::Helpers::MultiDelegator
@@ -34,10 +36,11 @@ module Karafka
           .to(STDOUT, file)
       end
 
-      # Makes sure the log directory exists
+      # Makes sure the log directory exists as long as we can write to it
       def ensure_dir_exists
-        dir = File.dirname(log_path)
-        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+        FileUtils.mkdir_p(File.dirname(log_path))
+      rescue Errno::EACCES
+        nil
       end
 
       # @return [Pathname] Path to a file to which we should log

data/lib/karafka/instrumentation/monitor.rb

@@ -6,13 +6,11 @@ module Karafka
     # Monitor is used to hookup external monitoring services to monitor how Karafka works
     # It provides a standardized API for checking incoming messages/enqueueing etc
     # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
-    # same time, which means that you might have for example file logging and newrelic at the same
+    # same time, which means that you might have for example file logging and NewRelic at the same
     # time
     # @note This class acts as a singleton because we are only permitted to have single monitor
     #   per running process (just as logger)
     class Monitor < Dry::Monitor::Notifications
-      include Singleton
-
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
       # @note Depending on Karafka extensions and additional engines, this might not be the
@@ -21,11 +19,14 @@ module Karafka
       #   Last 4 events are from WaterDrop but for convenience we use the same monitor for the
       #   whole karafka ecosystem
       BASE_EVENTS = %w[
-        params.params.parse
-        params.params.parse.error
+        params.params.deserialize
+        params.params.deserialize.error
+        connection.listener.before_fetch_loop
+        connection.listener.fetch_loop
         connection.listener.fetch_loop.error
         connection.client.fetch_loop.error
-        connection.delegator.call
+        connection.batch_delegator.call
+        connection.message_delegator.call
         fetcher.call.error
         backends.inline.process
         process.notice_signal
@@ -34,8 +35,12 @@ module Karafka
         async_producer.call.retry
         sync_producer.call.error
         sync_producer.call.retry
-        server.stop
-        server.stop.error
+        app.initializing
+        app.initialized
+        app.running
+        app.stopping
+        app.stopping.error
+        app.stopped
       ].freeze
 
       private_constant :BASE_EVENTS
@@ -52,7 +57,8 @@ module Karafka
       def subscribe(event_name_or_listener)
         return super unless event_name_or_listener.is_a?(String)
         return super if available_events.include?(event_name_or_listener)
-        raise Errors::UnregisteredMonitorEvent, event_name_or_listener
+
+        raise Errors::UnregisteredMonitorEventError, event_name_or_listener
       end
 
       # @return [Array<String>] names of available events to which we can subscribe

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Listener that sets a proc title with a nice descriptive value
+    class ProctitleListener
+      # Updates proc title to an initializing one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_initializing(_event)
+        setproctitle('initializing')
+      end
+
+      # Updates proc title to a running one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_running(_event)
+        setproctitle('running')
+      end
+
+      # Updates proc title to a stopping one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping(_event)
+        setproctitle('stopping')
+      end
+
+      private
+
+      # Sets a proper proc title with our constant prefix
+      # @param status [String] any status we want to set
+      def setproctitle(status)
+        ::Process.setproctitle(
+          "karafka #{Karafka::App.config.client_id} (#{status})"
+        )
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default listener that hooks up to our instrumentation and uses its events for logging
+    # It can be removed/replaced or anything without any harm to the Karafka app flow
+    class StdoutListener
+      # Log levels that we use in this particular listener
+      USED_LOG_LEVELS = %i[
+        debug
+        info
+        error
+        fatal
+      ].freeze
+
+      # Logs details about incoming batches and with which consumer we will consume them
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_batch_delegator_call(event)
+        consumer = event[:consumer]
+        topic = consumer.topic.name
+        kafka_messages = event[:kafka_batch].messages
+        info(
+          <<~MSG.chomp.tr("\n", ' ')
+            #{kafka_messages.count} messages
+            on #{topic} topic
+            delegated to #{consumer.class}
+          MSG
+        )
+      end
+
+      # Logs details about incoming message and with which consumer we will consume it
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_message_delegator_call(event)
+        consumer = event[:consumer]
+        topic = consumer.topic.name
+        info "1 message on #{topic} topic delegated to #{consumer.class}"
+      end
+
+      # Logs details about each received message value deserialization
+      # @param event [Dry::Events::Event] event details including payload
+      def on_params_params_deserialize(event)
+        # Keep in mind, that a caller here is a param object not a controller,
+        # so it returns a topic as a string, not a routing topic
+        debug(
+          <<~MSG.chomp.tr("\n", ' ')
+            Params deserialization for #{event[:caller].metadata.topic} topic
+            successful in #{event[:time]} ms
+          MSG
+        )
+      end
+
+      # Logs unsuccessful deserialization attempts of incoming data
+      # @param event [Dry::Events::Event] event details including payload
+      def on_params_params_deserialize_error(event)
+        topic = event[:caller].metadata.topic
+        error = event[:error]
+        error "Params deserialization error for #{topic} topic: #{error}"
+      end
+
+      # Logs errors that occurred in a listener fetch loop
+      # @param event [Dry::Events::Event] event details including payload
+      # @note It's an error as we can recover from it not a fatal
+      def on_connection_listener_fetch_loop_error(event)
+        error "Listener fetch loop error: #{event[:error]}"
+      end
+
+      # Logs errors that are related to the connection itself
+      # @param event [Dry::Events::Event] event details including payload
+      # @note Karafka will attempt to reconnect, so an error not a fatal
+      def on_connection_client_fetch_loop_error(event)
+        error "Client fetch loop error: #{event[:error]}"
+      end
+
+      # Logs info about crashed fetcher
+      # @param event [Dry::Events::Event] event details including payload
+      # @note If this happens, Karafka will shutdown as it means a critical error
+      #   in one of the threads
+      def on_fetcher_call_error(event)
+        fatal "Fetcher crash due to an error: #{event[:error]}"
+      end
+
+      # Logs info about processing of a certain dataset with an inline backend
+      # @param event [Dry::Events::Event] event details including payload
+      def on_backends_inline_process(event)
+        count = event[:caller].send(:params_batch).to_a.size
+        topic = event[:caller].topic.name
+        time = event[:time]
+        info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
+      end
+
+      # Logs info about system signals that Karafka received
+      # @param event [Dry::Events::Event] event details including payload
+      def on_process_notice_signal(event)
+        info "Received #{event[:signal]} system signal"
+      end
+
+      # Logs info about responder usage withing a controller flow
+      # @param event [Dry::Events::Event] event details including payload
+      def on_consumers_responders_respond_with(event)
+        calling = event[:caller]
+        responder = calling.topic.responder
+        data = event[:data]
+        info "Responded from #{calling.class} using #{responder} with following data #{data}"
+      end
+
+      # Logs info that we're initializing Karafka app
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_initializing(_event)
+        info "Initializing Karafka server #{::Process.pid}"
+      end
+
+      # Logs info that we're running Karafka app
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_running(_event)
+        info "Running Karafka server #{::Process.pid}"
+      end
+
+      # Logs info that we're going to stop the Karafka server
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping(_event)
+        # We use a separate thread as logging can't be called from trap context
+        Thread.new { info "Stopping Karafka server #{::Process.pid}" }
+      end
+
+      # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
+      #   forced exit
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping_error(_event)
+        # We use a separate thread as logging can't be called from trap context
+        Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
+      end
+
+      USED_LOG_LEVELS.each do |log_level|
+        define_method log_level do |*args|
+          Karafka.logger.send(log_level, *args)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/batch_metadata.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Simple batch metadata object that stores all non-message information received from Kafka
+    # cluster while fetching the data
+    # @note This metadata object refers to per batch metadata, not `#params.metadata`
+    BatchMetadata = Struct.new(
+      :batch_size,
+      :first_offset,
+      :highwater_mark_offset,
+      :unknown_last_offset,
+      :last_offset,
+      :offset_lag,
+      :deserializer,
+      :partition,
+      :topic,
+      keyword_init: true
+    ) do
+      # @return [Boolean] is the last offset known or unknown
+      def unknown_last_offset?
+        unknown_last_offset
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/batch_metadata.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    module Builders
+      # Builder for creating batch metadata object based on the batch informations
+      module BatchMetadata
+        class << self
+          # Creates metadata based on the kafka batch data
+          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
+          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
+          # @return [Karafka::Params::BatchMetadata] batch metadata object
+          def from_kafka_batch(kafka_batch, topic)
+            Karafka::Params::BatchMetadata.new(
+              batch_size: kafka_batch.messages.count,
+              first_offset: kafka_batch.first_offset,
+              highwater_mark_offset: kafka_batch.highwater_mark_offset,
+              unknown_last_offset: kafka_batch.unknown_last_offset?,
+              last_offset: kafka_batch.last_offset,
+              offset_lag: kafka_batch.offset_lag,
+              deserializer: topic.deserializer,
+              partition: kafka_batch.partition,
+              topic: topic.name
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/params/builders/params.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Params
+    # Due to the fact, that we create params related objects in couple contexts / places
+    # plus backends can build up them their own way we have this namespace.
+    # It allows to isolate actual params objects from their building process that can be
+    # context dependent.
+    module Builders
+      # Builder for params
+      module Params
+        class << self
+          # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
+          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
+          # @return [Karafka::Params::Params] params object with payload and message metadata
+          def from_kafka_message(kafka_message, topic)
+            metadata = Karafka::Params::Metadata.new(
+              create_time: kafka_message.create_time,
+              headers: kafka_message.headers || {},
+              is_control_record: kafka_message.is_control_record,
+              key: kafka_message.key,
+              offset: kafka_message.offset,
+              deserializer: topic.deserializer,
+              partition: kafka_message.partition,
+              receive_time: Time.now,
+              topic: topic.name
+            ).freeze
+
+            Karafka::Params::Params.new(
+              kafka_message.value,
+              metadata
+            )
+          end
+        end
+      end
+    end
+  end
+end