karafka 1.2.13 → 1.3.0.rc1

data/lib/karafka/contracts/consumer_group_topic.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Consumer group topic validation rules
+    class ConsumerGroupTopic < Dry::Validation::Contract
+      params do
+        required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:backend).filled(included_in?: %i[inline sidekiq])
+        required(:consumer).filled
+        required(:deserializer).filled
+        required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
+        required(:start_from_beginning).filled(:bool?)
+        required(:batch_consuming).filled(:bool?)
+      end
+    end
+  end
+end

data/lib/karafka/contracts/responder_usage.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Validator to check responder topic usage
+    class ResponderUsageTopic < Dry::Validation::Contract
+      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
+
+      params do
+        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:required).filled(:bool?)
+        required(:usage_count).filled(:int?, gteq?: 0)
+        required(:registered).filled(eql?: true)
+        required(:async).filled(:bool?)
+        required(:serializer).filled
+      end
+
+      rule(:required, :usage_count) do
+        key(:name).failure(:required_usage_count) if values[:required] && values[:usage_count] < 1
+      end
+    end
+
+    # Validator to check that everything in a responder flow matches responder rules
+    class ResponderUsage < Dry::Validation::Contract
+      include Dry::Core::Constants
+
+      # Contract for verifying the topic usage details
+      TOPIC_CONTRACT = ResponderUsageTopic.new.freeze
+
+      private_constant :TOPIC_CONTRACT
+
+      params do
+        required(:used_topics)
+        required(:registered_topics)
+      end
+
+      rule(:used_topics) do
+        (value || EMPTY_ARRAY).each do |used_topic|
+          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
+            key([:used_topics, used_topic, error.path[0]]).failure(error.text)
+          end
+        end
+      end
+
+      rule(:registered_topics) do
+        (value || EMPTY_ARRAY).each do |used_topic|
+          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
+            key([:registered_topics, used_topic, error.path[0]]).failure(error.text)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/contracts/server_cli_options.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Contract for validating correctness of the server cli command options
+    # We validate some basics + the list of consumer_groups on which we want to use, to make
+    # sure that all of them are defined, plus that a pidfile does not exist
+    class ServerCliOptions < Dry::Validation::Contract
+      params do
+        optional(:pid).filled(:str?)
+        optional(:daemon).filled(:bool?)
+        optional(:consumer_groups).value(:array, :filled?)
+      end
+
+      rule(:pid) do
+        key(:pid).failure(:pid_already_exists) if value && File.exist?(value)
+      end
+
+      rule(:consumer_groups) do
+        # If there were no consumer_groups declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        if !value.nil? &&
+           !(value - Karafka::App.config.internal.routing_builder.map(&:name)).empty?
+          key(:consumer_groups).failure(:consumer_groups_inclusion)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -6,17 +6,18 @@ module Karafka
     # Base class for all the Karafka internal errors
     BaseError = Class.new(StandardError)
 
-    # Should be raised when we attemp to parse incoming params but parsing fails
-    #   If this error (or its descendant) is detected, we will pass the raw message
-    #   into params and proceed further
-    ParserError = Class.new(BaseError)
+    # Should be raised when we have that that we cannot serialize
+    SerializationError = Class.new(BaseError)
+
+    # Should be raised when we tried to deserialize incoming data but we failed
+    DeserializationError = Class.new(BaseError)
 
     # Raised when router receives topic name which does not correspond with any routes
     # This can only happen in a case when:
     #   - you've received a message and we cannot match it with a consumer
     #   - you've changed the routing, so router can no longer associate your topic to
     #     any consumer
-    #   - or in a case when you do a lot of metaprogramming and you change routing/etc on runtime
+    #   - or in a case when you do a lot of meta-programming and you change routing/etc on runtime
     #
     # In case this happens, you will have to create a temporary route that will allow
     # you to "eat" everything from the Sidekiq queue.
@@ -25,26 +26,26 @@ module Karafka
 
     # Raised when we don't use or use responder not in the way it expected to based on the
     # topics usage definitions
-    InvalidResponderUsage = Class.new(BaseError)
+    InvalidResponderUsageError = Class.new(BaseError)
 
-    # Raised when options that we provide to the responder to respond aren't what the schema
+    # Raised when options that we provide to the responder to respond aren't what the contract
     # requires
-    InvalidResponderMessageOptions = Class.new(BaseError)
+    InvalidResponderMessageOptionsError = Class.new(BaseError)
 
-    # Raised when configuration doesn't match with validation schema
-    InvalidConfiguration = Class.new(BaseError)
+    # Raised when configuration doesn't match with validation contract
+    InvalidConfigurationError = Class.new(BaseError)
 
-    # Raised when we try to use Karafka CLI commands (except install) without a bootfile
-    MissingBootFile = Class.new(BaseError)
+    # Raised when we try to use Karafka CLI commands (except install) without a boot file
+    MissingBootFileError = Class.new(BaseError)
 
     # Raised when we want to read a persisted thread messages consumer but it is unavailable
     # This should never happen and if it does, please contact us
-    MissingClient = Class.new(BaseError)
+    MissingClientError = Class.new(BaseError)
 
     # Raised when want to hook up to an event that is not registered and supported
-    UnregisteredMonitorEvent = Class.new(BaseError)
+    UnregisteredMonitorEventError = Class.new(BaseError)
 
-    # Raised when we've waited enough for shutting down an unresponding process
-    ForcefulShutdown = Class.new(BaseError)
+    # Raised when we've waited enough for shutting down a non-responsive process
+    ForcefulShutdownError = Class.new(BaseError)
   end
 end

data/lib/karafka/fetcher.rb

@@ -5,39 +5,37 @@ module Karafka
   # @note Creating multiple fetchers will result in having multiple connections to the same
   #   topics, which means that if there are no partitions, it won't use them.
   class Fetcher
-    class << self
-      # Starts listening on all the listeners asynchronously
-      # Fetch loop should never end, which means that we won't create more actor clusters
-      # so we don't have to terminate them
-      def call
-        threads = listeners.map do |listener|
-          # We abort on exception because there should be an exception handling developed for
-          # each listener running in separate threads, so the exceptions should never leak
-          # and if that happens, it means that something really bad happened and we should stop
-          # the whole process
-          Thread
-            .new { listener.call }
-            .tap { |thread| thread.abort_on_exception = true }
-        end
-
-        # We aggregate threads here for a supervised shutdown process
-        threads.each { |thread| Karafka::Server.consumer_threads << thread }
-        threads.each(&:join)
-      # If anything crashes here, we need to raise the error and crush the runner because it means
-      # that something terrible happened
-      rescue StandardError => e
-        Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
-        Karafka::App.stop!
-        raise e
+    # Starts listening on all the listeners asynchronously
+    # Fetch loop should never end, which means that we won't create more actor clusters
+    # so we don't have to terminate them
+    def call
+      threads = listeners.map do |listener|
+        # We abort on exception because there should be an exception handling developed for
+        # each listener running in separate threads, so the exceptions should never leak
+        # and if that happens, it means that something really bad happened and we should stop
+        # the whole process
+        Thread
+          .new { listener.call }
+          .tap { |thread| thread.abort_on_exception = true }
       end
 
-      private
+      # We aggregate threads here for a supervised shutdown process
+      threads.each { |thread| Karafka::Server.consumer_threads << thread }
+      threads.each(&:join)
+    # If anything crashes here, we need to raise the error and crush the runner because it means
+    # that something terrible happened
+    rescue StandardError => e
+      Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
+      Karafka::App.stop!
+      raise e
+    end
+
+    private
 
-      # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
-      def listeners
-        @listeners ||= App.consumer_groups.active.map do |consumer_group|
-          Karafka::Connection::Listener.new(consumer_group)
-        end
+    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
+    def listeners
+      @listeners ||= App.consumer_groups.active.map do |consumer_group|
+        Karafka::Connection::Listener.new(consumer_group)
       end
     end
   end

data/lib/karafka/helpers/class_matcher.rb

@@ -8,7 +8,9 @@ module Karafka
     class ClassMatcher
       # Regexp used to remove any non classy like characters that might be in the consumer
       # class name (if defined dynamically, etc)
-      CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}
+      CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}.freeze
+
+      private_constant :CONSTANT_REGEXP
 
       # @param klass [Class] class to which we want to find a corresponding class
       # @param from [String] what type of object is it (based on postfix name part)
@@ -30,6 +32,7 @@ module Karafka
       def match
         return nil if name.empty?
         return nil unless scope.const_defined?(name)
+
         matching = scope.const_get(name)
         same_scope?(matching) ? matching : nil
       end
@@ -71,6 +74,7 @@ module Karafka
       def scope_of(klass)
         enclosing = klass.to_s.split('::')[0...-1]
         return ::Object if enclosing.empty?
+
         ::Object.const_get(enclosing.join('::'))
       end
 

data/lib/karafka/helpers/config_retriever.rb

@@ -5,7 +5,7 @@ module Karafka
     # A helper method that allows us to build methods that try to get a given
     # attribute from its instance value and if it fails, will fallback to
     # the default config or config.kafka value for a given attribute.
-    # It is used to simplify the checkings.
+    # It is used to simplify the checks.
     # @note Worth noticing, that the value might be equal to false, so even
     #   then we need to return it. That's why we check for nil?
     # @example Define config retried attribute for start_from_beginning

data/lib/karafka/helpers/inflector.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Inflector provides inflection for the whole Karafka framework with additional inflection
+    # caching (due to the fact, that Dry::Inflector is slow)
+    module Inflector
+      # What inflection engine do we want to use
+      ENGINE = Dry::Inflector.new
+
+      @map = Concurrent::Hash.new
+
+      private_constant :ENGINE
+
+      class << self
+        # @param string [String] string that we want to convert to our underscore format
+        # @return [String] inflected string
+        # @example
+        #   Karafka::Helpers::Inflector.map('Module/ControllerName') #=> 'module_controller_name'
+        def map(string)
+          @map[string] ||= ENGINE.underscore(string).tr('/', '_')
+        end
+      end
+    end
+  end
+end

data/lib/karafka/helpers/multi_delegator.rb

@@ -7,7 +7,6 @@ module Karafka
     # Multidelegator is used to delegate calls to multiple targets
     class MultiDelegator
       # @param targets to which we want to delegate methods
-      #
       def initialize(*targets)
         @targets = targets
       end

data/lib/karafka/instrumentation/logger.rb

@@ -5,8 +5,6 @@ module Karafka
     # Default logger for Event Delegator
     # @note It uses ::Logger features - providing basic logging
     class Logger < ::Logger
-      include Singleton
-
       # Map containing information about log level for given environment
       ENV_MAP = {
         'production' => ::Logger::ERROR,
@@ -16,7 +14,11 @@ module Karafka
         'default' => ::Logger::INFO
       }.freeze
 
+      private_constant :ENV_MAP
+
       # Creates a new instance of logger ensuring that it has a place to write to
+      # @param _args Any arguments that we don't care about but that are needed in order to
+      #   make this logger compatible with the default Ruby one
       def initialize(*_args)
         ensure_dir_exists
         super(target)
@@ -26,7 +28,7 @@ module Karafka
       private
 
       # @return [Karafka::Helpers::MultiDelegator] multi delegator instance
-      #   to which we will be writtng logs
+      #   to which we will be writing logs
       # We use this approach to log stuff to file and to the STDOUT at the same time
       def target
         Karafka::Helpers::MultiDelegator

data/lib/karafka/instrumentation/monitor.rb

@@ -6,13 +6,11 @@ module Karafka
     # Monitor is used to hookup external monitoring services to monitor how Karafka works
     # It provides a standardized API for checking incoming messages/enqueueing etc
     # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
-    # same time, which means that you might have for example file logging and newrelic at the same
+    # same time, which means that you might have for example file logging and NewRelic at the same
     # time
     # @note This class acts as a singleton because we are only permitted to have single monitor
     #   per running process (just as logger)
     class Monitor < Dry::Monitor::Notifications
-      include Singleton
-
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
       # @note Depending on Karafka extensions and additional engines, this might not be the
@@ -21,11 +19,14 @@ module Karafka
       #   Last 4 events are from WaterDrop but for convenience we use the same monitor for the
       #   whole karafka ecosystem
       BASE_EVENTS = %w[
-        params.params.parse
-        params.params.parse.error
+        params.params.deserialize
+        params.params.deserialize.error
+        connection.listener.before_fetch_loop
+        connection.listener.fetch_loop
         connection.listener.fetch_loop.error
         connection.client.fetch_loop.error
-        connection.delegator.call
+        connection.batch_delegator.call
+        connection.message_delegator.call
         fetcher.call.error
         backends.inline.process
         process.notice_signal
@@ -34,8 +35,12 @@ module Karafka
         async_producer.call.retry
         sync_producer.call.error
         sync_producer.call.retry
-        server.stop
-        server.stop.error
+        app.initializing
+        app.initialized
+        app.running
+        app.stopping
+        app.stopping.error
+        app.stopped
       ].freeze
 
       private_constant :BASE_EVENTS
@@ -52,7 +57,8 @@ module Karafka
       def subscribe(event_name_or_listener)
         return super unless event_name_or_listener.is_a?(String)
         return super if available_events.include?(event_name_or_listener)
-        raise Errors::UnregisteredMonitorEvent, event_name_or_listener
+
+        raise Errors::UnregisteredMonitorEventError, event_name_or_listener
       end
 
       # @return [Array<String>] names of available events to which we can subscribe

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Listener that sets a proc title with a nice descriptive value
+    class ProctitleListener
+      # Updates proc title to an initializing one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_initializing(_event)
+        setproctitle('initializing')
+      end
+
+      # Updates proc title to a running one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_running(_event)
+        setproctitle('running')
+      end
+
+      # Updates proc title to a stopping one
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping(_event)
+        setproctitle('stopping')
+      end
+
+      private
+
+      # Sets a proper proc title with our constant prefix
+      # @param status [String] any status we want to set
+      def setproctitle(status)
+        ::Process.setproctitle(
+          "karafka #{Karafka::App.config.client_id} (#{status})"
+        )
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default listener that hooks up to our instrumentation and uses its events for logging
+    # It can be removed/replaced or anything without any harm to the Karafka app flow
+    class StdoutListener
+      # Log levels that we use in this particular listener
+      USED_LOG_LEVELS = %i[
+        debug
+        info
+        error
+        fatal
+      ].freeze
+
+      # Logs details about incoming batches and with which consumer we will consume them
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_batch_delegator_call(event)
+        consumer = event[:consumer]
+        topic = consumer.topic.name
+        kafka_messages = event[:kafka_batch].messages
+        info(
+          <<~MSG.chomp.tr("\n", ' ')
+            #{kafka_messages.count} messages
+            on #{topic} topic
+            delegated to #{consumer.class}
+          MSG
+        )
+      end
+
+      # Logs details about incoming message and with which consumer we will consume it
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_message_delegator_call(event)
+        consumer = event[:consumer]
+        topic = consumer.topic.name
+        info "1 message on #{topic} topic delegated to #{consumer.class}"
+      end
+
+      # Logs details about each received message value deserialization
+      # @param event [Dry::Events::Event] event details including payload
+      def on_params_params_deserialize(event)
+        # Keep in mind, that a caller here is a param object not a controller,
+        # so it returns a topic as a string, not a routing topic
+        debug(
+          <<~MSG.chomp.tr("\n", ' ')
+            Params deserialization for #{event[:caller].topic} topic
+            successful in #{event[:time]} ms
+          MSG
+        )
+      end
+
+      # Logs unsuccessful deserialization attempts of incoming data
+      # @param event [Dry::Events::Event] event details including payload
+      def on_params_params_deserialize_error(event)
+        error "Params deserialization error for #{event[:caller].topic} topic: #{event[:error]}"
+      end
+
+      # Logs errors that occurred in a listener fetch loop
+      # @param event [Dry::Events::Event] event details including payload
+      # @note It's an error as we can recover from it not a fatal
+      def on_connection_listener_fetch_loop_error(event)
+        error "Listener fetch loop error: #{event[:error]}"
+      end
+
+      # Logs errors that are related to the connection itself
+      # @note Karafka will attempt to reconnect, so an error not a fatal
+      # @param event [Dry::Events::Event] event details including payload
+      def on_connection_client_fetch_loop_error(event)
+        error "Client fetch loop error: #{event[:error]}"
+      end
+
+      # Logs info about crashed fetcher
+      # @note If this happens, Karafka will shutdown as it means a critical error
+      #   in one of the threads
+      # @param event [Dry::Events::Event] event details including payload
+      def on_fetcher_call_error(event)
+        fatal "Fetcher crash due to an error: #{event[:error]}"
+      end
+
+      # Logs info about processing of a certain dataset with an inline backend
+      # @param event [Dry::Events::Event] event details including payload
+      def on_backends_inline_process(event)
+        count = event[:caller].send(:params_batch).to_a.size
+        topic = event[:caller].topic.name
+        time = event[:time]
+        info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
+      end
+
+      # Logs info about system signals that Karafka received
+      # @param event [Dry::Events::Event] event details including payload
+      def on_process_notice_signal(event)
+        info "Received #{event[:signal]} system signal"
+      end
+
+      # Logs info about responder usage withing a controller flow
+      # @param event [Dry::Events::Event] event details including payload
+      def on_consumers_responders_respond_with(event)
+        calling = event[:caller]
+        responder = calling.topic.responder
+        data = event[:data]
+        info "Responded from #{calling.class} using #{responder} with following data #{data}"
+      end
+
+      # Logs info that we're initializing Karafka app
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_initializing(_event)
+        info "Initializing Karafka server #{::Process.pid}"
+      end
+
+      # Logs info that we're running Karafka app
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_running(_event)
+        info "Running Karafka server #{::Process.pid}"
+      end
+
+      # Logs info that we're going to stop the Karafka server
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping(_event)
+        # We use a separate thread as logging can't be called from trap context
+        Thread.new { info "Stopping Karafka server #{::Process.pid}" }
+      end
+
+      # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
+      #   forced exit
+      # @param _event [Dry::Events::Event] event details including payload
+      def on_app_stopping_error(_event)
+        # We use a separate thread as logging can't be called from trap context
+        Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
+      end
+
+      USED_LOG_LEVELS.each do |log_level|
+        define_method log_level do |*args|
+          Karafka.logger.send(log_level, *args)
+        end
+      end
+    end
+  end
+end