karafka 1.1.0 → 1.2.1

data/lib/karafka/connection/delegator.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that delegates processing of messages for which we listen to a proper processor
+    module Delegator
+      class << self
+        # Delegates messages (does something with them)
+        # It will either schedule or run a proper processor action for messages
+        # @note This should be looped to obtain a constant delegating of new messages
+        # @note We catch all the errors here, to make sure that none failures
+        #   for a given consumption will affect other consumed messages
+        #   If we wouldn't catch it, it would propagate up until killing the thread
+        # @note It is a one huge method, because of performance reasons. It is much faster then
+        #   using send or invoking additional methods
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
+        def call(group_id, kafka_messages)
+          # @note We always get messages by topic and partition so we can take topic from the
+          # first one and it will be valid for all the messages
+          topic = Persistence::Topic.fetch(group_id, kafka_messages[0].topic)
+          consumer = Persistence::Consumer.fetch(topic, kafka_messages[0].partition)
+
+          Karafka.monitor.instrument(
+            'connection.delegator.call',
+            caller: self,
+            consumer: consumer,
+            kafka_messages: kafka_messages
+          ) do
+            # Depending on a case (persisted or not) we might use new consumer instance per
+            # each batch, or use the same one for all of them (for implementing buffering, etc.)
+            if topic.batch_consuming
+              consumer.params_batch = kafka_messages
+              consumer.call
+            else
+              kafka_messages.each do |kafka_message|
+                consumer.params_batch = [kafka_message]
+                consumer.call
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -7,8 +7,6 @@ module Karafka
     # @note Listener itself does nothing with the message - it will return to the block
     #   a raw Kafka::FetchedMessage
     class Listener
-      attr_reader :consumer_group
-
       # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
       #   on what topics and with what settings should we listen
       # @return [Karafka::Connection::Listener] listener instance
@@ -16,6 +14,17 @@ module Karafka
         @consumer_group = consumer_group
       end
 
+      # Runs prefetch callbacks and executes the main listener fetch loop
+      def call
+        Karafka::Callbacks.before_fetch_loop(
+          @consumer_group,
+          client
+        )
+        fetch_loop
+      end
+
+      private
+
       # Opens connection, gets messages and calls a block for each of the incoming messages
       # @yieldparam [String] consumer group id
       # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
@@ -26,25 +35,25 @@ module Karafka
       #   won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far
-      def fetch_loop(block)
-        consumer.fetch_loop do |raw_messages|
-          block.call(consumer_group.id, raw_messages)
+      def fetch_loop
+        client.fetch_loop do |raw_messages|
+          # @note What happens here is a delegation of processing to a proper processor based
+          #   on the incoming messages characteristics
+          Karafka::Connection::Delegator.call(@consumer_group.id, raw_messages)
         end
         # This is on purpose - see the notes for this method
         # rubocop:disable RescueException
       rescue Exception => e
+        Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
         # rubocop:enable RescueException
-        Karafka.monitor.notice_error(self.class, e)
-        @consumer&.stop
-        retry if @consumer
+        @client&.stop
+        retry if @client
       end
 
-      private
-
-      # @return [Karafka::Connection::Consumer] wrapped kafka consumer for a given topic
+      # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic
       #   consumption
-      def consumer
-        @consumer ||= Consumer.new(consumer_group)
+      def client
+        @client ||= Client.new(@consumer_group)
       end
     end
   end

data/lib/karafka/{controllers → consumers}/callbacks.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Controllers
+  module Consumers
     # Additional callbacks that can be used to trigger some actions on certain moments like
     # manual offset management, committing or anything else outside of a standard messages flow
     # They are not included by default, as we don't want to provide functionalities that are
@@ -10,7 +10,7 @@ module Karafka
     module Callbacks
       # Types of events on which we run callbacks
       TYPES = %i[
-        after_fetched
+        after_fetch
         after_poll
         before_poll
         before_stop
@@ -28,9 +28,9 @@ module Karafka
         end
       end
 
-      # @param controller_class [Class] controller class that we extend with callbacks
-      def self.included(controller_class)
-        controller_class.class_eval do
+      # @param consumer_class [Class] consumer class that we extend with callbacks
+      def self.included(consumer_class)
+        consumer_class.class_eval do
           extend ClassMethods
           include ActiveSupport::Callbacks
 
@@ -41,11 +41,11 @@ module Karafka
         end
       end
 
-      # Executes the default controller flow, runs callbacks and if not halted will call process
-      # method of a proper backend. This is here because it interacts with the default Karafka
-      # call flow and needs to be overwritten in order to support callbacks
+      # Executes the default consumer flow, runs callbacks and if not halted will call process
+      # method of a proper backend. It is here because it interacts with the default Karafka
+      # call flow and needs to be overwritten to support callbacks
       def call
-        run_callbacks :after_fetched do
+        run_callbacks :after_fetch do
           process
         end
       end

data/lib/karafka/consumers/includer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Additional functionalities for consumers
+  module Consumers
+    # Module used to inject functionalities into a given consumer class, based on the consumer
+    # topic and its settings
+    # We don't need all the behaviors in all the cases, so it is not worth having everything
+    # in all the cases all the time
+    module Includer
+      class << self
+        # @param consumer_class [Class] consumer class, that will get some functionalities
+        #   based on the topic under which it operates
+        def call(consumer_class)
+          topic = consumer_class.topic
+
+          bind_backend(consumer_class, topic)
+          bind_params(consumer_class, topic)
+          bind_responders(consumer_class, topic)
+        end
+
+        private
+
+        # Figures out backend for a given consumer class, based on the topic backend and
+        #   includes it into the consumer class
+        # @param consumer_class [Class] consumer class
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_backend(consumer_class, topic)
+          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
+          consumer_class.include backend
+        end
+
+        # Adds a single #params support for non batch processed topics
+        # @param consumer_class [Class] consumer class
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_params(consumer_class, topic)
+          return if topic.batch_consuming
+          consumer_class.include SingleParams
+        end
+
+        # Adds responders support for topics and consumers with responders defined for them
+        # @param consumer_class [Class] consumer class
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_responders(consumer_class, topic)
+          return unless topic.responder
+          consumer_class.include Responders
+        end
+      end
+    end
+  end
+end

data/lib/karafka/consumers/responders.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Feature that allows us to use responders flow in consumer
+    module Responders
+      # Responds with given data using given responder. This allows us to have a similar way of
+      # defining flows like synchronous protocols
+      # @param data Anything we want to pass to responder based on which we want to trigger further
+      #   Kafka responding
+      def respond_with(*data)
+        Karafka.monitor.instrument(
+          'consumers.responders.respond_with',
+          caller: self,
+          data: data
+        ) do
+          # @note we build a new instance of responder each time, as a long-running (persisted)
+          #   consumers can respond multiple times during the lifecycle
+          topic.responder.new(topic.parser).call(*data)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/{controllers → consumers}/single_params.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 module Karafka
-  module Controllers
-    # Params alias for single message consumption controllers
+  module Consumers
+    # Params alias for single message consumption consumers
     module SingleParams
       private
 
-      # @return [Karafka::Params::Params] params instance for non batch consumption controllers
+      # @return [Karafka::Params::Params] params instance for non batch consumption consumers
       def params
         params_batch.first
       end

data/lib/karafka/errors.rb

@@ -13,9 +13,9 @@ module Karafka
 
     # Raised when router receives topic name which does not correspond with any routes
     # This can only happen in a case when:
-    #   - you've received a message and we cannot match it with a controller
+    #   - you've received a message and we cannot match it with a consumer
     #   - you've changed the routing, so router can no longer associate your topic to
-    #     any controller
+    #     any consumer
     #   - or in a case when you do a lot of metaprogramming and you change routing/etc on runtime
     #
     # In case this happens, you will have to create a temporary route that will allow
@@ -27,6 +27,10 @@ module Karafka
     # topics usage definitions
     InvalidResponderUsage = Class.new(BaseError)
 
+    # Raised when options that we provide to the responder to respond aren't what the schema
+    # requires
+    InvalidResponderMessageOptions = Class.new(BaseError)
+
     # Raised when configuration doesn't match with validation schema
     InvalidConfiguration = Class.new(BaseError)
 
@@ -35,9 +39,15 @@ module Karafka
 
     # Raised when we want to read a persisted thread messages consumer but it is unavailable
     # This should never happen and if it does, please contact us
-    MissingConsumer = Class.new(BaseError)
+    MissingClient = Class.new(BaseError)
 
     # Raised when we attemp to pause a partition but the pause timeout is equal to 0
     InvalidPauseTimeout = Class.new(BaseError)
+
+    # Raised when want to hook up to an event that is not registered and supported
+    UnregisteredMonitorEvent = Class.new(BaseError)
+
+    # Raised when we've waited enough for shutting down an unresponding process
+    ForcefulShutdown = Class.new(BaseError)
   end
 end

data/lib/karafka/fetcher.rb

@@ -5,43 +5,39 @@ module Karafka
   # @note Creating multiple fetchers will result in having multiple connections to the same
   #   topics, which means that if there are no partitions, it won't use them.
   class Fetcher
-    # Starts listening on all the listeners asynchronously
-    # Fetch loop should never end, which means that we won't create more actor clusters
-    # so we don't have to terminate them
-    def fetch_loop
-      threads = listeners.map do |listener|
-        # We abort on exception because there should be an exception handling developed for
-        # each listener running in separate threads, so the exceptions should never leak
-        # and if that happens, it means that something really bad happened and we should stop
-        # the whole process
-        Thread
-          .new { listener.fetch_loop(processor) }
-          .tap { |thread| thread.abort_on_exception = true }
-      end
-
-      threads.each(&:join)
-    # If anything crashes here, we need to raise the error and crush the runner because it means
-    # that something really bad happened
-    rescue StandardError => e
-      Karafka.monitor.notice_error(self.class, e)
-      Karafka::App.stop!
-      raise e
-    end
+    class << self
+      # Starts listening on all the listeners asynchronously
+      # Fetch loop should never end, which means that we won't create more actor clusters
+      # so we don't have to terminate them
+      def call
+        threads = listeners.map do |listener|
+          # We abort on exception because there should be an exception handling developed for
+          # each listener running in separate threads, so the exceptions should never leak
+          # and if that happens, it means that something really bad happened and we should stop
+          # the whole process
+          Thread
+            .new { listener.call }
+            .tap { |thread| thread.abort_on_exception = true }
+        end
 
-    private
-
-    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
-    def listeners
-      @listeners ||= App.consumer_groups.active.map do |consumer_group|
-        Karafka::Connection::Listener.new(consumer_group)
+        # We aggregate threads here for a supervised shutdown process
+        threads.each { |thread| Karafka::Server.consumer_threads << thread }
+        threads.each(&:join)
+      # If anything crashes here, we need to raise the error and crush the runner because it means
+      # that something terrible happened
+      rescue StandardError => e
+        Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
+        Karafka::App.stop!
+        raise e
       end
-    end
 
-    # @return [Proc] proc that should be processed when a messages arrive
-    # @yieldparam messages [Array<Kafka::FetchedMessage>] messages from kafka (raw)
-    def processor
-      lambda do |group_id, messages|
-        Karafka::Connection::Processor.process(group_id, messages)
+      private
+
+      # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
+      def listeners
+        @listeners ||= App.consumer_groups.active.map do |consumer_group|
+          Karafka::Connection::Listener.new(consumer_group)
+        end
       end
     end
   end

data/lib/karafka/helpers/class_matcher.rb

@@ -4,20 +4,20 @@ module Karafka
   module Helpers
     # Class used to autodetect corresponding classes that are internally inside Karafka framework
     # It is used among others to match:
-    #   controller => responder
+    #   consumer => responder
     class ClassMatcher
-      # Regexp used to remove any non classy like characters that might be in the controller
+      # Regexp used to remove any non classy like characters that might be in the consumer
       # class name (if defined dynamically, etc)
       CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}
 
       # @param klass [Class] class to which we want to find a corresponding class
       # @param from [String] what type of object is it (based on postfix name part)
       # @param to [String] what are we looking for (based on a postfix name part)
-      # @example Controller that has a corresponding responder
-      #   matcher = Karafka::Helpers::ClassMatcher.new(SuperController, 'Controller', 'Responder')
+      # @example Consumer that has a corresponding responder
+      #   matcher = Karafka::Helpers::ClassMatcher.new(SuperConsumer, 'Consumer', 'Responder')
       #   matcher.match #=> SuperResponder
-      # @example Controller without a corresponding responder
-      #   matcher = Karafka::Helpers::ClassMatcher.new(Super2Controller, 'Controller', 'Responder')
+      # @example Consumer without a corresponding responder
+      #   matcher = Karafka::Helpers::ClassMatcher.new(Super2Consumer, 'Consumer', 'Responder')
       #   matcher.match #=> nil
       def initialize(klass, from:, to:)
         @klass = klass
@@ -36,9 +36,9 @@ module Karafka
 
       # @return [String] name of a new class that we're looking for
       # @note This method returns name of a class without a namespace
-      # @example From SuperController matching responder
+      # @example From SuperConsumer matching responder
       #   matcher.name #=> 'SuperResponder'
-      # @example From Namespaced::Super2Controller matching responder
+      # @example From Namespaced::Super2Consumer matching responder
       #   matcher.name #=> Super2Responder
       def name
         inflected = @klass.to_s.split('::').last.to_s

data/lib/karafka/helpers/config_retriever.rb

@@ -33,9 +33,9 @@ module Karafka
           return current_value unless current_value.nil?
 
           value = if Karafka::App.config.respond_to?(attribute)
-                    Karafka::App.config.public_send(attribute)
+                    Karafka::App.config.send(attribute)
                   else
-                    Karafka::App.config.kafka.public_send(attribute)
+                    Karafka::App.config.kafka.send(attribute)
                   end
 
           instance_variable_set(:"@#{attribute}", value)

data/lib/karafka/instrumentation/listener.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Default listener that hooks up to our instrumentation and uses its events for logging
+    # It can be removed/replaced or anything without any harm to the Karafka app flow
+    module Listener
+      # Log levels that we use in this particular listener
+      USED_LOG_LEVELS = %i[
+        debug
+        info
+        error
+        fatal
+      ].freeze
+
+      # Injects WaterDrop listener logger actions
+      extend WaterDrop::Instrumentation::Listener
+
+      class << self
+        # Logs details about incoming messages and with which consumer we will consume them
+        # @param event [Dry::Events::Event] event details including payload
+        def on_connection_delegator_call(event)
+          consumer = event[:consumer]
+          topic = consumer.topic.name
+          kafka_messages = event[:kafka_messages]
+          info "#{kafka_messages.count} messages on #{topic} topic delegated to #{consumer.class}"
+        end
+
+        # Logs details about each received message value parsing
+        # @param event [Dry::Events::Event] event details including payload
+        def on_params_params_parse(event)
+          # Keep in mind, that a caller here is a param object not a controller,
+          # so it returns a topic as a string, not a routing topic
+          debug "Params parsing for #{event[:caller].topic} topic successful in #{event[:time]} ms"
+        end
+
+        # Logs unsuccessful parsing attempts of incoming data
+        # @param event [Dry::Events::Event] event details including payload
+        def on_params_params_parse_error(event)
+          error "Params parsing error for #{event[:caller].topic} topic: #{event[:error]}"
+        end
+
+        # Logs errors that occured in a listener fetch loop
+        # @param event [Dry::Events::Event] event details including payload
+        # @note It's an error as we can recover from it not a fatal
+        def on_connection_listener_fetch_loop_error(event)
+          error "Listener fetch loop error: #{event[:error]}"
+        end
+
+        # Logs errors that are related to the connection itself
+        # @note Karafka will attempt to reconnect, so an error not a fatal
+        # @param event [Dry::Events::Event] event details including payload
+        def on_connection_client_fetch_loop_error(event)
+          error "Client fetch loop error: #{event[:error]}"
+        end
+
+        # Logs info about crashed fetcher
+        # @note If this happens, Karafka will shutdown as it means a critical error
+        #   in one of the threads
+        # @param event [Dry::Events::Event] event details including payload
+        def on_fetcher_call_error(event)
+          fatal "Fetcher crash due to an error: #{event[:error]}"
+        end
+
+        # Logs info about processing of a certain dataset with an inline backend
+        # @param event [Dry::Events::Event] event details including payload
+        def on_backends_inline_process(event)
+          count = event[:caller].send(:params_batch).to_a.size
+          topic = event[:caller].topic.name
+          time = event[:time]
+          info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
+        end
+
+        # Logs info about system signals that Karafka received
+        # @param event [Dry::Events::Event] event details including payload
+        def on_process_notice_signal(event)
+          info "Received #{event[:signal]} system signal"
+        end
+
+        # Logs info about responder usage withing a controller flow
+        # @param event [Dry::Events::Event] event details including payload
+        def on_consumers_responders_respond_with(event)
+          calling = event[:caller].class
+          responder = calling.topic.responder
+          data = event[:data]
+          info "Responded from #{calling} using #{responder} with following data #{data}"
+        end
+
+        # Logs info that we're going to stop the Karafka server
+        # @param _event [Dry::Events::Event] event details including payload
+        def on_server_stop(_event)
+          # We use a separate thread as logging can't be called from trap context
+          Thread.new { info "Stopping Karafka server #{::Process.pid}" }
+        end
+
+        # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
+        #   forced exit
+        # @param _event [Dry::Events::Event] event details including payload
+        def on_server_stop_error(_event)
+          # We use a separate thread as logging can't be called from trap context
+          Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
+        end
+
+        USED_LOG_LEVELS.each do |log_level|
+          define_method log_level do |*args|
+            Karafka.logger.send(log_level, *args)
+          end
+        end
+      end
+    end
+  end
+end