karafka 1.1.1 → 1.2.0

data/lib/karafka/base_responder.rb

@@ -62,6 +62,11 @@ module Karafka
     # Definitions of all topics that we want to be able to use in this responder should go here
     class_attribute :topics
 
+    # Schema that we can use to control and/or require some additional details upon options
+    # that are being passed to the producer. This can be in particular useful if we want to make
+    # sure that for example partition_key is always present.
+    class_attribute :options_schema
+
     attr_reader :messages_buffer
 
     class << self
@@ -108,7 +113,8 @@ module Karafka
     #   UsersCreatedResponder.new(MyParser).call(@created_user)
     def call(*data)
       respond(*data)
-      validate!
+      validate_usage!
+      validate_options!
       deliver!
     end
 
@@ -116,7 +122,7 @@ module Karafka
 
     # Checks if we met all the topics requirements. It will fail if we didn't send a message to
     # a registered required topic, etc.
-    def validate!
+    def validate_usage!
       registered_topics = self.class.topics.map do |name, topic|
         topic.to_h.merge!(
           usage_count: messages_buffer[name]&.count || 0
@@ -138,20 +144,26 @@ module Karafka
       raise Karafka::Errors::InvalidResponderUsage, result.errors
     end
 
+    # Checks if we met all the options requirements before sending them to the producer.
+    def validate_options!
+      return true unless self.class.options_schema
+
+      messages_buffer.each_value do |messages_set|
+        messages_set.each do |message_data|
+          result = self.class.options_schema.call(message_data.last)
+          next if result.success?
+          raise Karafka::Errors::InvalidResponderMessageOptions, result.errors
+        end
+      end
+    end
+
     # Takes all the messages from the buffer and delivers them one by one
     # @note This method is executed after the validation, so we're sure that
     #   what we send is legit and it will go to a proper topics
     def deliver!
-      messages_buffer.each do |topic, data_elements|
-        # We map this topic name, so it will match namespaced/etc topic in Kafka
-        # @note By default will not change topic (if default mapper used)
-        mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
-
+      messages_buffer.each_value do |data_elements|
         data_elements.each do |data, options|
-          producer(options).call(
-            data,
-            options.merge(topic: mapped_topic)
-          )
+          producer(options).call(data, options)
         end
       end
     end
@@ -170,10 +182,17 @@ module Karafka
     # @param options [Hash] options for waterdrop (e.g. partition_key)
     # @note Respond to does not accept multiple data arguments.
     def respond_to(topic, data, options = {})
-      Karafka.monitor.notice(self.class, topic: topic, data: data, options: options)
+      # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
+      # string topics
+      topic = topic.to_s
 
-      messages_buffer[topic.to_s] ||= []
-      messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
+      messages_buffer[topic] ||= []
+      messages_buffer[topic] << [
+        @parser_class.generate(data),
+        # We map this topic name, so it will match namespaced/etc topic in Kafka
+        # @note By default will not change topic (if default mapper used)
+        options.merge(topic: Karafka::App.config.topic_mapper.outgoing(topic))
+      ]
     end
 
     # @param options [Hash] options for waterdrop

data/lib/karafka/callbacks.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Additional callbacks that are used to trigger some things in given places during the
+  # system lifecycle
+  # @note Those callbacks aren't the same as consumer callbacks as they are not related to the
+  #   lifecycle of particular messages fetches but rather to the internal flow process.
+  #   They cannot be defined on a consumer callback level because for some of those,
+  #   there aren't consumers in the memory yet and/or they aren't per consumer thread
+  module Callbacks
+    # Types of system callbacks that we have that are not related to consumers
+    TYPES = %i[
+      after_init
+      before_fetch_loop
+    ].freeze
+
+    class << self
+      TYPES.each do |callback_type|
+        # Executes given callbacks set at a given moment with provided arguments
+        define_method callback_type do |*args|
+          Karafka::App
+            .config
+            .callbacks
+            .send(callback_type)
+            .each { |callback| callback.call(*args) }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/callbacks/config.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Callbacks
+    # Additional configuration required to store procs that we will execute upon callback trigger
+    module Config
+      # Builds up internal callback accumulators
+      # @param klass [Class] Class that we extend with callback config
+      def self.extended(klass)
+        # option internal [Hash] - optional - internal karafka configuration settings that should
+        #   never be changed by users directly
+        klass.setting :callbacks do
+          Callbacks::TYPES.each do |callback_type|
+            # option [Array<Proc>] an array of blocks that will be executed at a given moment
+            #   depending on the callback type
+            setting callback_type, []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/callbacks/dsl.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Callbacks
+    # App level dsl to define callbacks
+    module Dsl
+      Callbacks::TYPES.each do |callback_type|
+        # Allows us to define a block, that will be executed for a given moment
+        # @param [Block] block that should be executed after the initialization process
+        define_method callback_type do |&block|
+          config.callbacks.send(callback_type).push block
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/install.rb

@@ -9,8 +9,7 @@ module Karafka
 
       # Directories created by default
       INSTALL_DIRS = %w[
-        app/models
-        app/controllers
+        app/consumers
         app/responders
         config
         log
@@ -20,7 +19,7 @@ module Karafka
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
         'karafka.rb.example' => Karafka.boot_file.basename,
-        'application_controller.rb.example' => 'app/controllers/application_controller.rb',
+        'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
         'application_responder.rb.example' => 'app/responders/application_responder.rb'
       }.freeze
 

data/lib/karafka/cli/server.rb

@@ -35,7 +35,6 @@ module Karafka
         # won't alarm or start new system process up until the current one is finished
         ObjectSpace.define_finalizer(self, proc { send(:clean) })
 
-        # After we fork, we can boot celluloid again
         Karafka::Server.run
       end
 

data/lib/karafka/connection/{consumer.rb → client.rb}

@@ -2,37 +2,54 @@
 
 module Karafka
   module Connection
-    # Class used as a wrapper around Ruby-Kafka to simplify additional
+    # Class used as a wrapper around Ruby-Kafka client to simplify additional
     # features that we provide/might provide in future and to hide the internal implementation
-    class Consumer
-      # Creates a queue consumer that will pull the data from Kafka
+    class Client
+      extend Forwardable
+
+      def_delegator :kafka_consumer, :seek
+
+      # Creates a queue consumer client that will pull the data from Kafka
       # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
       #   we create a client
-      # @return [Karafka::Connection::Consumer] group consumer that can subscribe to
+      # @return [Karafka::Connection::Client] group consumer that can subscribe to
       #   multiple topics
       def initialize(consumer_group)
         @consumer_group = consumer_group
-        Persistence::Consumer.write(self)
+        Persistence::Client.write(self)
       end
 
       # Opens connection, gets messages and calls a block for each of the incoming messages
       # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
       # @note This will yield with raw messages - no preprocessing or reformatting.
       def fetch_loop
-        send(
-          consumer_group.batch_fetching ? :consume_each_batch : :consume_each_message
-        ) { |messages| yield(messages) }
-      rescue Kafka::ProcessingError => e
+        settings = ConfigAdapter.consuming(consumer_group)
+
+        if consumer_group.batch_fetching
+          kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
+        else
+          # always yield an array of messages, so we have consistent API (always a batch)
+          kafka_consumer.each_message(*settings) { |message| yield([message]) }
+        end
+      rescue Kafka::ProcessingError => error
         # If there was an error during consumption, we have to log it, pause current partition
         # and process other things
-        Karafka.monitor.notice_error(self.class, e.cause)
-        pause(e.topic, e.partition)
+        Karafka.monitor.instrument(
+          'connection.client.fetch_loop.error',
+          caller: self,
+          error: error.cause
+        )
+        pause(error.topic, error.partition)
         retry
         # This is on purpose - see the notes for this method
         # rubocop:disable RescueException
-      rescue Exception => e
+      rescue Exception => error
         # rubocop:enable RescueException
-        Karafka.monitor.notice_error(self.class, e)
+        Karafka.monitor.instrument(
+          'connection.client.fetch_loop.error',
+          caller: self,
+          error: error
+        )
         retry
       end
 
@@ -70,32 +87,11 @@ module Karafka
 
       attr_reader :consumer_group
 
-      # Consumes messages from Kafka in batches
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      def consume_each_batch
-        kafka_consumer.each_batch(
-          ConfigAdapter.consuming(consumer_group)
-        ) do |batch|
-          yield(batch.messages)
-        end
-      end
-
-      # Consumes messages from Kafka one by one
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      def consume_each_message
-        kafka_consumer.each_message(
-          ConfigAdapter.consuming(consumer_group)
-        ) do |message|
-          #   always yield an array of messages, so we have consistent API (always a batch)
-          yield([message])
-        end
-      end
-
       # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
       #   that is set up to consume from topics of a given consumer group
       def kafka_consumer
         @kafka_consumer ||= kafka.consumer(
-          ConfigAdapter.consumer(consumer_group)
+          *ConfigAdapter.consumer(consumer_group)
         ).tap do |consumer|
           consumer_group.topics.each do |topic|
             consumer.subscribe(*ConfigAdapter.subscription(topic))
@@ -114,7 +110,7 @@ module Karafka
       # @note We don't cache it internally because we cache kafka_consumer that uses kafka
       #   object instance
       def kafka
-        Kafka.new(ConfigAdapter.client(consumer_group))
+        Kafka.new(*ConfigAdapter.client(consumer_group))
       end
     end
   end

data/lib/karafka/connection/config_adapter.rb

@@ -14,7 +14,10 @@ module Karafka
       class << self
         # Builds all the configuration settings for Kafka.new method
         # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Hash] hash with all the settings required by Kafka.new method
+        # @return [Array<Hash>] Array with all the client arguments including hash with all
+        #   the settings required by Kafka.new method
+        # @note We return array, so we can inject any arguments we want, in case of changes in the
+        #   raw driver
         def client(_consumer_group)
           # This one is a default that takes all the settings except special
           # cases defined in the map
@@ -33,28 +36,33 @@ module Karafka
             settings[setting_name] = setting_value
           end
 
-          sanitize(settings)
+          settings_hash = sanitize(settings)
+
+          # Normalization for the way Kafka::Client accepts arguments from  0.5.3
+          [settings_hash.delete(:seed_brokers), settings_hash]
         end
 
         # Builds all the configuration settings for kafka#consumer method
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Hash] hash with all the settings required by Kafka#consumer method
+        # @return [Array<Hash>] array with all the consumer arguments including hash with all
+        #   the settings required by Kafka#consumer
         def consumer(consumer_group)
           settings = { group_id: consumer_group.id }
           settings = fetch_for(:consumer, consumer_group, settings)
-          sanitize(settings)
+          [sanitize(settings)]
         end
 
         # Builds all the configuration settings for kafka consumer consume_each_batch and
         #   consume_each_message methods
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Hash] hash with all the settings required by
+        # @return [Array<Hash>] Array with all the arguments required by consuming method
+        #   including hash with all the settings required by
         #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
         def consuming(consumer_group)
           settings = {
             automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
           }
-          sanitize(fetch_for(:consuming, consumer_group, settings))
+          [sanitize(fetch_for(:consuming, consumer_group, settings))]
         end
 
         # Builds all the configuration settings for kafka consumer#subscribe method

data/lib/karafka/connection/delegator.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that delegates processing of messages for which we listen to a proper processor
+    module Delegator
+      class << self
+        # Delegates messages (does something with them)
+        # It will either schedule or run a proper processor action for messages
+        # @note This should be looped to obtain a constant delegating of new messages
+        # @note We catch all the errors here, to make sure that none failures
+        #   for a given consumption will affect other consumed messages
+        #   If we wouldn't catch it, it would propagate up until killing the thread
+        # @note It is a one huge method, because of performance reasons. It is much faster then
+        #   using send or invoking additional methods
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
+        def call(group_id, kafka_messages)
+          # @note We always get messages by topic and partition so we can take topic from the
+          # first one and it will be valid for all the messages
+          topic = Persistence::Topic.fetch(group_id, kafka_messages[0].topic)
+          consumer = Persistence::Consumer.fetch(topic, kafka_messages[0].partition)
+
+          Karafka.monitor.instrument(
+            'connection.delegator.call',
+            caller: self,
+            consumer: consumer,
+            kafka_messages: kafka_messages
+          ) do
+            # Depending on a case (persisted or not) we might use new consumer instance per
+            # each batch, or use the same one for all of them (for implementing buffering, etc.)
+            if topic.batch_consuming
+              consumer.params_batch = kafka_messages
+              consumer.call
+            else
+              kafka_messages.each do |kafka_message|
+                consumer.params_batch = [kafka_message]
+                consumer.call
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -7,8 +7,6 @@ module Karafka
     # @note Listener itself does nothing with the message - it will return to the block
     #   a raw Kafka::FetchedMessage
     class Listener
-      attr_reader :consumer_group
-
       # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
       #   on what topics and with what settings should we listen
       # @return [Karafka::Connection::Listener] listener instance
@@ -16,6 +14,17 @@ module Karafka
         @consumer_group = consumer_group
       end
 
+      # Runs prefetch callbacks and executes the main listener fetch loop
+      def call
+        Karafka::Callbacks.before_fetch_loop(
+          @consumer_group,
+          client
+        )
+        fetch_loop
+      end
+
+      private
+
       # Opens connection, gets messages and calls a block for each of the incoming messages
       # @yieldparam [String] consumer group id
       # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
@@ -26,25 +35,25 @@ module Karafka
       #   won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far
-      def fetch_loop(block)
-        consumer.fetch_loop do |raw_messages|
-          block.call(consumer_group.id, raw_messages)
+      def fetch_loop
+        client.fetch_loop do |raw_messages|
+          # @note What happens here is a delegation of processing to a proper processor based
+          #   on the incoming messages characteristics
+          Karafka::Connection::Delegator.call(@consumer_group.id, raw_messages)
         end
         # This is on purpose - see the notes for this method
         # rubocop:disable RescueException
       rescue Exception => e
+        Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
         # rubocop:enable RescueException
-        Karafka.monitor.notice_error(self.class, e)
-        @consumer&.stop
-        retry if @consumer
+        @client&.stop
+        retry if @client
       end
 
-      private
-
-      # @return [Karafka::Connection::Consumer] wrapped kafka consumer for a given topic
+      # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic
       #   consumption
-      def consumer
-        @consumer ||= Consumer.new(consumer_group)
+      def client
+        @client ||= Client.new(@consumer_group)
       end
     end
   end