karafka 1.0.0 → 1.2.0

data/lib/karafka/callbacks/dsl.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Callbacks
+    # App level dsl to define callbacks
+    module Dsl
+      Callbacks::TYPES.each do |callback_type|
+        # Allows us to define a block, that will be executed for a given moment
+        # @param [Block] block that should be executed after the initialization process
+        define_method callback_type do |&block|
+          config.callbacks.send(callback_type).push block
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/base.rb

@@ -18,6 +18,8 @@ module Karafka
     #     end
     #   end
     class Base
+      include Thor::Shell
+
       # We can use it to call other cli methods via this object
       attr_reader :cli
 

data/lib/karafka/cli/flow.rb

@@ -15,7 +15,7 @@ module Karafka
           if any_topics
             puts "#{topic.name} =>"
 
-            topic.responder.topics.each do |_name, responder_topic|
+            topic.responder.topics.each_value do |responder_topic|
               features = []
               features << (responder_topic.required? ? 'always' : 'conditionally')
               features << (responder_topic.multiple_usage? ? 'one or more' : 'exactly once')

data/lib/karafka/cli/info.rb

@@ -15,9 +15,8 @@ module Karafka
           "Karafka framework version: #{Karafka::VERSION}",
           "Application client id: #{config.client_id}",
           "Backend: #{config.backend}",
+          "Batch fetching: #{config.batch_fetching}",
           "Batch consuming: #{config.batch_consuming}",
-          "Batch processing: #{config.batch_processing}",
-          "Number of threads: #{config.concurrency}",
           "Boot file: #{Karafka.boot_file}",
           "Environment: #{Karafka.env}",
           "Kafka seed brokers: #{config.kafka.seed_brokers}"

data/lib/karafka/cli/install.rb

@@ -9,8 +9,7 @@ module Karafka
 
       # Directories created by default
       INSTALL_DIRS = %w[
-        app/models
-        app/controllers
+        app/consumers
         app/responders
         config
         log
@@ -20,7 +19,7 @@ module Karafka
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
         'karafka.rb.example' => Karafka.boot_file.basename,
-        'application_controller.rb.example' => 'app/controllers/application_controller.rb',
+        'application_consumer.rb.example' => 'app/consumers/application_consumer.rb',
         'application_responder.rb.example' => 'app/responders/application_responder.rb'
       }.freeze
 

data/lib/karafka/cli/server.rb

@@ -20,24 +20,21 @@ module Karafka
 
         if cli.options[:daemon]
           FileUtils.mkdir_p File.dirname(cli.options[:pid])
-          # For some reason Celluloid spins threads that break forking
-          # Threads are not shutdown immediately so deamonization will stale until
-          # those threads are killed by Celluloid manager (via timeout)
-          # There's nothing initialized here yet, so instead we shutdown celluloid
-          # and run it again when we need (after fork)
-          Celluloid.shutdown
           daemonize
-          Celluloid.boot
         end
 
-        # Remove pidfile on shutdown
-        ObjectSpace.define_finalizer(String.new, proc { send(:clean) })
-
         # We assign active topics on a server level, as only server is expected to listen on
         # part of the topics
         Karafka::Server.consumer_groups = cli.options[:consumer_groups]
 
-        # After we fork, we can boot celluloid again
+        # Remove pidfile on stop, just before the server instance is going to be GCed
+        # We want to delay the moment in which the pidfile is removed as much as we can,
+        # so instead of removing it after the server stops running, we rely on the gc moment
+        # when this object gets removed (it is a bit later), so it is closer to the actual
+        # system process end. We do that, so monitoring and deployment tools that rely on pids
+        # won't alarm or start new system process up until the current one is finished
+        ObjectSpace.define_finalizer(self, proc { send(:clean) })
+
         Karafka::Server.run
       end
 
@@ -62,7 +59,7 @@ module Karafka
 
       # Removes a pidfile (if exist)
       def clean
-        FileUtils.rm_f(cli.options[:pid])
+        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
       end
     end
   end

data/lib/karafka/connection/client.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class used as a wrapper around Ruby-Kafka client to simplify additional
+    # features that we provide/might provide in future and to hide the internal implementation
+    class Client
+      extend Forwardable
+
+      def_delegator :kafka_consumer, :seek
+
+      # Creates a queue consumer client that will pull the data from Kafka
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
+      #   we create a client
+      # @return [Karafka::Connection::Client] group consumer that can subscribe to
+      #   multiple topics
+      def initialize(consumer_group)
+        @consumer_group = consumer_group
+        Persistence::Client.write(self)
+      end
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      # @note This will yield with raw messages - no preprocessing or reformatting.
+      def fetch_loop
+        settings = ConfigAdapter.consuming(consumer_group)
+
+        if consumer_group.batch_fetching
+          kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
+        else
+          # always yield an array of messages, so we have consistent API (always a batch)
+          kafka_consumer.each_message(*settings) { |message| yield([message]) }
+        end
+      rescue Kafka::ProcessingError => error
+        # If there was an error during consumption, we have to log it, pause current partition
+        # and process other things
+        Karafka.monitor.instrument(
+          'connection.client.fetch_loop.error',
+          caller: self,
+          error: error.cause
+        )
+        pause(error.topic, error.partition)
+        retry
+        # This is on purpose - see the notes for this method
+        # rubocop:disable RescueException
+      rescue Exception => error
+        # rubocop:enable RescueException
+        Karafka.monitor.instrument(
+          'connection.client.fetch_loop.error',
+          caller: self,
+          error: error
+        )
+        retry
+      end
+
+      # Gracefuly stops topic consumption
+      # @note Stopping running consumers without a really important reason is not recommended
+      #   as until all the consumers are stopped, the server will keep running serving only
+      #   part of the messages
+      def stop
+        @kafka_consumer&.stop
+        @kafka_consumer = nil
+      end
+
+      # Pauses fetching and consumption of a given topic partition
+      # @param topic [String] topic that we want to pause
+      # @param partition [Integer] number partition that we want to pause
+      def pause(topic, partition)
+        settings = ConfigAdapter.pausing(consumer_group)
+        timeout = settings[:timeout]
+        raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
+        kafka_consumer.pause(topic, partition, settings)
+      end
+
+      # Marks a given message as consumed and commit the offsets
+      # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
+      #   that offset commit happen asap in case of a crash
+      # @param [Karafka::Params::Params] params message that we want to mark as processed
+      def mark_as_consumed(params)
+        kafka_consumer.mark_message_as_processed(params)
+        # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
+        # before the automatic triggers have kicked in.
+        kafka_consumer.commit_offsets
+      end
+
+      private
+
+      attr_reader :consumer_group
+
+      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
+      #   that is set up to consume from topics of a given consumer group
+      def kafka_consumer
+        @kafka_consumer ||= kafka.consumer(
+          *ConfigAdapter.consumer(consumer_group)
+        ).tap do |consumer|
+          consumer_group.topics.each do |topic|
+            consumer.subscribe(*ConfigAdapter.subscription(topic))
+          end
+        end
+      rescue Kafka::ConnectionError
+        # If we would not wait it would totally spam log file with failed
+        # attempts if Kafka is down
+        sleep(consumer_group.reconnect_timeout)
+        # We don't log and just reraise - this will be logged
+        # down the road
+        raise
+      end
+
+      # @return [Kafka] returns a Kafka
+      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
+      #   object instance
+      def kafka
+        Kafka.new(*ConfigAdapter.client(consumer_group))
+      end
+    end
+  end
+end

data/lib/karafka/connection/config_adapter.rb

@@ -14,7 +14,10 @@ module Karafka
       class << self
         # Builds all the configuration settings for Kafka.new method
         # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Hash] hash with all the settings required by Kafka.new method
+        # @return [Array<Hash>] Array with all the client arguments including hash with all
+        #   the settings required by Kafka.new method
+        # @note We return array, so we can inject any arguments we want, in case of changes in the
+        #   raw driver
         def client(_consumer_group)
           # This one is a default that takes all the settings except special
           # cases defined in the map
@@ -33,33 +36,40 @@ module Karafka
             settings[setting_name] = setting_value
           end
 
-          sanitize(settings)
+          settings_hash = sanitize(settings)
+
+          # Normalization for the way Kafka::Client accepts arguments from  0.5.3
+          [settings_hash.delete(:seed_brokers), settings_hash]
         end
 
         # Builds all the configuration settings for kafka#consumer method
         # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Hash] hash with all the settings required by Kafka#consumer method
+        # @return [Array<Hash>] array with all the consumer arguments including hash with all
+        #   the settings required by Kafka#consumer
         def consumer(consumer_group)
           settings = { group_id: consumer_group.id }
-          settings = fetch_for(:consumer, settings)
-          sanitize(settings)
+          settings = fetch_for(:consumer, consumer_group, settings)
+          [sanitize(settings)]
         end
 
         # Builds all the configuration settings for kafka consumer consume_each_batch and
         #   consume_each_message methods
-        # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Hash] hash with all the settings required by
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array<Hash>] Array with all the arguments required by consuming method
+        #   including hash with all the settings required by
         #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
-        def consuming(_consumer_group)
-          sanitize(fetch_for(:consuming))
+        def consuming(consumer_group)
+          settings = {
+            automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
+          }
+          [sanitize(fetch_for(:consuming, consumer_group, settings))]
         end
 
         # Builds all the configuration settings for kafka consumer#subscribe method
         # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
         # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
         def subscription(topic)
-          settings = { start_from_beginning: topic.start_from_beginning }
-          settings = fetch_for(:subscription, settings)
+          settings = fetch_for(:subscription, topic)
           [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
         end
 
@@ -74,13 +84,19 @@ module Karafka
 
         # Fetches proper settings for a given map namespace
         # @param namespace_key [Symbol] namespace from attributes map config adapter hash
+        # @param route_layer [Object] route topic or consumer group
         # @param preexisting_settings [Hash] hash with some preexisting settings that might have
         #   been loaded in a different way
-        def fetch_for(namespace_key, preexisting_settings = {})
-          kafka_configs.each do |setting_name, setting_value|
+        def fetch_for(namespace_key, route_layer, preexisting_settings = {})
+          kafka_configs.each_key do |setting_name|
+            # Ignore settings that are not related to our namespace
             next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
+            # Ignore settings that are already initialized
+            # In case they are in preexisting settings fetched differently
             next if preexisting_settings.keys.include?(setting_name)
-            preexisting_settings[setting_name] = setting_value
+            # Fetch all the settings from a given layer object. Objects can handle the fallback
+            # to the kafka settings, so
+            preexisting_settings[setting_name] = route_layer.send(setting_name)
           end
 
           preexisting_settings

data/lib/karafka/connection/delegator.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that delegates processing of messages for which we listen to a proper processor
+    module Delegator
+      class << self
+        # Delegates messages (does something with them)
+        # It will either schedule or run a proper processor action for messages
+        # @note This should be looped to obtain a constant delegating of new messages
+        # @note We catch all the errors here, to make sure that none failures
+        #   for a given consumption will affect other consumed messages
+        #   If we wouldn't catch it, it would propagate up until killing the thread
+        # @note It is a one huge method, because of performance reasons. It is much faster then
+        #   using send or invoking additional methods
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
+        def call(group_id, kafka_messages)
+          # @note We always get messages by topic and partition so we can take topic from the
+          # first one and it will be valid for all the messages
+          topic = Persistence::Topic.fetch(group_id, kafka_messages[0].topic)
+          consumer = Persistence::Consumer.fetch(topic, kafka_messages[0].partition)
+
+          Karafka.monitor.instrument(
+            'connection.delegator.call',
+            caller: self,
+            consumer: consumer,
+            kafka_messages: kafka_messages
+          ) do
+            # Depending on a case (persisted or not) we might use new consumer instance per
+            # each batch, or use the same one for all of them (for implementing buffering, etc.)
+            if topic.batch_consuming
+              consumer.params_batch = kafka_messages
+              consumer.call
+            else
+              kafka_messages.each do |kafka_message|
+                consumer.params_batch = [kafka_message]
+                consumer.call
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -7,12 +7,6 @@ module Karafka
     # @note Listener itself does nothing with the message - it will return to the block
     #   a raw Kafka::FetchedMessage
     class Listener
-      include Celluloid
-
-      execute_block_on_receiver :fetch_loop
-
-      attr_reader :consumer_group
-
       # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
       #   on what topics and with what settings should we listen
       # @return [Karafka::Connection::Listener] listener instance
@@ -20,6 +14,17 @@ module Karafka
         @consumer_group = consumer_group
       end
 
+      # Runs prefetch callbacks and executes the main listener fetch loop
+      def call
+        Karafka::Callbacks.before_fetch_loop(
+          @consumer_group,
+          client
+        )
+        fetch_loop
+      end
+
+      private
+
       # Opens connection, gets messages and calls a block for each of the incoming messages
       # @yieldparam [String] consumer group id
       # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
@@ -30,28 +35,25 @@ module Karafka
       #   won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far
-      def fetch_loop(block)
-        messages_consumer.fetch_loop do |raw_messages|
-          block.call(consumer_group.id, raw_messages)
+      def fetch_loop
+        client.fetch_loop do |raw_messages|
+          # @note What happens here is a delegation of processing to a proper processor based
+          #   on the incoming messages characteristics
+          Karafka::Connection::Delegator.call(@consumer_group.id, raw_messages)
         end
         # This is on purpose - see the notes for this method
         # rubocop:disable RescueException
       rescue Exception => e
+        Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
         # rubocop:enable RescueException
-        Karafka.monitor.notice_error(self.class, e)
-        @messages_consumer&.stop
-        retry if @messages_consumer
+        @client&.stop
+        retry if @client
       end
 
-      private
-
-      # @return [Karafka::Connection::MessagesConsumer] wrapped kafka consumer for a given topic
+      # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic
       #   consumption
-      # @note It adds consumer into Karafka::Server consumers pool for graceful shutdown on exit
-      def messages_consumer
-        @messages_consumer ||= MessagesConsumer.new(consumer_group).tap do |consumer|
-          Karafka::Server.consumers << consumer if Karafka::Server.consumers
-        end
+      def client
+        @client ||= Client.new(@consumer_group)
       end
     end
   end

data/lib/karafka/consumers/callbacks.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Additional callbacks that can be used to trigger some actions on certain moments like
+    # manual offset management, committing or anything else outside of a standard messages flow
+    # They are not included by default, as we don't want to provide functionalities that are
+    # not required by users by default
+    # Please refer to the wiki callbacks page for more details on how to use them
+    module Callbacks
+      # Types of events on which we run callbacks
+      TYPES = %i[
+        after_fetch
+        after_poll
+        before_poll
+        before_stop
+      ].freeze
+
+      # Class methods needed to make callbacks run
+      module ClassMethods
+        TYPES.each do |type|
+          # A Creates a callback wrapper
+          # @param method_name [Symbol, String] method name or nil if we plan to provide a block
+          # @yield A block with a code that should be executed before scheduling
+          define_method type do |method_name = nil, &block|
+            set_callback type, :before, method_name ? method_name : block
+          end
+        end
+      end
+
+      # @param consumer_class [Class] consumer class that we extend with callbacks
+      def self.included(consumer_class)
+        consumer_class.class_eval do
+          extend ClassMethods
+          include ActiveSupport::Callbacks
+
+          # The call method is wrapped with a set of callbacks
+          # We won't run process if any of the callbacks throw abort
+          # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
+          TYPES.each { |type| define_callbacks type }
+        end
+      end
+
+      # Executes the default consumer flow, runs callbacks and if not halted will call process
+      # method of a proper backend. It is here because it interacts with the default Karafka
+      # call flow and needs to be overwritten to support callbacks
+      def call
+        run_callbacks :after_fetch do
+          process
+        end
+      end
+    end
+  end
+end