karafka 1.0.1 → 1.1.0.alpha1

data/lib/karafka/connection/{messages_consumer.rb → consumer.rb}

@@ -3,15 +3,16 @@
 module Karafka
   module Connection
     # Class used as a wrapper around Ruby-Kafka to simplify additional
-    # features that we provide/might provide in future
-    class MessagesConsumer
+    # features that we provide/might provide in future and to hide the internal implementation
+    class Consumer
       # Creates a queue consumer that will pull the data from Kafka
       # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
       #   we create a client
-      # @return [Karafka::Connection::MessagesConsumer] group consumer that can subscribe to
+      # @return [Karafka::Connection::Consumer] group consumer that can subscribe to
       #   multiple topics
       def initialize(consumer_group)
         @consumer_group = consumer_group
+        Persistence::Consumer.write(self)
       end
 
       # Opens connection, gets messages and calls a block for each of the incoming messages
@@ -19,10 +20,10 @@ module Karafka
       # @note This will yield with raw messages - no preprocessing or reformatting.
       def fetch_loop
         send(
-          consumer_group.batch_consuming ? :consume_each_batch : :consume_each_message
+          consumer_group.batch_fetching ? :consume_each_batch : :consume_each_message
         ) { |messages| yield(messages) }
       rescue Kafka::ProcessingError => e
-        # If there was an error during processing, we have to log it, pause current partition
+        # If there was an error during consumption, we have to log it, pause current partition
         # and process other things
         Karafka.monitor.notice_error(self.class, e.cause)
         pause(e.topic, e.partition)
@@ -36,25 +37,39 @@ module Karafka
       end
 
       # Gracefuly stops topic consumption
+      # @note Stopping running consumers without a really important reason is not recommended
+      #   as until all the consumers are stopped, the server will keep running serving only
+      #   part of the messages
       def stop
         @kafka_consumer&.stop
         @kafka_consumer = nil
       end
 
-      private
-
-      attr_reader :consumer_group
-
-      # Pauses processing of a given topic partition
+      # Pauses fetching and consumption of a given topic partition
       # @param topic [String] topic that we want to pause
       # @param partition [Integer] number partition that we want to pause
       def pause(topic, partition)
         settings = ConfigAdapter.pausing(consumer_group)
-        return false unless settings[:timeout].positive?
+        timeout = settings[:timeout]
+        raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
         kafka_consumer.pause(topic, partition, settings)
-        true
       end
 
+      # Marks a given message as consumed and commit the offsets
+      # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
+      #   that offset commit happen asap in case of a crash
+      # @param [Karafka::Params::Params] params message that we want to mark as processed
+      def mark_as_consumed(params)
+        kafka_consumer.mark_message_as_processed(params)
+        # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
+        # before the automatic triggers have kicked in.
+        kafka_consumer.commit_offsets
+      end
+
+      private
+
+      attr_reader :consumer_group
+
       # Consumes messages from Kafka in batches
       # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
       def consume_each_batch

data/lib/karafka/connection/listener.rb

@@ -7,10 +7,6 @@ module Karafka
     # @note Listener itself does nothing with the message - it will return to the block
     #   a raw Kafka::FetchedMessage
     class Listener
-      include Celluloid
-
-      execute_block_on_receiver :fetch_loop
-
       attr_reader :consumer_group
 
       # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
@@ -31,7 +27,7 @@ module Karafka
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far
       def fetch_loop(block)
-        messages_consumer.fetch_loop do |raw_messages|
+        consumer.fetch_loop do |raw_messages|
           block.call(consumer_group.id, raw_messages)
         end
         # This is on purpose - see the notes for this method
@@ -39,19 +35,16 @@ module Karafka
       rescue Exception => e
         # rubocop:enable RescueException
         Karafka.monitor.notice_error(self.class, e)
-        @messages_consumer&.stop
-        retry if @messages_consumer
+        @consumer&.stop
+        retry if @consumer
       end
 
       private
 
-      # @return [Karafka::Connection::MessagesConsumer] wrapped kafka consumer for a given topic
+      # @return [Karafka::Connection::Consumer] wrapped kafka consumer for a given topic
       #   consumption
-      # @note It adds consumer into Karafka::Server consumers pool for graceful shutdown on exit
-      def messages_consumer
-        @messages_consumer ||= MessagesConsumer.new(consumer_group).tap do |consumer|
-          Karafka::Server.consumers << consumer if Karafka::Server.consumers
-        end
+      def consumer
+        @consumer ||= Consumer.new(consumer_group)
       end
     end
   end

data/lib/karafka/connection/{messages_processor.rb → processor.rb}

@@ -3,14 +3,14 @@
 module Karafka
   module Connection
     # Class that consumes messages for which we listen
-    module MessagesProcessor
+    module Processor
       class << self
         # Processes messages (does something with them)
         # It will either schedule or run a proper controller action for messages
         # @note This should be looped to obtain a constant listening
         # @note We catch all the errors here, to make sure that none failures
         #   for a given consumption will affect other consumed messages
-        #   If we wouldn't catch it, it would propagate up until killing the Celluloid actor
+        #   If we wouldn't catch it, it would propagate up until killing the thread
         # @param group_id [String] group_id of a group from which a given message came
         # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
         def process(group_id, kafka_messages)
@@ -27,7 +27,7 @@ module Karafka
           # Depending on a case (persisted or not) we might use new controller instance per each
           # batch, or use the same instance for all of them (for implementing buffering, etc)
           send(
-            topic.batch_processing ? :process_batch : :process_each,
+            topic.batch_consuming ? :process_batch : :process_each,
             controller,
             kafka_messages
           )

data/lib/karafka/controllers/callbacks.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Controllers
+    # Additional callbacks that can be used to trigger some actions on certain moments like
+    # manual offset management, committing or anything else outside of a standard messages flow
+    # They are not included by default, as we don't want to provide functionalities that are
+    # not required by users by default
+    # Please refer to the wiki callbacks page for more details on how to use them
+    module Callbacks
+      # Types of events on which we run callbacks
+      TYPES = %i[
+        after_fetched
+        after_poll
+        before_poll
+        before_stop
+      ].freeze
+
+      # Class methods needed to make callbacks run
+      module ClassMethods
+        TYPES.each do |type|
+          # A Creates a callback wrapper
+          # @param method_name [Symbol, String] method name or nil if we plan to provide a block
+          # @yield A block with a code that should be executed before scheduling
+          define_method type do |method_name = nil, &block|
+            set_callback type, :before, method_name ? method_name : block
+          end
+        end
+      end
+
+      # @param controller_class [Class] controller class that we extend with callbacks
+      def self.included(controller_class)
+        controller_class.class_eval do
+          extend ClassMethods
+          include ActiveSupport::Callbacks
+
+          # The call method is wrapped with a set of callbacks
+          # We won't run process if any of the callbacks throw abort
+          # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
+          TYPES.each { |type| define_callbacks type }
+        end
+      end
+
+      # Executes the default controller flow, runs callbacks and if not halted will call process
+      # method of a proper backend. This is here because it interacts with the default Karafka
+      # call flow and needs to be overwritten in order to support callbacks
+      def call
+        run_callbacks :after_fetched do
+          process
+        end
+      end
+    end
+  end
+end

data/lib/karafka/controllers/includer.rb

@@ -34,7 +34,7 @@ module Karafka
         # @param controller_class [Class] controller class
         # @param topic [Karafka::Routing::Topic] topic of a controller class
         def bind_params(controller_class, topic)
-          return if topic.batch_processing
+          return if topic.batch_consuming
           controller_class.include SingleParams
         end
 

data/lib/karafka/controllers/single_params.rb

@@ -2,11 +2,11 @@
 
 module Karafka
   module Controllers
-    # Params alias for single message processing controllers
+    # Params alias for single message consumption controllers
     module SingleParams
       private
 
-      # @return [Karafka::Params::Params] params instance for non batch processed controllers
+      # @return [Karafka::Params::Params] params instance for non batch consumption controllers
       def params
         params_batch.first
       end

data/lib/karafka/errors.rb

@@ -32,5 +32,12 @@ module Karafka
 
     # Raised when we try to use Karafka CLI commands (except install) without a bootfile
     MissingBootFile = Class.new(BaseError)
+
+    # Raised when we want to read a persisted thread messages consumer but it is unavailable
+    # This should never happen and if it does, please contact us
+    MissingConsumer = Class.new(BaseError)
+
+    # Raised when we attemp to pause a partition but the pause timeout is equal to 0
+    InvalidPauseTimeout = Class.new(BaseError)
   end
 end

data/lib/karafka/fetcher.rb

@@ -9,14 +9,20 @@ module Karafka
     # Fetch loop should never end, which means that we won't create more actor clusters
     # so we don't have to terminate them
     def fetch_loop
-      futures = listeners.map do |listener|
-        listener.future.public_send(:fetch_loop, processor)
+      threads = listeners.map do |listener|
+        # We abort on exception because there should be an exception handling developed for
+        # each listener running in separate threads, so the exceptions should never leak
+        # and if that happens, it means that something really bad happened and we should stop
+        # the whole process
+        Thread
+          .new { listener.fetch_loop(processor) }
+          .tap { |thread| thread.abort_on_exception = true }
       end
 
-      futures.map(&:value)
+      threads.each(&:join)
     # If anything crashes here, we need to raise the error and crush the runner because it means
     # that something really bad happened
-    rescue => e
+    rescue StandardError => e
       Karafka.monitor.notice_error(self.class, e)
       Karafka::App.stop!
       raise e
@@ -35,7 +41,7 @@ module Karafka
     # @yieldparam messages [Array<Kafka::FetchedMessage>] messages from kafka (raw)
     def processor
       lambda do |group_id, messages|
-        Karafka::Connection::MessagesProcessor.process(group_id, messages)
+        Karafka::Connection::Processor.process(group_id, messages)
       end
     end
   end

data/lib/karafka/monitor.rb

@@ -13,7 +13,7 @@ module Karafka
     include Singleton
 
     # This method is executed in many important places in the code (during data flow), like
-    # the moment before #perform_async, etc. For full list just grep for 'monitor.notice'
+    # the moment before #consume_async, etc. For full list just grep for 'monitor.notice'
     # @param caller_class [Class] class of object that executed this call
     # @param options [Hash] hash with options that we passed to notice. It differs based
     #   on of who and when is calling
@@ -55,7 +55,7 @@ module Karafka
     def caller_exceptions_map
       @caller_exceptions_map ||= {
         error: [
-          Karafka::Connection::MessagesConsumer,
+          Karafka::Connection::Consumer,
           Karafka::Connection::Listener,
           Karafka::Params::Params
         ],

data/lib/karafka/params/params.rb

@@ -14,6 +14,7 @@ module Karafka
         partition
         offset
         key
+        create_time
       ].freeze
 
       # Params attributes that should be available via a method call invocation for Kafka
@@ -26,6 +27,7 @@ module Karafka
         partition
         offset
         key
+        create_time
       ].freeze
 
       class << self
@@ -46,7 +48,7 @@ module Karafka
           if message.is_a?(Hash)
             new(parser: parser).send(:merge!, message)
           else
-            # This happens inside Kafka::FetchedMessagesProcessor
+            # This happens inside Kafka::FetchedProcessor
             new(
               parser: parser,
               parsed: false,

data/lib/karafka/params/params_batch.rb

@@ -4,7 +4,7 @@ module Karafka
   module Params
     # Params batch represents a set of messages received from Kafka.
     # @note Params internally are lazy loaded before first use. That way we can skip parsing
-    #   process if we have after_received that rejects some incoming messages without using params
+    #   process if we have after_fetched that rejects some incoming messages without using params
     #   It can be also used when handling really heavy data (in terms of parsing).
     class ParamsBatch
       include Enumerable

data/lib/karafka/patches/dry_configurable.rb

@@ -29,5 +29,3 @@ module Karafka
     end
   end
 end
-
-::Dry::Configurable::Config.prepend(Karafka::Patches::DryConfigurable)

data/lib/karafka/patches/ruby_kafka.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Patches
+    # Batches for Ruby Kafka gem
+    module RubyKafka
+      # This patch allows us to inject business logic in between fetches and before the consumer
+      # stop, so we can perform stop commit or anything else that we need since
+      # ruby-kafka fetch loop does not allow that directly
+      # We don't wan't to use poll ruby-kafka api as it brings many more problems that we would
+      # have to take care of. That way, nothing like that ever happens but we get the control
+      # over the stopping process that we need (since we're the once that initiate it for each
+      # thread)
+      def consumer_loop
+        super do
+          controllers = Karafka::Persistence::Controller
+                        .all
+                        .values
+                        .flat_map(&:values)
+                        .select { |ctrl| ctrl.respond_to?(:run_callbacks) }
+
+          if Karafka::App.stopped?
+            controllers.each { |ctrl| ctrl.run_callbacks :before_stop }
+            Karafka::Persistence::Consumer.read.stop
+          else
+            controllers.each { |ctrl| ctrl.run_callbacks :before_poll }
+            yield
+            controllers.each { |ctrl| ctrl.run_callbacks :after_poll }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/consumer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Persistence layer to store current thread messages consumer for further use
+    class Consumer
+      # Thread.current key under which we store current thread messages consumer
+      PERSISTENCE_SCOPE = :consumer
+
+      # @param consumer [Karafka::Connection::Consumer] messages consumer of
+      #   a current thread
+      # @return [Karafka::Connection::Consumer] persisted messages consumer
+      def self.write(consumer)
+        Thread.current[PERSISTENCE_SCOPE] = consumer
+      end
+
+      # @return [Karafka::Connection::Consumer] persisted messages consumer
+      # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
+      #   but we try to use it anyway
+      def self.read
+        Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingConsumer)
+      end
+    end
+  end
+end

data/lib/karafka/persistence/controller.rb

@@ -8,15 +8,30 @@ module Karafka
     # topic and partition to store some additional details when the persistent mode
     # for a given topic is turned on
     class Controller
-      # Used to build (if block given) and/or fetch a current controller instance that will be used
-      #   to process messages from a given topic and partition
-      # @return [Karafka::BaseController] base controller descendant
-      # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
-      # @param partition [Integer] number of partition for which we want to cache
-      def self.fetch(topic, partition)
-        return yield unless topic.persistent
-        Thread.current[topic.id] ||= {}
-        Thread.current[topic.id][partition] ||= yield
+      # Thread.current scope under which we store controllers data
+      PERSISTENCE_SCOPE = :controllers
+
+      class << self
+        # @return [Hash] current thread persistence scope hash with all the controllers
+        def all
+          Thread.current[PERSISTENCE_SCOPE] ||= {}
+        end
+
+        # Used to build (if block given) and/or fetch a current controller instance that will be
+        #   used to process messages from a given topic and partition
+        # @return [Karafka::BaseController] base controller descendant
+        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
+        # @param partition [Integer] number of partition for which we want to cache
+        def fetch(topic, partition)
+          all[topic.id] ||= {}
+
+          # We always store a current instance
+          if topic.persistent
+            all[topic.id][partition] ||= yield
+          else
+            all[topic.id][partition] = yield
+          end
+        end
       end
     end
   end

data/lib/karafka/process.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Class used to catch signals from ruby Signal class in order to manage Karafka shutdown
+  # Class used to catch signals from ruby Signal class in order to manage Karafka stop
   # @note There might be only one process - this class is a singleton
   class Process
     include Singleton