karafka 1.1.0

data/lib/karafka/connection/listener.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # A single listener that listens to incoming messages from a single route
+    # @note It does not loop on itself - it needs to be executed in a loop
+    # @note Listener itself does nothing with the message - it will return to the block
+    #   a raw Kafka::FetchedMessage
+    class Listener
+      attr_reader :consumer_group
+
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
+      #   on what topics and with what settings should we listen
+      # @return [Karafka::Connection::Listener] listener instance
+      def initialize(consumer_group)
+        @consumer_group = consumer_group
+      end
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @yieldparam [String] consumer group id
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      # @note This will yield with a raw message - no preprocessing or reformatting
+      # @note We catch all the errors here, so they don't affect other listeners (or this one)
+      #   so we will be able to listen and consume other incoming messages.
+      #   Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
+      #   won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
+      #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
+      #   propagate this far
+      def fetch_loop(block)
+        consumer.fetch_loop do |raw_messages|
+          block.call(consumer_group.id, raw_messages)
+        end
+        # This is on purpose - see the notes for this method
+        # rubocop:disable RescueException
+      rescue Exception => e
+        # rubocop:enable RescueException
+        Karafka.monitor.notice_error(self.class, e)
+        @consumer&.stop
+        retry if @consumer
+      end
+
+      private
+
+      # @return [Karafka::Connection::Consumer] wrapped kafka consumer for a given topic
+      #   consumption
+      def consumer
+        @consumer ||= Consumer.new(consumer_group)
+      end
+    end
+  end
+end

data/lib/karafka/connection/processor.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that consumes messages for which we listen
+    module Processor
+      class << self
+        # Processes messages (does something with them)
+        # It will either schedule or run a proper controller action for messages
+        # @note This should be looped to obtain a constant listening
+        # @note We catch all the errors here, to make sure that none failures
+        #   for a given consumption will affect other consumed messages
+        #   If we wouldn't catch it, it would propagate up until killing the thread
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
+        def process(group_id, kafka_messages)
+          # @note We always get messages by topic and partition so we can take topic from the
+          # first one and it will be valid for all the messages
+          # We map from incoming topic name, as it might be namespaced, etc.
+          # @see topic_mapper internal docs
+          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(kafka_messages[0].topic)
+          topic = Routing::Router.find("#{group_id}_#{mapped_topic_name}")
+          controller = Persistence::Controller.fetch(topic, kafka_messages[0].partition) do
+            topic.controller.new
+          end
+
+          # Depending on a case (persisted or not) we might use new controller instance per each
+          # batch, or use the same instance for all of them (for implementing buffering, etc)
+          send(
+            topic.batch_consuming ? :process_batch : :process_each,
+            controller,
+            kafka_messages
+          )
+        end
+
+        private
+
+        # Processes whole batch in one request (all at once)
+        # @param controller [Karafka::BaseController] base controller descendant
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
+        def process_batch(controller, kafka_messages)
+          controller.params_batch = kafka_messages
+          Karafka.monitor.notice(self, kafka_messages)
+          controller.call
+        end
+
+        # Processes messages one by one (like with std http requests)
+        # @param controller [Karafka::BaseController] base controller descendant
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages from kafka
+        def process_each(controller, kafka_messages)
+          kafka_messages.each do |kafka_message|
+            # @note This is a simple trick - we just process one after another, but in order
+            # not to handle everywhere both cases (single vs batch), we just "fake" batching with
+            # a single message for each
+            process_batch(controller, [kafka_message])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/controllers/callbacks.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Controllers
+    # Additional callbacks that can be used to trigger some actions on certain moments like
+    # manual offset management, committing or anything else outside of a standard messages flow
+    # They are not included by default, as we don't want to provide functionalities that are
+    # not required by users by default
+    # Please refer to the wiki callbacks page for more details on how to use them
+    module Callbacks
+      # Types of events on which we run callbacks
+      TYPES = %i[
+        after_fetched
+        after_poll
+        before_poll
+        before_stop
+      ].freeze
+
+      # Class methods needed to make callbacks run
+      module ClassMethods
+        TYPES.each do |type|
+          # A Creates a callback wrapper
+          # @param method_name [Symbol, String] method name or nil if we plan to provide a block
+          # @yield A block with a code that should be executed before scheduling
+          define_method type do |method_name = nil, &block|
+            set_callback type, :before, method_name ? method_name : block
+          end
+        end
+      end
+
+      # @param controller_class [Class] controller class that we extend with callbacks
+      def self.included(controller_class)
+        controller_class.class_eval do
+          extend ClassMethods
+          include ActiveSupport::Callbacks
+
+          # The call method is wrapped with a set of callbacks
+          # We won't run process if any of the callbacks throw abort
+          # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
+          TYPES.each { |type| define_callbacks type }
+        end
+      end
+
+      # Executes the default controller flow, runs callbacks and if not halted will call process
+      # method of a proper backend. This is here because it interacts with the default Karafka
+      # call flow and needs to be overwritten in order to support callbacks
+      def call
+        run_callbacks :after_fetched do
+          process
+        end
+      end
+    end
+  end
+end

data/lib/karafka/controllers/includer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Additional functionalities for controllers
+  module Controllers
+    # Module used to inject functionalities into a given controller class, based on the controller
+    # topic and its settings
+    # We don't need all the behaviors in all the cases, so it is totally not worth having
+    # everything in all the cases all the time
+    module Includer
+      class << self
+        # @param controller_class [Class] controller class, that will get some functionalities
+        #   based on the topic under which it operates
+        def call(controller_class)
+          topic = controller_class.topic
+
+          bind_backend(controller_class, topic)
+          bind_params(controller_class, topic)
+          bind_responders(controller_class, topic)
+        end
+
+        private
+
+        # Figures out backend for a given controller class, based on the topic backend and
+        #   includes it into the controller class
+        # @param controller_class [Class] controller class
+        # @param topic [Karafka::Routing::Topic] topic of a controller class
+        def bind_backend(controller_class, topic)
+          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
+          controller_class.include backend
+        end
+
+        # Adds a single #params support for non batch processed topics
+        # @param controller_class [Class] controller class
+        # @param topic [Karafka::Routing::Topic] topic of a controller class
+        def bind_params(controller_class, topic)
+          return if topic.batch_consuming
+          controller_class.include SingleParams
+        end
+
+        # Adds responders support for topics and controllers with responders defined for them
+        # @param controller_class [Class] controller class
+        # @param topic [Karafka::Routing::Topic] topic of a controller class
+        def bind_responders(controller_class, topic)
+          return unless topic.responder
+          controller_class.include Responders
+        end
+      end
+    end
+  end
+end

data/lib/karafka/controllers/responders.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Controllers
+    # Feature that allows us to use responders flow in controller
+    module Responders
+      # Responds with given data using given responder. This allows us to have a similar way of
+      # defining flows like synchronous protocols
+      # @param data Anything we want to pass to responder based on which we want to trigger further
+      #   Kafka responding
+      def respond_with(*data)
+        Karafka.monitor.notice(self.class, data: data)
+        # @note we build a new instance of responder each time, as a long running (persisted)
+        #   controllers can respond multiple times during the lifecycle
+        topic.responder.new(topic.parser).call(*data)
+      end
+    end
+  end
+end

data/lib/karafka/controllers/single_params.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Controllers
+    # Params alias for single message consumption controllers
+    module SingleParams
+      private
+
+      # @return [Karafka::Params::Params] params instance for non batch consumption controllers
+      def params
+        params_batch.first
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace used to encapsulate all the internal errors of Karafka
+  module Errors
+    # Base class for all the Karafka internal errors
+    BaseError = Class.new(StandardError)
+
+    # Should be raised when we attemp to parse incoming params but parsing fails
+    #   If this error (or its descendant) is detected, we will pass the raw message
+    #   into params and proceed further
+    ParserError = Class.new(BaseError)
+
+    # Raised when router receives topic name which does not correspond with any routes
+    # This can only happen in a case when:
+    #   - you've received a message and we cannot match it with a controller
+    #   - you've changed the routing, so router can no longer associate your topic to
+    #     any controller
+    #   - or in a case when you do a lot of metaprogramming and you change routing/etc on runtime
+    #
+    # In case this happens, you will have to create a temporary route that will allow
+    # you to "eat" everything from the Sidekiq queue.
+    # @see https://github.com/karafka/karafka/issues/135
+    NonMatchingRouteError = Class.new(BaseError)
+
+    # Raised when we don't use or use responder not in the way it expected to based on the
+    # topics usage definitions
+    InvalidResponderUsage = Class.new(BaseError)
+
+    # Raised when configuration doesn't match with validation schema
+    InvalidConfiguration = Class.new(BaseError)
+
+    # Raised when we try to use Karafka CLI commands (except install) without a bootfile
+    MissingBootFile = Class.new(BaseError)
+
+    # Raised when we want to read a persisted thread messages consumer but it is unavailable
+    # This should never happen and if it does, please contact us
+    MissingConsumer = Class.new(BaseError)
+
+    # Raised when we attemp to pause a partition but the pause timeout is equal to 0
+    InvalidPauseTimeout = Class.new(BaseError)
+  end
+end

data/lib/karafka/fetcher.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Class used to run the Karafka consumer and handle shutting down, restarting etc
+  # @note Creating multiple fetchers will result in having multiple connections to the same
+  #   topics, which means that if there are no partitions, it won't use them.
+  class Fetcher
+    # Starts listening on all the listeners asynchronously
+    # Fetch loop should never end, which means that we won't create more actor clusters
+    # so we don't have to terminate them
+    def fetch_loop
+      threads = listeners.map do |listener|
+        # We abort on exception because there should be an exception handling developed for
+        # each listener running in separate threads, so the exceptions should never leak
+        # and if that happens, it means that something really bad happened and we should stop
+        # the whole process
+        Thread
+          .new { listener.fetch_loop(processor) }
+          .tap { |thread| thread.abort_on_exception = true }
+      end
+
+      threads.each(&:join)
+    # If anything crashes here, we need to raise the error and crush the runner because it means
+    # that something really bad happened
+    rescue StandardError => e
+      Karafka.monitor.notice_error(self.class, e)
+      Karafka::App.stop!
+      raise e
+    end
+
+    private
+
+    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
+    def listeners
+      @listeners ||= App.consumer_groups.active.map do |consumer_group|
+        Karafka::Connection::Listener.new(consumer_group)
+      end
+    end
+
+    # @return [Proc] proc that should be processed when a messages arrive
+    # @yieldparam messages [Array<Kafka::FetchedMessage>] messages from kafka (raw)
+    def processor
+      lambda do |group_id, messages|
+        Karafka::Connection::Processor.process(group_id, messages)
+      end
+    end
+  end
+end

data/lib/karafka/helpers/class_matcher.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # Class used to autodetect corresponding classes that are internally inside Karafka framework
+    # It is used among others to match:
+    #   controller => responder
+    class ClassMatcher
+      # Regexp used to remove any non classy like characters that might be in the controller
+      # class name (if defined dynamically, etc)
+      CONSTANT_REGEXP = %r{[?!=+\-\*/\^\|&\[\]<>%~\#\:\s\(\)]}
+
+      # @param klass [Class] class to which we want to find a corresponding class
+      # @param from [String] what type of object is it (based on postfix name part)
+      # @param to [String] what are we looking for (based on a postfix name part)
+      # @example Controller that has a corresponding responder
+      #   matcher = Karafka::Helpers::ClassMatcher.new(SuperController, 'Controller', 'Responder')
+      #   matcher.match #=> SuperResponder
+      # @example Controller without a corresponding responder
+      #   matcher = Karafka::Helpers::ClassMatcher.new(Super2Controller, 'Controller', 'Responder')
+      #   matcher.match #=> nil
+      def initialize(klass, from:, to:)
+        @klass = klass
+        @from = from
+        @to = to
+      end
+
+      # @return [Class] matched class
+      # @return [nil] nil if we couldn't find matching class
+      def match
+        return nil if name.empty?
+        return nil unless scope.const_defined?(name)
+        matching = scope.const_get(name)
+        same_scope?(matching) ? matching : nil
+      end
+
+      # @return [String] name of a new class that we're looking for
+      # @note This method returns name of a class without a namespace
+      # @example From SuperController matching responder
+      #   matcher.name #=> 'SuperResponder'
+      # @example From Namespaced::Super2Controller matching responder
+      #   matcher.name #=> Super2Responder
+      def name
+        inflected = @klass.to_s.split('::').last.to_s
+        inflected.gsub!(@from, @to)
+        inflected.gsub!(CONSTANT_REGEXP, '')
+        inflected
+      end
+
+      # @return [Class, Module] class or module in which we're looking for a matching
+      def scope
+        scope_of(@klass)
+      end
+
+      private
+
+      # @param klass [Class] class for which we want to extract it's enclosing class/module
+      # @return [Class, Module] enclosing class/module
+      # @return [::Object] object if it was a root class
+      #
+      # @example Non-namespaced class
+      #   scope_of(SuperClass) #=> Object
+      # @example Namespaced class
+      #   scope_of(Abc::SuperClass) #=> Abc
+      def scope_of(klass)
+        enclosing = klass.to_s.split('::')[0...-1]
+        return ::Object if enclosing.empty?
+        ::Object.const_get(enclosing.join('::'))
+      end
+
+      # @param matching [Class] class of which scope we want to check
+      # @return [Boolean] true if the scope of class is the same as scope of matching
+      def same_scope?(matching)
+        scope == scope_of(matching)
+      end
+    end
+  end
+end

data/lib/karafka/helpers/config_retriever.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Helpers
+    # A helper method that allows us to build methods that try to get a given
+    # attribute from its instance value and if it fails, will fallback to
+    # the default config or config.kafka value for a given attribute.
+    # It is used to simplify the checkings.
+    # @note Worth noticing, that the value might be equal to false, so even
+    #   then we need to return it. That's why we check for nil?
+    # @example Define config retried attribute for start_from_beginning
+    # class Test
+    #   extend Karafka::Helpers::ConfigRetriever
+    #   config_retriever_for :start_from_beginning
+    # end
+    #
+    # Test.new.start_from_beginning #=> false
+    # test_instance = Test.new
+    # test_instance.start_from_beginning = true
+    # test_instance.start_from_beginning #=> true
+    module ConfigRetriever
+      # Builds proper methods for setting and retrieving (with fallback) given attribute value
+      # @param attribute [Symbol] attribute name based on which we will build
+      #   accessor with fallback
+      def config_retriever_for(attribute)
+        attr_writer attribute unless method_defined? :"#{attribute}="
+
+        # Don't redefine if we already have accessor for a given element
+        return if method_defined? attribute
+
+        define_method attribute do
+          current_value = instance_variable_get(:"@#{attribute}")
+          return current_value unless current_value.nil?
+
+          value = if Karafka::App.config.respond_to?(attribute)
+                    Karafka::App.config.public_send(attribute)
+                  else
+                    Karafka::App.config.kafka.public_send(attribute)
+                  end
+
+          instance_variable_set(:"@#{attribute}", value)
+        end
+      end
+    end
+  end
+end