karafka 1.2.0

data/lib/karafka/connection/config_adapter.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related to Kafka connection
+  module Connection
+    # Mapper used to convert our internal settings into ruby-kafka settings
+    # Since ruby-kafka has more and more options and there are few "levels" on which
+    # we have to apply them (despite the fact, that in Karafka you configure all of it
+    # in one place), we have to remap it into what ruby-kafka driver requires
+    # @note The good thing about Kafka.new method is that it ignores all options that
+    #   do nothing. So we don't have to worry about injecting our internal settings
+    #   into the client and breaking stuff
+    module ConfigAdapter
+      class << self
+        # Builds all the configuration settings for Kafka.new method
+        # @param _consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array<Hash>] Array with all the client arguments including hash with all
+        #   the settings required by Kafka.new method
+        # @note We return array, so we can inject any arguments we want, in case of changes in the
+        #   raw driver
+        def client(_consumer_group)
+          # This one is a default that takes all the settings except special
+          # cases defined in the map
+          settings = {
+            logger: ::Karafka.logger,
+            client_id: ::Karafka::App.config.client_id
+          }
+
+          kafka_configs.each do |setting_name, setting_value|
+            # All options for config adapter should be ignored as we're just interested
+            # in what is left, as we want to pass all the options that are "typical"
+            # and not listed in the config_adapter special cases mapping. All the values
+            # from the config_adapter mapping go somewhere else, not to the client directly
+            next if AttributesMap.config_adapter.values.flatten.include?(setting_name)
+
+            settings[setting_name] = setting_value
+          end
+
+          settings_hash = sanitize(settings)
+
+          # Normalization for the way Kafka::Client accepts arguments from  0.5.3
+          [settings_hash.delete(:seed_brokers), settings_hash]
+        end
+
+        # Builds all the configuration settings for kafka#consumer method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array<Hash>] array with all the consumer arguments including hash with all
+        #   the settings required by Kafka#consumer
+        def consumer(consumer_group)
+          settings = { group_id: consumer_group.id }
+          settings = fetch_for(:consumer, consumer_group, settings)
+          [sanitize(settings)]
+        end
+
+        # Builds all the configuration settings for kafka consumer consume_each_batch and
+        #   consume_each_message methods
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array<Hash>] Array with all the arguments required by consuming method
+        #   including hash with all the settings required by
+        #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
+        def consuming(consumer_group)
+          settings = {
+            automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
+          }
+          [sanitize(fetch_for(:consuming, consumer_group, settings))]
+        end
+
+        # Builds all the configuration settings for kafka consumer#subscribe method
+        # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
+        # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
+        def subscription(topic)
+          settings = fetch_for(:subscription, topic)
+          [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
+        end
+
+        # Builds all the configuration settings required by kafka consumer#pause method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Hash] hash with all the settings required to pause kafka consumer
+        def pausing(consumer_group)
+          { timeout: consumer_group.pause_timeout }
+        end
+
+        private
+
+        # Fetches proper settings for a given map namespace
+        # @param namespace_key [Symbol] namespace from attributes map config adapter hash
+        # @param route_layer [Object] route topic or consumer group
+        # @param preexisting_settings [Hash] hash with some preexisting settings that might have
+        #   been loaded in a different way
+        def fetch_for(namespace_key, route_layer, preexisting_settings = {})
+          kafka_configs.each_key do |setting_name|
+            # Ignore settings that are not related to our namespace
+            next unless AttributesMap.config_adapter[namespace_key].include?(setting_name)
+            # Ignore settings that are already initialized
+            # In case they are in preexisting settings fetched differently
+            next if preexisting_settings.keys.include?(setting_name)
+            # Fetch all the settings from a given layer object. Objects can handle the fallback
+            # to the kafka settings, so
+            preexisting_settings[setting_name] = route_layer.send(setting_name)
+          end
+
+          preexisting_settings
+        end
+
+        # Removes nil containing keys from the final settings so it can use Kafkas driver
+        #   defaults for those
+        # @param settings [Hash] settings that may contain nil values
+        # @return [Hash] settings without nil using keys (non of karafka options should be nil)
+        def sanitize(settings)
+          settings.reject { |_key, value| value.nil? }
+        end
+
+        # @return [Hash] Kafka config details as a hash
+        def kafka_configs
+          ::Karafka::App.config.kafka.to_h
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/delegator.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that delegates processing of messages for which we listen to a proper processor
+    module Delegator
+      class << self
+        # Delegates messages (does something with them)
+        # It will either schedule or run a proper processor action for messages
+        # @note This should be looped to obtain a constant delegating of new messages
+        # @note We catch all the errors here, to make sure that none failures
+        #   for a given consumption will affect other consumed messages
+        #   If we wouldn't catch it, it would propagate up until killing the thread
+        # @note It is a one huge method, because of performance reasons. It is much faster then
+        #   using send or invoking additional methods
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
+        def call(group_id, kafka_messages)
+          # @note We always get messages by topic and partition so we can take topic from the
+          # first one and it will be valid for all the messages
+          topic = Persistence::Topic.fetch(group_id, kafka_messages[0].topic)
+          consumer = Persistence::Consumer.fetch(topic, kafka_messages[0].partition)
+
+          Karafka.monitor.instrument(
+            'connection.delegator.call',
+            caller: self,
+            consumer: consumer,
+            kafka_messages: kafka_messages
+          ) do
+            # Depending on a case (persisted or not) we might use new consumer instance per
+            # each batch, or use the same one for all of them (for implementing buffering, etc.)
+            if topic.batch_consuming
+              consumer.params_batch = kafka_messages
+              consumer.call
+            else
+              kafka_messages.each do |kafka_message|
+                consumer.params_batch = [kafka_message]
+                consumer.call
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # A single listener that listens to incoming messages from a single route
+    # @note It does not loop on itself - it needs to be executed in a loop
+    # @note Listener itself does nothing with the message - it will return to the block
+    #   a raw Kafka::FetchedMessage
+    class Listener
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
+      #   on what topics and with what settings should we listen
+      # @return [Karafka::Connection::Listener] listener instance
+      def initialize(consumer_group)
+        @consumer_group = consumer_group
+      end
+
+      # Runs prefetch callbacks and executes the main listener fetch loop
+      def call
+        Karafka::Callbacks.before_fetch_loop(
+          @consumer_group,
+          client
+        )
+        fetch_loop
+      end
+
+      private
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @yieldparam [String] consumer group id
+      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      # @note This will yield with a raw message - no preprocessing or reformatting
+      # @note We catch all the errors here, so they don't affect other listeners (or this one)
+      #   so we will be able to listen and consume other incoming messages.
+      #   Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
+      #   won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
+      #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
+      #   propagate this far
+      def fetch_loop
+        client.fetch_loop do |raw_messages|
+          # @note What happens here is a delegation of processing to a proper processor based
+          #   on the incoming messages characteristics
+          Karafka::Connection::Delegator.call(@consumer_group.id, raw_messages)
+        end
+        # This is on purpose - see the notes for this method
+        # rubocop:disable RescueException
+      rescue Exception => e
+        Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
+        # rubocop:enable RescueException
+        @client&.stop
+        retry if @client
+      end
+
+      # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic
+      #   consumption
+      def client
+        @client ||= Client.new(@consumer_group)
+      end
+    end
+  end
+end

data/lib/karafka/consumers/callbacks.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Additional callbacks that can be used to trigger some actions on certain moments like
+    # manual offset management, committing or anything else outside of a standard messages flow
+    # They are not included by default, as we don't want to provide functionalities that are
+    # not required by users by default
+    # Please refer to the wiki callbacks page for more details on how to use them
+    module Callbacks
+      # Types of events on which we run callbacks
+      TYPES = %i[
+        after_fetch
+        after_poll
+        before_poll
+        before_stop
+      ].freeze
+
+      # Class methods needed to make callbacks run
+      module ClassMethods
+        TYPES.each do |type|
+          # A Creates a callback wrapper
+          # @param method_name [Symbol, String] method name or nil if we plan to provide a block
+          # @yield A block with a code that should be executed before scheduling
+          define_method type do |method_name = nil, &block|
+            set_callback type, :before, method_name ? method_name : block
+          end
+        end
+      end
+
+      # @param consumer_class [Class] consumer class that we extend with callbacks
+      def self.included(consumer_class)
+        consumer_class.class_eval do
+          extend ClassMethods
+          include ActiveSupport::Callbacks
+
+          # The call method is wrapped with a set of callbacks
+          # We won't run process if any of the callbacks throw abort
+          # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
+          TYPES.each { |type| define_callbacks type }
+        end
+      end
+
+      # Executes the default consumer flow, runs callbacks and if not halted will call process
+      # method of a proper backend. It is here because it interacts with the default Karafka
+      # call flow and needs to be overwritten to support callbacks
+      def call
+        run_callbacks :after_fetch do
+          process
+        end
+      end
+    end
+  end
+end

data/lib/karafka/consumers/includer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Additional functionalities for consumers
+  module Consumers
+    # Module used to inject functionalities into a given consumer class, based on the consumer
+    # topic and its settings
+    # We don't need all the behaviors in all the cases, so it is not worth having everything
+    # in all the cases all the time
+    module Includer
+      class << self
+        # @param consumer_class [Class] consumer class, that will get some functionalities
+        #   based on the topic under which it operates
+        def call(consumer_class)
+          topic = consumer_class.topic
+
+          bind_backend(consumer_class, topic)
+          bind_params(consumer_class, topic)
+          bind_responders(consumer_class, topic)
+        end
+
+        private
+
+        # Figures out backend for a given consumer class, based on the topic backend and
+        #   includes it into the consumer class
+        # @param consumer_class [Class] consumer class
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_backend(consumer_class, topic)
+          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
+          consumer_class.include backend
+        end
+
+        # Adds a single #params support for non batch processed topics
+        # @param consumer_class [Class] consumer class
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_params(consumer_class, topic)
+          return if topic.batch_consuming
+          consumer_class.include SingleParams
+        end
+
+        # Adds responders support for topics and consumers with responders defined for them
+        # @param consumer_class [Class] consumer class
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_responders(consumer_class, topic)
+          return unless topic.responder
+          consumer_class.include Responders
+        end
+      end
+    end
+  end
+end

data/lib/karafka/consumers/responders.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Feature that allows us to use responders flow in consumer
+    module Responders
+      # Responds with given data using given responder. This allows us to have a similar way of
+      # defining flows like synchronous protocols
+      # @param data Anything we want to pass to responder based on which we want to trigger further
+      #   Kafka responding
+      def respond_with(*data)
+        Karafka.monitor.instrument(
+          'consumers.responders.respond_with',
+          caller: self,
+          data: data
+        ) do
+          # @note we build a new instance of responder each time, as a long-running (persisted)
+          #   consumers can respond multiple times during the lifecycle
+          topic.responder.new(topic.parser).call(*data)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/consumers/single_params.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Params alias for single message consumption consumers
+    module SingleParams
+      private
+
+      # @return [Karafka::Params::Params] params instance for non batch consumption consumers
+      def params
+        params_batch.first
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace used to encapsulate all the internal errors of Karafka
+  module Errors
+    # Base class for all the Karafka internal errors
+    BaseError = Class.new(StandardError)
+
+    # Should be raised when we attemp to parse incoming params but parsing fails
+    #   If this error (or its descendant) is detected, we will pass the raw message
+    #   into params and proceed further
+    ParserError = Class.new(BaseError)
+
+    # Raised when router receives topic name which does not correspond with any routes
+    # This can only happen in a case when:
+    #   - you've received a message and we cannot match it with a consumer
+    #   - you've changed the routing, so router can no longer associate your topic to
+    #     any consumer
+    #   - or in a case when you do a lot of metaprogramming and you change routing/etc on runtime
+    #
+    # In case this happens, you will have to create a temporary route that will allow
+    # you to "eat" everything from the Sidekiq queue.
+    # @see https://github.com/karafka/karafka/issues/135
+    NonMatchingRouteError = Class.new(BaseError)
+
+    # Raised when we don't use or use responder not in the way it expected to based on the
+    # topics usage definitions
+    InvalidResponderUsage = Class.new(BaseError)
+
+    # Raised when options that we provide to the responder to respond aren't what the schema
+    # requires
+    InvalidResponderMessageOptions = Class.new(BaseError)
+
+    # Raised when configuration doesn't match with validation schema
+    InvalidConfiguration = Class.new(BaseError)
+
+    # Raised when we try to use Karafka CLI commands (except install) without a bootfile
+    MissingBootFile = Class.new(BaseError)
+
+    # Raised when we want to read a persisted thread messages consumer but it is unavailable
+    # This should never happen and if it does, please contact us
+    MissingClient = Class.new(BaseError)
+
+    # Raised when we attemp to pause a partition but the pause timeout is equal to 0
+    InvalidPauseTimeout = Class.new(BaseError)
+
+    # Raised when want to hook up to an event that is not registered and supported
+    UnregisteredMonitorEvent = Class.new(BaseError)
+  end
+end

data/lib/karafka/fetcher.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Class used to run the Karafka consumer and handle shutting down, restarting etc
+  # @note Creating multiple fetchers will result in having multiple connections to the same
+  #   topics, which means that if there are no partitions, it won't use them.
+  class Fetcher
+    class << self
+      # Starts listening on all the listeners asynchronously
+      # Fetch loop should never end, which means that we won't create more actor clusters
+      # so we don't have to terminate them
+      def call
+        threads = listeners.map do |listener|
+          # We abort on exception because there should be an exception handling developed for
+          # each listener running in separate threads, so the exceptions should never leak
+          # and if that happens, it means that something really bad happened and we should stop
+          # the whole process
+          Thread
+            .new { listener.call }
+            .tap { |thread| thread.abort_on_exception = true }
+        end
+
+        # We aggregate threads here for a supervised shutdown process
+        threads.each { |thread| Karafka::Server.consumer_threads << thread }
+        threads.each(&:join)
+      # If anything crashes here, we need to raise the error and crush the runner because it means
+      # that something terrible happened
+      rescue StandardError => e
+        Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
+        Karafka::App.stop!
+        raise e
+      end
+
+      private
+
+      # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
+      def listeners
+        @listeners ||= App.consumer_groups.active.map do |consumer_group|
+          Karafka::Connection::Listener.new(consumer_group)
+        end
+      end
+    end
+  end
+end