karafka 1.4.13 → 2.0.0

data/lib/karafka/consumers/callbacks.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Additional callbacks that can be used to trigger some actions on certain moments like
-    # manual offset management, committing or anything else outside of a standard messages flow
-    # They are not included by default, as we don't want to provide functionalities that are
-    # not required by users by default
-    # Please refer to the wiki callbacks page for more details on how to use them
-    module Callbacks
-      # Types of events on which we run callbacks
-      TYPES = %i[
-        after_fetch
-        after_poll
-        before_poll
-        before_stop
-      ].freeze
-
-      private_constant :TYPES
-
-      # Class methods needed to make callbacks run
-      module ClassMethods
-        TYPES.each do |type|
-          # Creates a callback wrapper
-          #
-          # @param method_name [Symbol, String] method name or nil if we plan to provide a block
-          # @yield A block with a code that should be executed before scheduling
-          # @note We don't have to optimize the key fetching here as those are class methods that
-          #   are evaluated once upon start
-          define_method(type) do |method_name = nil, &block|
-            key = "consumers.#{Helpers::Inflector.map(to_s)}.#{type}"
-            Karafka::App.monitor.register_event(key)
-
-            Karafka::App.monitor.subscribe(key) do |event|
-              context = event[:context]
-
-              if method_name
-                context.send(method_name)
-              else
-                context.instance_eval(&block)
-              end
-            end
-          end
-        end
-      end
-
-      class << self
-        # @param consumer_class [Class] consumer class that we extend with callbacks
-        def included(consumer_class)
-          consumer_class.class_eval do
-            extend ClassMethods
-          end
-        end
-      end
-
-      # Executes the default consumer flow, runs callbacks and if not halted will call process
-      # method of a proper backend. It is here because it interacts with the default Karafka
-      # call flow and needs to be overwritten to support callbacks
-      def call
-        if self.class.respond_to?(:after_fetch)
-          Karafka::App.monitor.instrument(
-            "consumers.#{Helpers::Inflector.map(self.class.to_s)}.after_fetch",
-            context: self
-          )
-        end
-
-        process
-      end
-    end
-  end
-end

data/lib/karafka/consumers/includer.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Additional functionalities for consumers
-  module Consumers
-    # Module used to inject functionalities into a given consumer instance, based on the consumer
-    # topic and its settings
-    # We don't need all the behaviors in all the cases, so it is not worth having everything
-    # in all the cases all the time
-    module Includer
-      class << self
-        # @param consumer [Karafka::BaseConsumer] consumer instance, that will get some
-        #   functionalities based on the topic under which it operates
-        def call(consumer)
-          topic = consumer.topic
-
-          bind_backend(consumer, topic)
-          bind_params(consumer, topic)
-          bind_batch_metadata(consumer, topic)
-          bind_responders(consumer, topic)
-        end
-
-        private
-
-        # Figures out backend for a given consumer class, based on the topic backend and
-        #   includes it into the consumer class
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_backend(consumer, topic)
-          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
-          consumer.extend(backend)
-        end
-
-        # Adds a single #params support for non batch processed topics
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_params(consumer, topic)
-          return if topic.batch_consuming
-
-          consumer.extend(SingleParams)
-        end
-
-        # Adds an option to work with batch metadata for consumer instances that have
-        #   batch fetching enabled
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_batch_metadata(consumer, topic)
-          return unless topic.batch_fetching
-
-          consumer.extend(BatchMetadata)
-        end
-
-        # Adds responders support for topics and consumers with responders defined for them
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_responders(consumer, topic)
-          return unless topic.responder
-
-          consumer.extend(Responders)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/consumers/responders.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Feature that allows us to use responders flow in consumer
-    module Responders
-      # Responds with given data using given responder. This allows us to have a similar way of
-      # defining flows like synchronous protocols
-      # @param data Anything we want to pass to responder based on which we want to trigger further
-      #   Kafka responding
-      def respond_with(*data)
-        Karafka.monitor.instrument(
-          'consumers.responders.respond_with',
-          caller: self,
-          data: data
-        ) do
-          # @note we build a new instance of responder each time, as a long-running (persisted)
-          #   consumers can respond multiple times during the life-cycle
-          topic.responder.new.call(*data)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/consumers/single_params.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Params alias for single message consumption consumers
-    module SingleParams
-      private
-
-      # @return [Karafka::Params::Params] params instance for non batch consumption consumers
-      def params
-        params_batch.first
-      end
-    end
-  end
-end

data/lib/karafka/contracts/responder_usage.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Contracts
-    # Validator to check responder topic usage
-    class ResponderUsageTopic < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
-      params do
-        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
-        required(:required).filled(:bool?)
-        required(:usage_count).filled(:int?, gteq?: 0)
-        required(:registered).filled(eql?: true)
-        required(:async).filled(:bool?)
-        required(:serializer).filled
-      end
-
-      rule(:required, :usage_count) do
-        key(:name).failure(:required_usage_count) if values[:required] && values[:usage_count] < 1
-      end
-    end
-
-    # Validator to check that everything in a responder flow matches responder rules
-    class ResponderUsage < Dry::Validation::Contract
-      include Dry::Core::Constants
-
-      # Contract for verifying the topic usage details
-      TOPIC_CONTRACT = ResponderUsageTopic.new.freeze
-
-      private_constant :TOPIC_CONTRACT
-
-      params do
-        required(:used_topics)
-        required(:registered_topics)
-      end
-
-      rule(:used_topics) do
-        (value || EMPTY_ARRAY).each do |used_topic|
-          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
-            key([:used_topics, used_topic, error.path[0]]).failure(error.text)
-          end
-        end
-      end
-
-      rule(:registered_topics) do
-        (value || EMPTY_ARRAY).each do |used_topic|
-          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
-            key([:registered_topics, used_topic, error.path[0]]).failure(error.text)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/fetcher.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Class used to run the Karafka consumer and handle shutting down, restarting etc
-  # @note Creating multiple fetchers will result in having multiple connections to the same
-  #   topics, which means that if there are no partitions, it won't use them.
-  class Fetcher
-    # Starts listening on all the listeners asynchronously
-    # Fetch loop should never end, which means that we won't create more actor clusters
-    # so we don't have to terminate them
-    def call
-      threads = listeners.map do |listener|
-        # We abort on exception because there should be an exception handling developed for
-        # each listener running in separate threads, so the exceptions should never leak
-        # and if that happens, it means that something really bad happened and we should stop
-        # the whole process
-        Thread
-          .new { listener.call }
-          .tap { |thread| thread.abort_on_exception = true }
-      end
-
-      # We aggregate threads here for a supervised shutdown process
-      threads.each { |thread| Karafka::Server.consumer_threads << thread }
-      threads.each(&:join)
-    # If anything crashes here, we need to raise the error and crush the runner because it means
-    # that something terrible happened
-    rescue StandardError => e
-      Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
-      Karafka::App.stop!
-      raise e
-    end
-
-    private
-
-    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
-    def listeners
-      @listeners ||= App.consumer_groups.active.map do |consumer_group|
-        Karafka::Connection::Listener.new(consumer_group)
-      end
-    end
-  end
-end

data/lib/karafka/helpers/class_matcher.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Helpers
-    # Class used to autodetect corresponding classes that are internally inside Karafka framework
-    # It is used among others to match:
-    #   consumer => responder
-    class ClassMatcher
-      # Regexp used to remove any non classy like characters that might be in the consumer
-      # class name (if defined dynamically, etc)
-      CONSTANT_REGEXP = %r{[?!=+\-*/\^|&\[\]<>%~\#:\s()]}.freeze
-
-      private_constant :CONSTANT_REGEXP
-
-      # @param klass [Class] class to which we want to find a corresponding class
-      # @param from [String] what type of object is it (based on postfix name part)
-      # @param to [String] what are we looking for (based on a postfix name part)
-      # @example Consumer that has a corresponding responder
-      #   matcher = Karafka::Helpers::ClassMatcher.new(SuperConsumer, 'Consumer', 'Responder')
-      #   matcher.match #=> SuperResponder
-      # @example Consumer without a corresponding responder
-      #   matcher = Karafka::Helpers::ClassMatcher.new(Super2Consumer, 'Consumer', 'Responder')
-      #   matcher.match #=> nil
-      def initialize(klass, from:, to:)
-        @klass = klass
-        @from = from
-        @to = to
-      end
-
-      # @return [Class] matched class
-      # @return [nil] nil if we couldn't find matching class
-      def match
-        return nil if name.empty?
-        return nil unless scope.const_defined?(name)
-
-        matching = scope.const_get(name)
-        same_scope?(matching) ? matching : nil
-      end
-
-      # @return [String] name of a new class that we're looking for
-      # @note This method returns name of a class without a namespace
-      # @example From SuperConsumer matching responder
-      #   matcher.name #=> 'SuperResponder'
-      # @example From Namespaced::Super2Consumer matching responder
-      #   matcher.name #=> Super2Responder
-      def name
-        inflected = +@klass.to_s.split('::').last.to_s
-        # We inject the from into the name just in case it is missing as in a situation like
-        # that it would just sanitize the name without adding the "to" postfix.
-        # It could create cases when we want to build for example a responder to a consumer
-        # that does not have the "Consumer" postfix and would do nothing returning the same name.
-        # That would be bad as the matching classes shouldn't be matched to themselves.
-        inflected << @from unless inflected.include?(@from)
-        inflected.gsub!(@from, @to)
-        inflected.gsub!(CONSTANT_REGEXP, '')
-        inflected
-      end
-
-      # @return [Class, Module] class or module in which we're looking for a matching
-      def scope
-        scope_of(@klass)
-      end
-
-      private
-
-      # @param klass [Class] class for which we want to extract it's enclosing class/module
-      # @return [Class, Module] enclosing class/module
-      # @return [::Object] object if it was a root class
-      #
-      # @example Non-namespaced class
-      #   scope_of(SuperClass) #=> Object
-      # @example Namespaced class
-      #   scope_of(Abc::SuperClass) #=> Abc
-      def scope_of(klass)
-        enclosing = klass.to_s.split('::')[0...-1]
-        return ::Object if enclosing.empty?
-
-        ::Object.const_get(enclosing.join('::'))
-      end
-
-      # @param matching [Class] class of which scope we want to check
-      # @return [Boolean] true if the scope of class is the same as scope of matching
-      def same_scope?(matching)
-        scope == scope_of(matching)
-      end
-    end
-  end
-end

data/lib/karafka/helpers/config_retriever.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Helpers
-    # A helper method that allows us to build methods that try to get a given
-    # attribute from its instance value and if it fails, will fallback to
-    # the default config or config.kafka value for a given attribute.
-    # It is used to simplify the checks.
-    # @note Worth noticing, that the value might be equal to false, so even
-    #   then we need to return it. That's why we check for nil?
-    # @example Define config retried attribute for start_from_beginning
-    # class Test
-    #   extend Karafka::Helpers::ConfigRetriever
-    #   config_retriever_for :start_from_beginning
-    # end
-    #
-    # Test.new.start_from_beginning #=> false
-    # test_instance = Test.new
-    # test_instance.start_from_beginning = true
-    # test_instance.start_from_beginning #=> true
-    module ConfigRetriever
-      # Builds proper methods for setting and retrieving (with fallback) given attribute value
-      # @param attribute [Symbol] attribute name based on which we will build
-      #   accessor with fallback
-      def config_retriever_for(attribute)
-        attr_writer attribute unless method_defined? :"#{attribute}="
-
-        # Don't redefine if we already have accessor for a given element
-        return if method_defined? attribute
-
-        define_method attribute do
-          current_value = instance_variable_get(:"@#{attribute}")
-          return current_value unless current_value.nil?
-
-          value = if Karafka::App.config.respond_to?(attribute)
-                    Karafka::App.config.send(attribute)
-                  else
-                    Karafka::App.config.kafka.send(attribute)
-                  end
-
-          instance_variable_set(:"@#{attribute}", value)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/helpers/inflector.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Helpers
-    # Inflector provides inflection for the whole Karafka framework with additional inflection
-    # caching (due to the fact, that Dry::Inflector is slow)
-    module Inflector
-      # What inflection engine do we want to use
-      ENGINE = Dry::Inflector.new
-
-      @map = Concurrent::Hash.new
-
-      private_constant :ENGINE
-
-      class << self
-        # @param string [String] string that we want to convert to our underscore format
-        # @return [String] inflected string
-        # @example
-        #   Karafka::Helpers::Inflector.map('Module/ControllerName') #=> 'module_controller_name'
-        def map(string)
-          @map[string] ||= ENGINE.underscore(string).tr('/', '_')
-        end
-      end
-    end
-  end
-end

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -1,140 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Instrumentation
-    # Default listener that hooks up to our instrumentation and uses its events for logging
-    # It can be removed/replaced or anything without any harm to the Karafka app flow
-    class StdoutListener
-      # Log levels that we use in this particular listener
-      USED_LOG_LEVELS = %i[
-        debug
-        info
-        error
-        fatal
-      ].freeze
-
-      # Logs details about incoming batches and with which consumer we will consume them
-      # @param event [Dry::Events::Event] event details including payload
-      def on_connection_batch_delegator_call(event)
-        consumer = event[:consumer]
-        topic = consumer.topic.name
-        kafka_messages = event[:kafka_batch].messages
-        info(
-          <<~MSG.chomp.tr("\n", ' ')
-            #{kafka_messages.count} messages
-            on #{topic} topic
-            delegated to #{consumer.class}
-          MSG
-        )
-      end
-
-      # Logs details about incoming message and with which consumer we will consume it
-      # @param event [Dry::Events::Event] event details including payload
-      def on_connection_message_delegator_call(event)
-        consumer = event[:consumer]
-        topic = consumer.topic.name
-        info "1 message on #{topic} topic delegated to #{consumer.class}"
-      end
-
-      # Logs details about each received message value deserialization
-      # @param event [Dry::Events::Event] event details including payload
-      def on_params_params_deserialize(event)
-        # Keep in mind, that a caller here is a param object not a controller,
-        # so it returns a topic as a string, not a routing topic
-        debug(
-          <<~MSG.chomp.tr("\n", ' ')
-            Params deserialization for #{event[:caller].metadata.topic} topic
-            successful in #{event[:time]} ms
-          MSG
-        )
-      end
-
-      # Logs unsuccessful deserialization attempts of incoming data
-      # @param event [Dry::Events::Event] event details including payload
-      def on_params_params_deserialize_error(event)
-        topic = event[:caller].metadata.topic
-        error = event[:error]
-        error "Params deserialization error for #{topic} topic: #{error}"
-      end
-
-      # Logs errors that occurred in a listener fetch loop
-      # @param event [Dry::Events::Event] event details including payload
-      # @note It's an error as we can recover from it not a fatal
-      def on_connection_listener_fetch_loop_error(event)
-        error "Listener fetch loop error: #{event[:error]}"
-      end
-
-      # Logs errors that are related to the connection itself
-      # @param event [Dry::Events::Event] event details including payload
-      # @note Karafka will attempt to reconnect, so an error not a fatal
-      def on_connection_client_fetch_loop_error(event)
-        error "Client fetch loop error: #{event[:error]}"
-      end
-
-      # Logs info about crashed fetcher
-      # @param event [Dry::Events::Event] event details including payload
-      # @note If this happens, Karafka will shutdown as it means a critical error
-      #   in one of the threads
-      def on_fetcher_call_error(event)
-        fatal "Fetcher crash due to an error: #{event[:error]}"
-      end
-
-      # Logs info about processing of a certain dataset with an inline backend
-      # @param event [Dry::Events::Event] event details including payload
-      def on_backends_inline_process(event)
-        count = event[:caller].send(:params_batch).to_a.size
-        topic = event[:caller].topic.name
-        time = event[:time]
-        info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
-      end
-
-      # Logs info about system signals that Karafka received
-      # @param event [Dry::Events::Event] event details including payload
-      def on_process_notice_signal(event)
-        info "Received #{event[:signal]} system signal"
-      end
-
-      # Logs info about responder usage withing a controller flow
-      # @param event [Dry::Events::Event] event details including payload
-      def on_consumers_responders_respond_with(event)
-        calling = event[:caller]
-        responder = calling.topic.responder
-        data = event[:data]
-        info "Responded from #{calling.class} using #{responder} with following data #{data}"
-      end
-
-      # Logs info that we're initializing Karafka framework components
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_initializing(_event)
-        info "Initializing Karafka framework #{::Process.pid}"
-      end
-
-      # Logs info that we're running Karafka app
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_running(_event)
-        info "Running Karafka server #{::Process.pid}"
-      end
-
-      # Logs info that we're going to stop the Karafka server
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_stopping(_event)
-        # We use a separate thread as logging can't be called from trap context
-        Thread.new { info "Stopping Karafka server #{::Process.pid}" }
-      end
-
-      # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
-      #   forced exit
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_stopping_error(_event)
-        # We use a separate thread as logging can't be called from trap context
-        Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
-      end
-
-      USED_LOG_LEVELS.each do |log_level|
-        define_method log_level do |*args|
-          Karafka.logger.send(log_level, *args)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/batch_metadata.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Simple batch metadata object that stores all non-message information received from Kafka
-    # cluster while fetching the data
-    # @note This metadata object refers to per batch metadata, not `#params.metadata`
-    BatchMetadata = Struct.new(
-      :batch_size,
-      :first_offset,
-      :highwater_mark_offset,
-      :unknown_last_offset,
-      :last_offset,
-      :offset_lag,
-      :deserializer,
-      :partition,
-      :topic,
-      keyword_init: true
-    ) do
-      # @return [Boolean] is the last offset known or unknown
-      def unknown_last_offset?
-        unknown_last_offset
-      end
-    end
-  end
-end

data/lib/karafka/params/builders/batch_metadata.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    module Builders
-      # Builder for creating batch metadata object based on the batch informations
-      module BatchMetadata
-        class << self
-          # Creates metadata based on the kafka batch data
-          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
-          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
-          # @return [Karafka::Params::BatchMetadata] batch metadata object
-          def from_kafka_batch(kafka_batch, topic)
-            Karafka::Params::BatchMetadata.new(
-              batch_size: kafka_batch.messages.count,
-              first_offset: kafka_batch.first_offset,
-              highwater_mark_offset: kafka_batch.highwater_mark_offset,
-              unknown_last_offset: kafka_batch.unknown_last_offset?,
-              last_offset: kafka_batch.last_offset,
-              offset_lag: kafka_batch.offset_lag,
-              deserializer: topic.deserializer,
-              partition: kafka_batch.partition,
-              topic: topic.name
-            ).freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/builders/params.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Due to the fact, that we create params related objects in couple contexts / places
-    # plus backends can build up them their own way we have this namespace.
-    # It allows to isolate actual params objects from their building process that can be
-    # context dependent.
-    module Builders
-      # Builder for params
-      module Params
-        class << self
-          # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
-          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
-          # @return [Karafka::Params::Params] params object with payload and message metadata
-          def from_kafka_message(kafka_message, topic)
-            metadata = Karafka::Params::Metadata.new(
-              create_time: kafka_message.create_time,
-              headers: kafka_message.headers || {},
-              is_control_record: kafka_message.is_control_record,
-              key: kafka_message.key,
-              offset: kafka_message.offset,
-              deserializer: topic.deserializer,
-              partition: kafka_message.partition,
-              receive_time: Time.now,
-              topic: topic.name
-            ).freeze
-
-            Karafka::Params::Params.new(
-              kafka_message.value,
-              metadata
-            )
-          end
-        end
-      end
-    end
-  end
-end