karafka 1.4.12 → 2.2.10

data/lib/karafka/connection/batch_delegator.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class that delegates processing of batch received messages for which we listen to
-    # a proper processor
-    module BatchDelegator
-      class << self
-        # Delegates messages (does something with them)
-        # It will either schedule or run a proper processor action for messages
-        # @param group_id [String] group_id of a group from which a given message came
-        # @param kafka_batch [<Kafka::FetchedBatch>] raw messages fetched batch
-        # @note This should be looped to obtain a constant delegating of new messages
-        def call(group_id, kafka_batch)
-          topic = Persistence::Topics.fetch(group_id, kafka_batch.topic)
-          consumer = Persistence::Consumers.fetch(topic, kafka_batch.partition)
-
-          Karafka.monitor.instrument(
-            'connection.batch_delegator.call',
-            caller: self,
-            consumer: consumer,
-            kafka_batch: kafka_batch
-          ) do
-            # Due to how ruby-kafka is built, we have the metadata that is stored on the batch
-            # level only available for batch consuming
-            consumer.batch_metadata = Params::Builders::BatchMetadata.from_kafka_batch(
-              kafka_batch,
-              topic
-            )
-
-            kafka_messages = kafka_batch.messages
-
-            # Depending on a case (persisted or not) we might use new consumer instance per
-            # each batch, or use the same one for all of them (for implementing buffering, etc.)
-            if topic.batch_consuming
-              consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
-                kafka_messages,
-                topic
-              )
-              consumer.call
-            else
-              kafka_messages.each do |kafka_message|
-                consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
-                  [kafka_message],
-                  topic
-                )
-                consumer.call
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/connection/builder.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Builder used to construct Kafka client
-    module Builder
-      class << self
-        # Builds a Kafka::Client instance that we use to work with Kafka cluster
-        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which we want
-        #   to have a new Kafka client
-        # @return [::Kafka::Client] returns a Kafka client
-        def call(consumer_group)
-          settings = ApiAdapter.client(consumer_group)
-
-          Kafka.new(
-            settings[0],
-            **settings[1]
-          )
-        end
-      end
-    end
-  end
-end

data/lib/karafka/connection/message_delegator.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class that delegates processing of a single received message for which we listen to
-    # a proper processor
-    module MessageDelegator
-      class << self
-        # Delegates message (does something with it)
-        # It will either schedule or run a proper processor action for the incoming message
-        # @param group_id [String] group_id of a group from which a given message came
-        # @param kafka_message [<Kafka::FetchedMessage>] raw message from kafka
-        # @note This should be looped to obtain a constant delegating of new messages
-        def call(group_id, kafka_message)
-          topic = Persistence::Topics.fetch(group_id, kafka_message.topic)
-          consumer = Persistence::Consumers.fetch(topic, kafka_message.partition)
-
-          Karafka.monitor.instrument(
-            'connection.message_delegator.call',
-            caller: self,
-            consumer: consumer,
-            kafka_message: kafka_message
-          ) do
-            # @note We always get a single message within single delegator, which means that
-            # we don't care if user marked it as a batch consumed or not.
-            consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
-              [kafka_message],
-              topic
-            )
-            consumer.call
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/consumers/batch_metadata.rb

@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Brings the batch metadata into consumers that support batch_fetching
-    module BatchMetadata
-      attr_accessor :batch_metadata
-    end
-  end
-end

data/lib/karafka/consumers/callbacks.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Additional callbacks that can be used to trigger some actions on certain moments like
-    # manual offset management, committing or anything else outside of a standard messages flow
-    # They are not included by default, as we don't want to provide functionalities that are
-    # not required by users by default
-    # Please refer to the wiki callbacks page for more details on how to use them
-    module Callbacks
-      # Types of events on which we run callbacks
-      TYPES = %i[
-        after_fetch
-        after_poll
-        before_poll
-        before_stop
-      ].freeze
-
-      private_constant :TYPES
-
-      # Class methods needed to make callbacks run
-      module ClassMethods
-        TYPES.each do |type|
-          # Creates a callback wrapper
-          #
-          # @param method_name [Symbol, String] method name or nil if we plan to provide a block
-          # @yield A block with a code that should be executed before scheduling
-          # @note We don't have to optimize the key fetching here as those are class methods that
-          #   are evaluated once upon start
-          define_method(type) do |method_name = nil, &block|
-            key = "consumers.#{Helpers::Inflector.map(to_s)}.#{type}"
-            Karafka::App.monitor.register_event(key)
-
-            Karafka::App.monitor.subscribe(key) do |event|
-              context = event[:context]
-
-              if method_name
-                context.send(method_name)
-              else
-                context.instance_eval(&block)
-              end
-            end
-          end
-        end
-      end
-
-      class << self
-        # @param consumer_class [Class] consumer class that we extend with callbacks
-        def included(consumer_class)
-          consumer_class.class_eval do
-            extend ClassMethods
-          end
-        end
-      end
-
-      # Executes the default consumer flow, runs callbacks and if not halted will call process
-      # method of a proper backend. It is here because it interacts with the default Karafka
-      # call flow and needs to be overwritten to support callbacks
-      def call
-        if self.class.respond_to?(:after_fetch)
-          Karafka::App.monitor.instrument(
-            "consumers.#{Helpers::Inflector.map(self.class.to_s)}.after_fetch",
-            context: self
-          )
-        end
-
-        process
-      end
-    end
-  end
-end

data/lib/karafka/consumers/includer.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Additional functionalities for consumers
-  module Consumers
-    # Module used to inject functionalities into a given consumer instance, based on the consumer
-    # topic and its settings
-    # We don't need all the behaviors in all the cases, so it is not worth having everything
-    # in all the cases all the time
-    module Includer
-      class << self
-        # @param consumer [Karafka::BaseConsumer] consumer instance, that will get some
-        #   functionalities based on the topic under which it operates
-        def call(consumer)
-          topic = consumer.topic
-
-          bind_backend(consumer, topic)
-          bind_params(consumer, topic)
-          bind_batch_metadata(consumer, topic)
-          bind_responders(consumer, topic)
-        end
-
-        private
-
-        # Figures out backend for a given consumer class, based on the topic backend and
-        #   includes it into the consumer class
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_backend(consumer, topic)
-          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
-          consumer.extend(backend)
-        end
-
-        # Adds a single #params support for non batch processed topics
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_params(consumer, topic)
-          return if topic.batch_consuming
-
-          consumer.extend(SingleParams)
-        end
-
-        # Adds an option to work with batch metadata for consumer instances that have
-        #   batch fetching enabled
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_batch_metadata(consumer, topic)
-          return unless topic.batch_fetching
-
-          consumer.extend(BatchMetadata)
-        end
-
-        # Adds responders support for topics and consumers with responders defined for them
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_responders(consumer, topic)
-          return unless topic.responder
-
-          consumer.extend(Responders)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/consumers/responders.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Feature that allows us to use responders flow in consumer
-    module Responders
-      # Responds with given data using given responder. This allows us to have a similar way of
-      # defining flows like synchronous protocols
-      # @param data Anything we want to pass to responder based on which we want to trigger further
-      #   Kafka responding
-      def respond_with(*data)
-        Karafka.monitor.instrument(
-          'consumers.responders.respond_with',
-          caller: self,
-          data: data
-        ) do
-          # @note we build a new instance of responder each time, as a long-running (persisted)
-          #   consumers can respond multiple times during the life-cycle
-          topic.responder.new.call(*data)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/consumers/single_params.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Params alias for single message consumption consumers
-    module SingleParams
-      private
-
-      # @return [Karafka::Params::Params] params instance for non batch consumption consumers
-      def params
-        params_batch.first
-      end
-    end
-  end
-end

data/lib/karafka/contracts/consumer_group_topic.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Contracts
-    # Consumer group topic validation rules
-    class ConsumerGroupTopic < Dry::Validation::Contract
-      params do
-        required(:id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
-        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
-        required(:backend).filled(included_in?: %i[inline sidekiq])
-        required(:consumer).filled
-        required(:deserializer).filled
-        required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
-        required(:start_from_beginning).filled(:bool?)
-        required(:batch_consuming).filled(:bool?)
-      end
-    end
-  end
-end

data/lib/karafka/contracts/responder_usage.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Contracts
-    # Validator to check responder topic usage
-    class ResponderUsageTopic < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
-      params do
-        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
-        required(:required).filled(:bool?)
-        required(:usage_count).filled(:int?, gteq?: 0)
-        required(:registered).filled(eql?: true)
-        required(:async).filled(:bool?)
-        required(:serializer).filled
-      end
-
-      rule(:required, :usage_count) do
-        key(:name).failure(:required_usage_count) if values[:required] && values[:usage_count] < 1
-      end
-    end
-
-    # Validator to check that everything in a responder flow matches responder rules
-    class ResponderUsage < Dry::Validation::Contract
-      include Dry::Core::Constants
-
-      # Contract for verifying the topic usage details
-      TOPIC_CONTRACT = ResponderUsageTopic.new.freeze
-
-      private_constant :TOPIC_CONTRACT
-
-      params do
-        required(:used_topics)
-        required(:registered_topics)
-      end
-
-      rule(:used_topics) do
-        (value || EMPTY_ARRAY).each do |used_topic|
-          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
-            key([:used_topics, used_topic, error.path[0]]).failure(error.text)
-          end
-        end
-      end
-
-      rule(:registered_topics) do
-        (value || EMPTY_ARRAY).each do |used_topic|
-          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
-            key([:registered_topics, used_topic, error.path[0]]).failure(error.text)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/fetcher.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Class used to run the Karafka consumer and handle shutting down, restarting etc
-  # @note Creating multiple fetchers will result in having multiple connections to the same
-  #   topics, which means that if there are no partitions, it won't use them.
-  class Fetcher
-    # Starts listening on all the listeners asynchronously
-    # Fetch loop should never end, which means that we won't create more actor clusters
-    # so we don't have to terminate them
-    def call
-      threads = listeners.map do |listener|
-        # We abort on exception because there should be an exception handling developed for
-        # each listener running in separate threads, so the exceptions should never leak
-        # and if that happens, it means that something really bad happened and we should stop
-        # the whole process
-        Thread
-          .new { listener.call }
-          .tap { |thread| thread.abort_on_exception = true }
-      end
-
-      # We aggregate threads here for a supervised shutdown process
-      threads.each { |thread| Karafka::Server.consumer_threads << thread }
-      threads.each(&:join)
-    # If anything crashes here, we need to raise the error and crush the runner because it means
-    # that something terrible happened
-    rescue StandardError => e
-      Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
-      Karafka::App.stop!
-      raise e
-    end
-
-    private
-
-    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
-    def listeners
-      @listeners ||= App.consumer_groups.active.map do |consumer_group|
-        Karafka::Connection::Listener.new(consumer_group)
-      end
-    end
-  end
-end

data/lib/karafka/helpers/class_matcher.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Helpers
-    # Class used to autodetect corresponding classes that are internally inside Karafka framework
-    # It is used among others to match:
-    #   consumer => responder
-    class ClassMatcher
-      # Regexp used to remove any non classy like characters that might be in the consumer
-      # class name (if defined dynamically, etc)
-      CONSTANT_REGEXP = %r{[?!=+\-*/\^|&\[\]<>%~\#:\s()]}.freeze
-
-      private_constant :CONSTANT_REGEXP
-
-      # @param klass [Class] class to which we want to find a corresponding class
-      # @param from [String] what type of object is it (based on postfix name part)
-      # @param to [String] what are we looking for (based on a postfix name part)
-      # @example Consumer that has a corresponding responder
-      #   matcher = Karafka::Helpers::ClassMatcher.new(SuperConsumer, 'Consumer', 'Responder')
-      #   matcher.match #=> SuperResponder
-      # @example Consumer without a corresponding responder
-      #   matcher = Karafka::Helpers::ClassMatcher.new(Super2Consumer, 'Consumer', 'Responder')
-      #   matcher.match #=> nil
-      def initialize(klass, from:, to:)
-        @klass = klass
-        @from = from
-        @to = to
-      end
-
-      # @return [Class] matched class
-      # @return [nil] nil if we couldn't find matching class
-      def match
-        return nil if name.empty?
-        return nil unless scope.const_defined?(name)
-
-        matching = scope.const_get(name)
-        same_scope?(matching) ? matching : nil
-      end
-
-      # @return [String] name of a new class that we're looking for
-      # @note This method returns name of a class without a namespace
-      # @example From SuperConsumer matching responder
-      #   matcher.name #=> 'SuperResponder'
-      # @example From Namespaced::Super2Consumer matching responder
-      #   matcher.name #=> Super2Responder
-      def name
-        inflected = +@klass.to_s.split('::').last.to_s
-        # We inject the from into the name just in case it is missing as in a situation like
-        # that it would just sanitize the name without adding the "to" postfix.
-        # It could create cases when we want to build for example a responder to a consumer
-        # that does not have the "Consumer" postfix and would do nothing returning the same name.
-        # That would be bad as the matching classes shouldn't be matched to themselves.
-        inflected << @from unless inflected.include?(@from)
-        inflected.gsub!(@from, @to)
-        inflected.gsub!(CONSTANT_REGEXP, '')
-        inflected
-      end
-
-      # @return [Class, Module] class or module in which we're looking for a matching
-      def scope
-        scope_of(@klass)
-      end
-
-      private
-
-      # @param klass [Class] class for which we want to extract it's enclosing class/module
-      # @return [Class, Module] enclosing class/module
-      # @return [::Object] object if it was a root class
-      #
-      # @example Non-namespaced class
-      #   scope_of(SuperClass) #=> Object
-      # @example Namespaced class
-      #   scope_of(Abc::SuperClass) #=> Abc
-      def scope_of(klass)
-        enclosing = klass.to_s.split('::')[0...-1]
-        return ::Object if enclosing.empty?
-
-        ::Object.const_get(enclosing.join('::'))
-      end
-
-      # @param matching [Class] class of which scope we want to check
-      # @return [Boolean] true if the scope of class is the same as scope of matching
-      def same_scope?(matching)
-        scope == scope_of(matching)
-      end
-    end
-  end
-end

data/lib/karafka/helpers/config_retriever.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Helpers
-    # A helper method that allows us to build methods that try to get a given
-    # attribute from its instance value and if it fails, will fallback to
-    # the default config or config.kafka value for a given attribute.
-    # It is used to simplify the checks.
-    # @note Worth noticing, that the value might be equal to false, so even
-    #   then we need to return it. That's why we check for nil?
-    # @example Define config retried attribute for start_from_beginning
-    # class Test
-    #   extend Karafka::Helpers::ConfigRetriever
-    #   config_retriever_for :start_from_beginning
-    # end
-    #
-    # Test.new.start_from_beginning #=> false
-    # test_instance = Test.new
-    # test_instance.start_from_beginning = true
-    # test_instance.start_from_beginning #=> true
-    module ConfigRetriever
-      # Builds proper methods for setting and retrieving (with fallback) given attribute value
-      # @param attribute [Symbol] attribute name based on which we will build
-      #   accessor with fallback
-      def config_retriever_for(attribute)
-        attr_writer attribute unless method_defined? :"#{attribute}="
-
-        # Don't redefine if we already have accessor for a given element
-        return if method_defined? attribute
-
-        define_method attribute do
-          current_value = instance_variable_get(:"@#{attribute}")
-          return current_value unless current_value.nil?
-
-          value = if Karafka::App.config.respond_to?(attribute)
-                    Karafka::App.config.send(attribute)
-                  else
-                    Karafka::App.config.kafka.send(attribute)
-                  end
-
-          instance_variable_set(:"@#{attribute}", value)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/helpers/inflector.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Helpers
-    # Inflector provides inflection for the whole Karafka framework with additional inflection
-    # caching (due to the fact, that Dry::Inflector is slow)
-    module Inflector
-      # What inflection engine do we want to use
-      ENGINE = Dry::Inflector.new
-
-      @map = Concurrent::Hash.new
-
-      private_constant :ENGINE
-
-      class << self
-        # @param string [String] string that we want to convert to our underscore format
-        # @return [String] inflected string
-        # @example
-        #   Karafka::Helpers::Inflector.map('Module/ControllerName') #=> 'module_controller_name'
-        def map(string)
-          @map[string] ||= ENGINE.underscore(string).tr('/', '_')
-        end
-      end
-    end
-  end
-end