karafka 1.4.9 → 2.0.0.alpha1

data/lib/karafka/consumers/callbacks.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Additional callbacks that can be used to trigger some actions on certain moments like
-    # manual offset management, committing or anything else outside of a standard messages flow
-    # They are not included by default, as we don't want to provide functionalities that are
-    # not required by users by default
-    # Please refer to the wiki callbacks page for more details on how to use them
-    module Callbacks
-      # Types of events on which we run callbacks
-      TYPES = %i[
-        after_fetch
-        after_poll
-        before_poll
-        before_stop
-      ].freeze
-
-      private_constant :TYPES
-
-      # Class methods needed to make callbacks run
-      module ClassMethods
-        TYPES.each do |type|
-          # Creates a callback wrapper
-          #
-          # @param method_name [Symbol, String] method name or nil if we plan to provide a block
-          # @yield A block with a code that should be executed before scheduling
-          # @note We don't have to optimize the key fetching here as those are class methods that
-          #   are evaluated once upon start
-          define_method(type) do |method_name = nil, &block|
-            key = "consumers.#{Helpers::Inflector.map(to_s)}.#{type}"
-            Karafka::App.monitor.register_event(key)
-
-            Karafka::App.monitor.subscribe(key) do |event|
-              context = event[:context]
-
-              if method_name
-                context.send(method_name)
-              else
-                context.instance_eval(&block)
-              end
-            end
-          end
-        end
-      end
-
-      class << self
-        # @param consumer_class [Class] consumer class that we extend with callbacks
-        def included(consumer_class)
-          consumer_class.class_eval do
-            extend ClassMethods
-          end
-        end
-      end
-
-      # Executes the default consumer flow, runs callbacks and if not halted will call process
-      # method of a proper backend. It is here because it interacts with the default Karafka
-      # call flow and needs to be overwritten to support callbacks
-      def call
-        if self.class.respond_to?(:after_fetch)
-          Karafka::App.monitor.instrument(
-            "consumers.#{Helpers::Inflector.map(self.class.to_s)}.after_fetch",
-            context: self
-          )
-        end
-
-        process
-      end
-    end
-  end
-end

data/lib/karafka/consumers/includer.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Additional functionalities for consumers
-  module Consumers
-    # Module used to inject functionalities into a given consumer instance, based on the consumer
-    # topic and its settings
-    # We don't need all the behaviors in all the cases, so it is not worth having everything
-    # in all the cases all the time
-    module Includer
-      class << self
-        # @param consumer [Karafka::BaseConsumer] consumer instance, that will get some
-        #   functionalities based on the topic under which it operates
-        def call(consumer)
-          topic = consumer.topic
-
-          bind_backend(consumer, topic)
-          bind_params(consumer, topic)
-          bind_batch_metadata(consumer, topic)
-          bind_responders(consumer, topic)
-        end
-
-        private
-
-        # Figures out backend for a given consumer class, based on the topic backend and
-        #   includes it into the consumer class
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_backend(consumer, topic)
-          backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
-          consumer.extend(backend)
-        end
-
-        # Adds a single #params support for non batch processed topics
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_params(consumer, topic)
-          return if topic.batch_consuming
-
-          consumer.extend(SingleParams)
-        end
-
-        # Adds an option to work with batch metadata for consumer instances that have
-        #   batch fetching enabled
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_batch_metadata(consumer, topic)
-          return unless topic.batch_fetching
-
-          consumer.extend(BatchMetadata)
-        end
-
-        # Adds responders support for topics and consumers with responders defined for them
-        # @param consumer [Karafka::BaseConsumer] consumer instance
-        # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_responders(consumer, topic)
-          return unless topic.responder
-
-          consumer.extend(Responders)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/consumers/responders.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Feature that allows us to use responders flow in consumer
-    module Responders
-      # Responds with given data using given responder. This allows us to have a similar way of
-      # defining flows like synchronous protocols
-      # @param data Anything we want to pass to responder based on which we want to trigger further
-      #   Kafka responding
-      def respond_with(*data)
-        Karafka.monitor.instrument(
-          'consumers.responders.respond_with',
-          caller: self,
-          data: data
-        ) do
-          # @note we build a new instance of responder each time, as a long-running (persisted)
-          #   consumers can respond multiple times during the life-cycle
-          topic.responder.new.call(*data)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/consumers/single_params.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Consumers
-    # Params alias for single message consumption consumers
-    module SingleParams
-      private
-
-      # @return [Karafka::Params::Params] params instance for non batch consumption consumers
-      def params
-        params_batch.first
-      end
-    end
-  end
-end

data/lib/karafka/contracts/responder_usage.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Contracts
-    # Validator to check responder topic usage
-    class ResponderUsageTopic < Dry::Validation::Contract
-      config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
-
-      params do
-        required(:name).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
-        required(:required).filled(:bool?)
-        required(:usage_count).filled(:int?, gteq?: 0)
-        required(:registered).filled(eql?: true)
-        required(:async).filled(:bool?)
-        required(:serializer).filled
-      end
-
-      rule(:required, :usage_count) do
-        key(:name).failure(:required_usage_count) if values[:required] && values[:usage_count] < 1
-      end
-    end
-
-    # Validator to check that everything in a responder flow matches responder rules
-    class ResponderUsage < Dry::Validation::Contract
-      include Dry::Core::Constants
-
-      # Contract for verifying the topic usage details
-      TOPIC_CONTRACT = ResponderUsageTopic.new.freeze
-
-      private_constant :TOPIC_CONTRACT
-
-      params do
-        required(:used_topics)
-        required(:registered_topics)
-      end
-
-      rule(:used_topics) do
-        (value || EMPTY_ARRAY).each do |used_topic|
-          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
-            key([:used_topics, used_topic, error.path[0]]).failure(error.text)
-          end
-        end
-      end
-
-      rule(:registered_topics) do
-        (value || EMPTY_ARRAY).each do |used_topic|
-          TOPIC_CONTRACT.call(used_topic).errors.each do |error|
-            key([:registered_topics, used_topic, error.path[0]]).failure(error.text)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/fetcher.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Class used to run the Karafka consumer and handle shutting down, restarting etc
-  # @note Creating multiple fetchers will result in having multiple connections to the same
-  #   topics, which means that if there are no partitions, it won't use them.
-  class Fetcher
-    # Starts listening on all the listeners asynchronously
-    # Fetch loop should never end, which means that we won't create more actor clusters
-    # so we don't have to terminate them
-    def call
-      threads = listeners.map do |listener|
-        # We abort on exception because there should be an exception handling developed for
-        # each listener running in separate threads, so the exceptions should never leak
-        # and if that happens, it means that something really bad happened and we should stop
-        # the whole process
-        Thread
-          .new { listener.call }
-          .tap { |thread| thread.abort_on_exception = true }
-      end
-
-      # We aggregate threads here for a supervised shutdown process
-      threads.each { |thread| Karafka::Server.consumer_threads << thread }
-      threads.each(&:join)
-    # If anything crashes here, we need to raise the error and crush the runner because it means
-    # that something terrible happened
-    rescue StandardError => e
-      Karafka.monitor.instrument('fetcher.call.error', caller: self, error: e)
-      Karafka::App.stop!
-      raise e
-    end
-
-    private
-
-    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages
-    def listeners
-      @listeners ||= App.consumer_groups.active.map do |consumer_group|
-        Karafka::Connection::Listener.new(consumer_group)
-      end
-    end
-  end
-end

data/lib/karafka/helpers/class_matcher.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Helpers
-    # Class used to autodetect corresponding classes that are internally inside Karafka framework
-    # It is used among others to match:
-    #   consumer => responder
-    class ClassMatcher
-      # Regexp used to remove any non classy like characters that might be in the consumer
-      # class name (if defined dynamically, etc)
-      CONSTANT_REGEXP = %r{[?!=+\-*/\^|&\[\]<>%~\#:\s()]}.freeze
-
-      private_constant :CONSTANT_REGEXP
-
-      # @param klass [Class] class to which we want to find a corresponding class
-      # @param from [String] what type of object is it (based on postfix name part)
-      # @param to [String] what are we looking for (based on a postfix name part)
-      # @example Consumer that has a corresponding responder
-      #   matcher = Karafka::Helpers::ClassMatcher.new(SuperConsumer, 'Consumer', 'Responder')
-      #   matcher.match #=> SuperResponder
-      # @example Consumer without a corresponding responder
-      #   matcher = Karafka::Helpers::ClassMatcher.new(Super2Consumer, 'Consumer', 'Responder')
-      #   matcher.match #=> nil
-      def initialize(klass, from:, to:)
-        @klass = klass
-        @from = from
-        @to = to
-      end
-
-      # @return [Class] matched class
-      # @return [nil] nil if we couldn't find matching class
-      def match
-        return nil if name.empty?
-        return nil unless scope.const_defined?(name)
-
-        matching = scope.const_get(name)
-        same_scope?(matching) ? matching : nil
-      end
-
-      # @return [String] name of a new class that we're looking for
-      # @note This method returns name of a class without a namespace
-      # @example From SuperConsumer matching responder
-      #   matcher.name #=> 'SuperResponder'
-      # @example From Namespaced::Super2Consumer matching responder
-      #   matcher.name #=> Super2Responder
-      def name
-        inflected = +@klass.to_s.split('::').last.to_s
-        # We inject the from into the name just in case it is missing as in a situation like
-        # that it would just sanitize the name without adding the "to" postfix.
-        # It could create cases when we want to build for example a responder to a consumer
-        # that does not have the "Consumer" postfix and would do nothing returning the same name.
-        # That would be bad as the matching classes shouldn't be matched to themselves.
-        inflected << @from unless inflected.include?(@from)
-        inflected.gsub!(@from, @to)
-        inflected.gsub!(CONSTANT_REGEXP, '')
-        inflected
-      end
-
-      # @return [Class, Module] class or module in which we're looking for a matching
-      def scope
-        scope_of(@klass)
-      end
-
-      private
-
-      # @param klass [Class] class for which we want to extract it's enclosing class/module
-      # @return [Class, Module] enclosing class/module
-      # @return [::Object] object if it was a root class
-      #
-      # @example Non-namespaced class
-      #   scope_of(SuperClass) #=> Object
-      # @example Namespaced class
-      #   scope_of(Abc::SuperClass) #=> Abc
-      def scope_of(klass)
-        enclosing = klass.to_s.split('::')[0...-1]
-        return ::Object if enclosing.empty?
-
-        ::Object.const_get(enclosing.join('::'))
-      end
-
-      # @param matching [Class] class of which scope we want to check
-      # @return [Boolean] true if the scope of class is the same as scope of matching
-      def same_scope?(matching)
-        scope == scope_of(matching)
-      end
-    end
-  end
-end

data/lib/karafka/helpers/config_retriever.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Helpers
-    # A helper method that allows us to build methods that try to get a given
-    # attribute from its instance value and if it fails, will fallback to
-    # the default config or config.kafka value for a given attribute.
-    # It is used to simplify the checks.
-    # @note Worth noticing, that the value might be equal to false, so even
-    #   then we need to return it. That's why we check for nil?
-    # @example Define config retried attribute for start_from_beginning
-    # class Test
-    #   extend Karafka::Helpers::ConfigRetriever
-    #   config_retriever_for :start_from_beginning
-    # end
-    #
-    # Test.new.start_from_beginning #=> false
-    # test_instance = Test.new
-    # test_instance.start_from_beginning = true
-    # test_instance.start_from_beginning #=> true
-    module ConfigRetriever
-      # Builds proper methods for setting and retrieving (with fallback) given attribute value
-      # @param attribute [Symbol] attribute name based on which we will build
-      #   accessor with fallback
-      def config_retriever_for(attribute)
-        attr_writer attribute unless method_defined? :"#{attribute}="
-
-        # Don't redefine if we already have accessor for a given element
-        return if method_defined? attribute
-
-        define_method attribute do
-          current_value = instance_variable_get(:"@#{attribute}")
-          return current_value unless current_value.nil?
-
-          value = if Karafka::App.config.respond_to?(attribute)
-                    Karafka::App.config.send(attribute)
-                  else
-                    Karafka::App.config.kafka.send(attribute)
-                  end
-
-          instance_variable_set(:"@#{attribute}", value)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/helpers/inflector.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Helpers
-    # Inflector provides inflection for the whole Karafka framework with additional inflection
-    # caching (due to the fact, that Dry::Inflector is slow)
-    module Inflector
-      # What inflection engine do we want to use
-      ENGINE = Dry::Inflector.new
-
-      @map = Concurrent::Hash.new
-
-      private_constant :ENGINE
-
-      class << self
-        # @param string [String] string that we want to convert to our underscore format
-        # @return [String] inflected string
-        # @example
-        #   Karafka::Helpers::Inflector.map('Module/ControllerName') #=> 'module_controller_name'
-        def map(string)
-          @map[string] ||= ENGINE.underscore(string).tr('/', '_')
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/builders/batch_metadata.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    module Builders
-      # Builder for creating batch metadata object based on the batch informations
-      module BatchMetadata
-        class << self
-          # Creates metadata based on the kafka batch data
-          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
-          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
-          # @return [Karafka::Params::BatchMetadata] batch metadata object
-          def from_kafka_batch(kafka_batch, topic)
-            Karafka::Params::BatchMetadata.new(
-              batch_size: kafka_batch.messages.count,
-              first_offset: kafka_batch.first_offset,
-              highwater_mark_offset: kafka_batch.highwater_mark_offset,
-              unknown_last_offset: kafka_batch.unknown_last_offset?,
-              last_offset: kafka_batch.last_offset,
-              offset_lag: kafka_batch.offset_lag,
-              deserializer: topic.deserializer,
-              partition: kafka_batch.partition,
-              topic: topic.name
-            ).freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/builders/params.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Due to the fact, that we create params related objects in couple contexts / places
-    # plus backends can build up them their own way we have this namespace.
-    # It allows to isolate actual params objects from their building process that can be
-    # context dependent.
-    module Builders
-      # Builder for params
-      module Params
-        class << self
-          # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
-          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
-          # @return [Karafka::Params::Params] params object with payload and message metadata
-          def from_kafka_message(kafka_message, topic)
-            metadata = Karafka::Params::Metadata.new(
-              create_time: kafka_message.create_time,
-              headers: kafka_message.headers || {},
-              is_control_record: kafka_message.is_control_record,
-              key: kafka_message.key,
-              offset: kafka_message.offset,
-              deserializer: topic.deserializer,
-              partition: kafka_message.partition,
-              receive_time: Time.now,
-              topic: topic.name
-            ).freeze
-
-            Karafka::Params::Params.new(
-              kafka_message.value,
-              metadata
-            )
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/builders/params_batch.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    module Builders
-      # Builder for creating params batch instances
-      module ParamsBatch
-        class << self
-          # Creates params batch with params inside based on the incoming messages
-          # and the topic from which it comes
-          # @param kafka_messages [Array<Kafka::FetchedMessage>] raw fetched messages
-          # @param topic [Karafka::Routing::Topic] topic for which we're received messages
-          # @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
-          def from_kafka_messages(kafka_messages, topic)
-            params_array = kafka_messages.map do |message|
-              Karafka::Params::Builders::Params.from_kafka_message(message, topic)
-            end
-
-            Karafka::Params::ParamsBatch.new(params_array).freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/params_batch.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Params batch represents a set of messages received from Kafka.
-    # @note Params internally are lazy loaded before first use. That way we can skip
-    #   deserialization process if we have after_fetch that rejects some incoming messages
-    #   without using params It can be also used when handling really heavy data.
-    class ParamsBatch
-      include Enumerable
-
-      # @param params_array [Array<Karafka::Params::Params>] array with karafka params
-      # @return [Karafka::Params::ParamsBatch] lazy evaluated params batch object
-      def initialize(params_array)
-        @params_array = params_array
-      end
-
-      # @yieldparam [Karafka::Params::Params] each params instance
-      # @note Invocation of this method will not cause loading and deserializing each param after
-      #   another.
-      def each
-        @params_array.each { |param| yield(param) }
-      end
-
-      # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
-      #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
-      #   be deserialized
-      def deserialize!
-        each(&:payload)
-      end
-
-      # @return [Array<Object>] array with deserialized payloads. This method can be useful when
-      #   we don't care about metadata and just want to extract all the data payloads from the
-      #   batch
-      def payloads
-        map(&:payload)
-      end
-
-      # @return [Karafka::Params::Params] first element
-      def first
-        @params_array.first
-      end
-
-      # @return [Karafka::Params::Params] last element
-      def last
-        @params_array.last
-      end
-
-      # @return [Integer] number of messages in the batch
-      def size
-        @params_array.size
-      end
-
-      # @return [Array<Karafka::Params::Params>] pure array with params
-      def to_a
-        @params_array
-      end
-    end
-  end
-end

data/lib/karafka/patches/ruby_kafka.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for various other libs patches
-  module Patches
-    # Patches for Ruby Kafka gem
-    module RubyKafka
-      # This patch allows us to inject business logic in between fetches and before the consumer
-      # stop, so we can perform stop commit or anything else that we need since
-      # ruby-kafka fetch loop does not allow that directly
-      # We don't won't to use poll ruby-kafka api as it brings many more problems that we would
-      # have to take care of. That way, nothing like that ever happens but we get the control
-      # over the stopping process that we need (since we're the once that initiate it for each
-      # thread)
-      def consumer_loop
-        super do
-          consumers = Karafka::Persistence::Consumers
-                      .current
-                      .values
-                      .flat_map(&:values)
-                      .select { |consumer| consumer.class.respond_to?(:after_fetch) }
-
-          if Karafka::App.stopping?
-            publish_event(consumers, 'before_stop')
-            Karafka::Persistence::Client.read.stop
-          else
-            publish_event(consumers, 'before_poll')
-            yield
-            publish_event(consumers, 'after_poll')
-          end
-        end
-      end
-
-      private
-
-      # Notifies consumers about particular events happening
-      # @param consumers [Array<Object>] all consumers that want to be notified about an event
-      # @param event_name [String] name of the event that happened
-      def publish_event(consumers, event_name)
-        consumers.each do |consumer|
-          key = "consumers.#{Helpers::Inflector.map(consumer.class.to_s)}.#{event_name}"
-          Karafka::App.monitor.instrument(key, context: consumer)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/persistence/client.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Persistence
-    # Persistence layer to store current thread messages consumer client for further use
-    class Client
-      # Thread.current key under which we store current thread messages consumer client
-      PERSISTENCE_SCOPE = :client
-
-      private_constant :PERSISTENCE_SCOPE
-
-      class << self
-        # @param client [Karafka::Connection::Client] messages consumer client of
-        #   a current thread
-        # @return [Karafka::Connection::Client] persisted messages consumer client
-        def write(client)
-          Thread.current[PERSISTENCE_SCOPE] = client
-        end
-
-        # @return [Karafka::Connection::Client] persisted messages consumer client
-        # @raise [Karafka::Errors::MissingClientError] raised when no thread messages consumer
-        #   client but we try to use it anyway
-        def read
-          Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClientError)
-        end
-      end
-    end
-  end
-end