karafka 1.4.0 → 2.0.10

data/lib/karafka/attributes_map.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Both Karafka and Ruby-Kafka contain a lot of settings that can be applied on multiple
-  # levels. In Karafka that is on consumer group and on the topic level. In Ruby-Kafka it
-  # is on consumer, subscription and consumption levels. In order to maintain an order
-  # in managing those settings, this module was created. It contains details on what setting
-  # where should go and which layer (both on Karafka and Ruby-Kafka) is responsible for
-  # setting it and sending it forward
-  # @note Settings presented here cover all the settings that are being used across Karafka
-  module AttributesMap
-    class << self
-      # What settings should go where in ruby-kafka
-      # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
-      # @note All other settings will be passed to Kafka.new method invocation.
-      #   All elements in this hash are just edge cases
-      def api_adapter
-        {
-          consumer: %i[
-            session_timeout offset_commit_interval offset_commit_threshold
-            offset_retention_time heartbeat_interval fetcher_max_queue_size
-          ],
-          subscribe: %i[start_from_beginning max_bytes_per_partition],
-          consumption: %i[min_bytes max_bytes max_wait_time],
-          pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
-          # All the options that are under kafka config namespace, but are not used
-          # directly with kafka api, but from the Karafka user perspective, they are
-          # still related to kafka. They should not be proxied anywhere
-          ignored: %i[reconnect_timeout automatically_mark_as_consumed]
-        }
-      end
-
-      # @return [Array<Symbol>] properties that can be set on a per topic level
-      def topic
-        (api_adapter[:subscribe] + %i[
-          backend
-          name
-          deserializer
-          responder
-          batch_consuming
-        ]).uniq
-      end
-
-      # @return [Array<Symbol>] properties that can be set on a per consumer group level
-      # @note Note that there are settings directly extracted from the config kafka namespace
-      #   I did this that way, so I won't have to repeat same setting keys over and over again
-      #   Thanks to this solution, if any new setting is available for ruby-kafka, we just need
-      #   to add it to our configuration class and it will be handled automatically.
-      def consumer_group
-        # @note We don't ignore the api_adapter[:ignored] values as they should be ignored
-        #   only when proxying details go ruby-kafka. We use ignored fields internally in karafka
-        ignored_settings = api_adapter[:subscribe]
-        defined_settings = api_adapter.values.flatten
-        karafka_settings = %i[batch_fetching]
-
-        dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
-
-        (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
-      end
-    end
-  end
-end

data/lib/karafka/backends/inline.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for all different backends Karafka supports
-  module Backends
-    # Backend that just runs stuff asap without any scheduling
-    module Inline
-      private
-
-      # Executes consume code immediately (without enqueuing)
-      def process
-        Karafka.monitor.instrument('backends.inline.process', caller: self) { consume }
-      end
-    end
-  end
-end

data/lib/karafka/base_responder.rb

@@ -1,226 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Base responder from which all Karafka responders should inherit
-  # Similar to Rails responders concept. It allows us to design flow from one app to another
-  # by isolating what responses should be sent (and where) based on a given action
-  # It differs from Rails responders in the way it works: in std http request we can have one
-  # response, here we can have unlimited number of them
-  #
-  # It has a simple API for defining where should we respond (and if it is required)
-  #
-  # @example Basic usage (each registered topic is required to be used by default)
-  #   class Responder < BaseResponder
-  #     topic :new_action
-  #
-  #     def respond(data)
-  #       respond_to :new_action, data
-  #     end
-  #   end
-  #
-  # @example Responding to a topic with extra options
-  #   class Responder < BaseResponder
-  #     topic :new_action
-  #
-  #     def respond(data)
-  #       respond_to :new_action, data, partition_key: 'thing'
-  #     end
-  #   end
-  #
-  # @example Marking topic as not required (we won't have to use it)
-  #   class Responder < BaseResponder
-  #     topic :required_topic
-  #     topic :new_action, required: false
-  #
-  #     def respond(data)
-  #       respond_to :required_topic, data
-  #     end
-  #   end
-  #
-  # @example Multiple times used topic
-  #   class Responder < BaseResponder
-  #     topic :required_topic
-  #
-  #     def respond(data)
-  #       data.each do |subset|
-  #         respond_to :required_topic, subset
-  #       end
-  #     end
-  #   end
-  #
-  # @example Specify serializer for a topic
-  #   class Responder < BaseResponder
-  #     topic :xml_topic, serializer: MyXMLSerializer
-  #
-  #     def respond(data)
-  #       data.each do |subset|
-  #         respond_to :xml_topic, subset
-  #       end
-  #     end
-  #   end
-  #
-  # @example Accept multiple arguments to a respond method
-  #   class Responder < BaseResponder
-  #     topic :users_actions
-  #     topic :articles_viewed
-  #
-  #     def respond(user, article)
-  #       respond_to :users_actions, user
-  #       respond_to :articles_viewed, article
-  #     end
-  #   end
-  class BaseResponder
-    # Responder usage contract
-    CONTRACT = Karafka::Contracts::ResponderUsage.new.freeze
-
-    private_constant :CONTRACT
-
-    class << self
-      # Definitions of all topics that we want to be able to use in this responder should go here
-      attr_accessor :topics
-      # Contract that we can use to control and/or require some additional details upon options
-      # that are being passed to the producer. This can be in particular useful if we want to make
-      # sure that for example partition_key is always present.
-      attr_accessor :options_contract
-
-      # Registers a topic as on to which we will be able to respond
-      # @param topic_name [Symbol, String] name of topic to which we want to respond
-      # @param options [Hash] hash with optional configuration details
-      def topic(topic_name, options = {})
-        options[:serializer] ||= Karafka::App.config.serializer
-        options[:registered] = true
-        self.topics ||= {}
-        topic_obj = Responders::Topic.new(topic_name, options)
-        self.topics[topic_obj.name] = topic_obj
-      end
-
-      # A simple alias for easier standalone responder usage.
-      # Instead of building it with new.call it allows (in case of using JSON serializer)
-      # to just run it directly from the class level
-      # @param data Anything that we want to respond with
-      # @example Send user data with a responder
-      #   UsersCreatedResponder.call(@created_user)
-      def call(*data)
-        # Just in case there were no topics defined for a responder, we initialize with
-        # empty hash not to handle a nil case
-        self.topics ||= {}
-        new.call(*data)
-      end
-    end
-
-    attr_reader :messages_buffer
-
-    # Creates a responder object
-    # @return [Karafka::BaseResponder] base responder descendant responder
-    def initialize
-      @messages_buffer = {}
-    end
-
-    # Performs respond and validates that all the response requirement were met
-    # @param data Anything that we want to respond with
-    # @note We know that validators should be executed also before sending data to topics, however
-    #   the implementation gets way more complicated then, that's why we check after everything
-    #   was sent using responder
-    # @example Send user data with a responder
-    #   UsersCreatedResponder.new.call(@created_user)
-    # @example Send user data with a responder using non default Parser
-    #   UsersCreatedResponder.new(MyParser).call(@created_user)
-    def call(*data)
-      respond(*data)
-      validate_usage!
-      validate_options!
-      deliver!
-    end
-
-    private
-
-    # Checks if we met all the topics requirements. It will fail if we didn't send a message to
-    # a registered required topic, etc.
-    def validate_usage!
-      registered_topics = self.class.topics.map do |name, topic|
-        topic.to_h.merge!(
-          usage_count: messages_buffer[name]&.count || 0
-        )
-      end
-
-      used_topics = messages_buffer.map do |name, usage|
-        topic = self.class.topics[name] || Responders::Topic.new(name, registered: false)
-        topic.to_h.merge!(usage_count: usage.count)
-      end
-
-      result = CONTRACT.call(
-        registered_topics: registered_topics,
-        used_topics: used_topics
-      )
-
-      return if result.success?
-
-      raise Karafka::Errors::InvalidResponderUsageError, result.errors.to_h
-    end
-
-    # Checks if we met all the options requirements before sending them to the producer.
-    def validate_options!
-      return true unless self.class.options_contract
-
-      messages_buffer.each_value do |messages_set|
-        messages_set.each do |message_data|
-          result = self.class.options_contract.call(message_data.last)
-          next if result.success?
-
-          raise Karafka::Errors::InvalidResponderMessageOptionsError, result.errors.to_h
-        end
-      end
-    end
-
-    # Takes all the messages from the buffer and delivers them one by one
-    # @note This method is executed after the validation, so we're sure that
-    #   what we send is legit and it will go to a proper topics
-    def deliver!
-      messages_buffer.each_value do |data_elements|
-        data_elements.each do |data, options|
-          # We map this topic name, so it will match namespaced/etc topic in Kafka
-          # @note By default will not change topic (if default mapper used)
-          mapped_topic = Karafka::App.config.topic_mapper.outgoing(options[:topic])
-          external_options = options.merge(topic: mapped_topic)
-          producer(options).call(data, external_options)
-        end
-      end
-    end
-
-    # Method that needs to be implemented in a subclass. It should handle responding
-    #   on registered topics
-    # @param _data [Object] anything that we want to use to send to Kafka
-    # @raise [NotImplementedError] This method needs to be implemented in a subclass
-    def respond(*_data)
-      raise NotImplementedError, 'Implement this in a subclass'
-    end
-
-    # This method allow us to respond to a single topic with a given data. It can be used
-    # as many times as we need. Especially when we have 1:n flow
-    # @param topic [Symbol, String] topic to which we want to respond
-    # @param data [String, Object] string or object that we want to send
-    # @param options [Hash] options for waterdrop (e.g. partition_key).
-    # @note Respond to does not accept multiple data arguments.
-    def respond_to(topic, data, options = {})
-      # We normalize the format to string, as WaterDrop and Ruby-Kafka support only
-      # string topics
-      topic = topic.to_s
-
-      messages_buffer[topic] ||= []
-      messages_buffer[topic] << [
-        self.class.topics[topic].serializer.call(data),
-        options.merge(topic: topic)
-      ]
-    end
-
-    # @param options [Hash] options for waterdrop
-    # @return [Class] WaterDrop producer (sync or async based on the settings)
-    def producer(options)
-      if self.class.topics[options[:topic]].async?
-        WaterDrop::AsyncProducer
-      else
-        WaterDrop::SyncProducer
-      end
-    end
-  end
-end

data/lib/karafka/cli/flow.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Karafka framework Cli
-  class Cli < Thor
-    # Description of topics flow (incoming/outgoing)
-    class Flow < Base
-      desc 'Print application data flow (incoming => outgoing)'
-
-      # Print out all defined routes in alphabetical order
-      def call
-        topics.each do |topic|
-          any_topics = !topic.responder&.topics.nil?
-          log_messages = []
-
-          if any_topics
-            log_messages << "#{topic.name} =>"
-
-            topic.responder.topics.each_value do |responder_topic|
-              features = []
-              features << (responder_topic.required? ? 'always' : 'conditionally')
-
-              log_messages << format(responder_topic.name, "(#{features.join(', ')})")
-            end
-          else
-            log_messages << "#{topic.name} => (nothing)"
-          end
-
-          Karafka.logger.info(log_messages.join("\n"))
-        end
-      end
-
-      private
-
-      # @return [Array<Karafka::Routing::Topic>] all topics sorted in alphabetical order
-      def topics
-        Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
-      end
-
-      # Formats a given value with label in a nice way
-      # @param label [String] label describing value
-      # @param value [String] value that should be printed
-      def format(label, value)
-        "  - #{label}:                #{value}"
-      end
-    end
-  end
-end

data/lib/karafka/code_reloader.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Special type of a listener, that is not an instrumentation one, but one that triggers
-  # code reload in the development mode after each fetched batch (or message)
-  #
-  # Please refer to the development code reload sections for details on the benefits and downsides
-  # of the in-process code reloading
-  class CodeReloader
-    # This mutex is needed as we might have an application that has multiple consumer groups
-    # running in separate threads and we should not trigger reload before fully reloading the app
-    # in previous thread
-    MUTEX = Mutex.new
-
-    private_constant :MUTEX
-
-    # @param reloaders [Array<Object>] any code loaders that we use in this app. Whether it is
-    #  the Rails loader, Zeitwerk or anything else that allows reloading triggering
-    # @param block [Proc] yields given block just before reloading. This can be used to hook custom
-    #   reloading stuff, that ain't reloaders (for example for resetting dry-events registry)
-    def initialize(*reloaders, &block)
-      @reloaders = reloaders
-      @block = block
-    end
-
-    # Binds to the instrumentation events and triggers reload
-    # @param _event [Dry::Event] empty dry event
-    # @note Since we de-register all the user defined objects and redraw routes, it means that
-    #   we won't be able to do a multi-batch buffering in the development mode as each of the
-    #   batches will be buffered on a newly created "per fetch" instance.
-    def on_connection_listener_fetch_loop(_event)
-      reload
-    end
-
-    private
-
-    # Triggers reload of both standard and Rails reloaders as well as expires all internals of
-    # Karafka, so it can be rediscovered and rebuilt
-    def reload
-      MUTEX.synchronize do
-        if @reloaders[0].respond_to?(:execute)
-          reload_with_rails
-        else
-          reload_without_rails
-        end
-      end
-    end
-
-    # Rails reloading procedure
-    def reload_with_rails
-      updatable = @reloaders.select(&:updated?)
-
-      return if updatable.empty?
-
-      updatable.each(&:execute)
-      @block&.call
-      Karafka::App.reload
-    end
-
-    # Zeitwerk and other reloaders
-    def reload_without_rails
-      @reloaders.each(&:reload)
-      @block&.call
-      Karafka::App.reload
-    end
-  end
-end

data/lib/karafka/connection/api_adapter.rb

@@ -1,161 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for all the things related to Kafka connection
-  module Connection
-    # Mapper used to convert our internal settings into ruby-kafka settings based on their
-    # API requirements.
-    # Since ruby-kafka has more and more options and there are few "levels" on which
-    # we have to apply them (despite the fact, that in Karafka you configure all of it
-    # in one place), we have to remap it into what ruby-kafka driver requires
-    # @note The good thing about Kafka.new method is that it ignores all options that
-    #   do nothing. So we don't have to worry about injecting our internal settings
-    #   into the client and breaking stuff
-    module ApiAdapter
-      class << self
-        # Builds all the configuration settings for Kafka.new method
-        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array<Hash>] Array with all the client arguments including hash with all
-        #   the settings required by Kafka.new method
-        # @note We return array, so we can inject any arguments we want, in case of changes in the
-        #   raw driver
-        def client(consumer_group)
-          # This one is a default that takes all the settings except special
-          # cases defined in the map
-          settings = {
-            logger: ::Karafka.logger,
-            client_id: ::Karafka::App.config.client_id
-          }
-
-          kafka_configs.each_key do |setting_name|
-            # All options for config adapter should be ignored as we're just interested
-            # in what is left, as we want to pass all the options that are "typical"
-            # and not listed in the api_adapter special cases mapping. All the values
-            # from the api_adapter mapping go somewhere else, not to the client directly
-            next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
-
-            # Settings for each consumer group are either defined per consumer group or are
-            # inherited from the global/general settings level, thus we don't have to fetch them
-            # from the kafka settings as they are already on a consumer group level
-            settings[setting_name] = consumer_group.public_send(setting_name)
-          end
-
-          settings_hash = sanitize(settings)
-
-          # Normalization for the way Kafka::Client accepts arguments from  0.5.3
-          [settings_hash.delete(:seed_brokers), settings_hash]
-        end
-
-        # Builds all the configuration settings for kafka#consumer method
-        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array<Hash>] array with all the consumer arguments including hash with all
-        #   the settings required by Kafka#consumer
-        def consumer(consumer_group)
-          settings = { group_id: consumer_group.id }
-          settings = fetch_for(:consumer, consumer_group, settings)
-          [sanitize(settings)]
-        end
-
-        # Builds all the configuration settings for kafka consumer consume_each_batch and
-        #   consume_each_message methods
-        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array<Hash>] Array with all the arguments required by consuming method
-        #   including hash with all the settings required by
-        #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
-        def consumption(consumer_group)
-          [
-            sanitize(
-              fetch_for(
-                :consumption,
-                consumer_group,
-                automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
-              )
-            )
-          ]
-        end
-
-        # Builds all the configuration settings for kafka consumer#subscribe method
-        # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
-        # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
-        def subscribe(topic)
-          settings = fetch_for(:subscribe, topic)
-          [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
-        end
-
-        # Builds all the configuration settings required by kafka consumer#pause method
-        # @param topic [String] topic that we want to pause
-        # @param partition [Integer] number partition that we want to pause
-        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
-        # @return [Array] array with all the details required to pause kafka consumer
-        def pause(topic, partition, consumer_group)
-          [
-            Karafka::App.config.topic_mapper.outgoing(topic),
-            partition,
-            {
-              timeout: consumer_group.pause_timeout,
-              max_timeout: consumer_group.pause_max_timeout,
-              exponential_backoff: consumer_group.pause_exponential_backoff
-            }
-          ]
-        end
-
-        # Remaps topic details taking the topic mapper feature into consideration.
-        # @param params [Karafka::Params::Params] params instance
-        # @return [Array] array with all the details needed by ruby-kafka to mark message
-        #   as processed
-        # @note When default empty topic mapper is used, no need for any conversion as the
-        #   internal and external format are exactly the same
-        def mark_message_as_processed(params)
-          # Majority of users don't use custom topic mappers. No need to change anything when it
-          # is a default mapper that does not change anything. Only some cloud providers require
-          # topics to be remapped
-          return [params.metadata] if Karafka::App.config.topic_mapper.is_a?(
-            Karafka::Routing::TopicMapper
-          )
-
-          # @note We don't use tap as it is around 13% slower than non-dup version
-          dupped = params.metadata.dup
-          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.metadata.topic)
-          [dupped]
-        end
-
-        private
-
-        # Fetches proper settings for a given map namespace
-        # @param namespace_key [Symbol] namespace from attributes map config adapter hash
-        # @param route_layer [Object] route topic or consumer group
-        # @param preexisting_settings [Hash] hash with some preexisting settings that might have
-        #   been loaded in a different way
-        def fetch_for(namespace_key, route_layer, preexisting_settings = {})
-          kafka_configs.each_key do |setting_name|
-            # Ignore settings that are not related to our namespace
-            next unless AttributesMap.api_adapter[namespace_key].include?(setting_name)
-
-            # Ignore settings that are already initialized
-            # In case they are in preexisting settings fetched differently
-            next if preexisting_settings.key?(setting_name)
-
-            # Fetch all the settings from a given layer object. Objects can handle the fallback
-            # to the kafka settings, so
-            preexisting_settings[setting_name] = route_layer.send(setting_name)
-          end
-
-          preexisting_settings
-        end
-
-        # Removes nil containing keys from the final settings so it can use Kafkas driver
-        #   defaults for those
-        # @param settings [Hash] settings that may contain nil values
-        # @return [Hash] settings without nil using keys (non of karafka options should be nil)
-        def sanitize(settings)
-          settings.reject { |_key, value| value.nil? }
-        end
-
-        # @return [Hash] Kafka config details as a hash
-        def kafka_configs
-          ::Karafka::App.config.kafka.to_h
-        end
-      end
-    end
-  end
-end

data/lib/karafka/connection/batch_delegator.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class that delegates processing of batch received messages for which we listen to
-    # a proper processor
-    module BatchDelegator
-      class << self
-        # Delegates messages (does something with them)
-        # It will either schedule or run a proper processor action for messages
-        # @param group_id [String] group_id of a group from which a given message came
-        # @param kafka_batch [<Kafka::FetchedBatch>] raw messages fetched batch
-        # @note This should be looped to obtain a constant delegating of new messages
-        def call(group_id, kafka_batch)
-          topic = Persistence::Topics.fetch(group_id, kafka_batch.topic)
-          consumer = Persistence::Consumers.fetch(topic, kafka_batch.partition)
-
-          Karafka.monitor.instrument(
-            'connection.batch_delegator.call',
-            caller: self,
-            consumer: consumer,
-            kafka_batch: kafka_batch
-          ) do
-            # Due to how ruby-kafka is built, we have the metadata that is stored on the batch
-            # level only available for batch consuming
-            consumer.batch_metadata = Params::Builders::BatchMetadata.from_kafka_batch(
-              kafka_batch,
-              topic
-            )
-
-            kafka_messages = kafka_batch.messages
-
-            # Depending on a case (persisted or not) we might use new consumer instance per
-            # each batch, or use the same one for all of them (for implementing buffering, etc.)
-            if topic.batch_consuming
-              consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
-                kafka_messages,
-                topic
-              )
-              consumer.call
-            else
-              kafka_messages.each do |kafka_message|
-                consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
-                  [kafka_message],
-                  topic
-                )
-                consumer.call
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/connection/builder.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Builder used to construct Kafka client
-    module Builder
-      class << self
-        # Builds a Kafka::Client instance that we use to work with Kafka cluster
-        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which we want
-        #   to have a new Kafka client
-        # @return [::Kafka::Client] returns a Kafka client
-        def call(consumer_group)
-          Kafka.new(*ApiAdapter.client(consumer_group))
-        end
-      end
-    end
-  end
-end