karafka 1.4.12 → 2.2.10

data/lib/karafka/instrumentation/stdout_listener.rb

@@ -1,140 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Instrumentation
-    # Default listener that hooks up to our instrumentation and uses its events for logging
-    # It can be removed/replaced or anything without any harm to the Karafka app flow
-    class StdoutListener
-      # Log levels that we use in this particular listener
-      USED_LOG_LEVELS = %i[
-        debug
-        info
-        error
-        fatal
-      ].freeze
-
-      # Logs details about incoming batches and with which consumer we will consume them
-      # @param event [Dry::Events::Event] event details including payload
-      def on_connection_batch_delegator_call(event)
-        consumer = event[:consumer]
-        topic = consumer.topic.name
-        kafka_messages = event[:kafka_batch].messages
-        info(
-          <<~MSG.chomp.tr("\n", ' ')
-            #{kafka_messages.count} messages
-            on #{topic} topic
-            delegated to #{consumer.class}
-          MSG
-        )
-      end
-
-      # Logs details about incoming message and with which consumer we will consume it
-      # @param event [Dry::Events::Event] event details including payload
-      def on_connection_message_delegator_call(event)
-        consumer = event[:consumer]
-        topic = consumer.topic.name
-        info "1 message on #{topic} topic delegated to #{consumer.class}"
-      end
-
-      # Logs details about each received message value deserialization
-      # @param event [Dry::Events::Event] event details including payload
-      def on_params_params_deserialize(event)
-        # Keep in mind, that a caller here is a param object not a controller,
-        # so it returns a topic as a string, not a routing topic
-        debug(
-          <<~MSG.chomp.tr("\n", ' ')
-            Params deserialization for #{event[:caller].metadata.topic} topic
-            successful in #{event[:time]} ms
-          MSG
-        )
-      end
-
-      # Logs unsuccessful deserialization attempts of incoming data
-      # @param event [Dry::Events::Event] event details including payload
-      def on_params_params_deserialize_error(event)
-        topic = event[:caller].metadata.topic
-        error = event[:error]
-        error "Params deserialization error for #{topic} topic: #{error}"
-      end
-
-      # Logs errors that occurred in a listener fetch loop
-      # @param event [Dry::Events::Event] event details including payload
-      # @note It's an error as we can recover from it not a fatal
-      def on_connection_listener_fetch_loop_error(event)
-        error "Listener fetch loop error: #{event[:error]}"
-      end
-
-      # Logs errors that are related to the connection itself
-      # @param event [Dry::Events::Event] event details including payload
-      # @note Karafka will attempt to reconnect, so an error not a fatal
-      def on_connection_client_fetch_loop_error(event)
-        error "Client fetch loop error: #{event[:error]}"
-      end
-
-      # Logs info about crashed fetcher
-      # @param event [Dry::Events::Event] event details including payload
-      # @note If this happens, Karafka will shutdown as it means a critical error
-      #   in one of the threads
-      def on_fetcher_call_error(event)
-        fatal "Fetcher crash due to an error: #{event[:error]}"
-      end
-
-      # Logs info about processing of a certain dataset with an inline backend
-      # @param event [Dry::Events::Event] event details including payload
-      def on_backends_inline_process(event)
-        count = event[:caller].send(:params_batch).to_a.size
-        topic = event[:caller].topic.name
-        time = event[:time]
-        info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
-      end
-
-      # Logs info about system signals that Karafka received
-      # @param event [Dry::Events::Event] event details including payload
-      def on_process_notice_signal(event)
-        info "Received #{event[:signal]} system signal"
-      end
-
-      # Logs info about responder usage withing a controller flow
-      # @param event [Dry::Events::Event] event details including payload
-      def on_consumers_responders_respond_with(event)
-        calling = event[:caller]
-        responder = calling.topic.responder
-        data = event[:data]
-        info "Responded from #{calling.class} using #{responder} with following data #{data}"
-      end
-
-      # Logs info that we're initializing Karafka framework components
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_initializing(_event)
-        info "Initializing Karafka framework #{::Process.pid}"
-      end
-
-      # Logs info that we're running Karafka app
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_running(_event)
-        info "Running Karafka server #{::Process.pid}"
-      end
-
-      # Logs info that we're going to stop the Karafka server
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_stopping(_event)
-        # We use a separate thread as logging can't be called from trap context
-        Thread.new { info "Stopping Karafka server #{::Process.pid}" }
-      end
-
-      # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
-      #   forced exit
-      # @param _event [Dry::Events::Event] event details including payload
-      def on_app_stopping_error(_event)
-        # We use a separate thread as logging can't be called from trap context
-        Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
-      end
-
-      USED_LOG_LEVELS.each do |log_level|
-        define_method log_level do |*args|
-          Karafka.logger.send(log_level, *args)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/batch_metadata.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Simple batch metadata object that stores all non-message information received from Kafka
-    # cluster while fetching the data
-    # @note This metadata object refers to per batch metadata, not `#params.metadata`
-    BatchMetadata = Struct.new(
-      :batch_size,
-      :first_offset,
-      :highwater_mark_offset,
-      :unknown_last_offset,
-      :last_offset,
-      :offset_lag,
-      :deserializer,
-      :partition,
-      :topic,
-      keyword_init: true
-    ) do
-      # @return [Boolean] is the last offset known or unknown
-      def unknown_last_offset?
-        unknown_last_offset
-      end
-    end
-  end
-end

data/lib/karafka/params/builders/batch_metadata.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    module Builders
-      # Builder for creating batch metadata object based on the batch informations
-      module BatchMetadata
-        class << self
-          # Creates metadata based on the kafka batch data
-          # @param kafka_batch [Kafka::FetchedBatch] kafka batch details
-          # @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
-          # @return [Karafka::Params::BatchMetadata] batch metadata object
-          def from_kafka_batch(kafka_batch, topic)
-            Karafka::Params::BatchMetadata.new(
-              batch_size: kafka_batch.messages.count,
-              first_offset: kafka_batch.first_offset,
-              highwater_mark_offset: kafka_batch.highwater_mark_offset,
-              unknown_last_offset: kafka_batch.unknown_last_offset?,
-              last_offset: kafka_batch.last_offset,
-              offset_lag: kafka_batch.offset_lag,
-              deserializer: topic.deserializer,
-              partition: kafka_batch.partition,
-              topic: topic.name
-            ).freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/builders/params.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Due to the fact, that we create params related objects in couple contexts / places
-    # plus backends can build up them their own way we have this namespace.
-    # It allows to isolate actual params objects from their building process that can be
-    # context dependent.
-    module Builders
-      # Builder for params
-      module Params
-        class << self
-          # @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
-          # @param topic [Karafka::Routing::Topic] topic for which this message was fetched
-          # @return [Karafka::Params::Params] params object with payload and message metadata
-          def from_kafka_message(kafka_message, topic)
-            metadata = Karafka::Params::Metadata.new(
-              create_time: kafka_message.create_time,
-              headers: kafka_message.headers || {},
-              is_control_record: kafka_message.is_control_record,
-              key: kafka_message.key,
-              offset: kafka_message.offset,
-              deserializer: topic.deserializer,
-              partition: kafka_message.partition,
-              receive_time: Time.now,
-              topic: topic.name
-            ).freeze
-
-            Karafka::Params::Params.new(
-              kafka_message.value,
-              metadata
-            )
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/builders/params_batch.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    module Builders
-      # Builder for creating params batch instances
-      module ParamsBatch
-        class << self
-          # Creates params batch with params inside based on the incoming messages
-          # and the topic from which it comes
-          # @param kafka_messages [Array<Kafka::FetchedMessage>] raw fetched messages
-          # @param topic [Karafka::Routing::Topic] topic for which we're received messages
-          # @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
-          def from_kafka_messages(kafka_messages, topic)
-            params_array = kafka_messages.map do |message|
-              Karafka::Params::Builders::Params.from_kafka_message(message, topic)
-            end
-
-            Karafka::Params::ParamsBatch.new(params_array).freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/params/params_batch.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Params
-    # Params batch represents a set of messages received from Kafka.
-    # @note Params internally are lazy loaded before first use. That way we can skip
-    #   deserialization process if we have after_fetch that rejects some incoming messages
-    #   without using params It can be also used when handling really heavy data.
-    class ParamsBatch
-      include Enumerable
-
-      # @param params_array [Array<Karafka::Params::Params>] array with karafka params
-      # @return [Karafka::Params::ParamsBatch] lazy evaluated params batch object
-      def initialize(params_array)
-        @params_array = params_array
-      end
-
-      # @yieldparam [Karafka::Params::Params] each params instance
-      # @note Invocation of this method will not cause loading and deserializing each param after
-      #   another.
-      def each
-        @params_array.each { |param| yield(param) }
-      end
-
-      # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
-      #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
-      #   be deserialized
-      def deserialize!
-        each(&:payload)
-      end
-
-      # @return [Array<Object>] array with deserialized payloads. This method can be useful when
-      #   we don't care about metadata and just want to extract all the data payloads from the
-      #   batch
-      def payloads
-        map(&:payload)
-      end
-
-      # @return [Karafka::Params::Params] first element
-      def first
-        @params_array.first
-      end
-
-      # @return [Karafka::Params::Params] last element
-      def last
-        @params_array.last
-      end
-
-      # @return [Integer] number of messages in the batch
-      def size
-        @params_array.size
-      end
-
-      # @return [Array<Karafka::Params::Params>] pure array with params
-      def to_a
-        @params_array
-      end
-    end
-  end
-end

data/lib/karafka/patches/ruby_kafka.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace for various other libs patches
-  module Patches
-    # Patches for Ruby Kafka gem
-    module RubyKafka
-      # This patch allows us to inject business logic in between fetches and before the consumer
-      # stop, so we can perform stop commit or anything else that we need since
-      # ruby-kafka fetch loop does not allow that directly
-      # We don't won't to use poll ruby-kafka api as it brings many more problems that we would
-      # have to take care of. That way, nothing like that ever happens but we get the control
-      # over the stopping process that we need (since we're the once that initiate it for each
-      # thread)
-      def consumer_loop
-        super do
-          consumers = Karafka::Persistence::Consumers
-                      .current
-                      .values
-                      .flat_map(&:values)
-                      .select { |consumer| consumer.class.respond_to?(:after_fetch) }
-
-          if Karafka::App.stopping?
-            publish_event(consumers, 'before_stop')
-            Karafka::Persistence::Client.read.stop
-          else
-            publish_event(consumers, 'before_poll')
-            yield
-            publish_event(consumers, 'after_poll')
-          end
-        end
-      end
-
-      private
-
-      # Notifies consumers about particular events happening
-      # @param consumers [Array<Object>] all consumers that want to be notified about an event
-      # @param event_name [String] name of the event that happened
-      def publish_event(consumers, event_name)
-        consumers.each do |consumer|
-          key = "consumers.#{Helpers::Inflector.map(consumer.class.to_s)}.#{event_name}"
-          Karafka::App.monitor.instrument(key, context: consumer)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/persistence/client.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Persistence
-    # Persistence layer to store current thread messages consumer client for further use
-    class Client
-      # Thread.current key under which we store current thread messages consumer client
-      PERSISTENCE_SCOPE = :client
-
-      private_constant :PERSISTENCE_SCOPE
-
-      class << self
-        # @param client [Karafka::Connection::Client] messages consumer client of
-        #   a current thread
-        # @return [Karafka::Connection::Client] persisted messages consumer client
-        def write(client)
-          Thread.current[PERSISTENCE_SCOPE] = client
-        end
-
-        # @return [Karafka::Connection::Client] persisted messages consumer client
-        # @raise [Karafka::Errors::MissingClientError] raised when no thread messages consumer
-        #   client but we try to use it anyway
-        def read
-          Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClientError)
-        end
-      end
-    end
-  end
-end

data/lib/karafka/persistence/consumers.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module used to provide a persistent cache layer for Karafka components that need to be
-  # shared inside of a same thread
-  module Persistence
-    # Module used to provide a persistent cache across batch requests for a given
-    # topic and partition to store some additional details when the persistent mode
-    # for a given topic is turned on
-    class Consumers
-      # Thread.current scope under which we store consumers data
-      PERSISTENCE_SCOPE = :consumers
-
-      private_constant :PERSISTENCE_SCOPE
-
-      class << self
-        # @return [Hash] current thread's persistence scope hash with all the consumers
-        def current
-          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
-            hash[key] = Concurrent::Hash.new
-          end
-        end
-
-        # Used to build (if block given) and/or fetch a current consumer instance that will be
-        #   used to process messages from a given topic and partition
-        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
-        # @param partition [Integer] number of partition for which we want to cache
-        # @return [Karafka::BaseConsumer] base consumer descendant
-        def fetch(topic, partition)
-          current[topic][partition] ||= topic.consumer.new(topic)
-        end
-
-        # Removes all persisted instances of consumers from the consumer cache
-        # @note This is used to reload consumers instances when code reloading in development mode
-        #   is present. This should not be used in production.
-        def clear
-          Thread
-            .list
-            .select { |thread| thread[PERSISTENCE_SCOPE] }
-            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
-        end
-      end
-    end
-  end
-end

data/lib/karafka/persistence/topics.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Persistence
-    # Local cache for routing topics
-    # We use it in order not to build string instances and remap incoming topic upon each
-    # message / message batches received
-    class Topics
-      # Thread.current scope under which we store topics data
-      PERSISTENCE_SCOPE = :topics
-
-      private_constant :PERSISTENCE_SCOPE
-
-      class << self
-        # @return [Concurrent::Hash] hash with all the topics from given groups
-        def current
-          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
-            hash[key] = Concurrent::Hash.new
-          end
-        end
-
-        # @param group_id [String] group id for which we fetch a topic representation
-        # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
-        #   topic representation
-        # @return [Karafka::Routing::Topics] remapped topic representation that can be used further
-        #   on when working with given parameters
-        def fetch(group_id, raw_topic_name)
-          current[group_id][raw_topic_name] ||= begin
-            # We map from incoming topic name, as it might be namespaced, etc.
-            # @see topic_mapper internal docs
-            mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
-            Routing::Router.find("#{group_id}_#{mapped_topic_name}")
-          end
-        end
-
-        # Clears the whole topics cache for all the threads
-        # This is used for in-development code reloading as we need to get rid of all the
-        # preloaded and cached instances of objects to make it work
-        def clear
-          Thread
-            .list
-            .select { |thread| thread[PERSISTENCE_SCOPE] }
-            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
-        end
-      end
-    end
-  end
-end

data/lib/karafka/responders/builder.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Responders namespace encapsulates all the internal responder implementation parts
-  module Responders
-    # Responders builder is used for finding (based on the consumer class name) a responder
-    # that match the consumer. We use it when user does not provide a responder inside routing,
-    # but he still names responder with the same convention (and namespaces) as consumer
-    #
-    # @example Matching responder exists
-    #   Karafka::Responder::Builder(NewEventsConsumer).build #=> NewEventsResponder
-    # @example Matching responder does not exist
-    #   Karafka::Responder::Builder(NewBuildsConsumer).build #=> nil
-    class Builder
-      # @param consumer_class [Karafka::BaseConsumer, nil] descendant of
-      #   Karafka::BaseConsumer
-      # @example Tries to find a responder that matches a given consumer. If nothing found,
-      #   will return nil (nil is accepted, because it means that a given consumer don't
-      #   pipe stuff further on)
-      def initialize(consumer_class)
-        @consumer_class = consumer_class
-      end
-
-      # Tries to figure out a responder based on a consumer class name
-      # @return [Class] Responder class (not an instance)
-      # @return [nil] or nil if there's no matching responding class
-      def build
-        Helpers::ClassMatcher.new(
-          @consumer_class,
-          from: 'Consumer',
-          to: 'Responder'
-        ).match
-      end
-    end
-  end
-end

data/lib/karafka/responders/topic.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Responders
-    # Topic describes a single topic on which we want to respond with responding requirements
-    # @example Define topic (required by default)
-    #   Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
-    # @example Define optional topic
-    #   Karafka::Responders::Topic.new(:topic_name, required: false)
-    class Topic
-      # Name of the topic on which we want to respond
-      attr_reader :name
-
-      # @param name [Symbol, String] name of a topic on which we want to respond
-      # @param options [Hash] non-default options for this topic
-      # @return [Karafka::Responders::Topic] topic description object
-      def initialize(name, options)
-        @name = name.to_s
-        @options = options
-      end
-
-      # @return [Boolean] is this a required topic (if not, it is optional)
-      def required?
-        @options.key?(:required) ? @options[:required] : true
-      end
-
-      # @return [Boolean] was usage of this topic registered or not
-      def registered?
-        @options[:registered] == true
-      end
-
-      # @return [Class] Class to use to serialize messages for this topic
-      def serializer
-        @options[:serializer]
-      end
-
-      # @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
-      #   is safer and introduces less problems
-      def async?
-        @options.key?(:async) ? @options[:async] : false
-      end
-
-      # @return [Hash] hash with this topic attributes and options
-      def to_h
-        {
-          name: name,
-          required: required?,
-          registered: registered?,
-          serializer: serializer,
-          async: async?
-        }
-      end
-    end
-  end
-end

data/lib/karafka/routing/topic_mapper.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Routing
-    # Default topic mapper that does not remap things
-    # Mapper can be used for Kafka providers that require namespaced topic names. Instead of being
-    # provider dependent, we can then define mapper and use internally "pure" topic names in
-    # routes and responders
-    #
-    # @example Mapper for mapping prefixed topics
-    #   class MyMapper
-    #     PREFIX = "my_user_name."
-    #
-    #     def incoming(topic)
-    #       topic.to_s.gsub(PREFIX, '')
-    #     end
-    #
-    #     def outgoing(topic)
-    #       "#{PREFIX}#{topic}"
-    #     end
-    #   end
-    #
-    # @example Mapper for replacing "." with "_" in topic names
-    #   class MyMapper
-    #     PREFIX = "my_user_name."
-    #
-    #     def incoming(topic)
-    #       topic.to_s.gsub('.', '_')
-    #     end
-    #
-    #     def outgoing(topic)
-    #       topic.to_s.gsub('_', '.')
-    #     end
-    #   end
-    class TopicMapper
-      # @param topic [String, Symbol] topic
-      # @return [String, Symbol] same topic as on input
-      # @example
-      #   incoming('topic_name') #=> 'topic_name'
-      def incoming(topic)
-        topic
-      end
-
-      # @param topic [String, Symbol] topic
-      # @return [String, Symbol] same topic as on input
-      # @example
-      #   outgoing('topic_name') #=> 'topic_name'
-      def outgoing(topic)
-        topic
-      end
-    end
-  end
-end

data/lib/karafka/serialization/json/serializer.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module for all supported by default serialization and deserialization ways
-  module Serialization
-    module Json
-      # Default Karafka Json serializer for serializing data
-      class Serializer
-        # @param content [Object] any object that we want to convert to a json string
-        # @return [String] Valid JSON string containing serialized data
-        # @raise [Karafka::Errors::SerializationError] raised when we don't have a way to
-        #   serialize provided data to json
-        # @note When string is passed to this method, we assume that it is already a json
-        #   string and we don't serialize it again. This allows us to serialize data before
-        #   it is being forwarded to this serializer if we want to have a custom (not that simple)
-        #   json serialization
-        #
-        # @example From an ActiveRecord object
-        #   Serializer.call(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
-        # @example From a string (no changes)
-        #   Serializer.call("{\"a\":1}") #=> "{\"a\":1}"
-        def call(content)
-          return content if content.is_a?(String)
-          return content.to_json if content.respond_to?(:to_json)
-
-          raise Karafka::Errors::SerializationError, content
-        end
-      end
-    end
-  end
-end