karafka 1.3.0

data/lib/karafka/persistence/client.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Persistence layer to store current thread messages consumer client for further use
+    class Client
+      # Thread.current key under which we store current thread messages consumer client
+      PERSISTENCE_SCOPE = :client
+
+      private_constant :PERSISTENCE_SCOPE
+
+      class << self
+        # @param client [Karafka::Connection::Client] messages consumer client of
+        #   a current thread
+        # @return [Karafka::Connection::Client] persisted messages consumer client
+        def write(client)
+          Thread.current[PERSISTENCE_SCOPE] = client
+        end
+
+        # @return [Karafka::Connection::Client] persisted messages consumer client
+        # @raise [Karafka::Errors::MissingClientError] raised when no thread messages consumer
+        #   client but we try to use it anyway
+        def read
+          Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClientError)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/consumers.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module used to provide a persistent cache layer for Karafka components that need to be
+  # shared inside of a same thread
+  module Persistence
+    # Module used to provide a persistent cache across batch requests for a given
+    # topic and partition to store some additional details when the persistent mode
+    # for a given topic is turned on
+    class Consumers
+      # Thread.current scope under which we store consumers data
+      PERSISTENCE_SCOPE = :consumers
+
+      private_constant :PERSISTENCE_SCOPE
+
+      class << self
+        # @return [Hash] current thread's persistence scope hash with all the consumers
+        def current
+          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
+            hash[key] = Concurrent::Hash.new
+          end
+        end
+
+        # Used to build (if block given) and/or fetch a current consumer instance that will be
+        #   used to process messages from a given topic and partition
+        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
+        # @param partition [Integer] number of partition for which we want to cache
+        # @return [Karafka::BaseConsumer] base consumer descendant
+        def fetch(topic, partition)
+          current[topic][partition] ||= topic.consumer.new(topic)
+        end
+
+        # Removes all persisted instances of consumers from the consumer cache
+        # @note This is used to reload consumers instances when code reloading in development mode
+        #   is present. This should not be used in production.
+        def clear
+          Thread
+            .list
+            .select { |thread| thread[PERSISTENCE_SCOPE] }
+            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/topics.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Local cache for routing topics
+    # We use it in order not to build string instances and remap incoming topic upon each
+    # message / message batches received
+    class Topics
+      # Thread.current scope under which we store topics data
+      PERSISTENCE_SCOPE = :topics
+
+      private_constant :PERSISTENCE_SCOPE
+
+      class << self
+        # @return [Concurrent::Hash] hash with all the topics from given groups
+        def current
+          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
+            hash[key] = Concurrent::Hash.new
+          end
+        end
+
+        # @param group_id [String] group id for which we fetch a topic representation
+        # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
+        #   topic representation
+        # @return [Karafka::Routing::Topics] remapped topic representation that can be used further
+        #   on when working with given parameters
+        def fetch(group_id, raw_topic_name)
+          current[group_id][raw_topic_name] ||= begin
+            # We map from incoming topic name, as it might be namespaced, etc.
+            # @see topic_mapper internal docs
+            mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
+            Routing::Router.find("#{group_id}_#{mapped_topic_name}")
+          end
+        end
+
+        # Clears the whole topics cache for all the threads
+        # This is used for in-development code reloading as we need to get rid of all the
+        # preloaded and cached instances of objects to make it work
+        def clear
+          Thread
+            .list
+            .select { |thread| thread[PERSISTENCE_SCOPE] }
+            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/process.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Class used to catch signals from ruby Signal class in order to manage Karafka stop
+  # @note There might be only one process - this class is a singleton
+  class Process
+    # Signal types that we handle
+    HANDLED_SIGNALS = %i[
+      SIGINT
+      SIGQUIT
+      SIGTERM
+    ].freeze
+
+    HANDLED_SIGNALS.each do |signal|
+      # Assigns a callback that will happen when certain signal will be send
+      # to Karafka server instance
+      # @note It does not define the callback itself -it needs to be passed in a block
+      # @example Define an action that should be taken on_sigint
+      #   process.on_sigint do
+      #     Karafka.logger.info('Log something here')
+      #     exit
+      #   end
+      define_method :"on_#{signal.to_s.downcase}" do |&block|
+        @callbacks[signal] << block
+      end
+    end
+
+    # Creates an instance of process and creates empty hash for callbacks
+    def initialize
+      @callbacks = Hash.new { |hsh, key| hsh[key] = [] }
+    end
+
+    # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
+    # @note If there are no callbacks, this method will just ignore a given signal that was sent
+    def supervise
+      HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
+    end
+
+    private
+
+    # Traps a single signal and performs callbacks (if any) or just ignores this signal
+    # @param [Symbol] signal type that we want to catch
+    def trap_signal(signal)
+      trap(signal) do
+        notice_signal(signal)
+        (@callbacks[signal] || []).each(&:call)
+      end
+    end
+
+    # Informs monitoring about trapped signal
+    # @param [Symbol] signal type that we received
+    # @note We cannot perform logging from trap context, that's why
+    #   we have to spin up a new thread to do this
+    def notice_signal(signal)
+      Thread.new do
+        Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
+      end
+    end
+  end
+end

data/lib/karafka/responders/builder.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Responders namespace encapsulates all the internal responder implementation parts
+  module Responders
+    # Responders builder is used for finding (based on the consumer class name) a responder
+    # that match the consumer. We use it when user does not provide a responder inside routing,
+    # but he still names responder with the same convention (and namespaces) as consumer
+    #
+    # @example Matching responder exists
+    #   Karafka::Responder::Builder(NewEventsConsumer).build #=> NewEventsResponder
+    # @example Matching responder does not exist
+    #   Karafka::Responder::Builder(NewBuildsConsumer).build #=> nil
+    class Builder
+      # @param consumer_class [Karafka::BaseConsumer, nil] descendant of
+      #   Karafka::BaseConsumer
+      # @example Tries to find a responder that matches a given consumer. If nothing found,
+      #   will return nil (nil is accepted, because it means that a given consumer don't
+      #   pipe stuff further on)
+      def initialize(consumer_class)
+        @consumer_class = consumer_class
+      end
+
+      # Tries to figure out a responder based on a consumer class name
+      # @return [Class] Responder class (not an instance)
+      # @return [nil] or nil if there's no matching responding class
+      def build
+        Helpers::ClassMatcher.new(
+          @consumer_class,
+          from: 'Consumer',
+          to: 'Responder'
+        ).match
+      end
+    end
+  end
+end

data/lib/karafka/responders/topic.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Responders
+    # Topic describes a single topic on which we want to respond with responding requirements
+    # @example Define topic (required by default)
+    #   Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
+    # @example Define optional topic
+    #   Karafka::Responders::Topic.new(:topic_name, required: false)
+    class Topic
+      # Name of the topic on which we want to respond
+      attr_reader :name
+
+      # @param name [Symbol, String] name of a topic on which we want to respond
+      # @param options [Hash] non-default options for this topic
+      # @return [Karafka::Responders::Topic] topic description object
+      def initialize(name, options)
+        @name = name.to_s
+        @options = options
+      end
+
+      # @return [Boolean] is this a required topic (if not, it is optional)
+      def required?
+        @options.key?(:required) ? @options[:required] : true
+      end
+
+      # @return [Boolean] was usage of this topic registered or not
+      def registered?
+        @options[:registered] == true
+      end
+
+      # @return [Class] Class to use to serialize messages for this topic
+      def serializer
+        @options[:serializer]
+      end
+
+      # @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
+      #   is safer and introduces less problems
+      def async?
+        @options.key?(:async) ? @options[:async] : false
+      end
+
+      # @return [Hash] hash with this topic attributes and options
+      def to_h
+        {
+          name: name,
+          required: required?,
+          registered: registered?,
+          serializer: serializer,
+          async: async?
+        }
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Builder used as a DSL layer for building consumers and telling them which topics to consume
+    # @example Build a simple (most common) route
+    #   consumers do
+    #     topic :new_videos do
+    #       consumer NewVideosConsumer
+    #     end
+    #   end
+    class Builder < Concurrent::Array
+      # Consumer group consistency checking contract
+      CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
+
+      private_constant :CONTRACT
+
+      def initialize
+        @draws = Concurrent::Array.new
+      end
+
+      # Used to draw routes for Karafka
+      # @param block [Proc] block we will evaluate within the builder context
+      # @yield Evaluates provided block in a builder context so we can describe routes
+      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
+      #   doesn't match with the config contract
+      # @note After it is done drawing it will store and validate all the routes to make sure that
+      #   they are correct and that there are no topic/group duplications (this is forbidden)
+      # @example
+      #   draw do
+      #     topic :xyz do
+      #     end
+      #   end
+      def draw(&block)
+        @draws << block
+
+        instance_eval(&block)
+
+        each do |consumer_group|
+          hashed_group = consumer_group.to_h
+          validation_result = CONTRACT.call(hashed_group)
+          next if validation_result.success?
+
+          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
+        end
+      end
+
+      # @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
+      #   we want to use. Since Karafka supports multi-process setup, we need to be able
+      #   to pick only those consumer groups that should be active in our given process context
+      def active
+        select(&:active?)
+      end
+
+      # Clears the builder and the draws memory
+      def clear
+        @draws.clear
+        super
+      end
+
+      # Redraws all the routes for the in-process code reloading.
+      # @note This won't allow registration of new topics without process restart but will trigger
+      #   cache invalidation so all the classes, etc are re-fetched after code reload
+      def reload
+        draws = @draws.dup
+        clear
+        draws.each { |block| draw(&block) }
+      end
+
+      private
+
+      # Builds and saves given consumer group
+      # @param group_id [String, Symbol] name for consumer group
+      # @param block [Proc] proc that should be executed in the proxy context
+      def consumer_group(group_id, &block)
+        consumer_group = ConsumerGroup.new(group_id.to_s)
+        self << Proxy.new(consumer_group, &block).target
+      end
+
+      # @param topic_name [String, Symbol] name of a topic from which we want to consumer
+      # @param block [Proc] proc we want to evaluate in the topic context
+      def topic(topic_name, &block)
+        consumer_group(topic_name) do
+          topic(topic_name, &block).tap(&:build)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/consumer_group.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Object used to describe a single consumer group that is going to subscribe to
+    # given topics
+    # It is a part of Karafka's DSL
+    class ConsumerGroup
+      extend Helpers::ConfigRetriever
+
+      attr_reader :topics
+      attr_reader :id
+      attr_reader :name
+
+      # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
+      #   yet have an application client_id namespace, this will be added here by default.
+      #   We add it to make a multi-system development easier for people that don't use
+      #   kafka and don't understand the concept of consumer groups.
+      def initialize(name)
+        @name = name
+        @id = Karafka::App.config.consumer_mapper.call(name)
+        @topics = []
+      end
+
+      # @return [Boolean] true if this consumer group should be active in our current process
+      def active?
+        Karafka::Server.consumer_groups.include?(name)
+      end
+
+      # Builds a topic representation inside of a current consumer group route
+      # @param name [String, Symbol] name of topic to which we want to subscribe
+      # @param block [Proc] block that we want to evaluate in the topic context
+      # @return [Karafka::Routing::Topic] newly built topic instance
+      def topic=(name, &block)
+        topic = Topic.new(name, self)
+        @topics << Proxy.new(topic, &block).target.tap(&:build)
+        @topics.last
+      end
+
+      Karafka::AttributesMap.consumer_group.each do |attribute|
+        config_retriever_for(attribute)
+      end
+
+      # Hashed version of consumer group that can be used for validation purposes
+      # @return [Hash] hash with consumer group attributes including serialized to hash
+      # topics inside of it.
+      def to_h
+        result = {
+          topics: topics.map(&:to_h),
+          id: id
+        }
+
+        Karafka::AttributesMap.consumer_group.each do |attribute|
+          result[attribute] = public_send(attribute)
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/karafka/routing/consumer_mapper.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Default consumer mapper that builds consumer ids based on app id and consumer group name
+    # Different mapper can be used in case of preexisting consumer names or for applying
+    # other naming conventions not compatible with Karafka client_id + consumer name concept
+    #
+    # @example Mapper for using consumer groups without a client_id prefix
+    #   class MyMapper
+    #     def call(raw_consumer_group_name)
+    #       raw_consumer_group_name
+    #     end
+    #   end
+    #
+    # @example Mapper for replacing "_" with "." in topic names
+    #   class MyMapper
+    #     def call(raw_consumer_group_name)
+    #       [
+    #         Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s),
+    #         raw_consumer_group_name
+    #       ].join('_').gsub('_', '.')
+    #     end
+    #   end
+    class ConsumerMapper
+      # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
+      # @return [String] remapped final consumer group name
+      def call(raw_consumer_group_name)
+        client_name = Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s)
+        "#{client_name}_#{raw_consumer_group_name}"
+      end
+    end
+  end
+end