karafka 1.2.0

data/lib/karafka/routing/consumer_group.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Object used to describe a single consumer group that is going to subscribe to
+    # given topics
+    # It is a part of Karafka's DSL
+    class ConsumerGroup
+      extend Helpers::ConfigRetriever
+
+      attr_reader :topics
+      attr_reader :id
+      attr_reader :name
+
+      # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
+      #   yet have an application client_id namespace, this will be added here by default.
+      #   We add it to make a multi-system development easier for people that don't use
+      #   kafka and don't understand the concept of consumer groups.
+      def initialize(name)
+        @name = name
+        @id = Karafka::App.config.consumer_mapper.call(name)
+        @topics = []
+      end
+
+      # @return [Boolean] true if this consumer group should be active in our current process
+      def active?
+        Karafka::Server.consumer_groups.include?(name)
+      end
+
+      # Builds a topic representation inside of a current consumer group route
+      # @param name [String, Symbol] name of topic to which we want to subscribe
+      # @yield Evaluates a given block in a topic context
+      # @return [Karafka::Routing::Topic] newly built topic instance
+      def topic=(name, &block)
+        topic = Topic.new(name, self)
+        @topics << Proxy.new(topic, &block).target.tap(&:build)
+        @topics.last
+      end
+
+      Karafka::AttributesMap.consumer_group.each do |attribute|
+        config_retriever_for(attribute)
+      end
+
+      # Hashed version of consumer group that can be used for validation purposes
+      # @return [Hash] hash with consumer group attributes including serialized to hash
+      # topics inside of it.
+      def to_h
+        result = {
+          topics: topics.map(&:to_h),
+          id: id
+        }
+
+        Karafka::AttributesMap.consumer_group.each do |attribute|
+          result[attribute] = public_send(attribute)
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/karafka/routing/consumer_mapper.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Default consumer mapper that builds consumer ids based on app id and consumer group name
+    # Different mapper can be used in case of preexisting consumer names or for applying
+    # other naming conventions not compatible wiih Karafkas client_id + consumer name concept
+    #
+    # @example Mapper for using consumer groups without a client_id prefix
+    #   module MyMapper
+    #     def self.call(raw_consumer_group_name)
+    #       raw_consumer_group_name
+    #     end
+    #   end
+    #
+    # @example Mapper for replacing "_" with "." in topic names
+    #   module MyMapper
+    #     def self.call(raw_consumer_group_name)
+    #       [
+    #         Dry::Inflector.new.underscore(Karafka::App.config.client_id.to_s),
+    #         raw_consumer_group_name
+    #       ].join('_').gsub('_', '.')
+    #     end
+    #   end
+    module ConsumerMapper
+      # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
+      # @return [String] remapped final consumer group name
+      def self.call(raw_consumer_group_name)
+        client_name = Dry::Inflector.new.underscore(Karafka::App.config.client_id.to_s)
+        "#{client_name}_#{raw_consumer_group_name}"
+      end
+    end
+  end
+end

data/lib/karafka/routing/proxy.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Proxy is used as a translation layer in between the DSL and raw topic and consumer group
+    # objects.
+    class Proxy
+      attr_reader :target
+
+      # We should proxy only non ? and = methods as we want to have a regular dsl
+      IGNORED_POSTFIXES = %w[
+        ?
+        =
+        !
+      ].freeze
+
+      # @param target [Object] target object to which we proxy any DSL call
+      # @yield Evaluates block in the proxy context
+      def initialize(target, &block)
+        @target = target
+        instance_eval(&block)
+      end
+
+      # Translates the no "=" DSL of routing into elements assignments on target
+      def method_missing(method_name, *arguments, &block)
+        return super unless respond_to_missing?(method_name)
+        @target.public_send(:"#{method_name}=", *arguments, &block)
+      end
+
+      # Tells whether or not a given element exists on the target
+      def respond_to_missing?(method_name, include_private = false)
+        return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
+        @target.respond_to?(:"#{method_name}=", include_private) || super
+      end
+    end
+  end
+end

data/lib/karafka/routing/router.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all elements related to requests routing
+  module Routing
+    # Karafka framework Router for routing incoming messages to proper consumers
+    # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
+    #  structure so all the routes are being stored in a single level array
+    module Router
+      # Find a proper topic based on full topic id
+      # @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
+      #   routing topic
+      # @return [Karafka::Routing::Route] proper route details
+      # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
+      #   any route defined by user using routes.draw
+      def find(topic_id)
+        App.consumer_groups.each do |consumer_group|
+          consumer_group.topics.each do |topic|
+            return topic if topic.id == topic_id
+          end
+        end
+
+        raise(Errors::NonMatchingRouteError, topic_id)
+      end
+
+      module_function :find
+    end
+  end
+end

data/lib/karafka/routing/topic.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Topic stores all the details on how we should interact with Kafka given topic
+    # It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
+    # It is a part of Karafka's DSL
+    class Topic
+      extend Helpers::ConfigRetriever
+
+      attr_reader :id, :consumer_group
+      attr_accessor :consumer
+
+      # @param [String, Symbol] name of a topic on which we want to listen
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
+      def initialize(name, consumer_group)
+        @name = name.to_s
+        @consumer_group = consumer_group
+        @attributes = {}
+        # @note We use identifier related to the consumer group that owns a topic, because from
+        #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
+        #   have same topic name across mutliple Kafkas
+        @id = "#{consumer_group.id}_#{@name}"
+      end
+
+      # Initializes default values for all the options that support defaults if their values are
+      # not yet specified. This is need to be done (cannot be lazy loaded on first use) because
+      # everywhere except Karafka server command, those would not be initialized on time - for
+      # example for Sidekiq
+      def build
+        Karafka::AttributesMap.topic.each { |attr| send(attr) }
+        consumer&.topic = self
+        self
+      end
+
+      # @return [Class, nil] Class (not an instance) of a responder that should respond from
+      #   consumer back to Kafka (usefull for piping dataflows)
+      def responder
+        @responder ||= Karafka::Responders::Builder.new(consumer).build
+      end
+
+      Karafka::AttributesMap.topic.each do |attribute|
+        config_retriever_for(attribute)
+      end
+
+      # @return [Hash] hash with all the topic attributes
+      # @note This is being used when we validate the consumer_group and its topics
+      def to_h
+        map = Karafka::AttributesMap.topic.map do |attribute|
+          [attribute, public_send(attribute)]
+        end
+
+        Hash[map].merge!(
+          id: id,
+          consumer: consumer
+        )
+      end
+    end
+  end
+end

data/lib/karafka/routing/topic_mapper.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Default topic mapper that does not remap things
+    # Mapper can be used for Kafka providers that require namespaced topic names. Instead of being
+    # provider dependent, we can then define mapper and use internally "pure" topic names in
+    # routes and responders
+    #
+    # @example Mapper for mapping prefixed topics
+    #   module MyMapper
+    #     PREFIX = "my_user_name."
+    #
+    #     def incoming(topic)
+    #       topic.to_s.gsub(PREFIX, '')
+    #     end
+    #
+    #     def outgoing(topic)
+    #       "#{PREFIX}#{topic}"
+    #     end
+    #   end
+    #
+    # @example Mapper for replacing "." with "_" in topic names
+    #   module MyMapper
+    #     PREFIX = "my_user_name."
+    #
+    #     def incoming(topic)
+    #       topic.to_s.gsub('.', '_')
+    #     end
+    #
+    #     def outgoing(topic)
+    #       topic.to_s.gsub('_', '.')
+    #     end
+    #   end
+    module TopicMapper
+      class << self
+        # @param topic [String, Symbol] topic
+        # @return [String, Symbol] same topic as on input
+        # @example
+        #   incoming('topic_name') #=> 'topic_name'
+        def incoming(topic)
+          topic
+        end
+
+        # @param topic [String, Symbol] topic
+        # @return [String, Symbol] same topic as on input
+        # @example
+        #   outgoing('topic_name') #=> 'topic_name'
+        def outgoing(topic)
+          topic
+        end
+      end
+    end
+  end
+end

data/lib/karafka/schemas/config.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the validation schemas that we use to check input
+  module Schemas
+    # Regexp for validating format of groups and topics
+    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/
+
+    # Schema with validation rules for Karafka configuration details
+    # @note There are many more configuration options inside of the
+    #   Karafka::Setup::Config model, but we don't validate them here as they are
+    #   validated per each route (topic + consumer_group) because they can be overwritten,
+    #   so we validate all of that once all the routes are defined and ready
+    Config = Dry::Validation.Schema do
+      required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:shutdown_timeout) { none? | (int? & gteq?(0)) }
+      required(:consumer_mapper)
+      required(:topic_mapper)
+      required(:params_base_class).filled
+
+      optional(:backend).filled
+    end
+  end
+end

data/lib/karafka/schemas/consumer_group.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Schema for single full route (consumer group + topics) validation.
+    ConsumerGroup = Dry::Validation.Schema do
+      # Valid uri schemas of Kafka broker url
+      # The ||= is due to the behavior of require_all that resolves dependencies
+      # but someetimes loads things twice
+      URI_SCHEMES ||= %w[kafka kafka+ssl].freeze
+
+      # Available sasl scram mechanism of authentication (plus nil)
+      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
+
+      configure do
+        config.messages_file = File.join(
+          Karafka.gem_root, 'config', 'errors.yml'
+        )
+
+        # Uri validator to check if uri is in a Karafka acceptable format
+        # @param uri [String] uri we want to validate
+        # @return [Boolean] true if it is a valid uri, otherwise false
+        def broker_schema?(uri)
+          uri = URI.parse(uri)
+          URI_SCHEMES.include?(uri.scheme) && uri.port
+        rescue URI::InvalidURIError
+          false
+        end
+      end
+
+      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:seed_brokers).filled { each(:broker_schema?) }
+      required(:session_timeout).filled { int? | float? }
+      required(:pause_timeout).filled { (int? | float?) & gteq?(0) }
+      required(:offset_commit_interval) { int? | float? }
+      required(:offset_commit_threshold).filled(:int?)
+      required(:offset_retention_time) { none?.not > int? }
+      required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
+      required(:connect_timeout).filled { (int? | float?) & gt?(0) }
+      required(:socket_timeout).filled { (int? | float?) & gt?(0) }
+      required(:min_bytes).filled(:int?, gt?: 0)
+      required(:max_bytes).filled(:int?, gt?: 0)
+      required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
+      required(:batch_fetching).filled(:bool?)
+      required(:topics).filled { each { schema(ConsumerGroupTopic) } }
+
+      # Max wait time cannot exceed socket_timeout - wouldn't make sense
+      rule(
+        max_wait_time_limit: %i[max_wait_time socket_timeout]
+      ) do |max_wait_time, socket_timeout|
+        socket_timeout.int? > max_wait_time.lteq?(value(:socket_timeout))
+      end
+
+      %i[
+        ssl_ca_cert
+        ssl_ca_cert_file_path
+        ssl_client_cert
+        ssl_client_cert_key
+        sasl_gssapi_principal
+        sasl_gssapi_keytab
+        sasl_plain_authzid
+        sasl_plain_username
+        sasl_plain_password
+        sasl_scram_username
+        sasl_scram_password
+      ].each do |encryption_attribute|
+        optional(encryption_attribute).maybe(:str?)
+      end
+
+      optional(:ssl_ca_certs_from_system).maybe(:bool?)
+
+      # It's not with other encryptions as it has some more rules
+      optional(:sasl_scram_mechanism)
+        .maybe(:str?, included_in?: Karafka::Schemas::SASL_SCRAM_MECHANISMS)
+    end
+  end
+end

data/lib/karafka/schemas/consumer_group_topic.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Consumer group topic validation rules
+    ConsumerGroupTopic = Dry::Validation.Schema do
+      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:backend).filled(included_in?: %i[inline sidekiq])
+      required(:consumer).filled
+      required(:parser).filled
+      required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
+      required(:start_from_beginning).filled(:bool?)
+      required(:batch_consuming).filled(:bool?)
+      required(:persistent).filled(:bool?)
+    end
+  end
+end

data/lib/karafka/schemas/responder_usage.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Validator to check responder topic usage
+    ResponderUsageTopic = Dry::Validation.Schema do
+      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:required).filled(:bool?)
+      required(:multiple_usage).filled(:bool?)
+      required(:usage_count).filled(:int?, gteq?: 0)
+      required(:registered).filled(eql?: true)
+      required(:async).filled(:bool?)
+
+      rule(
+        required_usage: %i[required usage_count]
+      ) do |required, usage_count|
+        required.true? > usage_count.gteq?(1)
+      end
+
+      rule(
+        multiple_usage_permission: %i[multiple_usage usage_count]
+      ) do |multiple_usage, usage_count|
+        usage_count.gt?(1) > multiple_usage.true?
+      end
+
+      rule(
+        multiple_usage_block: %i[multiple_usage usage_count]
+      ) do |multiple_usage, usage_count|
+        multiple_usage.false? > usage_count.lteq?(1)
+      end
+    end
+
+    # Validator to check that everything in a responder flow matches responder rules
+    ResponderUsage = Dry::Validation.Schema do
+      required(:used_topics) { filled? > each { schema(ResponderUsageTopic) } }
+      required(:registered_topics) { filled? > each { schema(ResponderUsageTopic) } }
+    end
+  end
+end