karafka 1.1.0

data/lib/karafka/routing/router.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all elements related to requests routing
+  module Routing
+    # Karafka framework Router for routing incoming messages to proper controllers
+    # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
+    #  structure so all the routes are being stored in a single level array
+    module Router
+      # Find a proper topic based on full topic id
+      # @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
+      #   routing topic
+      # @return [Karafka::Routing::Route] proper route details
+      # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
+      #   any route defined by user using routes.draw
+      def find(topic_id)
+        App.consumer_groups.each do |consumer_group|
+          consumer_group.topics.each do |topic|
+            return topic if topic.id == topic_id
+          end
+        end
+
+        raise(Errors::NonMatchingRouteError, topic_id)
+      end
+
+      module_function :find
+    end
+  end
+end

data/lib/karafka/routing/topic.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Topic stores all the details on how we should interact with Kafka given topic
+    # It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
+    # It is a part of Karafka's DSL
+    class Topic
+      extend Helpers::ConfigRetriever
+
+      attr_reader :id, :consumer_group
+      attr_accessor :controller
+
+      # @param [String, Symbol] name of a topic on which we want to listen
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
+      def initialize(name, consumer_group)
+        @name = name.to_s
+        @consumer_group = consumer_group
+        @attributes = {}
+        # @note We use identifier related to the consumer group that owns a topic, because from
+        #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
+        #   have same topic name across mutliple Kafkas
+        @id = "#{consumer_group.id}_#{@name}"
+      end
+
+      # Initializes default values for all the options that support defaults if their values are
+      # not yet specified. This is need to be done (cannot be lazy loaded on first use) because
+      # everywhere except Karafka server command, those would not be initialized on time - for
+      # example for Sidekiq
+      def build
+        Karafka::AttributesMap.topic.each { |attr| send(attr) }
+        controller&.topic = self
+        self
+      end
+
+      # @return [Class, nil] Class (not an instance) of a responder that should respond from
+      #   controller back to Kafka (usefull for piping dataflows)
+      def responder
+        @responder ||= Karafka::Responders::Builder.new(controller).build
+      end
+
+      # @return [Class] Parser class (not instance) that we want to use to unparse Kafka messages
+      # @note If not provided - will use Json as default
+      def parser
+        @parser ||= Karafka::Parsers::Json
+      end
+
+      Karafka::AttributesMap.topic.each do |attribute|
+        config_retriever_for(attribute)
+      end
+
+      # @return [Hash] hash with all the topic attributes
+      # @note This is being used when we validate the consumer_group and its topics
+      def to_h
+        map = Karafka::AttributesMap.topic.map do |attribute|
+          [attribute, public_send(attribute)]
+        end
+
+        Hash[map].merge!(
+          id: id,
+          controller: controller
+        )
+      end
+    end
+  end
+end

data/lib/karafka/routing/topic_mapper.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Default topic mapper that does not remap things
+    # Mapper can be used for Kafka providers that require namespaced topic names. Instead of being
+    # provider dependent, we can then define mapper and use internally "pure" topic names in
+    # routes and responders
+    #
+    # @example Mapper for mapping prefixed topics
+    #   module MyMapper
+    #     PREFIX = "my_user_name."
+    #
+    #     def incoming(topic)
+    #       topic.to_s.gsub(PREFIX, '')
+    #     end
+    #
+    #     def outgoing(topic)
+    #       "#{PREFIX}#{topic}"
+    #     end
+    #   end
+    #
+    # @example Mapper for replacing "." with "_" in topic names
+    #   module MyMapper
+    #     PREFIX = "my_user_name."
+    #
+    #     def incoming(topic)
+    #       topic.to_s.gsub('.', '_')
+    #     end
+    #
+    #     def outgoing(topic)
+    #       topic.to_s.gsub('_', '.')
+    #     end
+    #   end
+    module TopicMapper
+      class << self
+        # @param topic [String, Symbol] topic
+        # @return [String, Symbol] same topic as on input
+        # @example
+        #   incoming('topic_name') #=> 'topic_name'
+        def incoming(topic)
+          topic
+        end
+
+        # @param topic [String, Symbol] topic
+        # @return [String, Symbol] same topic as on input
+        # @example
+        #   outgoing('topic_name') #=> 'topic_name'
+        def outgoing(topic)
+          topic
+        end
+      end
+    end
+  end
+end

data/lib/karafka/schemas/config.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the validation schemas that we use to check input
+  module Schemas
+    # Regexp for validating format of groups and topics
+    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/
+
+    # Schema with validation rules for Karafka configuration details
+    # @note There are many more configuration options inside of the
+    #   Karafka::Setup::Config model, but we don't validate them here as they are
+    #   validated per each route (topic + consumer_group) because they can be overwritten,
+    #   so we validate all of that once all the routes are defined and ready
+    Config = Dry::Validation.Schema do
+      required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:consumer_mapper)
+      required(:topic_mapper)
+      optional(:backend).filled
+    end
+  end
+end

data/lib/karafka/schemas/consumer_group.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Schema for single full route (consumer group + topics) validation.
+    ConsumerGroup = Dry::Validation.Schema do
+      # Valid uri schemas of Kafka broker url
+      # The ||= is due to the behavior of require_all that resolves dependencies
+      # but someetimes loads things twice
+      URI_SCHEMES ||= %w[kafka kafka+ssl].freeze
+
+      configure do
+        config.messages_file = File.join(
+          Karafka.gem_root, 'config', 'errors.yml'
+        )
+
+        # Uri validator to check if uri is in a Karafka acceptable format
+        # @param uri [String] uri we want to validate
+        # @return [Boolean] true if it is a valid uri, otherwise false
+        def broker_schema?(uri)
+          uri = URI.parse(uri)
+          URI_SCHEMES.include?(uri.scheme) && uri.port
+        rescue URI::InvalidURIError
+          return false
+        end
+      end
+
+      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:seed_brokers).filled { each(:broker_schema?) }
+      required(:session_timeout).filled { int? | float? }
+      required(:pause_timeout).filled { (int? | float?) & gteq?(0) }
+      required(:offset_commit_interval) { int? | float? }
+      required(:offset_commit_threshold).filled(:int?)
+      required(:offset_retention_time) { none?.not > int? }
+      required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
+      required(:connect_timeout).filled { (int? | float?) & gt?(0) }
+      required(:socket_timeout).filled { (int? | float?) & gt?(0) }
+      required(:min_bytes).filled(:int?, gt?: 0)
+      required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
+      required(:batch_fetching).filled(:bool?)
+      required(:topics).filled { each { schema(ConsumerGroupTopic) } }
+
+      # Max wait time cannot exceed socket_timeout - wouldn't make sense
+      rule(
+        max_wait_time_limit: %i[max_wait_time socket_timeout]
+      ) do |max_wait_time, socket_timeout|
+        socket_timeout.int? > max_wait_time.lteq?(value(:socket_timeout))
+      end
+
+      %i[
+        ssl_ca_cert
+        ssl_ca_cert_file_path
+        ssl_client_cert
+        ssl_client_cert_key
+        sasl_plain_authzid
+        sasl_plain_username
+        sasl_plain_password
+        sasl_gssapi_principal
+        sasl_gssapi_keytab
+      ].each do |encryption_attribute|
+        optional(encryption_attribute).maybe(:str?)
+      end
+    end
+  end
+end

data/lib/karafka/schemas/consumer_group_topic.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Consumer group topic validation rules
+    ConsumerGroupTopic = Dry::Validation.Schema do
+      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:backend).filled(included_in?: %i[inline sidekiq])
+      required(:controller).filled
+      required(:parser).filled
+      required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
+      required(:start_from_beginning).filled(:bool?)
+      required(:batch_consuming).filled(:bool?)
+      required(:persistent).filled(:bool?)
+    end
+  end
+end

data/lib/karafka/schemas/responder_usage.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Validator to check responder topic usage
+    ResponderUsageTopic = Dry::Validation.Schema do
+      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:required).filled(:bool?)
+      required(:multiple_usage).filled(:bool?)
+      required(:usage_count).filled(:int?, gteq?: 0)
+      required(:registered).filled(eql?: true)
+      required(:async).filled(:bool?)
+
+      rule(
+        required_usage: %i[required usage_count]
+      ) do |required, usage_count|
+        required.true? > usage_count.gteq?(1)
+      end
+
+      rule(
+        multiple_usage_permission: %i[multiple_usage usage_count]
+      ) do |multiple_usage, usage_count|
+        usage_count.gt?(1) > multiple_usage.true?
+      end
+
+      rule(
+        multiple_usage_block: %i[multiple_usage usage_count]
+      ) do |multiple_usage, usage_count|
+        multiple_usage.false? > usage_count.lteq?(1)
+      end
+    end
+
+    # Validator to check that everything in a responder flow matches responder rules
+    ResponderUsage = Dry::Validation.Schema do
+      required(:used_topics) { filled? > each { schema(ResponderUsageTopic) } }
+      required(:registered_topics) { filled? > each { schema(ResponderUsageTopic) } }
+    end
+  end
+end

data/lib/karafka/schemas/server_cli_options.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Schema for validating correctness of the server cli command options
+    # We validate some basics + the list of consumer_groups on which we want to use, to make
+    # sure that all of them are defined, plus that a pidfile does not exist
+    ServerCliOptions = Dry::Validation.Schema do
+      configure do
+        option :consumer_groups
+
+        def self.messages
+          super.merge(
+            en: {
+              errors: {
+                consumer_groups_inclusion: 'Unknown consumer group.',
+                pid_existence: 'Pidfile already exists.'
+              }
+            }
+          )
+        end
+      end
+
+      optional(:pid).filled(:str?)
+      optional(:daemon).filled(:bool?)
+      optional(:consumer_groups).filled(:array?)
+
+      validate(consumer_groups_inclusion: :consumer_groups) do |consumer_groups|
+        # If there were no consumer_groups declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        if consumer_groups.nil?
+          true
+        else
+          (consumer_groups - Karafka::Routing::Builder.instance.map(&:name)).empty?
+        end
+      end
+
+      validate(pid_existence: :pid) do |pid|
+        pid ? !File.exist?(pid) : true
+      end
+    end
+  end
+end

data/lib/karafka/server.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Karafka consuming server class
+  class Server
+    class << self
+      # Set of consuming threads. Each consumer thread contains a single consumer
+      attr_accessor :consumer_threads
+
+      # Writer for list of consumer groups that we want to consume in our current process context
+      attr_writer :consumer_groups
+
+      # Method which runs app
+      def run
+        @consumer_threads = Concurrent::Array.new
+        bind_on_sigint
+        bind_on_sigquit
+        bind_on_sigterm
+        start_supervised
+      end
+
+      # @return [Array<String>] array with names of consumer groups that should be consumed in a
+      #   current server context
+      def consumer_groups
+        # If not specified, a server will listed on all the topics
+        @consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
+      end
+
+      private
+
+      # @return [Karafka::Process] process wrapper instance used to catch system signal calls
+      def process
+        Karafka::Process.instance
+      end
+
+      # What should happen when we decide to quit with sigint
+      def bind_on_sigint
+        process.on_sigint { Karafka::App.stop! }
+      end
+
+      # What should happen when we decide to quit with sigquit
+      def bind_on_sigquit
+        process.on_sigquit { Karafka::App.stop! }
+      end
+
+      # What should happen when we decide to quit with sigterm
+      def bind_on_sigterm
+        process.on_sigterm { Karafka::App.stop! }
+      end
+
+      # Starts Karafka with a supervision
+      # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
+      # finish loop (and it won't happen until we explicitily want to stop)
+      def start_supervised
+        process.supervise do
+          Karafka::App.run!
+          Karafka::Fetcher.new.fetch_loop
+        end
+      end
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module containing all Karafka setup related elements like configuration settings,
+  # config validations and configurators for external gems integration
+  module Setup
+    # Configurator for setting up all the framework details that are required to make it work
+    # @note If you want to do some configurations after all of this is done, please add to
+    #   karafka/config a proper file (needs to inherit from Karafka::Setup::Configurators::Base
+    #   and implement setup method) after that everything will happen automatically
+    # @note This config object allows to create a 1 level nestings (nodes) only. This should be
+    #   enough and will still keep the code simple
+    # @see Karafka::Setup::Configurators::Base for more details about configurators api
+    class Config
+      extend Dry::Configurable
+
+      # Available settings
+      # option client_id [String] kafka client_id - used to provide
+      #   default Kafka groups namespaces and identify that app in kafka
+      setting :client_id
+      # What backend do we want to use to process messages
+      setting :backend, :inline
+      # option logger [Instance] logger that we want to use
+      setting :logger, -> { ::Karafka::Logger.instance }
+      # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
+      setting :monitor, -> { ::Karafka::Monitor.instance }
+      # Mapper used to remap consumer groups ids, so in case users migrate from other tools
+      # or they need to maintain their own internal consumer group naming conventions, they
+      # can easily do it, replacing the default client_id + consumer name pattern concept
+      setting :consumer_mapper, -> { Routing::ConsumerMapper }
+      # Mapper used to remap names of topics, so we can have a clean internal topic namings
+      # despite using any Kafka provider that uses namespacing, etc
+      # It needs to implement two methods:
+      #   - #incoming - for remapping from the incoming message to our internal format
+      #   - #outgoing - for remapping from internal topic name into outgoing message
+      setting :topic_mapper, -> { Routing::TopicMapper }
+      # If batch_fetching is true, we will fetch kafka messages in batches instead of 1 by 1
+      # @note Fetching does not equal consuming, see batch_consuming description for details
+      setting :batch_fetching, true
+      # If batch_consuming is true, we will have access to #params_batch instead of #params.
+      # #params_batch will contain params received from Kafka (may be more than 1) so we can
+      # process them in batches
+      setting :batch_consuming, false
+      # Should we operate in a single controller instance across multiple batches of messages,
+      # from the same partition or should we build a new instance for each incoming batch.
+      # Disabling that can be useful when you want to build a new controller instance for each
+      # incoming batch. It's disabled by default, not to create more objects that needed on
+      # each batch
+      setting :persistent, true
+
+      # option kafka [Hash] - optional - kafka configuration options
+      setting :kafka do
+        # Array with at least one host
+        setting :seed_brokers
+        # option session_timeout [Integer] the number of seconds after which, if a client
+        #   hasn't contacted the Kafka cluster, it will be kicked out of the group.
+        setting :session_timeout, 30
+        # Time that a given partition will be paused from fetching messages, when message
+        # consumption fails. It allows us to process other partitions, while the error is being
+        # resolved and also "slows" things down, so it prevents from "eating" up all messages and
+        # consuming them with failed code
+        setting :pause_timeout, 10
+        # option offset_commit_interval [Integer] the interval between offset commits,
+        #   in seconds.
+        setting :offset_commit_interval, 10
+        # option offset_commit_threshold [Integer] the number of messages that can be
+        #   processed before their offsets are committed. If zero, offset commits are
+        #   not triggered by message consumption.
+        setting :offset_commit_threshold, 0
+        # option heartbeat_interval [Integer] the interval between heartbeats; must be less
+        #   than the session window.
+        setting :heartbeat_interval, 10
+        # option max_bytes_per_partition [Integer] the maximum amount of data fetched
+        #   from a single partition at a time.
+        setting :max_bytes_per_partition, 1_048_576
+        #  whether to consume messages starting at the beginning or to just consume new messages
+        setting :start_from_beginning, true
+        # option min_bytes [Integer] the minimum number of bytes to read before
+        #   returning messages from the server; if `max_wait_time` is reached, this
+        #   is ignored.
+        setting :min_bytes, 1
+        # option max_wait_time [Integer, Float] max_wait_time is the maximum number of seconds to
+        #   wait before returning data from a single message fetch. By setting this high you also
+        #   increase the fetching throughput - and by setting it low you set a bound on latency.
+        #   This configuration overrides `min_bytes`, so you'll _always_ get data back within the
+        #   time specified. The default value is one second. If you want to have at most five
+        #   seconds of latency, set `max_wait_time` to 5. You should make sure
+        #   max_wait_time * num brokers + heartbeat_interval is less than session_timeout.
+        setting :max_wait_time, 1
+        # option automatically_mark_as_consumed [Boolean] should we automatically mark received
+        # messages as consumed (processed) after non-error consumption
+        setting :automatically_mark_as_consumed, true
+        # option reconnect_timeout [Integer] How long should we wait before trying to reconnect to
+        # Kafka cluster that went down (in seconds)
+        setting :reconnect_timeout, 5
+        # option offset_retention_time [Integer] The length of the retention window, known as
+        #   offset retention time
+        setting :offset_retention_time, nil
+        # option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
+        # a broker for the first time. When ruby-kafka initializes, it needs to connect to at
+        # least one host.
+        setting :connect_timeout, 10
+        # option socket_timeout [Integer] Sets the number of seconds to wait when reading from or
+        # writing to a socket connection to a broker. After this timeout expires the connection
+        # will be killed. Note that some Kafka operations are by definition long-running, such as
+        # waiting for new messages to arrive in a partition, so don't set this value too low
+        setting :socket_timeout, 30
+
+        # SSL authentication related settings
+        # option ca_cert [String] SSL CA certificate
+        setting :ssl_ca_cert, nil
+        # option ssl_ca_cert_file_path [String] SSL CA certificate file path
+        setting :ssl_ca_cert_file_path, nil
+        # option ssl_client_cert [String] SSL client certificate
+        setting :ssl_client_cert, nil
+        # option ssl_client_cert_key [String] SSL client certificate password
+        setting :ssl_client_cert_key, nil
+        # option sasl_gssapi_principal [String] sasl principal
+        setting :sasl_gssapi_principal, nil
+        # option sasl_gssapi_keytab [String] sasl keytab
+        setting :sasl_gssapi_keytab, nil
+        # option sasl_plain_authzid [String] The authorization identity to use
+        setting :sasl_plain_authzid, ''
+        # option sasl_plain_username [String] The username used to authenticate
+        setting :sasl_plain_username, nil
+        # option sasl_plain_password [String] The password used to authenticate
+        setting :sasl_plain_password, nil
+      end
+
+      class << self
+        # Configurating method
+        # @yield Runs a block of code providing a config singleton instance to it
+        # @yieldparam [Karafka::Setup::Config] Karafka config instance
+        def setup
+          configure do |config|
+            yield(config)
+          end
+        end
+
+        # Everything that should be initialized after the setup
+        # Components are in karafka/config directory and are all loaded one by one
+        # If you want to configure a next component, please add a proper file to config dir
+        def setup_components
+          Configurators::Base.descendants.each do |klass|
+            klass.new(config).setup
+          end
+        end
+
+        # Validate config based on ConfigurationSchema
+        # @return [Boolean] true if configuration is valid
+        # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
+        #   doesn't match with ConfigurationSchema
+        def validate!
+          validation_result = Karafka::Schemas::Config.call(config.to_h)
+
+          return true if validation_result.success?
+
+          raise Errors::InvalidConfiguration, validation_result.errors
+        end
+      end
+    end
+  end
+end