karafka 1.0.0 → 1.2.0

data/lib/karafka/routing/builder.rb

@@ -6,7 +6,7 @@ module Karafka
     # @example Build a simple (most common) route
     #   consumers do
     #     topic :new_videos do
-    #       controller NewVideosController
+    #       consumer NewVideosConsumer
     #     end
     #   end
     class Builder < Array
@@ -28,7 +28,7 @@ module Karafka
           hashed_group = consumer_group.to_h
           validation_result = Karafka::Schemas::ConsumerGroup.call(hashed_group)
           return if validation_result.success?
-          raise Errors::InvalidConfiguration, [validation_result.errors, hashed_group]
+          raise Errors::InvalidConfiguration, validation_result.errors
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -18,7 +18,7 @@ module Karafka
       #   kafka and don't understand the concept of consumer groups.
       def initialize(name)
         @name = name
-        @id = "#{Karafka::App.config.client_id.to_s.underscore}_#{@name}"
+        @id = Karafka::App.config.consumer_mapper.call(name)
         @topics = []
       end
 

data/lib/karafka/routing/consumer_mapper.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Default consumer mapper that builds consumer ids based on app id and consumer group name
+    # Different mapper can be used in case of preexisting consumer names or for applying
+    # other naming conventions not compatible wiih Karafkas client_id + consumer name concept
+    #
+    # @example Mapper for using consumer groups without a client_id prefix
+    #   module MyMapper
+    #     def self.call(raw_consumer_group_name)
+    #       raw_consumer_group_name
+    #     end
+    #   end
+    #
+    # @example Mapper for replacing "_" with "." in topic names
+    #   module MyMapper
+    #     def self.call(raw_consumer_group_name)
+    #       [
+    #         Dry::Inflector.new.underscore(Karafka::App.config.client_id.to_s),
+    #         raw_consumer_group_name
+    #       ].join('_').gsub('_', '.')
+    #     end
+    #   end
+    module ConsumerMapper
+      # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
+      # @return [String] remapped final consumer group name
+      def self.call(raw_consumer_group_name)
+        client_name = Dry::Inflector.new.underscore(Karafka::App.config.client_id.to_s)
+        "#{client_name}_#{raw_consumer_group_name}"
+      end
+    end
+  end
+end

data/lib/karafka/routing/router.rb

@@ -3,7 +3,7 @@
 module Karafka
   # Namespace for all elements related to requests routing
   module Routing
-    # Karafka framework Router for routing incoming messages to proper controllers
+    # Karafka framework Router for routing incoming messages to proper consumers
     # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
     #  structure so all the routes are being stored in a single level array
     module Router

data/lib/karafka/routing/topic.rb

@@ -9,7 +9,7 @@ module Karafka
       extend Helpers::ConfigRetriever
 
       attr_reader :id, :consumer_group
-      attr_accessor :controller
+      attr_accessor :consumer
 
       # @param [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
@@ -29,20 +29,14 @@ module Karafka
       # example for Sidekiq
       def build
         Karafka::AttributesMap.topic.each { |attr| send(attr) }
-        controller&.topic = self
+        consumer&.topic = self
         self
       end
 
       # @return [Class, nil] Class (not an instance) of a responder that should respond from
-      #   controller back to Kafka (usefull for piping dataflows)
+      #   consumer back to Kafka (usefull for piping dataflows)
       def responder
-        @responder ||= Karafka::Responders::Builder.new(controller).build
-      end
-
-      # @return [Class] Parser class (not instance) that we want to use to unparse Kafka messages
-      # @note If not provided - will use Json as default
-      def parser
-        @parser ||= Karafka::Parsers::Json
+        @responder ||= Karafka::Responders::Builder.new(consumer).build
       end
 
       Karafka::AttributesMap.topic.each do |attribute|
@@ -58,7 +52,7 @@ module Karafka
 
         Hash[map].merge!(
           id: id,
-          controller: controller
+          consumer: consumer
         )
       end
     end

data/lib/karafka/routing/{mapper.rb → topic_mapper.rb}

@@ -2,7 +2,7 @@
 
 module Karafka
   module Routing
-    # Default routes mapper that does not remap things
+    # Default topic mapper that does not remap things
     # Mapper can be used for Kafka providers that require namespaced topic names. Instead of being
     # provider dependent, we can then define mapper and use internally "pure" topic names in
     # routes and responders
@@ -32,7 +32,7 @@ module Karafka
     #       topic.to_s.gsub('_', '.')
     #     end
     #   end
-    module Mapper
+    module TopicMapper
       class << self
         # @param topic [String, Symbol] topic
         # @return [String, Symbol] same topic as on input

data/lib/karafka/schemas/config.rb

@@ -13,13 +13,12 @@ module Karafka
     #   so we validate all of that once all the routes are defined and ready
     Config = Dry::Validation.Schema do
       required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:shutdown_timeout) { none? | (int? & gteq?(0)) }
+      required(:consumer_mapper)
+      required(:topic_mapper)
+      required(:params_base_class).filled
 
       optional(:backend).filled
-
-      optional(:connection_pool).schema do
-        required(:size).filled
-        optional(:timeout).filled(:int?)
-      end
     end
   end
 end

data/lib/karafka/schemas/consumer_group.rb

@@ -2,33 +2,46 @@
 
 module Karafka
   module Schemas
-    # Consumer group topic validation rules
-    ConsumerGroupTopic = Dry::Validation.Schema do
-      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:backend).filled(included_in?: %i[inline sidekiq])
-      required(:controller).filled
-      required(:parser).filled
-      required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
-      required(:start_from_beginning).filled(:bool?)
-      required(:batch_processing).filled(:bool?)
-      required(:persistent).filled(:bool?)
-    end
-
     # Schema for single full route (consumer group + topics) validation.
     ConsumerGroup = Dry::Validation.Schema do
+      # Valid uri schemas of Kafka broker url
+      # The ||= is due to the behavior of require_all that resolves dependencies
+      # but someetimes loads things twice
+      URI_SCHEMES ||= %w[kafka kafka+ssl].freeze
+
+      # Available sasl scram mechanism of authentication (plus nil)
+      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
+
+      configure do
+        config.messages_file = File.join(
+          Karafka.gem_root, 'config', 'errors.yml'
+        )
+
+        # Uri validator to check if uri is in a Karafka acceptable format
+        # @param uri [String] uri we want to validate
+        # @return [Boolean] true if it is a valid uri, otherwise false
+        def broker_schema?(uri)
+          uri = URI.parse(uri)
+          URI_SCHEMES.include?(uri.scheme) && uri.port
+        rescue URI::InvalidURIError
+          false
+        end
+      end
+
       required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
-      required(:seed_brokers).filled(:array?)
-      required(:session_timeout).filled(:int?)
-      required(:pause_timeout).filled(:int?, gteq?: 0)
-      required(:offset_commit_interval).filled(:int?)
+      required(:seed_brokers).filled { each(:broker_schema?) }
+      required(:session_timeout).filled { int? | float? }
+      required(:pause_timeout).filled { (int? | float?) & gteq?(0) }
+      required(:offset_commit_interval) { int? | float? }
       required(:offset_commit_threshold).filled(:int?)
       required(:offset_retention_time) { none?.not > int? }
-      required(:heartbeat_interval).filled(:int?, gteq?: 0)
-      required(:connect_timeout).filled(:int?, gt?: 0)
-      required(:socket_timeout).filled(:int?, gt?: 0)
-      required(:max_wait_time).filled(:int?, gteq?: 0)
-      required(:batch_consuming).filled(:bool?)
+      required(:heartbeat_interval).filled { (int? | float?) & gteq?(0) }
+      required(:connect_timeout).filled { (int? | float?) & gt?(0) }
+      required(:socket_timeout).filled { (int? | float?) & gt?(0) }
+      required(:min_bytes).filled(:int?, gt?: 0)
+      required(:max_bytes).filled(:int?, gt?: 0)
+      required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
+      required(:batch_fetching).filled(:bool?)
       required(:topics).filled { each { schema(ConsumerGroupTopic) } }
 
       # Max wait time cannot exceed socket_timeout - wouldn't make sense
@@ -43,14 +56,22 @@ module Karafka
         ssl_ca_cert_file_path
         ssl_client_cert
         ssl_client_cert_key
+        sasl_gssapi_principal
+        sasl_gssapi_keytab
         sasl_plain_authzid
         sasl_plain_username
         sasl_plain_password
-        sasl_gssapi_principal
-        sasl_gssapi_keytab
+        sasl_scram_username
+        sasl_scram_password
       ].each do |encryption_attribute|
         optional(encryption_attribute).maybe(:str?)
       end
+
+      optional(:ssl_ca_certs_from_system).maybe(:bool?)
+
+      # It's not with other encryptions as it has some more rules
+      optional(:sasl_scram_mechanism)
+        .maybe(:str?, included_in?: Karafka::Schemas::SASL_SCRAM_MECHANISMS)
     end
   end
 end

data/lib/karafka/schemas/consumer_group_topic.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Schemas
+    # Consumer group topic validation rules
+    ConsumerGroupTopic = Dry::Validation.Schema do
+      required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:backend).filled(included_in?: %i[inline sidekiq])
+      required(:consumer).filled
+      required(:parser).filled
+      required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
+      required(:start_from_beginning).filled(:bool?)
+      required(:batch_consuming).filled(:bool?)
+      required(:persistent).filled(:bool?)
+    end
+  end
+end

data/lib/karafka/schemas/responder_usage.rb

@@ -9,6 +9,7 @@ module Karafka
       required(:multiple_usage).filled(:bool?)
       required(:usage_count).filled(:int?, gteq?: 0)
       required(:registered).filled(eql?: true)
+      required(:async).filled(:bool?)
 
       rule(
         required_usage: %i[required usage_count]

data/lib/karafka/server.rb

@@ -3,17 +3,22 @@
 module Karafka
   # Karafka consuming server class
   class Server
+    @consumer_threads = Concurrent::Array.new
+
+    # How long should we sleep between checks on shutting down consumers
+    SUPERVISION_SLEEP = 1
+    # What system exit code should we use when we terminated forcefully
+    FORCEFUL_EXIT_CODE = 2
+
     class << self
-      # We need to store reference to all the consumers in the main server thread,
-      # So we can have access to them later on and be able to stop them on exit
-      attr_reader :consumers
+      # Set of consuming threads. Each consumer thread contains a single consumer
+      attr_accessor :consumer_threads
 
       # Writer for list of consumer groups that we want to consume in our current process context
       attr_writer :consumer_groups
 
       # Method which runs app
       def run
-        @consumers = Concurrent::Array.new
         bind_on_sigint
         bind_on_sigquit
         bind_on_sigterm
@@ -36,29 +41,17 @@ module Karafka
 
       # What should happen when we decide to quit with sigint
       def bind_on_sigint
-        process.on_sigint do
-          Karafka::App.stop!
-          consumers.map(&:stop)
-          Kernel.exit
-        end
+        process.on_sigint { stop_supervised }
       end
 
       # What should happen when we decide to quit with sigquit
       def bind_on_sigquit
-        process.on_sigquit do
-          Karafka::App.stop!
-          consumers.map(&:stop)
-          Kernel.exit
-        end
+        process.on_sigquit { stop_supervised }
       end
 
       # What should happen when we decide to quit with sigterm
       def bind_on_sigterm
-        process.on_sigterm do
-          Karafka::App.stop!
-          consumers.map(&:stop)
-          Kernel.exit
-        end
+        process.on_sigterm { stop_supervised }
       end
 
       # Starts Karafka with a supervision
@@ -67,8 +60,34 @@ module Karafka
       def start_supervised
         process.supervise do
           Karafka::App.run!
-          Karafka::Fetcher.new.fetch_loop
+          Karafka::Fetcher.call
+        end
+      end
+
+      # Stops Karafka with a supervision (as long as there is a shutdown timeout)
+      # If consumers won't stop in a given timeframe, it will force them to exit
+      def stop_supervised
+        Karafka.monitor.instrument('server.stop', {})
+
+        Karafka::App.stop!
+        # If there is no shutdown timeout, we don't exit and wait until all the consumers
+        # had done their work
+        return unless Karafka::App.config.shutdown_timeout
+
+        # If there is a timeout, we check every 1 second (for the timeout period) if all
+        # the threads finished their work and if so, we can just return and normal
+        # shutdown process will take place
+        Karafka::App.config.shutdown_timeout.to_i.times do
+          return if consumer_threads.count(&:alive?).zero?
+          sleep SUPERVISION_SLEEP
         end
+
+        Karafka.monitor.instrument('server.stop.error', {})
+        # We're done waiting, lets kill them!
+        consumer_threads.each(&:terminate)
+
+        # exit is not within the instrumentation as it would not trigger due to exit
+        Kernel.exit FORCEFUL_EXIT_CODE
       end
     end
   end

data/lib/karafka/setup/config.rb

@@ -13,6 +13,7 @@ module Karafka
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
       extend Dry::Configurable
+      extend Callbacks::Config
 
       # Available settings
       # option client_id [String] kafka client_id - used to provide
@@ -21,39 +22,46 @@ module Karafka
       # What backend do we want to use to process messages
       setting :backend, :inline
       # option logger [Instance] logger that we want to use
-      setting :logger, -> { ::Karafka::Logger.instance }
+      setting :logger, -> { ::Karafka::Instrumentation::Logger.instance }
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
-      setting :monitor, -> { ::Karafka::Monitor.instance }
+      setting :monitor, -> { ::Karafka::Instrumentation::Monitor.instance }
+      # Mapper used to remap consumer groups ids, so in case users migrate from other tools
+      # or they need to maintain their own internal consumer group naming conventions, they
+      # can easily do it, replacing the default client_id + consumer name pattern concept
+      setting :consumer_mapper, -> { Routing::ConsumerMapper }
       # Mapper used to remap names of topics, so we can have a clean internal topic namings
       # despite using any Kafka provider that uses namespacing, etc
       # It needs to implement two methods:
       #   - #incoming - for remapping from the incoming message to our internal format
       #   - #outgoing - for remapping from internal topic name into outgoing message
-      setting :topic_mapper, -> { Routing::Mapper }
-      # If batch_consuming is true, we will consume kafka messages in batches instead of 1 by 1
-      # @note Consuming does not equal processing, see batch_processing description for details
-      setting :batch_consuming, true
-      # If batch_processing is true, we will have access to #params_batch instead of #params.
+      setting :topic_mapper, -> { Routing::TopicMapper }
+      # Default parser for parsing and unparsing incoming and outgoing data
+      setting :parser, -> { Karafka::Parsers::Json }
+      # If batch_fetching is true, we will fetch kafka messages in batches instead of 1 by 1
+      # @note Fetching does not equal consuming, see batch_consuming description for details
+      setting :batch_fetching, true
+      # If batch_consuming is true, we will have access to #params_batch instead of #params.
       # #params_batch will contain params received from Kafka (may be more than 1) so we can
       # process them in batches
-      setting :batch_processing, false
-      # Should we operate in a single controller instance across multiple batches of messages,
-      # from the same partition or should we build a new instance for each incoming batch.
-      # Disabling that can be useful when you want to build a new controller instance for each
-      # incoming batch. It's disabled by default, not to create more objects that needed on
-      # each batch
+      setting :batch_consuming, false
+      # Should we operate in a single consumer instance across multiple batches of messages,
+      # from the same partition or should we build a new one for each incoming batch.
+      # Disabling that can be useful when you want to create a new consumer instance for each
+      # incoming batch. It's disabled by default, not to create more objects that needed
+      # on each batch
       setting :persistent, true
-      # Connection pool options are used for producer (Waterdrop) - by default it will adapt to
-      # number of active actors
-      setting :connection_pool do
-        # Connection pool size for producers. If you use sidekiq or any other multi threaded
-        # backend, you might want to tune it to match number of threads of your background
-        # processing engine
-        setting :size, -> { ::Karafka::App.consumer_groups.active.count }
-        # How long should we wait for a working resource from the pool before rising timeout
-        # With a proper connection pool size, this should never happen
-        setting :timeout, 5
-      end
+      # option shutdown_timeout [Integer, nil] the number of seconds after which Karafka no
+      #   longer wait for the consumers to stop gracefully but instead we force
+      #   terminate everything.
+      # @note Keep in mind, that if your business logic
+      # @note If set to nil, it won't forcefully shutdown the process at all.
+      setting :shutdown_timeout, 60
+      # option params_base_class [Class] base class for params class initialization
+      #   This can be either a Hash or a HashWithIndifferentAccess depending on your
+      #   requirements. Note, that by using HashWithIndifferentAccess, you remove some of the
+      #   performance in favor of convenience. This can be useful especially if you already use
+      #   it with Rails, etc
+      setting :params_base_class, Hash
 
       # option kafka [Hash] - optional - kafka configuration options
       setting :kafka do
@@ -62,17 +70,17 @@ module Karafka
         # option session_timeout [Integer] the number of seconds after which, if a client
         #   hasn't contacted the Kafka cluster, it will be kicked out of the group.
         setting :session_timeout, 30
-        # Time that a given partition will be paused from processing messages, when message
-        # processing fails. It allows us to process other partitions, while the error is being
+        # Time that a given partition will be paused from fetching messages, when message
+        # consumption fails. It allows us to process other partitions, while the error is being
         # resolved and also "slows" things down, so it prevents from "eating" up all messages and
-        # processing them with failed code
+        # consuming them with failed code
         setting :pause_timeout, 10
         # option offset_commit_interval [Integer] the interval between offset commits,
         #   in seconds.
         setting :offset_commit_interval, 10
         # option offset_commit_threshold [Integer] the number of messages that can be
         #   processed before their offsets are committed. If zero, offset commits are
-        #   not triggered by message processing.
+        #   not triggered by message consumption.
         setting :offset_commit_threshold, 0
         # option heartbeat_interval [Integer] the interval between heartbeats; must be less
         #   than the session window.
@@ -86,9 +94,20 @@ module Karafka
         #   returning messages from the server; if `max_wait_time` is reached, this
         #   is ignored.
         setting :min_bytes, 1
-        # option max_wait_time [Integer, Float] the maximum duration of time to wait before
-        #   returning messages from the server, in seconds.
-        setting :max_wait_time, 5
+        # option max_bytes [Integer] the maximum number of bytes to read before returning messages
+        #   from each broker.
+        setting :max_bytes, 10_485_760
+        # option max_wait_time [Integer, Float] max_wait_time is the maximum number of seconds to
+        #   wait before returning data from a single message fetch. By setting this high you also
+        #   increase the fetching throughput - and by setting it low you set a bound on latency.
+        #   This configuration overrides `min_bytes`, so you'll _always_ get data back within the
+        #   time specified. The default value is one second. If you want to have at most five
+        #   seconds of latency, set `max_wait_time` to 5. You should make sure
+        #   max_wait_time * num brokers + heartbeat_interval is less than session_timeout.
+        setting :max_wait_time, 1
+        # option automatically_mark_as_consumed [Boolean] should we automatically mark received
+        # messages as consumed (processed) after non-error consumption
+        setting :automatically_mark_as_consumed, true
         # option reconnect_timeout [Integer] How long should we wait before trying to reconnect to
         # Kafka cluster that went down (in seconds)
         setting :reconnect_timeout, 5
@@ -103,50 +122,54 @@ module Karafka
         # writing to a socket connection to a broker. After this timeout expires the connection
         # will be killed. Note that some Kafka operations are by definition long-running, such as
         # waiting for new messages to arrive in a partition, so don't set this value too low
-        setting :socket_timeout, 10
+        setting :socket_timeout, 30
+
         # SSL authentication related settings
-        # option ca_cert [String] SSL CA certificate
+        # option ca_cert [String, nil] SSL CA certificate
         setting :ssl_ca_cert, nil
-        # option ssl_ca_cert_file_path [String] SSL CA certificate file path
+        # option ssl_ca_cert_file_path [String, nil] SSL CA certificate file path
         setting :ssl_ca_cert_file_path, nil
-        # option client_cert [String] SSL client certificate
+        # option ssl_ca_certs_from_system [Boolean] Use the CA certs from your system's default
+        #   certificate store
+        setting :ssl_ca_certs_from_system, false
+        # option ssl_client_cert [String, nil] SSL client certificate
         setting :ssl_client_cert, nil
-        # option client_cert_key [String] SSL client certificate password
+        # option ssl_client_cert_key [String, nil] SSL client certificate password
         setting :ssl_client_cert_key, nil
-        # option sasl_gssapi_principal [String] sasl principal
+        # option sasl_gssapi_principal [String, nil] sasl principal
         setting :sasl_gssapi_principal, nil
-        # option sasl_gssapi_keytab [String] sasl keytab
+        # option sasl_gssapi_keytab [String, nil] sasl keytab
         setting :sasl_gssapi_keytab, nil
         # option sasl_plain_authzid [String] The authorization identity to use
         setting :sasl_plain_authzid, ''
-        # option sasl_plain_username [String] The username used to authenticate
+        # option sasl_plain_username [String, nil] The username used to authenticate
         setting :sasl_plain_username, nil
-        # option sasl_plain_password [String] The password used to authenticate
+        # option sasl_plain_password [String, nil] The password used to authenticate
         setting :sasl_plain_password, nil
+        # option sasl_scram_username [String, nil] The username used to authenticate
+        setting :sasl_scram_username, nil
+        # option sasl_scram_password [String, nil] The password used to authenticate
+        setting :sasl_scram_password, nil
+        # option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
+        setting :sasl_scram_mechanism, nil
       end
 
-      # This is configured automatically, don't overwrite it!
-      # Each consumer group requires separate thread, so number of threads should be equal to
-      # number of consumer groups
-      setting :concurrency, -> { ::Karafka::App.consumer_groups.count }
-
       class << self
         # Configurating method
         # @yield Runs a block of code providing a config singleton instance to it
         # @yieldparam [Karafka::Setup::Config] Karafka config instance
         def setup
-          configure do |config|
-            yield(config)
-          end
+          configure { |config| yield(config) }
         end
 
         # Everything that should be initialized after the setup
         # Components are in karafka/config directory and are all loaded one by one
         # If you want to configure a next component, please add a proper file to config dir
         def setup_components
-          Configurators::Base.descendants.each do |klass|
-            klass.new(config).setup
-          end
+          [
+            Configurators::Params,
+            Configurators::WaterDrop
+          ].each { |klass| klass.setup(config) }
         end
 
         # Validate config based on ConfigurationSchema