karafka 1.1.2 → 1.2.0.beta1

data/lib/karafka/params/params_batch.rb

@@ -4,7 +4,7 @@ module Karafka
   module Params
     # Params batch represents a set of messages received from Kafka.
     # @note Params internally are lazy loaded before first use. That way we can skip parsing
-    #   process if we have after_fetched that rejects some incoming messages without using params
+    #   process if we have after_fetch that rejects some incoming messages without using params
     #   It can be also used when handling really heavy data (in terms of parsing).
     class ParamsBatch
       include Enumerable
@@ -13,7 +13,7 @@ module Karafka
       # @param messages_batch [Array<Kafka::FetchedMessage>] messages batch
       # @param topic_parser [Class] topic parser for unparsing messages values
       def initialize(messages_batch, topic_parser)
-        @params_batch = messages_batch.map do |message|
+        @params_batch = messages_batch.map! do |message|
           Karafka::Params::Params.build(message, topic_parser)
         end
       end

data/lib/karafka/patches/dry_configurable.rb

@@ -19,11 +19,15 @@ module Karafka
       private
 
       # Method that rebuilds a given accessor, so when it consists a proc value, it will
-      # evaluate it upon return
+      # evaluate it upon return for blocks that don't require any arguments, otherwise
+      # it will return the block
       # @param method_name [Symbol] name of an accessor that we want to rebuild
       def rebuild(method_name)
         define_singleton_method method_name do
-          super().is_a?(Proc) ? super().call : super()
+          value = super()
+          return value unless value.is_a?(Proc)
+          return value unless value.parameters.empty?
+          value.call
         end
       end
     end

data/lib/karafka/patches/ruby_kafka.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Patches
-    # Batches for Ruby Kafka gem
+    # Patches for Ruby Kafka gem
     module RubyKafka
       # This patch allows us to inject business logic in between fetches and before the consumer
       # stop, so we can perform stop commit or anything else that we need since
@@ -13,19 +13,19 @@ module Karafka
       # thread)
       def consumer_loop
         super do
-          controllers = Karafka::Persistence::Controller
-                        .all
-                        .values
-                        .flat_map(&:values)
-                        .select { |ctrl| ctrl.respond_to?(:run_callbacks) }
+          consumers = Karafka::Persistence::Consumer
+                      .all
+                      .values
+                      .flat_map(&:values)
+                      .select { |ctrl| ctrl.respond_to?(:run_callbacks) }
 
           if Karafka::App.stopped?
-            controllers.each { |ctrl| ctrl.run_callbacks :before_stop }
-            Karafka::Persistence::Consumer.read.stop
+            consumers.each { |ctrl| ctrl.run_callbacks :before_stop }
+            Karafka::Persistence::Client.read.stop
           else
-            controllers.each { |ctrl| ctrl.run_callbacks :before_poll }
+            consumers.each { |ctrl| ctrl.run_callbacks :before_poll }
             yield
-            controllers.each { |ctrl| ctrl.run_callbacks :after_poll }
+            consumers.each { |ctrl| ctrl.run_callbacks :after_poll }
           end
         end
       end

data/lib/karafka/persistence/client.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Persistence layer to store current thread messages consumer client for further use
+    class Client
+      # Thread.current key under which we store current thread messages consumer client
+      PERSISTENCE_SCOPE = :client
+
+      # @param client [Karafka::Connection::Client] messages consumer client of
+      #   a current thread
+      # @return [Karafka::Connection::Client] persisted messages consumer client
+      def self.write(client)
+        Thread.current[PERSISTENCE_SCOPE] = client
+      end
+
+      # @return [Karafka::Connection::Client] persisted messages consumer client
+      # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
+      #   client but we try to use it anyway
+      def self.read
+        Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClient)
+      end
+    end
+  end
+end

data/lib/karafka/persistence/consumer.rb

@@ -1,24 +1,37 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Module used to provide a persistent cache layer for Karafka components that need to be
+  # shared inside of a same thread
   module Persistence
-    # Persistence layer to store current thread messages consumer for further use
+    # Module used to provide a persistent cache across batch requests for a given
+    # topic and partition to store some additional details when the persistent mode
+    # for a given topic is turned on
     class Consumer
-      # Thread.current key under which we store current thread messages consumer
-      PERSISTENCE_SCOPE = :consumer
+      # Thread.current scope under which we store consumers data
+      PERSISTENCE_SCOPE = :consumers
 
-      # @param consumer [Karafka::Connection::Consumer] messages consumer of
-      #   a current thread
-      # @return [Karafka::Connection::Consumer] persisted messages consumer
-      def self.write(consumer)
-        Thread.current[PERSISTENCE_SCOPE] = consumer
-      end
+      class << self
+        # @return [Hash] current thread persistence scope hash with all the consumers
+        def all
+          # @note This does not need to be threadsafe (Hash) as it is always executed in a
+          # current thread context
+          Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
+        end
 
-      # @return [Karafka::Connection::Consumer] persisted messages consumer
-      # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
-      #   but we try to use it anyway
-      def self.read
-        Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingConsumer)
+        # Used to build (if block given) and/or fetch a current consumer instance that will be
+        #   used to process messages from a given topic and partition
+        # @return [Karafka::BaseConsumer] base consumer descendant
+        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
+        # @param partition [Integer] number of partition for which we want to cache
+        def fetch(topic, partition)
+          # We always store a current instance for callback reasons
+          if topic.persistent
+            all[topic][partition] ||= topic.consumer.new
+          else
+            all[topic][partition] = topic.consumer.new
+          end
+        end
       end
     end
   end

data/lib/karafka/persistence/topic.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Local cache for routing topics
+    # We use it in order not to build string instances and remap incoming topic upon each
+    # message / message batches received
+    class Topic
+      # Thread.current scope under which we store topics data
+      PERSISTENCE_SCOPE = :topics
+
+      # @param group_id [String] group id for which we fetch a topic representation
+      # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
+      #   topic representation
+      # @return [Karafka::Routing::Topic] remapped topic representation that can be used further
+      #   on when working with given parameters
+      def self.fetch(group_id, raw_topic_name)
+        Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
+
+        Thread.current[PERSISTENCE_SCOPE][group_id][raw_topic_name] ||= begin
+          # We map from incoming topic name, as it might be namespaced, etc.
+          # @see topic_mapper internal docs
+          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
+          Routing::Router.find("#{group_id}_#{mapped_topic_name}")
+        end
+      end
+    end
+  end
+end

data/lib/karafka/process.rb

@@ -8,7 +8,9 @@ module Karafka
 
     # Signal types that we handle
     HANDLED_SIGNALS = %i[
-      SIGINT SIGQUIT SIGTERM
+      SIGINT
+      SIGQUIT
+      SIGTERM
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|
@@ -27,8 +29,7 @@ module Karafka
 
     # Creates an instance of process and creates empty hash for callbacks
     def initialize
-      @callbacks = {}
-      HANDLED_SIGNALS.each { |signal| @callbacks[signal] = [] }
+      @callbacks = Hash.new { |hsh, key| hsh[key] = [] }
     end
 
     # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
@@ -56,7 +57,7 @@ module Karafka
     #   we have to spin up a new thread to do this
     def notice_signal(signal)
       Thread.new do
-        Karafka.monitor.notice(self.class, signal: signal)
+        Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
       end
     end
   end

data/lib/karafka/responders/builder.rb

@@ -3,30 +3,31 @@
 module Karafka
   # Responders namespace encapsulates all the internal responder implementation parts
   module Responders
-    # Responders builder is used to find (based on the controller class name) a responder that
-    # match the controller. This is used when user does not provide a responder inside routing
-    # but he still names responder with the same convention (and namespaces) as controller
+    # Responders builder is used to finding (based on the consumer class name) a responder
+    # that match the consumer. We use it when user does not provide a responder inside routing,
+    # but he still names responder with the same convention (and namespaces) as consumer
+    #
     # @example Matching responder exists
-    #   Karafka::Responder::Builder(NewEventsController).build #=> NewEventsResponder
+    #   Karafka::Responder::Builder(NewEventsConsumer).build #=> NewEventsResponder
     # @example Matching responder does not exist
-    #   Karafka::Responder::Builder(NewBuildsController).build #=> nil
+    #   Karafka::Responder::Builder(NewBuildsConsumer).build #=> nil
     class Builder
-      # @param controller_class [Karafka::BaseController, nil] descendant of
-      #   Karafka::BaseController
-      # @example Tries to find a responder that matches a given controller. If nothing found,
-      #   will return nil (nil is accepted, because it means that a given controller don't
+      # @param consumer_class [Karafka::BaseConsumer, nil] descendant of
+      #   Karafka::BaseConsumer
+      # @example Tries to find a responder that matches a given consumer. If nothing found,
+      #   will return nil (nil is accepted, because it means that a given consumer don't
       #   pipe stuff further on)
-      def initialize(controller_class)
-        @controller_class = controller_class
+      def initialize(consumer_class)
+        @consumer_class = consumer_class
       end
 
-      # Tries to figure out a responder based on a controller class name
+      # Tries to figure out a responder based on a consumer class name
       # @return [Class] Responder class (not an instance)
       # @return [nil] or nil if there's no matching responding class
       def build
         Helpers::ClassMatcher.new(
-          @controller_class,
-          from: 'Controller',
+          @consumer_class,
+          from: 'Consumer',
           to: 'Responder'
         ).match
       end

data/lib/karafka/routing/builder.rb

@@ -6,7 +6,7 @@ module Karafka
     # @example Build a simple (most common) route
     #   consumers do
     #     topic :new_videos do
-    #       controller NewVideosController
+    #       consumer NewVideosConsumer
     #     end
     #   end
     class Builder < Array

data/lib/karafka/routing/consumer_mapper.rb

@@ -17,7 +17,7 @@ module Karafka
     #   module MyMapper
     #     def self.call(raw_consumer_group_name)
     #       [
-    #         Karafka::App.config.client_id.to_s.underscope,
+    #         Dry::Inflector.new.underscore(Karafka::App.config.client_id.to_s),
     #         raw_consumer_group_name
     #       ].join('_').gsub('_', '.')
     #     end
@@ -26,7 +26,8 @@ module Karafka
       # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
       # @return [String] remapped final consumer group name
       def self.call(raw_consumer_group_name)
-        "#{Karafka::App.config.client_id.to_s.underscore}_#{raw_consumer_group_name}"
+        client_name = Dry::Inflector.new.underscore(Karafka::App.config.client_id.to_s)
+        "#{client_name}_#{raw_consumer_group_name}"
       end
     end
   end

data/lib/karafka/routing/router.rb

@@ -3,7 +3,7 @@
 module Karafka
   # Namespace for all elements related to requests routing
   module Routing
-    # Karafka framework Router for routing incoming messages to proper controllers
+    # Karafka framework Router for routing incoming messages to proper consumers
     # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
     #  structure so all the routes are being stored in a single level array
     module Router

data/lib/karafka/routing/topic.rb

@@ -9,7 +9,7 @@ module Karafka
       extend Helpers::ConfigRetriever
 
       attr_reader :id, :consumer_group
-      attr_accessor :controller
+      attr_accessor :consumer
 
       # @param [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
@@ -29,14 +29,14 @@ module Karafka
       # example for Sidekiq
       def build
         Karafka::AttributesMap.topic.each { |attr| send(attr) }
-        controller&.topic = self
+        consumer&.topic = self
         self
       end
 
       # @return [Class, nil] Class (not an instance) of a responder that should respond from
-      #   controller back to Kafka (usefull for piping dataflows)
+      #   consumer back to Kafka (usefull for piping dataflows)
       def responder
-        @responder ||= Karafka::Responders::Builder.new(controller).build
+        @responder ||= Karafka::Responders::Builder.new(consumer).build
       end
 
       Karafka::AttributesMap.topic.each do |attribute|
@@ -52,7 +52,7 @@ module Karafka
 
         Hash[map].merge!(
           id: id,
-          controller: controller
+          consumer: consumer
         )
       end
     end

data/lib/karafka/schemas/config.rb

@@ -13,8 +13,11 @@ module Karafka
     #   so we validate all of that once all the routes are defined and ready
     Config = Dry::Validation.Schema do
       required(:client_id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
+      required(:shutdown_timeout) { none? | (int? & gteq?(0)) }
       required(:consumer_mapper)
       required(:topic_mapper)
+      required(:params_base_class).filled
+
       optional(:backend).filled
     end
   end

data/lib/karafka/schemas/consumer_group.rb

@@ -9,6 +9,9 @@ module Karafka
       # but someetimes loads things twice
       URI_SCHEMES ||= %w[kafka kafka+ssl].freeze
 
+      # Available sasl scram mechanism of authentication (plus nil)
+      SASL_SCRAM_MECHANISMS ||= %w[sha256 sha512].freeze
+
       configure do
         config.messages_file = File.join(
           Karafka.gem_root, 'config', 'errors.yml'
@@ -36,6 +39,7 @@ module Karafka
       required(:connect_timeout).filled { (int? | float?) & gt?(0) }
       required(:socket_timeout).filled { (int? | float?) & gt?(0) }
       required(:min_bytes).filled(:int?, gt?: 0)
+      required(:max_bytes).filled(:int?, gt?: 0)
       required(:max_wait_time).filled { (int? | float?) & gteq?(0) }
       required(:batch_fetching).filled(:bool?)
       required(:topics).filled { each { schema(ConsumerGroupTopic) } }
@@ -52,14 +56,22 @@ module Karafka
         ssl_ca_cert_file_path
         ssl_client_cert
         ssl_client_cert_key
+        sasl_gssapi_principal
+        sasl_gssapi_keytab
         sasl_plain_authzid
         sasl_plain_username
         sasl_plain_password
-        sasl_gssapi_principal
-        sasl_gssapi_keytab
+        sasl_scram_username
+        sasl_scram_password
       ].each do |encryption_attribute|
         optional(encryption_attribute).maybe(:str?)
       end
+
+      optional(:ssl_ca_certs_from_system).maybe(:bool?)
+
+      # It's not with other encryptions as it has some more rules
+      optional(:sasl_scram_mechanism)
+        .maybe(:str?, included_in?: Karafka::Schemas::SASL_SCRAM_MECHANISMS)
     end
   end
 end

data/lib/karafka/schemas/consumer_group_topic.rb

@@ -7,7 +7,7 @@ module Karafka
       required(:id).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
       required(:name).filled(:str?, format?: Karafka::Schemas::TOPIC_REGEXP)
       required(:backend).filled(included_in?: %i[inline sidekiq])
-      required(:controller).filled
+      required(:consumer).filled
       required(:parser).filled
       required(:max_bytes_per_partition).filled(:int?, gteq?: 0)
       required(:start_from_beginning).filled(:bool?)

data/lib/karafka/server.rb

@@ -3,6 +3,13 @@
 module Karafka
   # Karafka consuming server class
   class Server
+    @consumer_threads = Concurrent::Array.new
+
+    # How long should we sleep between checks on shutting down consumers
+    SUPERVISION_SLEEP = 1
+    # What system exit code should we use when we terminated forcefully
+    FORCEFUL_EXIT_CODE = 2
+
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
       attr_accessor :consumer_threads
@@ -12,7 +19,6 @@ module Karafka
 
       # Method which runs app
       def run
-        @consumer_threads = Concurrent::Array.new
         bind_on_sigint
         bind_on_sigquit
         bind_on_sigterm
@@ -35,17 +41,17 @@ module Karafka
 
       # What should happen when we decide to quit with sigint
       def bind_on_sigint
-        process.on_sigint { Karafka::App.stop! }
+        process.on_sigint { stop_supervised }
       end
 
       # What should happen when we decide to quit with sigquit
       def bind_on_sigquit
-        process.on_sigquit { Karafka::App.stop! }
+        process.on_sigquit { stop_supervised }
       end
 
       # What should happen when we decide to quit with sigterm
       def bind_on_sigterm
-        process.on_sigterm { Karafka::App.stop! }
+        process.on_sigterm { stop_supervised }
       end
 
       # Starts Karafka with a supervision
@@ -54,8 +60,30 @@ module Karafka
       def start_supervised
         process.supervise do
           Karafka::App.run!
-          Karafka::Fetcher.new.fetch_loop
+          Karafka::Fetcher.call
+        end
+      end
+
+      # Stops Karafka with a supervision (as long as there is a shutdown timeout)
+      # If consumers won't stop in a given timeframe, it will force them to exit
+      def stop_supervised
+        Karafka::App.stop!
+
+        # If there is no shutdown timeout, we don't exit and wait until all the consumers
+        # had done their work
+        return unless Karafka::App.config.shutdown_timeout
+
+        # If there is a timeout, we check every 1 second (for the timeout period) if all
+        # the threads finished their work and if so, we can just return and normal
+        # shutdown process will take place
+        Karafka::App.config.shutdown_timeout.to_i.times do
+          return if consumer_threads.count(&:alive?).zero?
+          sleep SUPERVISION_SLEEP
         end
+
+        # We're done waiting, lets kill them!
+        consumer_threads.each(&:terminate)
+        Kernel.exit FORCEFUL_EXIT_CODE
       end
     end
   end