karafka 1.1.0 → 1.3.0

data/lib/karafka/routing/proxy.rb

@@ -14,22 +14,31 @@ module Karafka
         !
       ].freeze
 
+      private_constant :IGNORED_POSTFIXES
+
       # @param target [Object] target object to which we proxy any DSL call
-      # @yield Evaluates block in the proxy context
+      # @param block [Proc] block that we want to evaluate in the proxy context
       def initialize(target, &block)
         @target = target
         instance_eval(&block)
       end
 
       # Translates the no "=" DSL of routing into elements assignments on target
+      # @param method_name [Symbol] name of the missing method
+      # @param arguments [Array] array with it's arguments
+      # @param block [Proc] block provided to the method
       def method_missing(method_name, *arguments, &block)
         return super unless respond_to_missing?(method_name)
+
         @target.public_send(:"#{method_name}=", *arguments, &block)
       end
 
       # Tells whether or not a given element exists on the target
+      # @param method_name [Symbol] name of the missing method
+      # @param include_private [Boolean] should we include private in the check as well
       def respond_to_missing?(method_name, include_private = false)
         return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
+
         @target.respond_to?(:"#{method_name}=", include_private) || super
       end
     end

data/lib/karafka/routing/router.rb

@@ -3,7 +3,7 @@
 module Karafka
   # Namespace for all elements related to requests routing
   module Routing
-    # Karafka framework Router for routing incoming messages to proper controllers
+    # Karafka framework Router for routing incoming messages to proper consumers
     # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
     #  structure so all the routes are being stored in a single level array
     module Router

data/lib/karafka/routing/topic.rb

@@ -7,9 +7,12 @@ module Karafka
     # It is a part of Karafka's DSL
     class Topic
       extend Helpers::ConfigRetriever
+      extend Forwardable
 
       attr_reader :id, :consumer_group
-      attr_accessor :controller
+      attr_accessor :consumer
+
+      def_delegator :@consumer_group, :batch_fetching
 
       # @param [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
@@ -19,7 +22,7 @@ module Karafka
         @attributes = {}
         # @note We use identifier related to the consumer group that owns a topic, because from
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
-        #   have same topic name across mutliple Kafkas
+        #   have same topic name across multiple Kafkas
         @id = "#{consumer_group.id}_#{@name}"
       end
 
@@ -29,20 +32,13 @@ module Karafka
       # example for Sidekiq
       def build
         Karafka::AttributesMap.topic.each { |attr| send(attr) }
-        controller&.topic = self
         self
       end
 
       # @return [Class, nil] Class (not an instance) of a responder that should respond from
-      #   controller back to Kafka (usefull for piping dataflows)
+      #   consumer back to Kafka (useful for piping data flows)
       def responder
-        @responder ||= Karafka::Responders::Builder.new(controller).build
-      end
-
-      # @return [Class] Parser class (not instance) that we want to use to unparse Kafka messages
-      # @note If not provided - will use Json as default
-      def parser
-        @parser ||= Karafka::Parsers::Json
+        @responder ||= Karafka::Responders::Builder.new(consumer).build
       end
 
       Karafka::AttributesMap.topic.each do |attribute|
@@ -58,7 +54,7 @@ module Karafka
 
         Hash[map].merge!(
           id: id,
-          controller: controller
+          consumer: consumer
         )
       end
     end

data/lib/karafka/routing/topic_mapper.rb

@@ -8,7 +8,7 @@ module Karafka
     # routes and responders
     #
     # @example Mapper for mapping prefixed topics
-    #   module MyMapper
+    #   class MyMapper
     #     PREFIX = "my_user_name."
     #
     #     def incoming(topic)
@@ -21,7 +21,7 @@ module Karafka
     #   end
     #
     # @example Mapper for replacing "." with "_" in topic names
-    #   module MyMapper
+    #   class MyMapper
     #     PREFIX = "my_user_name."
     #
     #     def incoming(topic)
@@ -32,23 +32,21 @@ module Karafka
     #       topic.to_s.gsub('_', '.')
     #     end
     #   end
-    module TopicMapper
-      class << self
-        # @param topic [String, Symbol] topic
-        # @return [String, Symbol] same topic as on input
-        # @example
-        #   incoming('topic_name') #=> 'topic_name'
-        def incoming(topic)
-          topic
-        end
+    class TopicMapper
+      # @param topic [String, Symbol] topic
+      # @return [String, Symbol] same topic as on input
+      # @example
+      #   incoming('topic_name') #=> 'topic_name'
+      def incoming(topic)
+        topic
+      end
 
-        # @param topic [String, Symbol] topic
-        # @return [String, Symbol] same topic as on input
-        # @example
-        #   outgoing('topic_name') #=> 'topic_name'
-        def outgoing(topic)
-          topic
-        end
+      # @param topic [String, Symbol] topic
+      # @return [String, Symbol] same topic as on input
+      # @example
+      #   outgoing('topic_name') #=> 'topic_name'
+      def outgoing(topic)
+        topic
       end
     end
   end

data/lib/karafka/serialization/json/deserializer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module for all supported by default serialization and deserialization ways
+  module Serialization
+    # Namespace for json ser/der
+    module Json
+      # Default Karafka Json deserializer for loading JSON data
+      class Deserializer
+        # @param params [Karafka::Params::Params] Full params object that we want to deserialize
+        # @return [Hash] hash with deserialized JSON data
+        # @example
+        #   params = {
+        #     'payload' => "{\"a\":1}",
+        #     'topic' => 'my-topic',
+        #     'headers' => { 'message_type' => :test }
+        #   }
+        #   Deserializer.call(params) #=> { 'a' => 1 }
+        def call(params)
+          ::MultiJson.load(params['payload'])
+        rescue ::MultiJson::ParseError => e
+          raise ::Karafka::Errors::DeserializationError, e
+        end
+      end
+    end
+  end
+end

data/lib/karafka/serialization/json/serializer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module for all supported by default serialization and deserialization ways
+  module Serialization
+    module Json
+      # Default Karafka Json serializer for serializing data
+      class Serializer
+        # @param content [Object] any object that we want to convert to a json string
+        # @return [String] Valid JSON string containing serialized data
+        # @raise [Karafka::Errors::SerializationError] raised when we don't have a way to
+        #   serialize provided data to json
+        # @note When string is passed to this method, we assume that it is already a json
+        #   string and we don't serialize it again. This allows us to serialize data before
+        #   it is being forwarded to this serializer if we want to have a custom (not that simple)
+        #   json serialization
+        #
+        # @example From an ActiveRecord object
+        #   Serializer.call(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
+        # @example From a string (no changes)
+        #   Serializer.call("{\"a\":1}") #=> "{\"a\":1}"
+        def call(content)
+          return content if content.is_a?(String)
+          return content.to_json if content.respond_to?(:to_json)
+
+          raise Karafka::Errors::SerializationError, content
+        end
+      end
+    end
+  end
+end

data/lib/karafka/server.rb

@@ -3,6 +3,18 @@
 module Karafka
   # Karafka consuming server class
   class Server
+    @consumer_threads = Concurrent::Array.new
+
+    # How long should we sleep between checks on shutting down consumers
+    SUPERVISION_SLEEP = 0.1
+    # What system exit code should we use when we terminated forcefully
+    FORCEFUL_EXIT_CODE = 2
+    # This factor allows us to calculate how many times we have to sleep before
+    # a forceful shutdown
+    SUPERVISION_CHECK_FACTOR = (1 / SUPERVISION_SLEEP)
+
+    private_constant :SUPERVISION_SLEEP, :FORCEFUL_EXIT_CODE, :SUPERVISION_CHECK_FACTOR
+
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
       attr_accessor :consumer_threads
@@ -12,11 +24,10 @@ module Karafka
 
       # Method which runs app
       def run
-        @consumer_threads = Concurrent::Array.new
-        bind_on_sigint
-        bind_on_sigquit
-        bind_on_sigterm
-        start_supervised
+        process.on_sigint { stop_supervised }
+        process.on_sigquit { stop_supervised }
+        process.on_sigterm { stop_supervised }
+        run_supervised
       end
 
       # @return [Array<String>] array with names of consumer groups that should be consumed in a
@@ -30,32 +41,42 @@ module Karafka
 
       # @return [Karafka::Process] process wrapper instance used to catch system signal calls
       def process
-        Karafka::Process.instance
+        Karafka::App.config.internal.process
       end
 
-      # What should happen when we decide to quit with sigint
-      def bind_on_sigint
-        process.on_sigint { Karafka::App.stop! }
+      # Starts Karafka with a supervision
+      # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
+      # finish loop (and it won't happen until we explicitly want to stop)
+      def run_supervised
+        process.supervise
+        Karafka::App.run!
+        Karafka::App.config.internal.fetcher.call
       end
 
-      # What should happen when we decide to quit with sigquit
-      def bind_on_sigquit
-        process.on_sigquit { Karafka::App.stop! }
-      end
+      # Stops Karafka with a supervision (as long as there is a shutdown timeout)
+      # If consumers won't stop in a given time frame, it will force them to exit
+      def stop_supervised
+        Karafka::App.stop!
 
-      # What should happen when we decide to quit with sigterm
-      def bind_on_sigterm
-        process.on_sigterm { Karafka::App.stop! }
-      end
+        # We check from time to time (for the timeout period) if all the threads finished
+        # their work and if so, we can just return and normal shutdown process will take place
+        (Karafka::App.config.shutdown_timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
+          if consumer_threads.count(&:alive?).zero?
+            Thread.new { Karafka.monitor.instrument('app.stopped') }.join
+            return
+          end
 
-      # Starts Karafka with a supervision
-      # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
-      # finish loop (and it won't happen until we explicitily want to stop)
-      def start_supervised
-        process.supervise do
-          Karafka::App.run!
-          Karafka::Fetcher.new.fetch_loop
+          sleep SUPERVISION_SLEEP
         end
+
+        raise Errors::ForcefulShutdownError
+      rescue Errors::ForcefulShutdownError => e
+        Thread.new { Karafka.monitor.instrument('app.stopping.error', error: e) }.join
+        # We're done waiting, lets kill them!
+        consumer_threads.each(&:terminate)
+
+        # exit! is not within the instrumentation as it would not trigger due to exit
+        Kernel.exit! FORCEFUL_EXIT_CODE
       end
     end
   end

data/lib/karafka/setup/config.rb

@@ -8,12 +8,17 @@ module Karafka
     # @note If you want to do some configurations after all of this is done, please add to
     #   karafka/config a proper file (needs to inherit from Karafka::Setup::Configurators::Base
     #   and implement setup method) after that everything will happen automatically
-    # @note This config object allows to create a 1 level nestings (nodes) only. This should be
+    # @note This config object allows to create a 1 level nesting (nodes) only. This should be
     #   enough and will still keep the code simple
     # @see Karafka::Setup::Configurators::Base for more details about configurators api
     class Config
       extend Dry::Configurable
 
+      # Contract for checking the config provided by the user
+      CONTRACT = Karafka::Contracts::Config.new.freeze
+
+      private_constant :CONTRACT
+
       # Available settings
       # option client_id [String] kafka client_id - used to provide
       #   default Kafka groups namespaces and identify that app in kafka
@@ -21,19 +26,23 @@ module Karafka
       # What backend do we want to use to process messages
       setting :backend, :inline
       # option logger [Instance] logger that we want to use
-      setting :logger, -> { ::Karafka::Logger.instance }
+      setting :logger, ::Karafka::Instrumentation::Logger.new
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
-      setting :monitor, -> { ::Karafka::Monitor.instance }
+      setting :monitor, ::Karafka::Instrumentation::Monitor.new
       # Mapper used to remap consumer groups ids, so in case users migrate from other tools
       # or they need to maintain their own internal consumer group naming conventions, they
       # can easily do it, replacing the default client_id + consumer name pattern concept
-      setting :consumer_mapper, -> { Routing::ConsumerMapper }
-      # Mapper used to remap names of topics, so we can have a clean internal topic namings
+      setting :consumer_mapper, Routing::ConsumerMapper.new
+      # Mapper used to remap names of topics, so we can have a clean internal topic naming
       # despite using any Kafka provider that uses namespacing, etc
       # It needs to implement two methods:
       #   - #incoming - for remapping from the incoming message to our internal format
       #   - #outgoing - for remapping from internal topic name into outgoing message
-      setting :topic_mapper, -> { Routing::TopicMapper }
+      setting :topic_mapper, Routing::TopicMapper.new
+      # Default serializer for converting whatever we want to send to kafka to json
+      setting :serializer, Karafka::Serialization::Json::Serializer.new
+      # Default deserializer for converting incoming data into ruby objects
+      setting :deserializer, Karafka::Serialization::Json::Deserializer.new
       # If batch_fetching is true, we will fetch kafka messages in batches instead of 1 by 1
       # @note Fetching does not equal consuming, see batch_consuming description for details
       setting :batch_fetching, true
@@ -41,25 +50,28 @@ module Karafka
       # #params_batch will contain params received from Kafka (may be more than 1) so we can
       # process them in batches
       setting :batch_consuming, false
-      # Should we operate in a single controller instance across multiple batches of messages,
-      # from the same partition or should we build a new instance for each incoming batch.
-      # Disabling that can be useful when you want to build a new controller instance for each
-      # incoming batch. It's disabled by default, not to create more objects that needed on
-      # each batch
-      setting :persistent, true
+      # option shutdown_timeout [Integer, nil] the number of seconds after which Karafka no
+      #   longer wait for the consumers to stop gracefully but instead we force terminate
+      #   everything.
+      setting :shutdown_timeout, 60
 
       # option kafka [Hash] - optional - kafka configuration options
       setting :kafka do
         # Array with at least one host
-        setting :seed_brokers
+        setting :seed_brokers, %w[kafka://127.0.0.1:9092]
         # option session_timeout [Integer] the number of seconds after which, if a client
         #   hasn't contacted the Kafka cluster, it will be kicked out of the group.
         setting :session_timeout, 30
         # Time that a given partition will be paused from fetching messages, when message
         # consumption fails. It allows us to process other partitions, while the error is being
         # resolved and also "slows" things down, so it prevents from "eating" up all messages and
-        # consuming them with failed code
+        # consuming them with failed code. Use `nil` if you want to pause forever and never retry.
         setting :pause_timeout, 10
+        # option pause_max_timeout [Integer, nil] the maximum number of seconds to pause for,
+        #   or `nil` if no maximum should be enforced.
+        setting :pause_max_timeout, nil
+        # option pause_exponential_backoff [Boolean] whether to enable exponential backoff
+        setting :pause_exponential_backoff, false
         # option offset_commit_interval [Integer] the interval between offset commits,
         #   in seconds.
         setting :offset_commit_interval, 10
@@ -70,6 +82,13 @@ module Karafka
         # option heartbeat_interval [Integer] the interval between heartbeats; must be less
         #   than the session window.
         setting :heartbeat_interval, 10
+        # option offset_retention_time [Integer] The length of the retention window, known as
+        #   offset retention time
+        setting :offset_retention_time, nil
+        # option fetcher_max_queue_size [Integer] max number of items in the fetch queue that
+        #   are stored for further processing. Note, that each item in the queue represents a
+        #   response from a single broker
+        setting :fetcher_max_queue_size, 10
         # option max_bytes_per_partition [Integer] the maximum amount of data fetched
         #   from a single partition at a time.
         setting :max_bytes_per_partition, 1_048_576
@@ -79,6 +98,9 @@ module Karafka
         #   returning messages from the server; if `max_wait_time` is reached, this
         #   is ignored.
         setting :min_bytes, 1
+        # option max_bytes [Integer] the maximum number of bytes to read before returning messages
+        #   from each broker.
+        setting :max_bytes, 10_485_760
         # option max_wait_time [Integer, Float] max_wait_time is the maximum number of seconds to
         #   wait before returning data from a single message fetch. By setting this high you also
         #   increase the fetching throughput - and by setting it low you set a bound on latency.
@@ -93,9 +115,6 @@ module Karafka
         # option reconnect_timeout [Integer] How long should we wait before trying to reconnect to
         # Kafka cluster that went down (in seconds)
         setting :reconnect_timeout, 5
-        # option offset_retention_time [Integer] The length of the retention window, known as
-        #   offset retention time
-        setting :offset_retention_time, nil
         # option connect_timeout [Integer] Sets the number of seconds to wait while connecting to
         # a broker for the first time. When ruby-kafka initializes, it needs to connect to at
         # least one host.
@@ -107,55 +126,94 @@ module Karafka
         setting :socket_timeout, 30
 
         # SSL authentication related settings
-        # option ca_cert [String] SSL CA certificate
+        # option ca_cert [String, nil] SSL CA certificate
         setting :ssl_ca_cert, nil
-        # option ssl_ca_cert_file_path [String] SSL CA certificate file path
+        # option ssl_ca_cert_file_path [String, nil] SSL CA certificate file path
         setting :ssl_ca_cert_file_path, nil
-        # option ssl_client_cert [String] SSL client certificate
+        # option ssl_ca_certs_from_system [Boolean] Use the CA certs from your system's default
+        #   certificate store
+        setting :ssl_ca_certs_from_system, false
+        # option ssl_verify_hostname [Boolean] Verify the hostname for client certs
+        setting :ssl_verify_hostname, true
+        # option ssl_client_cert [String, nil] SSL client certificate
         setting :ssl_client_cert, nil
-        # option ssl_client_cert_key [String] SSL client certificate password
+        # option ssl_client_cert_key [String, nil] SSL client certificate password
         setting :ssl_client_cert_key, nil
-        # option sasl_gssapi_principal [String] sasl principal
+        # option sasl_gssapi_principal [String, nil] sasl principal
         setting :sasl_gssapi_principal, nil
-        # option sasl_gssapi_keytab [String] sasl keytab
+        # option sasl_gssapi_keytab [String, nil] sasl keytab
         setting :sasl_gssapi_keytab, nil
         # option sasl_plain_authzid [String] The authorization identity to use
         setting :sasl_plain_authzid, ''
-        # option sasl_plain_username [String] The username used to authenticate
+        # option sasl_plain_username [String, nil] The username used to authenticate
         setting :sasl_plain_username, nil
-        # option sasl_plain_password [String] The password used to authenticate
+        # option sasl_plain_password [String, nil] The password used to authenticate
         setting :sasl_plain_password, nil
+        # option sasl_scram_username [String, nil] The username used to authenticate
+        setting :sasl_scram_username, nil
+        # option sasl_scram_password [String, nil] The password used to authenticate
+        setting :sasl_scram_password, nil
+        # option sasl_scram_mechanism [String, nil] Scram mechanism, either 'sha256' or 'sha512'
+        setting :sasl_scram_mechanism, nil
+        # option sasl_over_ssl [Boolean] whether to enforce SSL with SASL
+        setting :sasl_over_ssl, true
+        # option ssl_client_cert_chain [String, nil] client cert chain or nil if not used
+        setting :ssl_client_cert_chain, nil
+        # option ssl_client_cert_key_password [String, nil] the password required to read
+        #   the ssl_client_cert_key
+        setting :ssl_client_cert_key_password, nil
+        # @param sasl_oauth_token_provider [Object, nil] OAuthBearer Token Provider instance that
+        #   implements method token.
+        setting :sasl_oauth_token_provider, nil
+      end
+
+      # Namespace for internal settings that should not be modified
+      # It's a temporary step to "declassify" several things internally before we move to a
+      # non global state
+      setting :internal do
+        # option routing_builder [Karafka::Routing::Builder] builder instance
+        setting :routing_builder, Routing::Builder.new
+        # option status [Karafka::Status] app status
+        setting :status, Status.new
+        # option process [Karafka::Process] process status
+        # @note In the future, we need to have a single process representation for all the karafka
+        #   instances
+        setting :process, Process.new
+        # option fetcher [Karafka::Fetcher] fetcher instance
+        setting :fetcher, Fetcher.new
+        # option configurators [Array<Object>] all configurators that we want to run after
+        #   the setup
+        setting :configurators, [Configurators::WaterDrop.new]
       end
 
       class << self
-        # Configurating method
+        # Configuring method
         # @yield Runs a block of code providing a config singleton instance to it
         # @yieldparam [Karafka::Setup::Config] Karafka config instance
         def setup
-          configure do |config|
-            yield(config)
-          end
+          configure { |config| yield(config) }
         end
 
         # Everything that should be initialized after the setup
         # Components are in karafka/config directory and are all loaded one by one
         # If you want to configure a next component, please add a proper file to config dir
         def setup_components
-          Configurators::Base.descendants.each do |klass|
-            klass.new(config).setup
-          end
+          config
+            .internal
+            .configurators
+            .each { |configurator| configurator.call(config) }
         end
 
-        # Validate config based on ConfigurationSchema
+        # Validate config based on the config contract
         # @return [Boolean] true if configuration is valid
-        # @raise [Karafka::Errors::InvalidConfiguration] raised when configuration
-        #   doesn't match with ConfigurationSchema
+        # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
+        #   doesn't match with the config contract
         def validate!
-          validation_result = Karafka::Schemas::Config.call(config.to_h)
+          validation_result = CONTRACT.call(config.to_h)
 
           return true if validation_result.success?
 
-          raise Errors::InvalidConfiguration, validation_result.errors
+          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
         end
       end
     end