karafka 1.2.13 → 1.3.0.rc1

data/lib/karafka/process.rb

@@ -4,8 +4,6 @@ module Karafka
   # Class used to catch signals from ruby Signal class in order to manage Karafka stop
   # @note There might be only one process - this class is a singleton
   class Process
-    include Singleton
-
     # Signal types that we handle
     HANDLED_SIGNALS = %i[
       SIGINT

data/lib/karafka/responders/topic.rb

@@ -7,8 +7,6 @@ module Karafka
     #   Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
     # @example Define optional topic
     #   Karafka::Responders::Topic.new(:topic_name, required: false)
-    # @example Define topic that on which we want to respond multiple times
-    #   Karafka::Responders::Topic.new(:topic_name, multiple_usage: true)
     class Topic
       # Name of the topic on which we want to respond
       attr_reader :name
@@ -26,16 +24,16 @@ module Karafka
         @options.key?(:required) ? @options[:required] : true
       end
 
-      # @return [Boolean] do we expect to use it multiple times in a single respond flow
-      def multiple_usage?
-        @options[:multiple_usage] || false
-      end
-
       # @return [Boolean] was usage of this topic registered or not
       def registered?
         @options[:registered] == true
       end
 
+      # @return [Class] Class to use to serialize messages for this topic
+      def serializer
+        @options[:serializer]
+      end
+
       # @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
       #   is safer and introduces less problems
       def async?
@@ -46,9 +44,9 @@ module Karafka
       def to_h
         {
           name: name,
-          multiple_usage: multiple_usage?,
           required: required?,
           registered: registered?,
+          serializer: serializer,
           async: async?
         }
       end

data/lib/karafka/routing/builder.rb

@@ -9,26 +9,39 @@ module Karafka
     #       consumer NewVideosConsumer
     #     end
     #   end
-    class Builder < Array
-      include Singleton
+    class Builder < Concurrent::Array
+      # Consumer group consistency checking contract
+      CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
+
+      private_constant :CONTRACT
+
+      def initialize
+        @draws = Concurrent::Array.new
+      end
 
       # Used to draw routes for Karafka
       # @note After it is done drawing it will store and validate all the routes to make sure that
       #   they are correct and that there are no topic/group duplications (this is forbidden)
+      # @param block [Proc] block we will evaluate within the builder context
       # @yield Evaluates provided block in a builder context so we can describe routes
+      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
+      #   doesn't match with the config contract
       # @example
       #   draw do
       #     topic :xyz do
       #     end
       #   end
       def draw(&block)
+        @draws << block
+
         instance_eval(&block)
 
         each do |consumer_group|
           hashed_group = consumer_group.to_h
-          validation_result = Karafka::Schemas::ConsumerGroup.call(hashed_group)
-          return if validation_result.success?
-          raise Errors::InvalidConfiguration, validation_result.errors
+          validation_result = CONTRACT.call(hashed_group)
+          next if validation_result.success?
+
+          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
         end
       end
 
@@ -39,18 +52,33 @@ module Karafka
         select(&:active?)
       end
 
+      # Clears the builder and the draws memory
+      def clear
+        @draws.clear
+        super
+      end
+
+      # Redraws all the routes for the in-process code reloading.
+      # @note This won't allow registration of new topics without process restart but will trigger
+      #   cache invalidation so all the classes, etc are re-fetched after code reload
+      def reload
+        draws = @draws.dup
+        clear
+        draws.each { |block| draw(&block) }
+      end
+
       private
 
       # Builds and saves given consumer group
       # @param group_id [String, Symbol] name for consumer group
-      # @yield Evaluates a given block in a consumer group context
+      # @param block [Proc] proc that should be executed in the proxy context
       def consumer_group(group_id, &block)
         consumer_group = ConsumerGroup.new(group_id.to_s)
         self << Proxy.new(consumer_group, &block).target
       end
 
       # @param topic_name [String, Symbol] name of a topic from which we want to consumer
-      # @yield Evaluates a given block in a topic context
+      # @param block [Proc] proc we want to evaluate in the topic context
       def topic(topic_name, &block)
         consumer_group(topic_name) do
           topic(topic_name, &block).tap(&:build)

data/lib/karafka/routing/consumer_group.rb

@@ -29,7 +29,7 @@ module Karafka
 
       # Builds a topic representation inside of a current consumer group route
       # @param name [String, Symbol] name of topic to which we want to subscribe
-      # @yield Evaluates a given block in a topic context
+      # @param block [Proc] block that we want to evaluate in the topic context
       # @return [Karafka::Routing::Topic] newly built topic instance
       def topic=(name, &block)
         topic = Topic.new(name, self)

data/lib/karafka/routing/consumer_mapper.rb

@@ -4,29 +4,29 @@ module Karafka
   module Routing
     # Default consumer mapper that builds consumer ids based on app id and consumer group name
     # Different mapper can be used in case of preexisting consumer names or for applying
-    # other naming conventions not compatible wiih Karafkas client_id + consumer name concept
+    # other naming conventions not compatible with Karafka client_id + consumer name concept
     #
     # @example Mapper for using consumer groups without a client_id prefix
-    #   module MyMapper
-    #     def self.call(raw_consumer_group_name)
+    #   class MyMapper
+    #     def call(raw_consumer_group_name)
     #       raw_consumer_group_name
     #     end
     #   end
     #
     # @example Mapper for replacing "_" with "." in topic names
-    #   module MyMapper
-    #     def self.call(raw_consumer_group_name)
+    #   class MyMapper
+    #     def call(raw_consumer_group_name)
     #       [
-    #         Dry::Inflector.new.underscore(Karafka::App.config.client_id.to_s),
+    #         Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s),
     #         raw_consumer_group_name
     #       ].join('_').gsub('_', '.')
     #     end
     #   end
-    module ConsumerMapper
+    class ConsumerMapper
       # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
       # @return [String] remapped final consumer group name
-      def self.call(raw_consumer_group_name)
-        client_name = Dry::Inflector.new.underscore(Karafka::App.config.client_id.to_s)
+      def call(raw_consumer_group_name)
+        client_name = Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s)
         "#{client_name}_#{raw_consumer_group_name}"
       end
     end

data/lib/karafka/routing/proxy.rb

@@ -14,22 +14,31 @@ module Karafka
         !
       ].freeze
 
+      private_constant :IGNORED_POSTFIXES
+
       # @param target [Object] target object to which we proxy any DSL call
-      # @yield Evaluates block in the proxy context
+      # @param block [Proc] block that we want to evaluate in the proxy context
       def initialize(target, &block)
         @target = target
         instance_eval(&block)
       end
 
       # Translates the no "=" DSL of routing into elements assignments on target
+      # @param method_name [Symbol] name of the missing method
+      # @param arguments [Array] array with it's arguments
+      # @param block [Proc] block provided to the method
       def method_missing(method_name, *arguments, &block)
         return super unless respond_to_missing?(method_name)
+
         @target.public_send(:"#{method_name}=", *arguments, &block)
       end
 
       # Tells whether or not a given element exists on the target
+      # @param method_name [Symbol] name of the missing method
+      # @param include_private [Boolean] should we include private in the check as well
       def respond_to_missing?(method_name, include_private = false)
         return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
+
         @target.respond_to?(:"#{method_name}=", include_private) || super
       end
     end

data/lib/karafka/routing/topic.rb

@@ -7,10 +7,13 @@ module Karafka
     # It is a part of Karafka's DSL
     class Topic
       extend Helpers::ConfigRetriever
+      extend Forwardable
 
       attr_reader :id, :consumer_group
       attr_accessor :consumer
 
+      def_delegator :@consumer_group, :batch_fetching
+
       # @param [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
       def initialize(name, consumer_group)
@@ -19,7 +22,7 @@ module Karafka
         @attributes = {}
         # @note We use identifier related to the consumer group that owns a topic, because from
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
-        #   have same topic name across mutliple Kafkas
+        #   have same topic name across multiple Kafkas
         @id = "#{consumer_group.id}_#{@name}"
       end
 
@@ -29,12 +32,11 @@ module Karafka
       # example for Sidekiq
       def build
         Karafka::AttributesMap.topic.each { |attr| send(attr) }
-        consumer&.topic = self
         self
       end
 
       # @return [Class, nil] Class (not an instance) of a responder that should respond from
-      #   consumer back to Kafka (usefull for piping dataflows)
+      #   consumer back to Kafka (useful for piping data flows)
       def responder
         @responder ||= Karafka::Responders::Builder.new(consumer).build
       end

data/lib/karafka/routing/topic_mapper.rb

@@ -8,7 +8,7 @@ module Karafka
     # routes and responders
     #
     # @example Mapper for mapping prefixed topics
-    #   module MyMapper
+    #   class MyMapper
     #     PREFIX = "my_user_name."
     #
     #     def incoming(topic)
@@ -21,7 +21,7 @@ module Karafka
     #   end
     #
     # @example Mapper for replacing "." with "_" in topic names
-    #   module MyMapper
+    #   class MyMapper
     #     PREFIX = "my_user_name."
     #
     #     def incoming(topic)
@@ -32,23 +32,21 @@ module Karafka
     #       topic.to_s.gsub('_', '.')
     #     end
     #   end
-    module TopicMapper
-      class << self
-        # @param topic [String, Symbol] topic
-        # @return [String, Symbol] same topic as on input
-        # @example
-        #   incoming('topic_name') #=> 'topic_name'
-        def incoming(topic)
-          topic
-        end
+    class TopicMapper
+      # @param topic [String, Symbol] topic
+      # @return [String, Symbol] same topic as on input
+      # @example
+      #   incoming('topic_name') #=> 'topic_name'
+      def incoming(topic)
+        topic
+      end
 
-        # @param topic [String, Symbol] topic
-        # @return [String, Symbol] same topic as on input
-        # @example
-        #   outgoing('topic_name') #=> 'topic_name'
-        def outgoing(topic)
-          topic
-        end
+      # @param topic [String, Symbol] topic
+      # @return [String, Symbol] same topic as on input
+      # @example
+      #   outgoing('topic_name') #=> 'topic_name'
+      def outgoing(topic)
+        topic
       end
     end
   end

data/lib/karafka/serialization/json/deserializer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module for all supported by default serialization and deserialization ways
+  module Serialization
+    # Namespace for json ser/der
+    module Json
+      # Default Karafka Json deserializer for loading JSON data
+      class Deserializer
+        # @param params [Karafka::Params::Params] Full params object that we want to deserialize
+        # @return [Hash] hash with deserialized JSON data
+        # @example
+        #   params = {
+        #     'payload' => "{\"a\":1}",
+        #     'topic' => 'my-topic',
+        #     'headers' => { 'message_type' => :test }
+        #   }
+        #   Deserializer.call(params) #=> { 'a' => 1 }
+        def call(params)
+          ::MultiJson.load(params['payload'])
+        rescue ::MultiJson::ParseError => e
+          raise ::Karafka::Errors::DeserializationError, e
+        end
+      end
+    end
+  end
+end

data/lib/karafka/serialization/json/serializer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module for all supported by default serialization and deserialization ways
+  module Serialization
+    module Json
+      # Default Karafka Json serializer for serializing data
+      class Serializer
+        # @param content [Object] any object that we want to convert to a json string
+        # @return [String] Valid JSON string containing serialized data
+        # @raise [Karafka::Errors::SerializationError] raised when we don't have a way to
+        #   serialize provided data to json
+        # @note When string is passed to this method, we assume that it is already a json
+        #   string and we don't serialize it again. This allows us to serialize data before
+        #   it is being forwarded to this serializer if we want to have a custom (not that simple)
+        #   json serialization
+        #
+        # @example From an ActiveRecord object
+        #   Serializer.call(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
+        # @example From a string (no changes)
+        #   Serializer.call("{\"a\":1}") #=> "{\"a\":1}"
+        def call(content)
+          return content if content.is_a?(String)
+          return content.to_json if content.respond_to?(:to_json)
+
+          raise Karafka::Errors::SerializationError, content
+        end
+      end
+    end
+  end
+end

data/lib/karafka/server.rb

@@ -6,9 +6,14 @@ module Karafka
     @consumer_threads = Concurrent::Array.new
 
     # How long should we sleep between checks on shutting down consumers
-    SUPERVISION_SLEEP = 1
+    SUPERVISION_SLEEP = 0.1
     # What system exit code should we use when we terminated forcefully
     FORCEFUL_EXIT_CODE = 2
+    # This factor allows us to calculate how many times we have to sleep before
+    # a forceful shutdown
+    SUPERVISION_CHECK_FACTOR = (1 / SUPERVISION_SLEEP)
+
+    private_constant :SUPERVISION_SLEEP, :FORCEFUL_EXIT_CODE, :SUPERVISION_CHECK_FACTOR
 
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
@@ -22,7 +27,7 @@ module Karafka
         process.on_sigint { stop_supervised }
         process.on_sigquit { stop_supervised }
         process.on_sigterm { stop_supervised }
-        start_supervised
+        run_supervised
       end
 
       # @return [Array<String>] array with names of consumer groups that should be consumed in a
@@ -36,49 +41,42 @@ module Karafka
 
       # @return [Karafka::Process] process wrapper instance used to catch system signal calls
       def process
-        Karafka::Process.instance
+        Karafka::App.config.internal.process
       end
 
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
-      # finish loop (and it won't happen until we explicitily want to stop)
-      def start_supervised
+      # finish loop (and it won't happen until we explicitly want to stop)
+      def run_supervised
         process.supervise
         Karafka::App.run!
-        Karafka::Fetcher.call
+        Karafka::App.config.internal.fetcher.call
       end
 
       # Stops Karafka with a supervision (as long as there is a shutdown timeout)
-      # If consumers won't stop in a given timeframe, it will force them to exit
+      # If consumers won't stop in a given time frame, it will force them to exit
       def stop_supervised
-        # Because this is called in the trap context, there is a chance that instrumentation
-        # listeners contain things that aren't allowed from within a trap context.
-        # To bypass that (instead of telling users not to do things they need to)
-        # we spin up a thread to instrument server.stop and server.stop.error and wait until
-        # they're finished
-        Thread.new { Karafka.monitor.instrument('server.stop', {}) }.join
-
         Karafka::App.stop!
-        # If there is no shutdown timeout, we don't exit and wait until all the consumers
-        # had done their work
-        return unless Karafka::App.config.shutdown_timeout
 
-        # If there is a timeout, we check every 1 second (for the timeout period) if all
-        # the threads finished their work and if so, we can just return and normal
-        # shutdown process will take place
-        Karafka::App.config.shutdown_timeout.to_i.times do
-          return if consumer_threads.count(&:alive?).zero?
+        # We check from time to time (for the timeout period) if all the threads finished
+        # their work and if so, we can just return and normal shutdown process will take place
+        (Karafka::App.config.shutdown_timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
+          if consumer_threads.count(&:alive?).zero?
+            Thread.new { Karafka.monitor.instrument('app.stopped') }.join
+            return
+          end
+
           sleep SUPERVISION_SLEEP
         end
 
-        raise Errors::ForcefulShutdown
-      rescue Errors::ForcefulShutdown => error
-        Thread.new { Karafka.monitor.instrument('server.stop.error', error: error) }.join
+        raise Errors::ForcefulShutdownError
+      rescue Errors::ForcefulShutdownError => e
+        Thread.new { Karafka.monitor.instrument('app.stopping.error', error: e) }.join
         # We're done waiting, lets kill them!
         consumer_threads.each(&:terminate)
 
-        # exit is not within the instrumentation as it would not trigger due to exit
-        Kernel.exit FORCEFUL_EXIT_CODE
+        # exit! is not within the instrumentation as it would not trigger due to exit
+        Kernel.exit! FORCEFUL_EXIT_CODE
       end
     end
   end