karafka 1.4.15 → 2.0.0.alpha1

data/lib/karafka/routing/subscription_group.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Object representing a set of single consumer group topics that can be subscribed together
+    # with one connection.
+    #
+    # @note One subscription group will always belong to one consumer group, but one consumer
+    #   group can have multiple subscription groups.
+    class SubscriptionGroup
+      attr_reader :id, :topics
+
+      # @param topics [Array<Topic>] all the topics that share the same key settings
+      # @return [SubscriptionGroup] built subscription group
+      def initialize(topics)
+        @id = SecureRandom.uuid
+        @topics = topics
+        freeze
+      end
+
+      # @return [String] consumer group id
+      def consumer_group_id
+        kafka['group.id']
+      end
+
+      # @return [Integer] max messages fetched in a single go
+      def max_messages
+        @topics.first.max_messages
+      end
+
+      # @return [Integer] max milliseconds we can wait for incoming messages
+      def max_wait_time
+        @topics.first.max_wait_time
+      end
+
+      # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
+      #   topics but they lack the group.id (unless explicitly) provided. To make it compatible
+      #   with our routing engine, we inject it before it will go to the consumer
+      def kafka
+        kafka = @topics.first.kafka.dup
+
+        kafka['client.id'] ||= Karafka::App.config.client_id
+        kafka['group.id'] ||= @topics.first.consumer_group.id
+        kafka['auto.offset.reset'] ||= 'earliest'
+        # Karafka manages the offsets based on the processing state, thus we do not rely on the
+        # rdkafka offset auto-storing
+        kafka['enable.auto.offset.store'] = 'false'
+        kafka.freeze
+        kafka
+      end
+    end
+  end
+end

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # rdkafka allows us to group topics subscriptions when they have same settings.
+    # This builder groups topics from a single consumer group into subscription groups that can be
+    # subscribed with one rdkafka connection.
+    # This way we save resources as having several rdkafka consumers under the hood is not the
+    # cheapest thing in a bigger system.
+    #
+    # In general, if we can, we try to subscribe to as many topics with one rdkafka connection as
+    # possible, but if not possible, we divide.
+    class SubscriptionGroupsBuilder
+      # Keys used to build up a hash for subscription groups distribution.
+      # In order to be able to use the same rdkafka connection for several topics, those keys need
+      # to have same values.
+      DISTRIBUTION_KEYS = %i[
+        kafka
+        max_messages
+        max_wait_time
+      ].freeze
+
+      private_constant :DISTRIBUTION_KEYS
+
+      # @param topics [Array<Topic>] array with topics based on which we want to build subscription
+      #   groups
+      # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
+      def call(topics)
+        topics
+          .map { |topic| [checksum(topic), topic] }
+          .group_by(&:first)
+          .values
+          .map { |value| value.map(&:last) }
+          .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
+      end
+
+      private
+
+      # @param topic [Karafka::Routing::Topic] topic for which we compute the grouping checksum
+      # @return [Integer] checksum that we can use to check if topics have the same set of
+      #   settings based on which we group
+      def checksum(topic)
+        accu = {}
+
+        DISTRIBUTION_KEYS.each { |key| accu[key] = topic.public_send(key) }
+
+        accu.hash
+      end
+    end
+  end
+end

data/lib/karafka/routing/topic.rb

@@ -2,17 +2,23 @@
 
 module Karafka
   module Routing
-    # Topic stores all the details on how we should interact with Kafka given topic
+    # Topic stores all the details on how we should interact with Kafka given topic.
     # It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
-    # It is a part of Karafka's DSL
+    # It is a part of Karafka's DSL.
     class Topic
-      extend Helpers::ConfigRetriever
-      extend Forwardable
+      attr_reader :id, :name, :consumer_group
+      attr_writer :consumer
 
-      attr_reader :id, :consumer_group
-      attr_accessor :consumer
+      # Attributes we can inherit from the root unless they were defined on this level
+      INHERITABLE_ATTRIBUTES = %i[
+        kafka
+        deserializer
+        manual_offset_management
+        max_messages
+        max_wait_time
+      ].freeze
 
-      def_delegator :@consumer_group, :batch_fetching
+      private_constant :INHERITABLE_ATTRIBUTES
 
       # @param [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
@@ -22,40 +28,56 @@ module Karafka
         @attributes = {}
         # @note We use identifier related to the consumer group that owns a topic, because from
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
-        #   have same topic name across multiple Kafkas
+        #   have same topic name across multiple consumer groups
         @id = "#{consumer_group.id}_#{@name}"
       end
 
-      # Initializes default values for all the options that support defaults if their values are
-      # not yet specified. This is need to be done (cannot be lazy loaded on first use) because
-      # everywhere except Karafka server command, those would not be initialized on time - for
-      # example for Sidekiq
-      def build
-        Karafka::AttributesMap.topic.each { |attr| send(attr) }
-        self
-      end
+      INHERITABLE_ATTRIBUTES.each do |attribute|
+        attr_writer attribute
+
+        define_method attribute do
+          current_value = instance_variable_get(:"@#{attribute}")
+
+          return current_value unless current_value.nil?
 
-      # @return [Class, nil] Class (not an instance) of a responder that should respond from
-      #   consumer back to Kafka (useful for piping data flows)
-      def responder
-        @responder ||= Karafka::Responders::Builder.new(consumer).build
+          value = Karafka::App.config.send(attribute)
+
+          instance_variable_set(:"@#{attribute}", value)
+        end
       end
 
-      Karafka::AttributesMap.topic.each do |attribute|
-        config_retriever_for(attribute)
+      # @return [Class] consumer class that we should use
+      def consumer
+        if Karafka::App.config.consumer_persistence
+          # When persistence of consumers is on, no need to reload them
+          @consumer
+        else
+          # In order to support code reload without having to change the topic api, we re-fetch the
+          # class of a consumer based on its class name. This will support all the cases where the
+          # consumer class is defined with a name. It won't support code reload for anonymous
+          # consumer classes, but this is an edge case
+          begin
+            ::Object.const_get(@consumer.to_s)
+          rescue NameError
+            # It will only fail if the in case of anonymous classes
+            @consumer
+          end
+        end
       end
 
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h
-        map = Karafka::AttributesMap.topic.map do |attribute|
+        map = INHERITABLE_ATTRIBUTES.map do |attribute|
           [attribute, public_send(attribute)]
         end
 
         Hash[map].merge!(
           id: id,
-          consumer: consumer
-        )
+          name: name,
+          consumer: consumer,
+          consumer_group_id: consumer_group.id
+        ).freeze
       end
     end
   end

data/lib/karafka/runner.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Class used to run the Karafka listeners in separate threads
+  class Runner
+    # Starts listening on all the listeners asynchronously
+    # Fetch loop should never end. If they do, it is a critical error
+    def call
+      # Despite possibility of having several independent listeners, we aim to have one queue for
+      # jobs across and one workers poll for that
+      jobs_queue = Processing::JobsQueue.new
+
+      workers = Processing::WorkersBatch.new(jobs_queue)
+      Karafka::Server.workers = workers
+
+      threads = listeners(jobs_queue).map do |listener|
+        # We abort on exception because there should be an exception handling developed for
+        # each listener running in separate threads, so the exceptions should never leak
+        # and if that happens, it means that something really bad happened and we should stop
+        # the whole process
+        Thread
+          .new { listener.call }
+          .tap { |thread| thread.abort_on_exception = true }
+      end
+
+      # We aggregate threads here for a supervised shutdown process
+      Karafka::Server.consumer_threads = threads
+
+      # All the listener threads need to finish
+      threads.each(&:join)
+      # All the workers need to stop processing anything before we can stop the runner completely
+      workers.each(&:join)
+    # If anything crashes here, we need to raise the error and crush the runner because it means
+    # that something terrible happened
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        caller: self,
+        error: e,
+        type: 'runner.call.error'
+      )
+      Karafka::App.stop!
+      raise e
+    end
+
+    private
+
+    # @param jobs_queue [Processing::JobsQueue] the main processing queue
+    # @return [Array<Karafka::Connection::Listener>] listeners that will consume messages for each
+    #   of the subscription groups
+    def listeners(jobs_queue)
+      App
+        .subscription_groups
+        .map do |subscription_group|
+          Karafka::Connection::Listener.new(subscription_group, jobs_queue)
+        end
+    end
+  end
+end

data/lib/karafka/serialization/json/deserializer.rb

@@ -1,25 +1,16 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Module for all supported by default serialization and deserialization ways
+  # Module for all supported by default serialization and deserialization ways.
   module Serialization
-    # Namespace for json ser/der
+    # Namespace for json serializers and deserializers.
     module Json
-      # Default Karafka Json deserializer for loading JSON data
+      # Default Karafka Json deserializer for loading JSON data.
       class Deserializer
-        # @param params [Karafka::Params::Params] Full params object that we want to deserialize
+        # @param message [Karafka::Messages::Message] Message object that we want to deserialize
         # @return [Hash] hash with deserialized JSON data
-        # @example
-        #   params = {
-        #     'payload' => "{\"a\":1}",
-        #     'topic' => 'my-topic',
-        #     'headers' => { 'message_type' => :test }
-        #   }
-        #   Deserializer.call(params) #=> { 'a' => 1 }
-        def call(params)
-          params.raw_payload.nil? ? nil : ::JSON.parse(params.raw_payload)
-        rescue ::JSON::ParserError => e
-          raise ::Karafka::Errors::DeserializationError, e
+        def call(message)
+          message.raw_payload.nil? ? nil : ::JSON.parse(message.raw_payload)
         end
       end
     end

data/lib/karafka/server.rb

@@ -3,8 +3,6 @@
 module Karafka
   # Karafka consuming server class
   class Server
-    @consumer_threads = Concurrent::Array.new
-
     # How long should we sleep between checks on shutting down consumers
     SUPERVISION_SLEEP = 0.1
     # What system exit code should we use when we terminated forcefully
@@ -19,53 +17,70 @@ module Karafka
       # Set of consuming threads. Each consumer thread contains a single consumer
       attr_accessor :consumer_threads
 
+      # Set of workers
+      attr_accessor :workers
+
       # Writer for list of consumer groups that we want to consume in our current process context
       attr_writer :consumer_groups
 
       # Method which runs app
       def run
-        process.on_sigint { stop_supervised }
-        process.on_sigquit { stop_supervised }
-        process.on_sigterm { stop_supervised }
-        run_supervised
+        process.on_sigint { stop }
+        process.on_sigquit { stop }
+        process.on_sigterm { stop }
+
+        # Start is blocking until stop is called and when we stop, it will wait until
+        # all of the things are ready to stop
+        start
+
+        # We always need to wait for Karafka to stop here since we should wait for the stop running
+        # in a separate thread (or trap context) to indicate everything is closed
+        Thread.pass until Karafka::App.stopped?
+      # Try its best to shutdown underlying components before re-raising
+      # rubocop:disable Lint/RescueException
+      rescue Exception => e
+        # rubocop:enable Lint/RescueException
+        stop
+
+        raise e
       end
 
       # @return [Array<String>] array with names of consumer groups that should be consumed in a
       #   current server context
       def consumer_groups
-        # If not specified, a server will listed on all the topics
+        # If not specified, a server will listen on all the topics
         @consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
       end
 
-      private
-
-      # @return [Karafka::Process] process wrapper instance used to catch system signal calls
-      def process
-        Karafka::App.config.internal.process
-      end
-
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitly want to stop)
-      def run_supervised
+      def start
         process.supervise
         Karafka::App.run!
-        Karafka::App.config.internal.fetcher.call
+        Karafka::Runner.new.call
       end
 
       # Stops Karafka with a supervision (as long as there is a shutdown timeout)
-      # If consumers won't stop in a given time frame, it will force them to exit
-      def stop_supervised
+      # If consumers or workers won't stop in a given time frame, it will force them to exit
+      #
+      # @note This method is not async. It should not be executed from the workers as it will
+      #   lock them forever. If you need to run Karafka shutdown from within workers threads,
+      #   please start a separate thread to do so.
+      def stop
         Karafka::App.stop!
 
-        # See https://github.com/dry-rb/dry-configurable/issues/93
         timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
 
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
-        (timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
-          if consumer_threads.count(&:alive?).zero?
-            Thread.new { Karafka.monitor.instrument('app.stopped') }.join
+        # We divide it by 1000 because we use time in ms.
+        ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
+          if consumer_threads.count(&:alive?).zero? &&
+             workers.count(&:alive?).zero?
+
+            Thread.new { Karafka::App.producer.close }.join
+
             return
           end
 
@@ -74,12 +89,34 @@ module Karafka
 
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
-        Thread.new { Karafka.monitor.instrument('app.stopping.error', error: e) }.join
-        # We're done waiting, lets kill them!
-        consumer_threads.each(&:terminate)
+        thread = Thread.new do
+          Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            type: 'app.stopping.error'
+          )
+
+          # We're done waiting, lets kill them!
+          workers.each(&:terminate)
+          consumer_threads.each(&:terminate)
+
+          Karafka::App.producer.close
+        end
+
+        thread.join
 
         # exit! is not within the instrumentation as it would not trigger due to exit
         Kernel.exit! FORCEFUL_EXIT_CODE
+      ensure
+        Karafka::App.stopped!
+      end
+
+      private
+
+      # @return [Karafka::Process] process wrapper instance used to catch system signal calls
+      def process
+        Karafka::App.config.internal.process
       end
     end
   end