karafka 2.0.0 → 2.1.0

data/lib/karafka/routing/proxy.rb

@@ -7,15 +7,6 @@ module Karafka
     class Proxy
       attr_reader :target
 
-      # We should proxy only non ? and = methods as we want to have a regular dsl
-      IGNORED_POSTFIXES = %w[
-        ?
-        =
-        !
-      ].freeze
-
-      private_constant :IGNORED_POSTFIXES
-
       # @param target [Object] target object to which we proxy any DSL call
       # @param block [Proc] block that we want to evaluate in the proxy context
       def initialize(target, &block)
@@ -23,23 +14,30 @@ module Karafka
         instance_eval(&block)
       end
 
-      # Translates the no "=" DSL of routing into elements assignments on target
-      # @param method_name [Symbol] name of the missing method
-      # @param arguments [Array] array with it's arguments
-      # @param block [Proc] block provided to the method
-      def method_missing(method_name, *arguments, &block)
-        return super unless respond_to_missing?(method_name)
+      # Ruby 2.7.0 to 2.7.2 do not have arg forwarding, so we fallback to the old way
+      arg_forwarding = RUBY_VERSION < '3.0' ? '*args, &block' : '...'
 
-        @target.public_send(:"#{method_name}=", *arguments, &block)
-      end
+      class_eval <<~RUBY, __FILE__, __LINE__ + 1
+        # Translates the no "=" DSL of routing into elements assignments on target
+        # @param method_name [Symbol] name of the missing method
+        def method_missing(method_name, #{arg_forwarding})
+          return super unless respond_to_missing?(method_name)
+
+          if @target.respond_to?(:"\#{method_name}=")
+            @target.public_send(:"\#{method_name}=", #{arg_forwarding})
+          else
+            @target.public_send(method_name, #{arg_forwarding})
+          end
+        end
+      RUBY
 
       # Tells whether or not a given element exists on the target
       # @param method_name [Symbol] name of the missing method
       # @param include_private [Boolean] should we include private in the check as well
       def respond_to_missing?(method_name, include_private = false)
-        return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
-
-        @target.respond_to?(:"#{method_name}=", include_private) || super
+        @target.respond_to?(:"#{method_name}=", include_private) ||
+          @target.respond_to?(method_name, include_private) ||
+          super
       end
     end
   end

data/lib/karafka/routing/router.rb

@@ -14,16 +14,41 @@ module Karafka
       # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
       #   any route defined by user using routes.draw
       def find(topic_id)
+        find_by(id: topic_id) || raise(Errors::NonMatchingRouteError, topic_id)
+      end
+
+      # Finds first reference of a given topic based on provided lookup attribute
+      # @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
+      # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
+      def find_by(lookup)
         App.consumer_groups.each do |consumer_group|
           consumer_group.topics.each do |topic|
-            return topic if topic.id == topic_id
+            return topic if lookup.all? do |attribute, value|
+              topic.public_send(attribute) == value
+            end
           end
         end
 
-        raise(Errors::NonMatchingRouteError, topic_id)
+        nil
+      end
+
+      # Finds the topic by name (in any consumer group) and if not present, will built a new
+      # representation of the topic with the defaults and default deserializer.
+      #
+      # This is used in places where we may operate on topics that are not part of the routing
+      # but we want to do something on them (display data, iterate over, etc)
+      # @param name [String] name of the topic we are looking for
+      # @return [Karafka::Routing::Topic]
+      #
+      # @note Please note, that in case of a new topic, it will have a newly built consumer group
+      #   as well, that is not part of the routing.
+      def find_or_initialize_by_name(name)
+        find_by(name: name) || Topic.new(name, ConsumerGroup.new(name))
       end
 
       module_function :find
+      module_function :find_by
+      module_function :find_or_initialize_by_name
     end
   end
 end

data/lib/karafka/routing/subscription_group.rb

@@ -8,13 +8,36 @@ module Karafka
     # @note One subscription group will always belong to one consumer group, but one consumer
     #   group can have multiple subscription groups.
     class SubscriptionGroup
-      attr_reader :id, :topics
+      attr_reader :id, :name, :topics, :kafka, :consumer_group
 
+      # Numeric for counting groups
+      GROUP_COUNT = Concurrent::AtomicFixnum.new
+
+      private_constant :GROUP_COUNT
+
+      class << self
+        # Generates new subscription group id that will be used in case of anonymous subscription
+        #   groups
+        # @return [String] hex(6) compatible reproducible id
+        def id
+          ::Digest::MD5.hexdigest(
+            GROUP_COUNT.increment.to_s
+          )[0..11]
+        end
+      end
+
+      # @param position [Integer] position of this subscription group in all the subscriptions
+      #   groups array. We need to have this value for sake of static group memberships, where
+      #   we need a "in-between" restarts unique identifier
       # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
-      def initialize(topics)
-        @id = SecureRandom.uuid
+      def initialize(position, topics)
+        @name = topics.first.subscription_group
+        @consumer_group = topics.first.consumer_group
+        @id = "#{@name}_#{position}"
+        @position = position
         @topics = topics
+        @kafka = build_kafka
         freeze
       end
 
@@ -33,12 +56,27 @@ module Karafka
         @topics.first.max_wait_time
       end
 
+      # @return [Boolean] is this subscription group one of active once
+      def active?
+        Karafka::App.config.internal.routing.activity_manager.active?(:subscription_groups, name)
+      end
+
+      private
+
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
       #   topics but they lack the group.id (unless explicitly) provided. To make it compatible
       #   with our routing engine, we inject it before it will go to the consumer
-      def kafka
-        kafka = @topics.first.kafka.dup
+      def build_kafka
+        kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
+
+        # If we use static group memberships, there can be a case, where same instance id would
+        # be set on many subscription groups as the group instance id from Karafka perspective is
+        # set per config. Each instance even if they are subscribed to different topics needs to
+        # have if fully unique. To make sure of that, we just add extra postfix at the end that
+        # increments.
+        group_instance_id = kafka.fetch(:'group.instance.id', false)
 
+        kafka[:'group.instance.id'] = "#{group_instance_id}_#{@position}" if group_instance_id
         kafka[:'client.id'] ||= Karafka::App.config.client_id
         kafka[:'group.id'] ||= @topics.first.consumer_group.id
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -19,10 +19,15 @@ module Karafka
         max_messages
         max_wait_time
         initial_offset
+        subscription_group
       ].freeze
 
       private_constant :DISTRIBUTION_KEYS
 
+      def initialize
+        @position = -1
+      end
+
       # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
       #   subscription groups
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
@@ -33,7 +38,7 @@ module Karafka
           .values
           .map { |value| value.map(&:last) }
           .map { |topics_array| Routing::Topics.new(topics_array) }
-          .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
+          .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
       end
 
       private

data/lib/karafka/routing/topic.rb

@@ -8,12 +8,12 @@ module Karafka
     class Topic
       attr_reader :id, :name, :consumer_group
       attr_writer :consumer
+      attr_accessor :subscription_group
 
       # Attributes we can inherit from the root unless they were defined on this level
       INHERITABLE_ATTRIBUTES = %i[
         kafka
         deserializer
-        manual_offset_management
         max_messages
         max_wait_time
         initial_offset
@@ -27,6 +27,7 @@ module Karafka
         @name = name.to_s
         @consumer_group = consumer_group
         @attributes = {}
+        @active = true
         # @note We use identifier related to the consumer group that owns a topic, because from
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
         #   have same topic name across multiple consumer groups
@@ -66,6 +67,12 @@ module Karafka
         end
       end
 
+      # Allows to disable topic by invoking this method and setting it to `false`.
+      # @param active [Boolean] should this topic be consumed or not
+      def active(active)
+        @active = active
+      end
+
       # @return [Class] consumer class that we should use
       # @note This is just an alias to the `#consumer` method. We however want to use it internally
       #   instead of referencing the `#consumer`. We use this to indicate that this method returns
@@ -75,9 +82,12 @@ module Karafka
         consumer
       end
 
-      # @return [Boolean] true if this topic offset is handled by the end user
-      def manual_offset_management?
-        manual_offset_management
+      # @return [Boolean] should this topic be in use
+      def active?
+        # Never active if disabled via routing
+        return false unless @active
+
+        Karafka::App.config.internal.routing.activity_manager.active?(:topics, name)
       end
 
       # @return [Hash] hash with all the topic attributes
@@ -90,8 +100,10 @@ module Karafka
         Hash[map].merge!(
           id: id,
           name: name,
+          active: active?,
           consumer: consumer,
-          consumer_group_id: consumer_group.id
+          consumer_group_id: consumer_group.id,
+          subscription_group: subscription_group
         ).freeze
       end
     end

data/lib/karafka/routing/topics.rb

@@ -23,6 +23,14 @@ module Karafka
         @accumulator.each(&block)
       end
 
+      # Allows us to remove elements from the topics
+      #
+      # Block to decide what to delete
+      # @param block [Proc]
+      def delete_if(&block)
+        @accumulator.delete_if(&block)
+      end
+
       # Finds topic by its name
       #
       # @param topic_name [String] topic name

data/lib/karafka/runner.rb

@@ -19,6 +19,7 @@ module Karafka
       # We aggregate threads here for a supervised shutdown process
       Karafka::Server.workers = workers
       Karafka::Server.listeners = listeners
+      Karafka::Server.jobs_queue = jobs_queue
 
       # All the listener threads need to finish
       listeners.each(&:join)

data/lib/karafka/serialization/json/deserializer.rb

@@ -10,6 +10,7 @@ module Karafka
         # @param message [Karafka::Messages::Message] Message object that we want to deserialize
         # @return [Hash] hash with deserialized JSON data
         def call(message)
+          # nil payload can be present for example for tombstone messages
           message.raw_payload.nil? ? nil : ::JSON.parse(message.raw_payload)
         end
       end

data/lib/karafka/server.rb

@@ -20,17 +20,27 @@ module Karafka
       # Set of workers
       attr_accessor :workers
 
-      # Writer for list of consumer groups that we want to consume in our current process context
-      attr_writer :consumer_groups
+      # Jobs queue
+      attr_accessor :jobs_queue
 
       # Method which runs app
       def run
-        # Since we do a lot of threading and queuing, we don't want to stop from the trap context
-        # as some things may not work there as expected, that is why we spawn a separate thread to
-        # handle the stopping process
-        process.on_sigint { Thread.new { stop } }
-        process.on_sigquit { Thread.new { stop } }
-        process.on_sigterm { Thread.new { stop } }
+        self.listeners = []
+        self.workers = []
+
+        # We need to validate this prior to running because it may be executed also from the
+        # embedded
+        # We cannot validate this during the start because config needs to be populated and routes
+        # need to be defined.
+        Contracts::ServerCliOptions.new.validate!(
+          Karafka::App.config.internal.routing.activity_manager.to_h
+        )
+
+        process.on_sigint { stop }
+        process.on_sigquit { stop }
+        process.on_sigterm { stop }
+        process.on_sigtstp { quiet }
+        process.supervise
 
         # Start is blocking until stop is called and when we stop, it will wait until
         # all of the things are ready to stop
@@ -38,9 +48,9 @@ module Karafka
 
         # We always need to wait for Karafka to stop here since we should wait for the stop running
         # in a separate thread (or trap context) to indicate everything is closed
-        # Since `#start` is blocking, we were get here only after the runner is done. This will
+        # Since `#start` is blocking, we will get here only after the runner is done. This will
         # not add any performance degradation because of that.
-        Thread.pass until Karafka::App.stopped?
+        sleep(0.1) until Karafka::App.terminated?
       # Try its best to shutdown underlying components before re-raising
       # rubocop:disable Lint/RescueException
       rescue Exception => e
@@ -50,18 +60,10 @@ module Karafka
         raise e
       end
 
-      # @return [Array<String>] array with names of consumer groups that should be consumed in a
-      #   current server context
-      def consumer_groups
-        # If not specified, a server will listen on all the topics
-        @consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
-      end
-
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitly want to stop)
       def start
-        process.supervise
         Karafka::App.run!
         Karafka::Runner.new.call
       end
@@ -73,6 +75,11 @@ module Karafka
       #   lock them forever. If you need to run Karafka shutdown from within workers threads,
       #   please start a separate thread to do so.
       def stop
+        # Initialize the stopping process only if Karafka was running
+        return if Karafka::App.stopping?
+        return if Karafka::App.stopped?
+        return if Karafka::App.terminated?
+
         Karafka::App.stop!
 
         timeout = Karafka::App.config.shutdown_timeout
@@ -81,13 +88,7 @@ module Karafka
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
-          if listeners.count(&:alive?).zero? &&
-             workers.count(&:alive?).zero?
-
-            Karafka::App.producer.close
-
-            return
-          end
+          return if listeners.count(&:alive?).zero? && workers.count(&:alive?).zero?
 
           sleep SUPERVISION_SLEEP
         end
@@ -108,12 +109,41 @@ module Karafka
         # This can cause memory leaks and crashes.
         listeners.each(&:shutdown)
 
-        Karafka::App.producer.close
+        # We also do not forcefully terminate everything when running in the embedded mode,
+        # otherwise we would overwrite the shutdown process of the process that started Karafka
+        return unless process.supervised?
 
         # exit! is not within the instrumentation as it would not trigger due to exit
-        Kernel.exit! FORCEFUL_EXIT_CODE
+        Kernel.exit!(FORCEFUL_EXIT_CODE)
       ensure
-        Karafka::App.stopped!
+        # We need to check if it wasn't an early exit to make sure that only on stop invocation
+        # can change the status after everything is closed
+        if timeout
+          Karafka::App.stopped!
+
+          # We close producer as the last thing as it can be used in the notification pipeline
+          # to dispatch state changes, etc
+          Karafka::App.producer.close
+
+          Karafka::App.terminate!
+        end
+      end
+
+      # Quiets the Karafka server.
+      #
+      # Karafka will stop processing but won't quit the consumer group, so no rebalance will be
+      # triggered until final shutdown.
+      def quiet
+        # We don't have to safe-guard it with check states as the state transitions work only
+        # in one direction
+        Karafka::App.quiet!
+
+        # We need one more thread to monitor the process and move to quieted once everything
+        # is quiet and no processing is happening anymore
+        Thread.new do
+          sleep(0.1) until listeners.coordinators.all?(&:finished?)
+          Karafka::App.quieted!
+        end
       end
 
       private