karafka 1.4.12 → 2.2.10

data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # This feature validation contracts
+        module Contracts
+          # Rules around manual offset management settings
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :manual_offset_management do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/topic.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Topic extensions to be able to manage manual offset management settings
+        module Topic
+          # @param active [Boolean] should we stop managing the offset in Karafka and make the user
+          #   responsible for marking messages as consumed.
+          # @return [Config] defined config
+          #
+          # @note Since this feature supports only one setting (active), we can use the old API
+          # where the boolean would be an argument
+          def manual_offset_management(active = false)
+            @manual_offset_management ||= Config.new(active: active)
+          end
+
+          # @return [Boolean] is manual offset management enabled for a given topic
+          def manual_offset_management?
+            manual_offset_management.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus manual offset
+          #   management namespace settings
+          def to_h
+            super.merge(
+              manual_offset_management: manual_offset_management.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # All the things needed to be able to manage manual offset management from the routing
+      # perspective.
+      #
+      # Manual offset management allows users to completely disable automatic management of the
+      # offset. This can be used for implementing long-living window operations and other things
+      # where we do not want to commit the offset with each batch.
+      #
+      # Not all the Karafka and Karafka Pro features may be compatible with this feature being on.
+      class ManualOffsetManagement < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/proxy.rb

@@ -7,39 +7,40 @@ module Karafka
     class Proxy
       attr_reader :target
 
-      # We should proxy only non ? and = methods as we want to have a regular dsl
-      IGNORED_POSTFIXES = %w[
-        ?
-        =
-        !
-      ].freeze
-
-      private_constant :IGNORED_POSTFIXES
-
       # @param target [Object] target object to which we proxy any DSL call
+      # @param defaults [Proc] defaults for target that should be applicable after the proper
+      #   proxy context (if needed)
       # @param block [Proc] block that we want to evaluate in the proxy context
-      def initialize(target, &block)
+      def initialize(target, defaults = ->(_) {}, &block)
         @target = target
         instance_eval(&block)
+        instance_eval(&defaults)
       end
 
-      # Translates the no "=" DSL of routing into elements assignments on target
-      # @param method_name [Symbol] name of the missing method
-      # @param arguments [Array] array with it's arguments
-      # @param block [Proc] block provided to the method
-      def method_missing(method_name, *arguments, &block)
-        return super unless respond_to_missing?(method_name)
+      # Ruby 2.7.0 to 2.7.2 do not have arg forwarding, so we fallback to the old way
+      arg_forwarding = RUBY_VERSION < '3.0' ? '*args, &block' : '...'
 
-        @target.public_send(:"#{method_name}=", *arguments, &block)
-      end
+      class_eval <<~RUBY, __FILE__, __LINE__ + 1
+        # Translates the no "=" DSL of routing into elements assignments on target
+        # @param method_name [Symbol] name of the missing method
+        def method_missing(method_name, #{arg_forwarding})
+          return super unless respond_to_missing?(method_name)
+
+          if @target.respond_to?(:"\#{method_name}=")
+            @target.public_send(:"\#{method_name}=", #{arg_forwarding})
+          else
+            @target.public_send(method_name, #{arg_forwarding})
+          end
+        end
+      RUBY
 
       # Tells whether or not a given element exists on the target
       # @param method_name [Symbol] name of the missing method
       # @param include_private [Boolean] should we include private in the check as well
       def respond_to_missing?(method_name, include_private = false)
-        return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
-
-        @target.respond_to?(:"#{method_name}=", include_private) || super
+        @target.respond_to?(:"#{method_name}=", include_private) ||
+          @target.respond_to?(method_name, include_private) ||
+          super
       end
     end
   end

data/lib/karafka/routing/router.rb

@@ -7,23 +7,37 @@ module Karafka
     # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
     #  structure so all the routes are being stored in a single level array
     module Router
-      # Find a proper topic based on full topic id
-      # @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
-      #   routing topic
-      # @return [Karafka::Routing::Route] proper route details
-      # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
-      #   any route defined by user using routes.draw
-      def find(topic_id)
+      # Finds first reference of a given topic based on provided lookup attribute
+      # @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
+      # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
+      def find_by(lookup)
         App.consumer_groups.each do |consumer_group|
           consumer_group.topics.each do |topic|
-            return topic if topic.id == topic_id
+            return topic if lookup.all? do |attribute, value|
+              topic.public_send(attribute) == value
+            end
           end
         end
 
-        raise(Errors::NonMatchingRouteError, topic_id)
+        nil
       end
 
-      module_function :find
+      # Finds the topic by name (in any consumer group) and if not present, will built a new
+      # representation of the topic with the defaults and default deserializer.
+      #
+      # This is used in places where we may operate on topics that are not part of the routing
+      # but we want to do something on them (display data, iterate over, etc)
+      # @param name [String] name of the topic we are looking for
+      # @return [Karafka::Routing::Topic]
+      #
+      # @note Please note, that in case of a new topic, it will have a newly built consumer group
+      #   as well, that is not part of the routing.
+      def find_or_initialize_by_name(name)
+        find_by(name: name) || Topic.new(name, ConsumerGroup.new(name))
+      end
+
+      module_function :find_by
+      module_function :find_or_initialize_by_name
     end
   end
 end

data/lib/karafka/routing/subscription_group.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Object representing a set of single consumer group topics that can be subscribed together
+    # with one connection.
+    #
+    # @note One subscription group will always belong to one consumer group, but one consumer
+    #   group can have multiple subscription groups.
+    class SubscriptionGroup
+      attr_reader :id, :name, :topics, :kafka, :consumer_group
+
+      # Numeric for counting groups
+      GROUP_COUNT = Concurrent::AtomicFixnum.new
+
+      private_constant :GROUP_COUNT
+
+      class << self
+        # Generates new subscription group id that will be used in case of anonymous subscription
+        #   groups
+        # @return [String] hex(6) compatible reproducible id
+        def id
+          ::Digest::MD5.hexdigest(
+            GROUP_COUNT.increment.to_s
+          )[0..11]
+        end
+      end
+
+      # @param position [Integer] position of this subscription group in all the subscriptions
+      #   groups array. We need to have this value for sake of static group memberships, where
+      #   we need a "in-between" restarts unique identifier
+      # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
+      # @return [SubscriptionGroup] built subscription group
+      def initialize(position, topics)
+        @name = topics.first.subscription_group_name
+        @consumer_group = topics.first.consumer_group
+        # We include the consumer group id here because we want to have unique ids of subscription
+        # groups across the system. Otherwise user could set the same name for multiple
+        # subscription groups in many consumer groups effectively having same id for different
+        # entities
+        @id = "#{@consumer_group.id}_#{@name}_#{position}"
+        @position = position
+        @topics = topics
+        @kafka = build_kafka
+        freeze
+      end
+
+      # @return [String] consumer group id
+      def consumer_group_id
+        kafka[:'group.id']
+      end
+
+      # @return [Integer] max messages fetched in a single go
+      def max_messages
+        @topics.first.max_messages
+      end
+
+      # @return [Integer] max milliseconds we can wait for incoming messages
+      def max_wait_time
+        @topics.first.max_wait_time
+      end
+
+      # @return [Boolean] is this subscription group one of active once
+      def active?
+        Karafka::App.config.internal.routing.activity_manager.active?(:subscription_groups, name)
+      end
+
+      # @return [Array<String>] names of topics to which we should subscribe.
+      #
+      # @note Most of the time it should not include inactive topics but in case of pattern
+      #   matching the matcher topics become inactive down the road, hence we filter out so
+      #   they are later removed.
+      def subscriptions
+        topics.select(&:active?).map(&:subscription_name)
+      end
+
+      # @return [String] id of the subscription group
+      # @note This is an alias for displaying in places where we print the stringified version.
+      def to_s
+        id
+      end
+
+      private
+
+      # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
+      #   topics but they lack the group.id (unless explicitly) provided. To make it compatible
+      #   with our routing engine, we inject it before it will go to the consumer
+      def build_kafka
+        kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
+
+        # If we use static group memberships, there can be a case, where same instance id would
+        # be set on many subscription groups as the group instance id from Karafka perspective is
+        # set per config. Each instance even if they are subscribed to different topics needs to
+        # have it fully unique. To make sure of that, we just add extra postfix at the end that
+        # increments.
+        group_instance_id = kafka.fetch(:'group.instance.id', false)
+
+        kafka[:'group.instance.id'] = "#{group_instance_id}_#{@position}" if group_instance_id
+        kafka[:'client.id'] ||= Karafka::App.config.client_id
+        kafka[:'group.id'] ||= @consumer_group.id
+        kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
+        # Karafka manages the offsets based on the processing state, thus we do not rely on the
+        # rdkafka offset auto-storing
+        kafka[:'enable.auto.offset.store'] = false
+        kafka.freeze
+        kafka
+      end
+    end
+  end
+end

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # rdkafka allows us to group topics subscriptions when they have same settings.
+    # This builder groups topics from a single consumer group into subscription groups that can be
+    # subscribed with one rdkafka connection.
+    # This way we save resources as having several rdkafka consumers under the hood is not the
+    # cheapest thing in a bigger system.
+    #
+    # In general, if we can, we try to subscribe to as many topics with one rdkafka connection as
+    # possible, but if not possible, we divide.
+    class SubscriptionGroupsBuilder
+      # Keys used to build up a hash for subscription groups distribution.
+      # In order to be able to use the same rdkafka connection for several topics, those keys need
+      # to have same values.
+      DISTRIBUTION_KEYS = %i[
+        kafka
+        max_messages
+        max_wait_time
+        initial_offset
+        subscription_group_name
+      ].freeze
+
+      private_constant :DISTRIBUTION_KEYS
+
+      def initialize
+        @position = -1
+      end
+
+      # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
+      #   subscription groups
+      # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
+      def call(topics)
+        topics
+          .map { |topic| [checksum(topic), topic] }
+          .group_by(&:first)
+          .values
+          .map { |value| value.map(&:last) }
+          .map { |topics_array| Routing::Topics.new(topics_array) }
+          .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
+          .tap do |subscription_groups|
+            subscription_groups.each do |subscription_group|
+              subscription_group.topics.each do |topic|
+                topic.subscription_group = subscription_group
+              end
+            end
+          end
+      end
+
+      private
+
+      # @param topic [Karafka::Routing::Topic] topic for which we compute the grouping checksum
+      # @return [Integer] checksum that we can use to check if topics have the same set of
+      #   settings based on which we group
+      def checksum(topic)
+        accu = {}
+
+        DISTRIBUTION_KEYS.each { |key| accu[key] = topic.public_send(key) }
+
+        accu.hash
+      end
+    end
+  end
+end

data/lib/karafka/routing/topic.rb

@@ -2,17 +2,33 @@
 
 module Karafka
   module Routing
-    # Topic stores all the details on how we should interact with Kafka given topic
+    # Topic stores all the details on how we should interact with Kafka given topic.
     # It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
-    # It is a part of Karafka's DSL
+    # It is a part of Karafka's DSL.
     class Topic
-      extend Helpers::ConfigRetriever
-      extend Forwardable
+      attr_reader :id, :name, :consumer_group
+      attr_writer :consumer
 
-      attr_reader :id, :consumer_group
-      attr_accessor :consumer
+      attr_accessor :subscription_group_name
 
-      def_delegator :@consumer_group, :batch_fetching
+      # Full subscription group reference can be built only when we have knowledge about the
+      # whole routing tree, this is why it is going to be set later on
+      attr_accessor :subscription_group
+
+      # Attributes we can inherit from the root unless they were defined on this level
+      INHERITABLE_ATTRIBUTES = %i[
+        kafka
+        deserializer
+        max_messages
+        max_wait_time
+        initial_offset
+        consumer_persistence
+        pause_timeout
+        pause_max_timeout
+        pause_with_exponential_backoff
+      ].freeze
+
+      private_constant :INHERITABLE_ATTRIBUTES
 
       # @param [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
@@ -20,42 +36,89 @@ module Karafka
         @name = name.to_s
         @consumer_group = consumer_group
         @attributes = {}
+        @active = true
         # @note We use identifier related to the consumer group that owns a topic, because from
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
-        #   have same topic name across multiple Kafkas
+        #   have same topic name across multiple consumer groups
         @id = "#{consumer_group.id}_#{@name}"
       end
 
-      # Initializes default values for all the options that support defaults if their values are
-      # not yet specified. This is need to be done (cannot be lazy loaded on first use) because
-      # everywhere except Karafka server command, those would not be initialized on time - for
-      # example for Sidekiq
-      def build
-        Karafka::AttributesMap.topic.each { |attr| send(attr) }
-        self
+      INHERITABLE_ATTRIBUTES.each do |attribute|
+        attr_writer attribute
+
+        define_method attribute do
+          current_value = instance_variable_get(:"@#{attribute}")
+
+          return current_value unless current_value.nil?
+
+          value = Karafka::App.config.send(attribute)
+
+          instance_variable_set(:"@#{attribute}", value)
+        end
       end
 
-      # @return [Class, nil] Class (not an instance) of a responder that should respond from
-      #   consumer back to Kafka (useful for piping data flows)
-      def responder
-        @responder ||= Karafka::Responders::Builder.new(consumer).build
+      # @return [String] name of subscription that will go to librdkafka
+      def subscription_name
+        name
       end
 
-      Karafka::AttributesMap.topic.each do |attribute|
-        config_retriever_for(attribute)
+      # @return [Class] consumer class that we should use
+      def consumer
+        if consumer_persistence
+          # When persistence of consumers is on, no need to reload them
+          @consumer
+        else
+          # In order to support code reload without having to change the topic api, we re-fetch the
+          # class of a consumer based on its class name. This will support all the cases where the
+          # consumer class is defined with a name. It won't support code reload for anonymous
+          # consumer classes, but this is an edge case
+          begin
+            ::Object.const_get(@consumer.to_s)
+          rescue NameError
+            # It will only fail if the in case of anonymous classes
+            @consumer
+          end
+        end
+      end
+
+      # Allows to disable topic by invoking this method and setting it to `false`.
+      # @param active [Boolean] should this topic be consumed or not
+      def active(active)
+        @active = active
+      end
+
+      # @return [Class] consumer class that we should use
+      # @note This is just an alias to the `#consumer` method. We however want to use it internally
+      #   instead of referencing the `#consumer`. We use this to indicate that this method returns
+      #   class and not an instance. In the routing we want to keep the `#consumer Consumer`
+      #   routing syntax, but for references outside, we should use this one.
+      def consumer_class
+        consumer
+      end
+
+      # @return [Boolean] should this topic be in use
+      def active?
+        # Never active if disabled via routing
+        return false unless @active
+
+        Karafka::App.config.internal.routing.activity_manager.active?(:topics, name)
       end
 
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h
-        map = Karafka::AttributesMap.topic.map do |attribute|
+        map = INHERITABLE_ATTRIBUTES.map do |attribute|
           [attribute, public_send(attribute)]
         end
 
         Hash[map].merge!(
           id: id,
-          consumer: consumer
-        )
+          name: name,
+          active: active?,
+          consumer: consumer,
+          consumer_group_id: consumer_group.id,
+          subscription_group_name: subscription_group_name
+        ).freeze
       end
     end
   end

data/lib/karafka/routing/topics.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Abstraction layer on top of groups of topics
+    class Topics
+      include Enumerable
+      extend Forwardable
+
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
+
+      # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
+      def initialize(topics_array)
+        @accumulator = topics_array.dup
+      end
+
+      # Yields each topic
+      #
+      # @param [Proc] block we want to yield with on each topic
+      def each(&block)
+        @accumulator.each(&block)
+      end
+
+      # Allows us to remove elements from the topics
+      #
+      # Block to decide what to delete
+      # @param block [Proc]
+      def delete_if(&block)
+        @accumulator.delete_if(&block)
+      end
+
+      # Finds topic by its name
+      #
+      # @param topic_name [String] topic name
+      # @return [Karafka::Routing::Topic]
+      # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+      #   please create an issue.
+      def find(topic_name)
+        @accumulator.find { |topic| topic.name == topic_name } ||
+          raise(Karafka::Errors::TopicNotFoundError, topic_name)
+      end
+    end
+  end
+end

data/lib/karafka/runner.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Class used to run the Karafka listeners in separate threads
+  class Runner
+    # Starts listening on all the listeners asynchronously and handles the jobs queue closing
+    # after listeners are done with their work.
+    def call
+      # Despite possibility of having several independent listeners, we aim to have one queue for
+      # jobs across and one workers poll for that
+      jobs_queue = Processing::JobsQueue.new
+
+      workers = Processing::WorkersBatch.new(jobs_queue)
+      listeners = Connection::ListenersBatch.new(jobs_queue)
+
+      workers.each(&:async_call)
+      listeners.each(&:async_call)
+
+      # We aggregate threads here for a supervised shutdown process
+      Karafka::Server.workers = workers
+      Karafka::Server.listeners = listeners
+      Karafka::Server.jobs_queue = jobs_queue
+
+      # All the listener threads need to finish
+      listeners.each(&:join)
+
+      # We close the jobs queue only when no listener threads are working.
+      # This ensures, that everything was closed prior to us not accepting anymore jobs and that
+      # no more jobs will be enqueued. Since each listener waits for jobs to finish, once those
+      # are done, we can close.
+      jobs_queue.close
+
+      # All the workers need to stop processing anything before we can stop the runner completely
+      # This ensures that even async long-running jobs have time to finish before we are done
+      # with everything. One thing worth keeping in mind though: It is the end user responsibility
+      # to handle the shutdown detection in their long-running processes. Otherwise if timeout
+      # is exceeded, there will be a forced shutdown.
+      workers.each(&:join)
+    # If anything crashes here, we need to raise the error and crush the runner because it means
+    # that something terrible happened
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        caller: self,
+        error: e,
+        type: 'runner.call.error'
+      )
+      Karafka::App.stop!
+      raise e
+    end
+  end
+end

data/lib/karafka/serialization/json/deserializer.rb

@@ -1,25 +1,17 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Module for all supported by default serialization and deserialization ways
+  # Module for all supported by default serialization and deserialization ways.
   module Serialization
-    # Namespace for json ser/der
+    # Namespace for json serializers and deserializers.
     module Json
-      # Default Karafka Json deserializer for loading JSON data
+      # Default Karafka Json deserializer for loading JSON data.
       class Deserializer
-        # @param params [Karafka::Params::Params] Full params object that we want to deserialize
+        # @param message [Karafka::Messages::Message] Message object that we want to deserialize
         # @return [Hash] hash with deserialized JSON data
-        # @example
-        #   params = {
-        #     'payload' => "{\"a\":1}",
-        #     'topic' => 'my-topic',
-        #     'headers' => { 'message_type' => :test }
-        #   }
-        #   Deserializer.call(params) #=> { 'a' => 1 }
-        def call(params)
-          params.raw_payload.nil? ? nil : ::JSON.parse(params.raw_payload)
-        rescue ::JSON::ParserError => e
-          raise ::Karafka::Errors::DeserializationError, e
+        def call(message)
+          # nil payload can be present for example for tombstone messages
+          message.raw_payload.nil? ? nil : ::JSON.parse(message.raw_payload)
         end
       end
     end