karafka 2.2.14 → 2.3.0.alpha1

data/lib/karafka/pro/routing/features/periodic_job.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature allowing to run consumer operations even when no data is present on periodic
+        # interval.
+        # This allows for advanced window-based operations regardless of income of new data and
+        # other advanced cases where the consumer is needed even when no data is coming
+        class PeriodicJob < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -21,6 +21,7 @@ module Karafka
             :active,
             :partitioner,
             :max_partitions,
+            :offset_metadata_strategy,
             keyword_init: true
           ) { alias_method :active?, :active }
         end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -32,6 +32,7 @@ module Karafka
                 required(:active) { |val| [true, false].include?(val) }
                 required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
                 required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
+                required(:offset_metadata_strategy) { |val| %i[exact current].include?(val) }
               end
 
               # When virtual partitions are defined, partitioner needs to respond to `#call` and it

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -23,16 +23,21 @@ module Karafka
             #   create more work than workers. When less, can ensure we have spare resources to
             #   process other things in parallel.
             # @param partitioner [nil, #call] nil or callable partitioner
+            # @param offset_metadata_strategy [Symbol] how we should match the metadata for the
+            #   offset. `:exact` will match the offset matching metadata and `:current` will select
+            #   the most recently reported metadata
             # @return [VirtualPartitions] method that allows to set the virtual partitions details
             #   during the routing configuration and then allows to retrieve it
             def virtual_partitions(
               max_partitions: Karafka::App.config.concurrency,
-              partitioner: nil
+              partitioner: nil,
+              offset_metadata_strategy: :current
             )
               @virtual_partitions ||= Config.new(
                 active: !partitioner.nil?,
                 max_partitions: max_partitions,
-                partitioner: partitioner
+                partitioner: partitioner,
+                offset_metadata_strategy: offset_metadata_strategy
               )
             end
 

data/lib/karafka/process.rb

@@ -25,9 +25,11 @@ module Karafka
       #     Karafka.logger.info('Log something here')
       #     exit
       #   end
-      define_method :"on_#{signal.to_s.downcase}" do |&block|
-        @callbacks[signal] << block
-      end
+      class_eval <<~RUBY, __FILE__, __LINE__ + 1
+        def on_#{signal.to_s.downcase}(&block)
+          @callbacks[:#{signal}] << block
+        end
+      RUBY
     end
 
     # Creates an instance of process and creates empty hash for callbacks

data/lib/karafka/processing/coordinator.rb

@@ -10,8 +10,12 @@ module Karafka
     #   listener thread, but we go with thread-safe by default for all not to worry about potential
     #   future mistakes.
     class Coordinator
+      extend Forwardable
+
       attr_reader :pause_tracker, :seek_offset, :topic, :partition
 
+      def_delegators :@pause_tracker, :attempt, :paused?
+
       # @param topic [Karafka::Routing::Topic]
       # @param partition [Integer]
       # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
@@ -149,7 +153,7 @@ module Karafka
 
       # @return [Boolean] are we in a pause that was initiated by the user
       def manual_pause?
-        @pause_tracker.paused? && @manual_pause
+        paused? && @manual_pause
       end
 
       # Marks seek as manual for coordination purposes

data/lib/karafka/processing/executor.rb

@@ -97,16 +97,6 @@ module Karafka
       # This may include house-keeping or other state management changes that can occur but that
       # not mean there are any new messages available for the end user to process
       def idle
-        # Initializes the messages set in case idle operation would happen before any processing
-        # This prevents us from having no messages object at all as the messages object and
-        # its metadata may be used for statistics
-        consumer.messages ||= Messages::Builders::Messages.call(
-          [],
-          topic,
-          partition,
-          Time.now
-        )
-
         consumer.on_idle
       end
 
@@ -170,10 +160,26 @@ module Karafka
           consumer.client = @client
           consumer.producer = ::Karafka::App.producer
           consumer.coordinator = @coordinator
+          # Since we have some message-less flows (idle, etc), we initialize consumer with empty
+          # messages set. In production we have persistent consumers, so this is not a performance
+          # overhead as this will happen only once per consumer lifetime
+          consumer.messages = empty_messages
 
           consumer
         end
       end
+
+      # Initializes the messages set in case given operation would happen before any processing
+      # This prevents us from having no messages object at all as the messages object and
+      # its metadata may be used for statistics
+      def empty_messages
+        Messages::Builders::Messages.call(
+          [],
+          topic,
+          partition,
+          Time.now
+        )
+      end
     end
   end
 end

data/lib/karafka/processing/executors_buffer.rb

@@ -13,6 +13,7 @@ module Karafka
         @client = client
         # We need two layers here to keep track of topics, partitions and processing groups
         @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
+        @executor_class = Karafka::App.config.internal.processing.executor_class
       end
 
       # Finds or creates an executor based on the provided details
@@ -21,15 +22,28 @@ module Karafka
       # @param partition [Integer] partition number
       # @param parallel_key [String] parallel group key
       # @param coordinator [Karafka::Processing::Coordinator]
-      # @return [Executor] consumer executor
+      # @return [Executor, Pro::Processing::Executor] consumer executor
       def find_or_create(topic, partition, parallel_key, coordinator)
-        @buffer[topic][partition][parallel_key] ||= Executor.new(
+        @buffer[topic][partition][parallel_key] ||= @executor_class.new(
           @subscription_group.id,
           @client,
           coordinator
         )
       end
 
+      # Finds all existing executors for given topic partition or creates one for it
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @param coordinator [Karafka::Processing::Coordinator]
+      # @return [Array<Executor, Pro::Processing::Executor>]
+      def find_all_or_create(topic, partition, coordinator)
+        existing = find_all(topic, partition)
+
+        return existing unless existing.empty?
+
+        [find_or_create(topic, partition, 0, coordinator)]
+      end
+
       # Revokes executors of a given topic partition, so they won't be used anymore for incoming
       # messages
       #
@@ -43,7 +57,8 @@ module Karafka
       #
       # @param topic [String] topic name
       # @param partition [Integer] partition number
-      # @return [Array<Executor>] executors in use for this topic + partition
+      # @return [Array<Executor, Pro::Processing::Executor>] executors in use for this
+      #   topic + partition
       def find_all(topic, partition)
         @buffer[topic][partition].values
       end
@@ -52,7 +67,7 @@ module Karafka
       # info
       # @yieldparam [Routing::Topic] karafka routing topic object
       # @yieldparam [Integer] partition number
-      # @yieldparam [Executor] given executor
+      # @yieldparam [Executor, Pro::Processing::Executor] given executor
       def each
         @buffer.each_value do |partitions|
           partitions.each_value do |executors|

data/lib/karafka/processing/schedulers/default.rb

@@ -13,16 +13,17 @@ module Karafka
 
         # Schedules jobs in the fifo order
         #
-        # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+        # @param jobs_array [Array<Karafka::Processing::Jobs::Consume>] jobs we want to schedule
         def on_schedule_consumption(jobs_array)
           jobs_array.each do |job|
             @queue << job
           end
         end
 
-        # Both revocation and shutdown jobs can also run in fifo by default
+        # Revocation, shutdown and idle jobs can also run in fifo by default
         alias on_schedule_revocation on_schedule_consumption
         alias on_schedule_shutdown on_schedule_consumption
+        alias on_schedule_idle on_schedule_consumption
 
         # This scheduler does not have anything to manage as it is a pass through and has no state
         def on_manage

data/lib/karafka/processing/strategies/default.rb

@@ -41,6 +41,9 @@ module Karafka
         #   already processed but rather at the next one. This applies to both sync and async
         #   versions of this method.
         def mark_as_consumed(message)
+          # seek offset can be nil only in case `#seek` was invoked with offset reset request
+          # In case like this we ignore marking
+          return true if coordinator.seek_offset.nil?
           # Ignore earlier offsets than the one we already committed
           return true if coordinator.seek_offset > message.offset
           return false if revoked?
@@ -57,6 +60,9 @@ module Karafka
         # @return [Boolean] true if we were able to mark the offset, false otherwise.
         #   False indicates that we were not able and that we have lost the partition.
         def mark_as_consumed!(message)
+          # seek offset can be nil only in case `#seek` was invoked with offset reset request
+          # In case like this we ignore marking
+          return true if coordinator.seek_offset.nil?
           # Ignore earlier offsets than the one we already committed
           return true if coordinator.seek_offset > message.offset
           return false if revoked?

data/lib/karafka/processing/strategies/dlq.rb

@@ -14,6 +14,42 @@ module Karafka
           dead_letter_queue
         ].freeze
 
+        # Override of the standard `#mark_as_consumed` in order to handle the pause tracker
+        # reset in case DLQ is marked as fully independent. When DLQ is marked independent,
+        # any offset marking causes the pause count tracker to reset. This is useful when
+        # the error is not due to the collective batch operations state but due to intermediate
+        # "crawling" errors that move with it
+        #
+        # @see `Strategies::Default#mark_as_consumed` for more details
+        # @param message [Messages::Message]
+        def mark_as_consumed(message)
+          # If we are not retrying pause count is already 0, no need to try to reset the state
+          return super unless retrying?
+          # If we do not use independent marking on DLQ, we just mark as consumed
+          return super unless topic.dead_letter_queue.independent?
+          # If we were not able to mark no need to reset
+          return false unless super
+
+          coordinator.pause_tracker.reset
+
+          true
+        end
+
+        # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
+        # DLQ was configured with the `independent` flag.
+        #
+        # @see `Strategies::Default#mark_as_consumed!` for more details
+        # @param message [Messages::Message]
+        def mark_as_consumed!(message)
+          return super unless retrying?
+          return super unless topic.dead_letter_queue.independent?
+          return false unless super
+
+          coordinator.pause_tracker.reset
+
+          true
+        end
+
         # When manual offset management is on, we do not mark anything as consumed automatically
         # and we rely on the user to figure things out
         def handle_after_consume

data/lib/karafka/routing/builder.rb

@@ -109,10 +109,20 @@ module Karafka
       # subscription group customization
       # @param subscription_group_name [String, Symbol] subscription group id. When not provided,
       #   a random uuid will be used
+      # @param args [Array] any extra arguments accepted by the subscription group builder
       # @param block [Proc] further topics definitions
-      def subscription_group(subscription_group_name = SubscriptionGroup.id, &block)
+      def subscription_group(
+        subscription_group_name = SubscriptionGroup.id,
+        **args,
+        &block
+      )
         consumer_group('app') do
-          target.public_send(:subscription_group=, subscription_group_name.to_s, &block)
+          target.public_send(
+            :subscription_group=,
+            subscription_group_name.to_s,
+            **args,
+            &block
+          )
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -14,7 +14,7 @@ module Karafka
       # It allows us to store the "current" subscription group defined in the routing
       # This subscription group id is then injected into topics, so we can compute the subscription
       # groups
-      attr_accessor :current_subscription_group_name
+      attr_accessor :current_subscription_group_details
 
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
@@ -26,7 +26,7 @@ module Karafka
         @topics = Topics.new([])
         # Initialize the subscription group so there's always a value for it, since even if not
         # defined directly, a subscription group will be created
-        @current_subscription_group_name = SubscriptionGroup.id
+        @current_subscription_group_details = { name: SubscriptionGroup.id }
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
@@ -48,7 +48,7 @@ module Karafka
         built_topic = @topics.last
         # We overwrite it conditionally in case it was not set by the user inline in the topic
         # block definition
-        built_topic.subscription_group_name ||= current_subscription_group_name
+        built_topic.subscription_group_details ||= current_subscription_group_details
         built_topic
       end
 
@@ -59,13 +59,13 @@ module Karafka
       def subscription_group=(name = SubscriptionGroup.id, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
-        @current_subscription_group_name = name.to_s
+        @current_subscription_group_details = { name: name.to_s }
 
         Proxy.new(self, &block)
 
         # We need to reset the current subscription group after it is used, so it won't leak
         # outside to other topics that would be defined without a defined subscription group
-        @current_subscription_group_name = SubscriptionGroup.id
+        @current_subscription_group_details = { name: SubscriptionGroup.id }
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on

data/lib/karafka/routing/features/base.rb

@@ -13,17 +13,46 @@ module Karafka
         class << self
           # Extends topic and builder with given feature API
           def activate
-            Topic.prepend(self::Topic) if const_defined?('Topic', false)
-            Topics.prepend(self::Topics) if const_defined?('Topics', false)
-            ConsumerGroup.prepend(self::ConsumerGroup) if const_defined?('ConsumerGroup', false)
-            Proxy.prepend(self::Proxy) if const_defined?('Proxy', false)
-            Builder.prepend(self::Builder) if const_defined?('Builder', false)
-            Builder.prepend(Base::Expander.new(self)) if const_defined?('Contracts', false)
+            if const_defined?('Topic', false)
+              Topic.prepend(self::Topic)
+            end
+
+            if const_defined?('Topics', false)
+              Topics.prepend(self::Topics)
+            end
+
+            if const_defined?('ConsumerGroup', false)
+              ConsumerGroup.prepend(self::ConsumerGroup)
+            end
+
+            if const_defined?('Proxy', false)
+              Proxy.prepend(self::Proxy)
+            end
+
+            if const_defined?('Builder', false)
+              Builder.prepend(self::Builder)
+            end
+
+            if const_defined?('Contracts', false)
+              Builder.prepend(Base::Expander.new(self))
+            end
+
+            if const_defined?('SubscriptionGroup', false)
+              SubscriptionGroup.prepend(self::SubscriptionGroup)
+            end
+
+            if const_defined?('SubscriptionGroupsBuilder', false)
+              SubscriptionGroupsBuilder.prepend(self::SubscriptionGroupsBuilder)
+            end
           end
 
-          # Loads all the features and activates them
+          # Loads all the features and activates them once
           def load_all
+            return if @loaded
+
             features.each(&:activate)
+
+            @loaded = true
           end
 
           # @param config [Karafka::Core::Configurable::Node] app config that we can alter with
@@ -41,11 +70,18 @@ module Karafka
 
           private
 
-          # @return [Array<Class>] all available routing features
+          # @return [Array<Class>] all available routing features that are direct descendants of
+          #   the features base.Approach with using `#superclass` prevents us from accidentally
+          #   loading Pro components
           def features
             ObjectSpace
               .each_object(Class)
               .select { |klass| klass < self }
+              # Ensures, that Pro components are only loaded when we operate in Pro mode. Since
+              # outside of specs Zeitwerk does not require them at all, they will not be loaded
+              # anyhow, but for specs this needs to be done as RSpec requires all files to be
+              # present
+              .reject { |klass| Karafka.pro? ? false : klass.superclass != self }
               .sort_by(&:to_s)
           end
 

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -11,8 +11,13 @@ module Karafka
           :max_retries,
           # To what topic the skipped messages should be moved
           :topic,
+          # Should retries be handled collectively on a batch or independently per message
+          :independent,
           keyword_init: true
-        ) { alias_method :active?, :active }
+        ) do
+          alias_method :active?, :active
+          alias_method :independent?, :independent
+        end
       end
     end
   end

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -18,6 +18,7 @@ module Karafka
 
             nested :dead_letter_queue do
               required(:active) { |val| [true, false].include?(val) }
+              required(:independent) { |val| [true, false].include?(val) }
               required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
             end
 

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -14,12 +14,19 @@ module Karafka
           # @param max_retries [Integer] after how many retries should we move data to dlq
           # @param topic [String, false] where the messages should be moved if failing or false
           #   if we do not want to move it anywhere and just skip
+          # @param independent [Boolean] needs to be true in order for each marking as consumed
+          #   in a retry flow to reset the errors counter
           # @return [Config] defined config
-          def dead_letter_queue(max_retries: DEFAULT_MAX_RETRIES, topic: nil)
+          def dead_letter_queue(
+            max_retries: DEFAULT_MAX_RETRIES,
+            topic: nil,
+            independent: false
+          )
             @dead_letter_queue ||= Config.new(
               active: !topic.nil?,
               max_retries: max_retries,
-              topic: topic
+              topic: topic,
+              independent: independent
             )
           end
 

data/lib/karafka/routing/subscription_group.rb

@@ -37,7 +37,8 @@ module Karafka
       # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
       def initialize(position, topics)
-        @name = topics.first.subscription_group_name
+        @details = topics.first.subscription_group_details
+        @name = @details.fetch(:name)
         @consumer_group = topics.first.consumer_group
         # We include the consumer group id here because we want to have unique ids of subscription
         # groups across the system. Otherwise user could set the same name for multiple
@@ -47,7 +48,6 @@ module Karafka
         @position = position
         @topics = topics
         @kafka = build_kafka
-        freeze
       end
 
       # @return [String] consumer group id

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -19,7 +19,7 @@ module Karafka
         max_messages
         max_wait_time
         initial_offset
-        subscription_group_name
+        subscription_group_details
       ].freeze
 
       private_constant :DISTRIBUTION_KEYS
@@ -37,7 +37,7 @@ module Karafka
           .group_by(&:first)
           .values
           .map { |value| value.map(&:last) }
-          .map { |topics_array| Routing::Topics.new(topics_array) }
+          .flat_map { |value| expand(value) }
           .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
           .tap do |subscription_groups|
             subscription_groups.each do |subscription_group|
@@ -60,6 +60,15 @@ module Karafka
 
         accu.hash
       end
+
+      # Hook for optional expansion of groups based on subscription group features
+      #
+      # @param topics_array [Array<Routing::Topic>] group of topics that have the same settings
+      #   and can use the same connection
+      # @return [Array<Array<Routing::Topics>>] expanded groups
+      def expand(topics_array)
+        [Routing::Topics.new(topics_array)]
+      end
     end
   end
 end

data/lib/karafka/routing/topic.rb

@@ -9,7 +9,7 @@ module Karafka
       attr_reader :id, :name, :consumer_group
       attr_writer :consumer
 
-      attr_accessor :subscription_group_name
+      attr_accessor :subscription_group_details
 
       # Full subscription group reference can be built only when we have knowledge about the
       # whole routing tree, this is why it is going to be set later on
@@ -46,15 +46,13 @@ module Karafka
       INHERITABLE_ATTRIBUTES.each do |attribute|
         attr_writer attribute
 
-        define_method attribute do
-          current_value = instance_variable_get(:"@#{attribute}")
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          def #{attribute}
+            return @#{attribute} unless @#{attribute}.nil?
 
-          return current_value unless current_value.nil?
-
-          value = Karafka::App.config.send(attribute)
-
-          instance_variable_set(:"@#{attribute}", value)
-        end
+            @#{attribute} = Karafka::App.config.send(:#{attribute})
+          end
+        RUBY
       end
 
       # @return [String] name of subscription that will go to librdkafka
@@ -117,7 +115,7 @@ module Karafka
           active: active?,
           consumer: consumer,
           consumer_group_id: consumer_group.id,
-          subscription_group_name: subscription_group_name
+          subscription_group_details: subscription_group_details
         ).freeze
       end
     end

data/lib/karafka/runner.rb

@@ -3,6 +3,11 @@
 module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
+    def initialize
+      @manager = App.config.internal.connection.manager
+      @conductor = App.config.internal.connection.conductor
+    end
+
     # Starts listening on all the listeners asynchronously and handles the jobs queue closing
     # after listeners are done with their work.
     def call
@@ -13,16 +18,21 @@ module Karafka
       workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)
 
+      # Register all the listeners so they can be started and managed
+      @manager.register(listeners)
+
       workers.each(&:async_call)
-      listeners.each(&:async_call)
 
       # We aggregate threads here for a supervised shutdown process
       Karafka::Server.workers = workers
       Karafka::Server.listeners = listeners
       Karafka::Server.jobs_queue = jobs_queue
 
-      # All the listener threads need to finish
-      listeners.each(&:join)
+      until @manager.done?
+        @conductor.wait
+
+        @manager.control
+      end
 
       # We close the jobs queue only when no listener threads are working.
       # This ensures, that everything was closed prior to us not accepting anymore jobs and that

data/lib/karafka/server.rb

@@ -88,7 +88,10 @@ module Karafka
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
-          return if listeners.count(&:alive?).zero? && workers.count(&:alive?).zero?
+          all_listeners_stopped = listeners.all?(&:stopped?)
+          all_workers_stopped = workers.none?(&:alive?)
+
+          return if all_listeners_stopped && all_workers_stopped
 
           sleep SUPERVISION_SLEEP
         end
@@ -104,7 +107,7 @@ module Karafka
 
         # We're done waiting, lets kill them!
         workers.each(&:terminate)
-        listeners.each(&:terminate)
+        listeners.active.each(&:terminate)
         # We always need to shutdown clients to make sure we do not force the GC to close consumer.
         # This can cause memory leaks and crashes.
         listeners.each(&:shutdown)
@@ -137,13 +140,6 @@ module Karafka
         # We don't have to safe-guard it with check states as the state transitions work only
         # in one direction
         Karafka::App.quiet!
-
-        # We need one more thread to monitor the process and move to quieted once everything
-        # is quiet and no processing is happening anymore
-        Thread.new do
-          sleep(0.1) until listeners.coordinators.all?(&:finished?)
-          Karafka::App.quieted!
-        end
       end
 
       private