karafka 1.4.12 → 2.2.10

data/lib/karafka/processing/coordinators_buffer.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Coordinators builder used to build coordinators per topic partition
+    #
+    # It provides direct pauses access for revocation
+    #
+    # @note This buffer operates only from the listener loop, thus we do not have to make it
+    #   thread-safe.
+    class CoordinatorsBuffer
+      # @param topics [Karafka::Routing::Topics]
+      def initialize(topics)
+        @pauses_manager = Connection::PausesManager.new
+        @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
+        @coordinators = Hash.new { |h, k| h[k] = {} }
+        @topics = topics
+      end
+
+      # @param topic_name [String] topic name
+      # @param partition [Integer] partition number
+      def find_or_create(topic_name, partition)
+        @coordinators[topic_name][partition] ||= begin
+          routing_topic = @topics.find(topic_name)
+
+          @coordinator_class.new(
+            routing_topic,
+            partition,
+            @pauses_manager.fetch(routing_topic, partition)
+          )
+        end
+      end
+
+      # Resumes processing of partitions for which pause time has ended.
+      # @param block we want to run for resumed topic partitions
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      def resume(&block)
+        @pauses_manager.resume(&block)
+      end
+
+      # @param topic_name [String] topic name
+      # @param partition [Integer] partition number
+      def revoke(topic_name, partition)
+        return unless @coordinators[topic_name].key?(partition)
+
+        # The fact that we delete here does not change the fact that the executor still holds the
+        # reference to this coordinator. We delete it here, as we will no longer process any
+        # new stuff with it and we may need a new coordinator if we regain this partition, but the
+        # coordinator may still be in use
+        @coordinators[topic_name].delete(partition).revoke
+      end
+
+      # Clears coordinators and re-created the pauses manager
+      # This should be used only for critical errors recovery
+      def reset
+        @pauses_manager = Connection::PausesManager.new
+        @coordinators.clear
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace that encapsulates all the logic related to processing data.
+  module Processing
+    # Executors:
+    # - run consumers code (for `#call`) or run given preparation / teardown operations when needed
+    #   from separate threads.
+    # - they re-create consumer instances in case of partitions that were revoked and assigned
+    #   back.
+    #
+    # @note Executors are not removed after partition is revoked. They are not that big and will
+    #   be re-used in case of a re-claim
+    class Executor
+      extend Forwardable
+
+      def_delegators :@coordinator, :topic, :partition
+
+      # @return [String] unique id that we use to ensure, that we use for state tracking
+      attr_reader :id
+
+      # @return [String] subscription group id to which a given executor belongs
+      attr_reader :group_id
+
+      # @return [Karafka::Messages::Messages] messages batch
+      attr_reader :messages
+
+      # @return [Karafka::Processing::Coordinator] coordinator for this executor
+      attr_reader :coordinator
+
+      # @param group_id [String] id of the subscription group to which the executor belongs
+      # @param client [Karafka::Connection::Client] kafka client
+      # @param coordinator [Karafka::Processing::Coordinator]
+      def initialize(group_id, client, coordinator)
+        @id = SecureRandom.hex(6)
+        @group_id = group_id
+        @client = client
+        @coordinator = coordinator
+      end
+
+      # Allows us to prepare the consumer in the listener thread prior to the job being send to
+      # the queue. It also allows to run some code that is time sensitive and cannot wait in the
+      # queue as it could cause starvation.
+      #
+      # @param messages [Array<Karafka::Messages::Message>]
+      def before_enqueue(messages)
+        # Recreate consumer with each batch if persistence is not enabled
+        # We reload the consumers with each batch instead of relying on some external signals
+        # when needed for consistency. That way devs may have it on or off and not in this
+        # middle state, where re-creation of a consumer instance would occur only sometimes
+        @consumer = nil unless topic.consumer_persistence
+
+        # First we build messages batch...
+        consumer.messages = Messages::Builders::Messages.call(
+          messages,
+          topic,
+          partition,
+          # the moment we've received the batch or actually the moment we've enqueued it,
+          # but good enough
+          Time.now
+        )
+
+        consumer.on_before_enqueue
+      end
+
+      # Runs setup and warm-up code in the worker prior to running the consumption
+      def before_consume
+        consumer.on_before_consume
+      end
+
+      # Runs consumer data processing against given batch and handles failures and errors.
+      def consume
+        # We run the consumer client logic...
+        consumer.on_consume
+      end
+
+      # Runs consumer after consumption code
+      def after_consume
+        consumer.on_after_consume
+      end
+
+      # Runs consumer idle operations
+      # This may include house-keeping or other state management changes that can occur but that
+      # not mean there are any new messages available for the end user to process
+      def idle
+        # Initializes the messages set in case idle operation would happen before any processing
+        # This prevents us from having no messages object at all as the messages object and
+        # its metadata may be used for statistics
+        consumer.messages ||= Messages::Builders::Messages.call(
+          [],
+          topic,
+          partition,
+          Time.now
+        )
+
+        consumer.on_idle
+      end
+
+      # Runs the controller `#revoked` method that should be triggered when a given consumer is
+      # no longer needed due to partitions reassignment.
+      #
+      # @note Clearing the consumer will ensure, that if we get the partition back, it will be
+      #   handled with a consumer with a clean state.
+      #
+      # @note We run it only when consumer was present, because presence indicates, that at least
+      #   a single message has been consumed.
+      #
+      # @note We do not reset the consumer but we indicate need for recreation instead, because
+      #   after the revocation, there still may be `#after_consume` running that needs a given
+      #   consumer instance.
+      def revoked
+        consumer.on_revoked if @consumer
+      end
+
+      # Runs the controller `#shutdown` method that should be triggered when a given consumer is
+      # no longer needed as we're closing the process.
+      #
+      # @note While we do not need to clear the consumer here, it's a good habit to clean after
+      #   work is done.
+      def shutdown
+        # There is a case, where the consumer no longer exists because it was revoked, in case like
+        # that we do not build a new instance and shutdown should not be triggered.
+        consumer.on_shutdown if @consumer
+      end
+
+      private
+
+      # @return [Object] cached consumer instance
+      def consumer
+        @consumer ||= begin
+          topic = @coordinator.topic
+
+          strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(topic)
+          expansions = ::Karafka::App.config.internal.processing.expansions_selector.find(topic)
+
+          consumer = topic.consumer_class.new
+          # We use singleton class as the same consumer class may be used to process different
+          # topics with different settings
+          consumer.singleton_class.include(strategy)
+
+          # Specific features may expand consumer API beyond the injected strategy. The difference
+          # here is that strategy impacts the flow of states while extra APIs just provide some
+          # extra methods with informations, etc but do no deviate the flow behavior
+          expansions.each { |expansion| consumer.singleton_class.include(expansion) }
+
+          consumer.client = @client
+          consumer.producer = ::Karafka::App.producer
+          consumer.coordinator = @coordinator
+
+          consumer
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/executors_buffer.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Buffer for executors of a given subscription group. It wraps around the concept of building
+    # and caching them, so we can re-use them instead of creating new each time.
+    class ExecutorsBuffer
+      # @param client [Connection::Client]
+      # @param subscription_group [Routing::SubscriptionGroup]
+      # @return [ExecutorsBuffer]
+      def initialize(client, subscription_group)
+        @subscription_group = subscription_group
+        @client = client
+        # We need two layers here to keep track of topics, partitions and processing groups
+        @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
+      end
+
+      # Finds or creates an executor based on the provided details
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @param parallel_key [String] parallel group key
+      # @param coordinator [Karafka::Processing::Coordinator]
+      # @return [Executor] consumer executor
+      def find_or_create(topic, partition, parallel_key, coordinator)
+        @buffer[topic][partition][parallel_key] ||= Executor.new(
+          @subscription_group.id,
+          @client,
+          coordinator
+        )
+      end
+
+      # Revokes executors of a given topic partition, so they won't be used anymore for incoming
+      # messages
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def revoke(topic, partition)
+        @buffer[topic][partition].clear
+      end
+
+      # Finds all the executors available for a given topic partition
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @return [Array<Executor>] executors in use for this topic + partition
+      def find_all(topic, partition)
+        @buffer[topic][partition].values
+      end
+
+      # Iterates over all available executors and yields them together with topic and partition
+      # info
+      # @yieldparam [Routing::Topic] karafka routing topic object
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Executor] given executor
+      def each
+        @buffer.each do |_, partitions|
+          partitions.each do |_, executors|
+            executors.each do |_, executor|
+              yield(executor)
+            end
+          end
+        end
+      end
+
+      # Clears the executors buffer. Useful for critical errors recovery.
+      def clear
+        @buffer.clear
+      end
+    end
+  end
+end

data/lib/karafka/processing/expansions_selector.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Selector of appropriate topic setup based features enhancements.
+    #
+    # Those expansions to the consumer API are NOT about the flow of processing. For this we have
+    # strategies. Those are suppose to provide certain extra APIs that user can use to get some
+    # extra non-flow related functionalities.
+    class ExpansionsSelector
+      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+      #   expansions
+      # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance the
+      #   consumer
+      def find(topic)
+        expansions = []
+        expansions << Processing::InlineInsights::Consumer if topic.inline_insights?
+        expansions
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/consumer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Namespace of the Inline Insights feature "non routing" related components
+    #
+    # @note We use both `#insights` because it is the feature name but also `#statistics` to make
+    #   it consistent with the fact that we publish and operate on statistics. User can pick
+    #   whichever name they prefer.
+    module InlineInsights
+      # Module that adds extra methods to the consumer that allow us to fetch the insights
+      module Consumer
+        # @return [Hash] empty hash or hash with given partition insights if already present
+        # @note We cache insights on the consumer, as in some scenarios we may no longer have them
+        #   inside the Tracker, for example under involuntary revocation, incoming statistics may
+        #   no longer have lost partition insights. Since we want to be consistent during single
+        #   batch operations, we want to ensure, that if we have insights they are available
+        #   throughout the whole processing.
+        def insights
+          insights = Tracker.find(topic, partition)
+
+          # If we no longer have new insights but we still have them locally, we can use them
+          return @insights if @insights && insights.empty?
+          # If insights are still the same, we can use them
+          return @insights if @insights.equal?(insights)
+
+          # If we've received new insights that are not empty, we can cache them
+          @insights = insights
+        end
+
+        # @return [Boolean] true if there are insights to work with, otherwise false
+        def insights?
+          !insights.empty?
+        end
+
+        alias statistics insights
+        alias statistics? insights?
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/listener.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module InlineInsights
+      # Listener that adds statistics to our inline tracker
+      class Listener
+        # Adds statistics to the tracker
+        # @param event [Karafka::Core::Monitoring::Event] event with statistics
+        def on_statistics_emitted(event)
+          Tracker.add(
+            event[:consumer_group_id],
+            event[:statistics]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/tracker.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module InlineInsights
+      # Object used to track statistics coming from librdkafka in a way that can be accessible by
+      # the consumers
+      #
+      # We use a single tracker because we do not need state management here as our consumer groups
+      # clients identified by statistics name value are unique. On top of that, having a per
+      # process one that is a singleton allows us to use tracker easily also from other places like
+      # filtering API etc.
+      #
+      # @note We include cache of 5 minutes for revoked partitions to compensate for cases where
+      #   when using LRJ a lost partition data would not be present anymore, however we would still
+      #   be in the processing phase. Since those metrics are published with each `poll`, regular
+      #   processing is not a subject of this issue. For LRJ we keep the reference. The only case
+      #   where this could be switched midway is when LRJ is running for an extended period of time
+      #   after the involuntary revocation. Having a time based cache instead of tracking
+      #   simplifies the design as we do not have to deal with state tracking, especially since
+      #   we would have to track also operations running in a revoked state.
+      #
+      # @note This tracker keeps in memory data about all topics and partitions that it encounters
+      #   because in case of routing patterns, we may start getting statistics prior to registering
+      #   given topic via dynamic routing expansions. In such case we would not have insights
+      #   where they were actually available for us to use.
+      #
+      # @note Memory usage is negligible as long as we can evict expired data. Single metrics set
+      #   for a single partition contains around 4KB of data. This means, that in case of an
+      #   assignment of 1000 partitions, we use around 4MB of space for tracking those metrics.
+      class Tracker
+        include Singleton
+        include Karafka::Core::Helpers::Time
+
+        # Empty hash we want to return in any case where we could not locate appropriate topic
+        # partition statistics.
+        EMPTY_HASH = {}.freeze
+
+        # Empty array to save on memory allocations.
+        EMPTY_ARRAY = [].freeze
+
+        # 5 minutes of cache. We cache last result per consumer group topic partition so we are
+        # not affected by involuntary rebalances during LRJ execution.
+        TTL = 5 * 60 * 1_000
+
+        private_constant :EMPTY_HASH, :EMPTY_ARRAY, :TTL
+
+        class << self
+          extend Forwardable
+
+          def_delegators :instance, :find, :add, :exists?, :clear
+        end
+
+        def initialize
+          @accu = {}
+          @mutex = Mutex.new
+        end
+
+        # Adds each partition statistics into internal accumulator. Single statistics set may
+        # contain data from multiple topics and their partitions because a single client can
+        # operate on multiple topics and partitions.
+        #
+        # We iterate over those topics and partitions and store topics partitions data only.
+        #
+        # @param consumer_group_id [String] id of the consumer group for which statistics were
+        #   emitted.
+        # @param statistics [Hash] librdkafka enriched statistics
+        def add(consumer_group_id, statistics)
+          @mutex.synchronize do
+            statistics.fetch('topics', EMPTY_HASH).each do |topic_name, t_details|
+              t_details.fetch('partitions', EMPTY_HASH).each do |partition_id, p_details|
+                next unless track?(partition_id, p_details)
+
+                key = "#{consumer_group_id}_#{topic_name}_#{partition_id}"
+                @accu[key] = [monotonic_now, p_details]
+              end
+            end
+
+            evict
+          end
+        end
+
+        # Finds statistics about requested consumer group topic partition
+        #
+        # @param topic [Karafka::Routing::Topic]
+        # @param partition [Integer]
+        # @return [Hash] hash with given topic partition statistics or empty hash if not present
+        #
+        # @note We do not enclose it with a mutex mainly because the only thing that could happen
+        #   here that would be a race-condition is a miss that anyhow we need to support due to
+        #   how librdkafka ships metrics and a potential removal of data on heavily revoked LRJ.
+        def find(topic, partition)
+          key = "#{topic.consumer_group.id}_#{topic.name}_#{partition}"
+          @accu.fetch(key, EMPTY_ARRAY).last || EMPTY_HASH
+        end
+
+        # Clears the tracker
+        def clear
+          @mutex.synchronize { @accu.clear }
+        end
+
+        private
+
+        # Evicts expired data from the cache
+        def evict
+          @accu.delete_if { |_, details| monotonic_now - details.first > TTL }
+        end
+
+        # Should we track given partition
+        #
+        # We do not track stopped partitions and the once we do not work with actively
+        # @param partition_id [String] partition id as a string
+        # @param p_details [Hash] partition statistics details
+        # @return [Boolean] true if we should track given partition
+        def track?(partition_id, p_details)
+          return false if partition_id == '-1'
+
+          fetch_state = p_details.fetch('fetch_state')
+
+          return false if fetch_state == 'stopped'
+          return false if fetch_state == 'none'
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/base.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Namespace for all the jobs that are supposed to run in workers.
+    module Jobs
+      # Base class for all the jobs types that are suppose to run in workers threads.
+      # Each job can have 3 main entry-points: `#before_call`, `#call` and `#after_call`
+      # Only `#call` is required.
+      class Base
+        extend Forwardable
+
+        # @note Since one job has always one executor, we use the jobs id and group id as reference
+        def_delegators :executor, :id, :group_id
+
+        attr_reader :executor
+
+        # Creates a new job instance
+        def initialize
+          # All jobs are blocking by default and they can release the lock when blocking operations
+          # are done (if needed)
+          @non_blocking = false
+        end
+
+        # When redefined can run any code prior to the job being enqueued
+        # @note This will run in the listener thread and not in the worker
+        def before_enqueue; end
+
+        # When redefined can run any code that should run before executing the proper code
+        def before_call; end
+
+        # The main entry-point of a job
+        def call
+          raise NotImplementedError, 'Please implement in a subclass'
+        end
+
+        # When redefined can run any code that should run after executing the proper code
+        def after_call; end
+
+        # @return [Boolean] is this a non-blocking job
+        #
+        # @note Blocking job is a job, that will cause the job queue to wait until it is finished
+        #   before removing the lock on new jobs being added
+        #
+        # @note All the jobs are blocking by default
+        #
+        # @note Job **needs** to mark itself as non-blocking only **after** it is done with all
+        #   the blocking things (pausing partition, etc).
+        def non_blocking?
+          @non_blocking
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/consume.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # The main job type. It runs the executor that triggers given topic partition messages
+      # processing in an underlying consumer instance.
+      class Consume < Base
+        # @return [Array<Rdkafka::Consumer::Message>] array with messages
+        attr_reader :messages
+
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job
+        # @param messages [Karafka::Messages::Messages] karafka messages batch
+        # @return [Consume]
+        def initialize(executor, messages)
+          @executor = executor
+          @messages = messages
+          super()
+        end
+
+        # Runs all the preparation code on the executor that needs to happen before the job is
+        # enqueued.
+        def before_enqueue
+          executor.before_enqueue(@messages)
+        end
+
+        # Runs the before consumption preparations on the executor
+        def before_call
+          executor.before_consume
+        end
+
+        # Runs the given executor
+        def call
+          executor.consume
+        end
+
+        # Runs any error handling and other post-consumption stuff on the executor
+        def after_call
+          executor.after_consume
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/idle.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Type of job that we may use to run some extra handling that happens without the user
+      # related lifecycle event like consumption, revocation, etc.
+      class Idle < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job on an active consumer
+        # @return [Shutdown]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Run the idle work via the executor
+        def call
+          executor.idle
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/revoked.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Job that runs the revoked operation when we loose a partition on a consumer that lost it.
+      class Revoked < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
+        # @return [Revoked]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Runs the revoking job via an executor.
+        def call
+          executor.revoked
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/shutdown.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Job that runs on each active consumer upon process shutdown (one job per consumer).
+      class Shutdown < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
+        #   job on an active consumer
+        # @return [Shutdown]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Runs the shutdown job via an executor.
+        def call
+          executor.shutdown
+        end
+      end
+    end
+  end
+end