karafka 1.4.0 → 2.0.10

data/lib/karafka/pro/processing/partitioner.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro partitioner that can distribute work based on the virtual partitioner settings
+      class Partitioner < ::Karafka::Processing::Partitioner
+        # @param topic [String] topic name
+        # @param messages [Array<Karafka::Messages::Message>] karafka messages
+        # @yieldparam [Integer] group id
+        # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
+        def call(topic, messages)
+          ktopic = @subscription_group.topics.find(topic)
+
+          # We only partition work if we have a virtual partitioner and more than one thread to
+          # process the data. With one thread it is not worth partitioning the work as the work
+          # itself will be assigned to one thread (pointless work)
+          if ktopic.virtual_partitions? && ktopic.virtual_partitions.max_partitions > 1
+            # We need to reduce it to number of threads, so the group_id is not a direct effect
+            # of the end user action. Otherwise the persistence layer for consumers would cache
+            # it forever and it would cause memory leaks
+            groupings = messages
+                        .group_by { |msg| ktopic.virtual_partitions.partitioner.call(msg) }
+                        .values
+
+            # Reduce the number of virtual partitions to a size that matches the max_partitions
+            # As mentioned above we cannot use the partitioning keys directly as it could cause
+            # memory leaks
+            #
+            # The algorithm here is simple, we assume that the most costly in terms of processing,
+            # will be processing of the biggest group and we reduce the smallest once to have
+            # max of groups equal to max_partitions
+            while groupings.size > ktopic.virtual_partitions.max_partitions
+              groupings.sort_by! { |grouping| -grouping.size }
+
+              # Offset order needs to be maintained for virtual partitions
+              groupings << (groupings.pop + groupings.pop).sort_by!(&:offset)
+            end
+
+            groupings.each_with_index { |messages_group, index| yield(index, messages_group) }
+          else
+            # When no virtual partitioner, works as regular one
+            yield(0, messages)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/scheduler.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Optimizes scheduler that takes into consideration of execution time needed to process
+      # messages from given topics partitions. It uses the non-preemptive LJF algorithm
+      #
+      # This scheduler is designed to optimize execution times on jobs that perform IO operations
+      # as when taking IO into consideration, the can achieve optimized parallel processing.
+      #
+      # This scheduler can also work with virtual partitions.
+      #
+      # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
+      # default FIFO scheduler from the default Karafka scheduler
+      class Scheduler < ::Karafka::Processing::Scheduler
+        # Schedules jobs in the LJF order for consumption
+        #
+        # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+        # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+        #
+        def schedule_consumption(queue, jobs_array)
+          pt = PerformanceTracker.instance
+
+          ordered = []
+
+          jobs_array.each do |job|
+            messages = job.messages
+            message = messages.first
+
+            cost = pt.processing_time_p95(message.topic, message.partition) * messages.size
+
+            ordered << [job, cost]
+          end
+
+          ordered.sort_by!(&:last)
+          ordered.reverse!
+          ordered.map!(&:first)
+
+          ordered.each do |job|
+            queue << job
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/builder_extensions.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro routing components
+    module Routing
+      # Routing extensions for builder to be able to validate Pro components correct usage
+      module BuilderExtensions
+        # Validate consumer groups with pro contracts
+        # @param block [Proc] routing defining block
+        def draw(&block)
+          super
+
+          each do |consumer_group|
+            ::Karafka::Pro::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/topic_extensions.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro routing components
+    module Routing
+      # Routing extensions that allow to configure some extra PRO routing options
+      module TopicExtensions
+        # Internal representation of the virtual partitions settings and configuration
+        # This allows us to abstract away things in a nice manner
+        #
+        # For features with more options than just on/off we use this approach as it simplifies
+        # the code. We do not use it for all not to create unneeded complexity
+        VirtualPartitions = Struct.new(
+          :active,
+          :partitioner,
+          :max_partitions,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+
+        class << self
+          # @param base [Class] class we extend
+          def prepended(base)
+            base.attr_accessor :long_running_job
+          end
+        end
+
+        # @param max_partitions [Integer] max number of virtual partitions that can come out of the
+        #   single distribution flow. When set to more than the Karafka threading, will create
+        #   more work than workers. When less, can ensure we have spare resources to process other
+        #   things in parallel.
+        # @param partitioner [nil, #call] nil or callable partitioner
+        # @return [VirtualPartitions] method that allows to set the virtual partitions details
+        #   during the routing configuration and then allows to retrieve it
+        def virtual_partitions(
+          max_partitions: Karafka::App.config.concurrency,
+          partitioner: nil
+        )
+          @virtual_partitions ||= VirtualPartitions.new(
+            active: !partitioner.nil?,
+            max_partitions: max_partitions,
+            partitioner: partitioner
+          )
+        end
+
+        # @return [Boolean] are virtual partitions enabled for given topic
+        def virtual_partitions?
+          virtual_partitions.active?
+        end
+
+        # @return [Boolean] is a given job on a topic a long-running one
+        def long_running_job?
+          @long_running_job || false
+        end
+
+        # @return [Hash] hash with topic details and the extensions details
+        def to_h
+          super.merge(
+            virtual_partitions: virtual_partitions.to_h
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component.
+# All of the commercial components are present in the lib/karafka/pro directory of this repository
+# and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+module Karafka
+  # Namespace for pro components, licensed under the commercial license agreement.
+  module Pro
+  end
+end

data/lib/karafka/process.rb

@@ -9,6 +9,7 @@ module Karafka
       SIGINT
       SIGQUIT
       SIGTERM
+      SIGTTIN
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|

data/lib/karafka/processing/coordinator.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Basic coordinator that allows us to provide coordination objects into consumers.
+    #
+    # This is a wrapping layer to simplify management of work to be handled around consumption.
+    #
+    # @note This coordinator needs to be thread safe. Some operations are performed only in the
+    #   listener thread, but we go with thread-safe by default for all not to worry about potential
+    #   future mistakes.
+    class Coordinator
+      # @return [Karafka::TimeTrackers::Pause]
+      attr_reader :pause_tracker
+
+      attr_reader :seek_offset
+
+      # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
+      def initialize(pause_tracker)
+        @pause_tracker = pause_tracker
+        @revoked = false
+        @consumptions = {}
+        @running_jobs = 0
+        @mutex = Mutex.new
+      end
+
+      # Starts the coordinator for given consumption jobs
+      # @param messages [Array<Karafka::Messages::Message>] batch of message for which we are
+      #   going to coordinate work. Not used with regular coordinator.
+      def start(messages)
+        @mutex.synchronize do
+          @running_jobs = 0
+          # We need to clear the consumption results hash here, otherwise we could end up storing
+          # consumption results of consumer instances we no longer control
+          @consumptions.clear
+
+          # We set it on the first encounter and never again, because then the offset setting
+          # should be up to the consumers logic (our or the end user)
+          # Seek offset needs to be always initialized as for case where manual offset management
+          # is turned on, we need to have reference to the first offset even in case of running
+          # multiple batches without marking any messages as consumed. Rollback needs to happen to
+          # the last place we know of or the last message + 1 that was marked
+          @seek_offset ||= messages.first.offset
+        end
+      end
+
+      # @param offset [Integer] message offset
+      def seek_offset=(offset)
+        @mutex.synchronize { @seek_offset = offset }
+      end
+
+      # Increases number of jobs that we handle with this coordinator
+      def increment
+        @mutex.synchronize { @running_jobs += 1 }
+      end
+
+      # Decrements number of jobs we handle at the moment
+      def decrement
+        @mutex.synchronize do
+          @running_jobs -= 1
+
+          return @running_jobs unless @running_jobs.negative?
+
+          # This should never happen. If it does, something is heavily out of sync. Please reach
+          # out to us if you encounter this
+          raise Karafka::Errors::InvalidCoordinatorState, 'Was zero before decrementation'
+        end
+      end
+
+      # @param consumer [Object] karafka consumer (normal or pro)
+      # @return [Karafka::Processing::Result] result object which we can use to indicate
+      #   consumption processing state.
+      def consumption(consumer)
+        @mutex.synchronize do
+          @consumptions[consumer] ||= Processing::Result.new
+        end
+      end
+
+      # Is all the consumption done and finished successfully for this coordinator
+      def success?
+        @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
+      end
+
+      # Marks given coordinator for processing group as revoked
+      #
+      # This is invoked in two places:
+      #   - from the main listener loop when we detect revoked partitions
+      #   - from the consumer in case checkpointing fails
+      #
+      # This means, we can end up having consumer being aware that it was revoked prior to the
+      # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
+      # processed until revocation jobs are done.
+      def revoke
+        @mutex.synchronize { @revoked = true }
+      end
+
+      # @return [Boolean] is the partition we are processing revoked or not
+      def revoked?
+        @revoked
+      end
+    end
+  end
+end

data/lib/karafka/processing/coordinators_buffer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Coordinators builder used to build coordinators per topic partition
+    #
+    # It provides direct pauses access for revocation
+    #
+    # @note This buffer operates only from the listener loop, thus we do not have to make it
+    #   thread-safe.
+    class CoordinatorsBuffer
+      def initialize
+        @pauses_manager = Connection::PausesManager.new
+        @coordinator_class = ::Karafka::App.config.internal.processing.coordinator_class
+        @coordinators = Hash.new { |h, k| h[k] = {} }
+      end
+
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def find_or_create(topic, partition)
+        @coordinators[topic][partition] ||= @coordinator_class.new(
+          @pauses_manager.fetch(topic, partition)
+        )
+      end
+
+      # Resumes processing of partitions for which pause time has ended.
+      # @param block we want to run for resumed topic partitions
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      def resume(&block)
+        @pauses_manager.resume(&block)
+      end
+
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def revoke(topic, partition)
+        return unless @coordinators[topic].key?(partition)
+
+        # The fact that we delete here does not change the fact that the executor still holds the
+        # reference to this coordinator. We delete it here, as we will no longer process any
+        # new stuff with it and we may need a new coordinator if we regain this partition, but the
+        # coordinator may still be in use
+        @coordinators[topic].delete(partition).revoke
+      end
+
+      # Clears coordinators and re-created the pauses manager
+      # This should be used only for critical errors recovery
+      def reset
+        @pauses_manager = Connection::PausesManager.new
+        @coordinators.clear
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace that encapsulates all the logic related to processing data.
+  module Processing
+    # Executors:
+    # - run consumers code (for `#call`) or run given preparation / teardown operations when needed
+    #   from separate threads.
+    # - they re-create consumer instances in case of partitions that were revoked and assigned
+    #   back.
+    #
+    # @note Executors are not removed after partition is revoked. They are not that big and will
+    #   be re-used in case of a re-claim
+    class Executor
+      # @return [String] unique id that we use to ensure, that we use for state tracking
+      attr_reader :id
+
+      # @return [String] subscription group id to which a given executor belongs
+      attr_reader :group_id
+
+      # @return [Karafka::Messages::Messages] messages batch
+      attr_reader :messages
+
+      # Topic accessibility may be needed for the jobs builder to be able to build a proper job
+      # based on the topic settings defined by the end user
+      #
+      # @return [Karafka::Routing::Topic] topic of this executor
+      attr_reader :topic
+
+      # @param group_id [String] id of the subscription group to which the executor belongs
+      # @param client [Karafka::Connection::Client] kafka client
+      # @param topic [Karafka::Routing::Topic] topic for which this executor will run
+      def initialize(group_id, client, topic)
+        @id = SecureRandom.uuid
+        @group_id = group_id
+        @client = client
+        @topic = topic
+      end
+
+      # Allows us to prepare the consumer in the listener thread prior to the job being send to
+      # the queue. It also allows to run some code that is time sensitive and cannot wait in the
+      # queue as it could cause starvation.
+      #
+      # @param messages [Array<Karafka::Messages::Message>]
+      # @param coordinator [Karafka::Processing::Coordinator] coordinator for processing management
+      def before_enqueue(messages, coordinator)
+        # the moment we've received the batch or actually the moment we've enqueued it,
+        # but good enough
+        @enqueued_at = Time.now
+
+        # Recreate consumer with each batch if persistence is not enabled
+        # We reload the consumers with each batch instead of relying on some external signals
+        # when needed for consistency. That way devs may have it on or off and not in this
+        # middle state, where re-creation of a consumer instance would occur only sometimes
+        @consumer = nil unless ::Karafka::App.config.consumer_persistence
+
+        consumer.coordinator = coordinator
+
+        # First we build messages batch...
+        consumer.messages = Messages::Builders::Messages.call(
+          messages,
+          @topic,
+          @enqueued_at
+        )
+
+        consumer.on_before_enqueue
+      end
+
+      # Runs setup and warm-up code in the worker prior to running the consumption
+      def before_consume
+        consumer.on_before_consume
+      end
+
+      # Runs consumer data processing against given batch and handles failures and errors.
+      def consume
+        # We run the consumer client logic...
+        consumer.on_consume
+      end
+
+      # Runs consumer after consumption code
+      def after_consume
+        consumer.on_after_consume
+      end
+
+      # Runs the controller `#revoked` method that should be triggered when a given consumer is
+      # no longer needed due to partitions reassignment.
+      #
+      # @note Clearing the consumer will ensure, that if we get the partition back, it will be
+      #   handled with a consumer with a clean state.
+      #
+      # @note We run it only when consumer was present, because presence indicates, that at least
+      #   a single message has been consumed.
+      #
+      # @note We do not reset the consumer but we indicate need for recreation instead, because
+      #   after the revocation, there still may be `#after_consume` running that needs a given
+      #   consumer instance.
+      def revoked
+        consumer.on_revoked if @consumer
+      end
+
+      # Runs the controller `#shutdown` method that should be triggered when a given consumer is
+      # no longer needed as we're closing the process.
+      #
+      # @note While we do not need to clear the consumer here, it's a good habit to clean after
+      #   work is done.
+      def shutdown
+        # There is a case, where the consumer no longer exists because it was revoked, in case like
+        # that we do not build a new instance and shutdown should not be triggered.
+        consumer.on_shutdown if @consumer
+      end
+
+      private
+
+      # @return [Object] cached consumer instance
+      def consumer
+        @consumer ||= begin
+          consumer = @topic.consumer_class.new
+          consumer.topic = @topic
+          consumer.client = @client
+          consumer.producer = ::Karafka::App.producer
+          consumer
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/executors_buffer.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Buffer for executors of a given subscription group. It wraps around the concept of building
+    # and caching them, so we can re-use them instead of creating new each time.
+    class ExecutorsBuffer
+      # @param client [Connection::Client]
+      # @param subscription_group [Routing::SubscriptionGroup]
+      # @return [ExecutorsBuffer]
+      def initialize(client, subscription_group)
+        @subscription_group = subscription_group
+        @client = client
+        # We need two layers here to keep track of topics, partitions and processing groups
+        @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
+      end
+
+      # Finds or creates an executor based on the provided details
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @param parallel_key [String] parallel group key
+      # @return [Executor] consumer executor
+      def find_or_create(topic, partition, parallel_key)
+        ktopic = find_topic(topic)
+
+        @buffer[ktopic][partition][parallel_key] ||= Executor.new(
+          @subscription_group.id,
+          @client,
+          ktopic
+        )
+      end
+
+      # Revokes executors of a given topic partition, so they won't be used anymore for incoming
+      # messages
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      def revoke(topic, partition)
+        ktopic = find_topic(topic)
+
+        @buffer[ktopic][partition].clear
+      end
+
+      # Finds all the executors available for a given topic partition
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @return [Array<Executor>] executors in use for this topic + partition
+      def find_all(topic, partition)
+        ktopic = find_topic(topic)
+
+        @buffer[ktopic][partition].values
+      end
+
+      # Iterates over all available executors and yields them together with topic and partition
+      # info
+      # @yieldparam [Routing::Topic] karafka routing topic object
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Executor] given executor
+      def each
+        @buffer.each do |ktopic, partitions|
+          partitions.each do |partition, executors|
+            executors.each do |_parallel_key, executor|
+              # We skip the parallel key here as it does not serve any value when iterating
+              yield(ktopic, partition, executor)
+            end
+          end
+        end
+      end
+
+      # Clears the executors buffer. Useful for critical errors recovery.
+      def clear
+        @buffer.clear
+      end
+
+      private
+
+      # Finds topic based on its name
+      #
+      # @param topic [String] topic we're looking for
+      # @return [Karafka::Routing::Topic] topic we're interested in
+      def find_topic(topic)
+        @subscription_group.topics.find(topic) || raise(Errors::TopicNotFoundError, topic)
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs/base.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Namespace for all the jobs that are supposed to run in workers.
+    module Jobs
+      # Base class for all the jobs types that are suppose to run in workers threads.
+      # Each job can have 3 main entry-points: `#before_call`, `#call` and `#after_call`
+      # Only `#call` is required.
+      class Base
+        extend Forwardable
+
+        # @note Since one job has always one executor, we use the jobs id and group id as reference
+        def_delegators :executor, :id, :group_id
+
+        attr_reader :executor
+
+        # Creates a new job instance
+        def initialize
+          # All jobs are blocking by default and they can release the lock when blocking operations
+          # are done (if needed)
+          @non_blocking = false
+        end
+
+        # When redefined can run any code prior to the job being enqueued
+        # @note This will run in the listener thread and not in the worker
+        def before_enqueue; end
+
+        # When redefined can run any code that should run before executing the proper code
+        def before_call; end
+
+        # The main entry-point of a job
+        def call
+          raise NotImplementedError, 'Please implement in a subclass'
+        end
+
+        # When redefined can run any code that should run after executing the proper code
+        def after_call; end
+
+        # @return [Boolean] is this a non-blocking job
+        #
+        # @note Blocking job is a job, that will cause the job queue to wait until it is finished
+        #   before removing the lock on new jobs being added
+        #
+        # @note All the jobs are blocking by default
+        #
+        # @note Job **needs** to mark itself as non-blocking only **after** it is done with all
+        #   the blocking things (pausing partition, etc).
+        def non_blocking?
+          @non_blocking
+        end
+      end
+    end
+  end
+end