karafka 1.4.12 → 2.2.10

data/lib/karafka/pro/processing/filters_applier.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Applier for all filters we want to have. Whether related to limiting messages based
+      # on the payload or any other things.
+      #
+      # From the outside world perspective, this encapsulates all the filters.
+      # This means that this is the API we expose as a single filter, allowing us to control
+      # the filtering via many filters easily.
+      class FiltersApplier
+        # @return [Array] registered filters array. Useful if we want to inject internal context
+        #   aware filters.
+        attr_reader :filters
+
+        # @param coordinator [Pro::Coordinator] pro coordinator
+        def initialize(coordinator)
+          # Builds filters out of their factories
+          # We build it that way (providing topic and partition) because there may be a case where
+          # someone wants to have a specific logic that is per topic or partition. Like for example
+          # a case where there is a cache bypassing revocations for topic partition.
+          #
+          # We provide full Karafka routing topic here and not the name only, in case the filter
+          # would be customized based on other topic settings (like VPs, etc)
+          #
+          # This setup allows for biggest flexibility also because topic object holds the reference
+          # to the subscription group and consumer group
+          @filters = coordinator.topic.filtering.factories.map do |factory|
+            factory.call(coordinator.topic, coordinator.partition)
+          end
+        end
+
+        # @param messages [Array<Karafka::Messages::Message>] array with messages from the
+        #   partition
+        def apply!(messages)
+          return unless active?
+
+          @filters.each { |filter| filter.apply!(messages) }
+        end
+
+        # @return [Boolean] did we filter out any messages during filtering run
+        def applied?
+          return false unless active?
+
+          !applied.empty?
+        end
+
+        # @return [Symbol] consumer post-filtering action that should be taken
+        def action
+          return :skip unless applied?
+
+          # The highest priority is on a potential backoff from any of the filters because it is
+          # the less risky (delay and continue later)
+          return :pause if applied.any? { |filter| filter.action == :pause }
+
+          # If none of the filters wanted to pause, we can check for any that would want to seek
+          # and if there is any, we can go with this strategy
+          return :seek if applied.any? { |filter| filter.action == :seek }
+
+          :skip
+        end
+
+        # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
+        #   filters to satisfy all of them.
+        def timeout
+          applied.map(&:timeout).compact.min || 0
+        end
+
+        # The first message we do need to get next time we poll. We use the minimum not to jump
+        # accidentally by over any.
+        # @return [Karafka::Messages::Message, nil] cursor message or nil if none
+        # @note Cursor message can also return the offset in the time format
+        def cursor
+          return nil unless active?
+
+          applied.map(&:cursor).compact.min_by(&:offset)
+        end
+
+        private
+
+        # @return [Boolean] is filtering active
+        def active?
+          !@filters.empty?
+        end
+
+        # @return [Array<Object>] filters that applied any sort of messages limiting
+        def applied
+          @filters.select(&:applied?)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro components related to processing part of Karafka
+    module Processing
+      # Pro jobs
+      module Jobs
+        # The main job type in a non-blocking variant.
+        # This variant works "like" the regular consumption but does not block the queue.
+        #
+        # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
+        # if would block.
+        #
+        # @note It needs to be working with a proper consumer that will handle the partition
+        #   management. This layer of the framework knows nothing about Kafka messages consumption.
+        class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
+          # Makes this job non-blocking from the start
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Consume`
+          def initialize(*args)
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro components related to processing part of Karafka
+    module Processing
+      # Pro jobs
+      module Jobs
+        # The revoked job type in a non-blocking variant.
+        # This variant works "like" the regular revoked but does not block the queue.
+        #
+        # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
+        # in scenarios where there are more jobs than threads, without this being async we
+        # would potentially stop polling
+        class RevokedNonBlocking < ::Karafka::Processing::Jobs::Revoked
+          # Makes this job non-blocking from the start
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Revoked`
+          def initialize(*args)
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro jobs builder that supports lrj
+      class JobsBuilder < ::Karafka::Processing::JobsBuilder
+        # @param executor [Karafka::Processing::Executor]
+        def idle(executor)
+          Karafka::Processing::Jobs::Idle.new(executor)
+        end
+
+        # @param executor [Karafka::Processing::Executor]
+        # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+        # @return [Karafka::Processing::Jobs::Consume] blocking job
+        # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
+        def consume(executor, messages)
+          if executor.topic.long_running_job?
+            Jobs::ConsumeNonBlocking.new(executor, messages)
+          else
+            super
+          end
+        end
+
+        # @param executor [Karafka::Processing::Executor]
+        # @return [Karafka::Processing::Jobs::Revoked] revocation job for non LRJ
+        # @return [Karafka::Processing::Jobs::RevokedNonBlocking] revocation job that is
+        #   non-blocking, so when revocation job is scheduled for LRJ it also will not block
+        def revoked(executor)
+          if executor.topic.long_running_job?
+            Jobs::RevokedNonBlocking.new(executor)
+          else
+            super
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/partitioner.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro partitioner that can distribute work based on the virtual partitioner settings
+      class Partitioner < ::Karafka::Processing::Partitioner
+        # @param topic [String] topic name
+        # @param messages [Array<Karafka::Messages::Message>] karafka messages
+        # @param coordinator [Karafka::Pro::Processing::Coordinator] processing coordinator that
+        #   will be used with those messages
+        # @yieldparam [Integer] group id
+        # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
+        def call(topic, messages, coordinator)
+          ktopic = @subscription_group.topics.find(topic)
+
+          # We only partition work if we have:
+          # - a virtual partitioner
+          # - more than one thread to process the data
+          # - collective is not collapsed via coordinator
+          #
+          # With one thread it is not worth partitioning the work as the work itself will be
+          # assigned to one thread (pointless work)
+          #
+          # We collapse the partitioning on errors because we "regain" full ordering on a batch
+          # that potentially contains the data that caused the error.
+          #
+          # This is great because it allows us to run things without the parallelization that adds
+          # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
+          if ktopic.virtual_partitions? &&
+             ktopic.virtual_partitions.max_partitions > 1 &&
+             !coordinator.collapsed?
+            # We need to reduce it to the max concurrency, so the group_id is not a direct effect
+            # of the end user action. Otherwise the persistence layer for consumers would cache
+            # it forever and it would cause memory leaks
+            #
+            # This also needs to be consistent because the aggregation here needs to warrant, that
+            # the same partitioned message will always be assigned to the same virtual partition.
+            # Otherwise in case of a window aggregation with VP spanning across several polls, the
+            # data could not be complete.
+            groupings = messages.group_by do |msg|
+              key = ktopic.virtual_partitions.partitioner.call(msg).to_s.sum
+
+              key % ktopic.virtual_partitions.max_partitions
+            end
+
+            groupings.each do |key, messages_group|
+              yield(key, messages_group)
+            end
+          else
+            # When no virtual partitioner, works as regular one
+            yield(0, messages)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/scheduler.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Optimizes scheduler that takes into consideration of execution time needed to process
+      # messages from given topics partitions. It uses the non-preemptive LJF algorithm
+      #
+      # This scheduler is designed to optimize execution times on jobs that perform IO operations
+      # as when taking IO into consideration, the can achieve optimized parallel processing.
+      #
+      # This scheduler can also work with virtual partitions.
+      #
+      # Aside from consumption jobs, other jobs do not run often, thus we can leave them with
+      # default FIFO scheduler from the default Karafka scheduler
+      class Scheduler < ::Karafka::Processing::Scheduler
+        # Schedules jobs in the LJF order for consumption
+        #
+        # @param queue [Karafka::Processing::JobsQueue] queue where we want to put the jobs
+        # @param jobs_array [Array<Karafka::Processing::Jobs::Base>] jobs we want to schedule
+        #
+        def schedule_consumption(queue, jobs_array)
+          perf_tracker = PerformanceTracker.instance
+
+          ordered = []
+
+          jobs_array.each do |job|
+            ordered << [
+              job,
+              processing_cost(perf_tracker, job)
+            ]
+          end
+
+          ordered.sort_by!(&:last)
+          ordered.reverse!
+          ordered.map!(&:first)
+
+          ordered.each do |job|
+            queue << job
+          end
+        end
+
+        private
+
+        # @param perf_tracker [PerformanceTracker]
+        # @param job [Karafka::Processing::Jobs::Base] job we will be processing
+        # @return [Numeric] estimated cost of processing this job
+        def processing_cost(perf_tracker, job)
+          if job.is_a?(::Karafka::Processing::Jobs::Consume)
+            messages = job.messages
+            message = messages.first
+
+            perf_tracker.processing_time_p95(message.topic, message.partition) * messages.size
+          else
+            # LJF will set first the most expensive, but we want to run the zero cost jobs
+            # related to the lifecycle always first. That is why we "emulate" that they
+            # the longest possible jobs that anyone can run
+            Float::INFINITY
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module DlqFtrLrjMom
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::DlqMom
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    # no need to check for manual seek because AJ consumer is internal and
+                    # fully controlled by us
+                    seek(coordinator.seek_offset, false)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module DlqFtrLrjMomVp
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::DlqMomVp
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Since we have VP here we do not commit intermediate offsets and need to commit
+                  # them here. We do commit in collapsed mode but this is generalized.
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    # no need to check for manual seek because AJ consumer is internal and
+                    # fully controlled by us
+                    seek(coordinator.seek_offset, false)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # - Aj
+          # - Dlq
+          # - Ftr
+          # - Mom
+          module DlqFtrMom
+            include Strategies::Dlq::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              manual_offset_management
+            ].freeze
+
+            # We write our own custom handler for after consume here, because we mark as consumed
+            # per each job in the AJ consumer itself (for this strategy). This means, that for DLQ
+            # dispatch, we can mark this message as consumed as well.
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # - Aj
+          # - Dlq
+          # - Ftr
+          # - Mom
+          # - VP
+          module DlqFtrMomVp
+            include Strategies::Aj::DlqMomVp
+            include Strategies::Aj::DlqFtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # AJ VP does not early stop on shutdown, hence here we can mark as consumed at the
+            # end of all VPs
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message)
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end