karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/expansions_selector.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Pro selector of appropriate topic setup based features enhancements.
-      class ExpansionsSelector < Karafka::Processing::ExpansionsSelector
-        # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
-        #   expansions
-        # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance
-        #   the consumer
-        def find(topic)
-          # Start with the non-pro expansions
-          expansions = super
-          expansions << Pro::Processing::Piping::Consumer
-          expansions << Pro::Processing::OffsetMetadata::Consumer if topic.offset_metadata?
-          expansions << Pro::Processing::AdaptiveIterator::Consumer if topic.adaptive_iterator?
-          expansions << Pro::Processing::PeriodicJob::Consumer if topic.periodic_job?
-          expansions
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -1,87 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Filters
-        # A filter that allows us to delay processing by pausing until time is right.
-        class Delayer < Base
-          # @param delay [Integer] ms delay / minimum age of each message we want to process
-          def initialize(delay)
-            super()
-
-            @delay = delay
-          end
-
-          # Removes too young messages
-          #
-          # @param messages [Array<Karafka::Messages::Message>]
-          def apply!(messages)
-            @applied = false
-            @cursor = nil
-
-            # Time on message is in seconds with ms precision, so we need to convert the ttl that
-            # is in ms to this format
-            border = Time.now.utc - (@delay / 1_000.0)
-
-            messages.delete_if do |message|
-              too_young = message.timestamp > border
-
-              if too_young
-                @applied = true
-
-                @cursor ||= message
-              end
-
-              @applied
-            end
-          end
-
-          # @return [Integer] timeout delay in ms
-          def timeout
-            return 0 unless @cursor
-
-            timeout = (@delay / 1_000.0) - (::Time.now.utc - @cursor.timestamp)
-
-            (timeout <= 0) ? 0 : timeout * 1_000
-          end
-
-          # @return [Symbol] action to take on post-filtering
-          def action
-            return :skip unless applied?
-
-            (timeout <= 0) ? :seek : :pause
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Filters
-        # Delayer that checks if we have appropriate insights available. If not, pauses for
-        # 5 seconds so the insights can be loaded from the broker.
-        #
-        # In case it would take more than five seconds to load insights, it will just pause again
-        #
-        # This filter ensures, that we always have inline insights that a consumer can use
-        #
-        # It is relevant in most cases only during the process start, when first poll may not
-        # yield statistics yet but will give some data.
-        class InlineInsightsDelayer < Base
-          # Minimum how long should we pause when there are no metrics
-          PAUSE_TIMEOUT = 5_000
-
-          private_constant :PAUSE_TIMEOUT
-
-          # @param topic [Karafka::Routing::Topic]
-          # @param partition [Integer] partition
-          def initialize(topic, partition)
-            super()
-            @topic = topic
-            @partition = partition
-          end
-
-          # Pauses if inline insights would not be available. Does nothing otherwise
-          #
-          # @param messages [Array<Karafka::Messages::Message>]
-          def apply!(messages)
-            @applied = false
-            @cursor = messages.first
-
-            # Nothing to do if there were no messages
-            # This can happen when we chain filters
-            return unless @cursor
-
-            insights = Karafka::Processing::InlineInsights::Tracker.find(
-              @topic,
-              @partition
-            )
-
-            # If insights are available, also nothing to do here and we can just process
-            return unless insights.empty?
-
-            messages.clear
-
-            @applied = true
-          end
-
-          # @return [Integer, nil] ms timeout in case of pause or nil if not delaying
-          def timeout
-            (@cursor && applied?) ? PAUSE_TIMEOUT : nil
-          end
-
-          # Pause when we had to back-off or skip if delay is not needed
-          def action
-            applied? ? :pause : :skip
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/filters/throttler.rb

@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Namespace containing Pro out of the box filters used by various strategies
-      module Filters
-        # Throttler used to limit number of messages we can process in a given time interval
-        # The tricky thing is, that even if we throttle on 100 messages, if we've reached 100, we
-        # still need to indicate, that we throttle despite not receiving 101. Otherwise we will
-        # not pause the partition and will fetch more data that we should not process.
-        #
-        # This is a special type of a filter that always throttles and makes us wait / seek if
-        # anything is applied out.
-        class Throttler < Base
-          # @param limit [Integer] how many messages we can process in a given time
-          # @param interval [Integer] interval in milliseconds for which we want to process
-          def initialize(limit, interval)
-            super()
-
-            @limit = limit
-            @interval = interval
-            @requests = Hash.new { |h, k| h[k] = 0 }
-          end
-
-          # Limits number of messages to a range that we can process (if needed) and keeps track
-          # of how many messages we've processed in a given time
-          # @param messages [Array<Karafka::Messages::Message>] limits the number of messages to
-          #   number we can accept in the context of throttling constraints
-          def apply!(messages)
-            @applied = false
-            @cursor = nil
-            @time = monotonic_now
-            @requests.delete_if { |timestamp, _| timestamp < (@time - @interval) }
-            values = @requests.values.sum
-            accepted = 0
-
-            messages.delete_if do |message|
-              # +1 because of current
-              @applied = (values + accepted + 1) > @limit
-
-              @cursor = message if @applied && @cursor.nil?
-
-              next true if @applied
-
-              accepted += 1
-
-              false
-            end
-
-            @requests[@time] += accepted
-          end
-
-          # @return [Symbol] action to take upon throttler reaching certain state
-          def action
-            if applied?
-              timeout.zero? ? :seek : :pause
-            else
-              :skip
-            end
-          end
-
-          # @return [Integer] minimum number of milliseconds to wait before getting more messages
-          #   so we are no longer throttled and so we can process at least one message
-          def timeout
-            timeout = @interval - (monotonic_now - @time)
-            [timeout, 0].max
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/filters/virtual_limiter.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Filters
-        # Removes messages that are already marked as consumed in the virtual offset manager
-        # This should operate only when using virtual partitions.
-        #
-        # This cleaner prevents us from duplicated processing of messages that were virtually
-        # marked as consumed even if we could not mark them as consumed in Kafka. This allows us
-        # to limit reprocessing when errors occur drastically when operating with virtual
-        # partitions
-        #
-        # @note It should be registered only when VPs are used
-        class VirtualLimiter < Base
-          # @param manager [Processing::VirtualOffsetManager]
-          # @param collapser [Processing::Collapser]
-          def initialize(manager, collapser)
-            @manager = manager
-            @collapser = collapser
-
-            super()
-          end
-
-          # Remove messages that we already marked as virtually consumed. Does nothing if not in
-          # the collapsed mode.
-          #
-          # @param messages [Array<Karafka::Messages::Message>]
-          def apply!(messages)
-            return unless @collapser.collapsed?
-
-            marked = @manager.marked
-
-            messages.delete_if { |message| marked.include?(message.offset) }
-          end
-
-          # @return [nil] This filter does not deal with pausing, so timeout is always nil
-          def timeout
-            nil
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Jobs
-        # Non-Blocking version of the Eofed job
-        # We use this version for LRJ topics for cases where saturated resources would not allow
-        # to run this job for extended period of time. Under such scenarios, if we would not use
-        # a non-blocking one, we would reach max.poll.interval.ms.
-        class EofedNonBlocking < Karafka::Processing::Jobs::Eofed
-          self.action = :eofed
-
-          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Eofed`
-          def initialize(*args)
-            super
-            @non_blocking = true
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -1,90 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Pro jobs builder that supports lrj
-      class JobsBuilder < Karafka::Processing::JobsBuilder
-        # @param executor [Karafka::Pro::Processing::Executor]
-        def idle(executor)
-          Karafka::Processing::Jobs::Idle.new(executor)
-        end
-
-        # @param executor [Karafka::Pro::Processing::Executor]
-        # @param messages [Karafka::Messages::Messages] messages batch to be consumed
-        # @return [Karafka::Processing::Jobs::Consume] blocking job
-        # @return [Karafka::Pro::Processing::Jobs::ConsumeNonBlocking] non blocking for lrj
-        def consume(executor, messages)
-          if executor.topic.long_running_job?
-            Jobs::ConsumeNonBlocking.new(executor, messages)
-          else
-            super
-          end
-        end
-
-        # @param executor [Karafka::Pro::Processing::Executor]
-        # @return [Karafka::Processing::Jobs::Eofed] eofed job for non LRJ
-        # @return [Karafka::Processing::Jobs::EofedBlocking] eofed job that is
-        #   non-blocking, so when revocation job is scheduled for LRJ it also will not block
-        def eofed(executor)
-          if executor.topic.long_running_job?
-            Jobs::EofedNonBlocking.new(executor)
-          else
-            super
-          end
-        end
-
-        # @param executor [Karafka::Pro::Processing::Executor]
-        # @return [Karafka::Processing::Jobs::Revoked] revocation job for non LRJ
-        # @return [Karafka::Processing::Jobs::RevokedNonBlocking] revocation job that is
-        #   non-blocking, so when revocation job is scheduled for LRJ it also will not block
-        def revoked(executor)
-          if executor.topic.long_running_job?
-            Jobs::RevokedNonBlocking.new(executor)
-          else
-            super
-          end
-        end
-
-        # @param executor [Karafka::Pro::Processing::Executor]
-        # @return [Jobs::Periodic] Periodic job
-        # @return [Jobs::PeriodicNonBlocking] Periodic non-blocking job
-        def periodic(executor)
-          if executor.topic.long_running_job?
-            Jobs::PeriodicNonBlocking.new(executor)
-          else
-            Jobs::Periodic.new(executor)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/offset_metadata/consumer.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Offset Metadata support on the processing side
-      module OffsetMetadata
-        # Extra API methods for offset metadata fetching
-        # @note Part of this feature API is embedded directly into the strategies because it alters
-        #   how marking methods (`#mark_as_consumed` and `#mark_as_consumed!`) operate. Because
-        #   of that, they had to be embedded into the strategies.
-        module Consumer
-          # @param cache [Boolean] should we use cached result if present (true by default)
-          # @return [false, Object] false in case we do not own the partition anymore or
-          #   deserialized metadata based on the deserializer
-          # @note Caching is on as the assumption here is, that most of the time user will be
-          #   interested only in the offset metadata that "came" from the time prior to the
-          #   rebalance. That is because the rest of the metadata (current) is created and
-          #   controlled by the user himself, thus there is no need to retrieve it. In case this
-          #   is not true and user wants to always get the Kafka metadata, `cache` value of this
-          #   feature can be set to false.
-          def offset_metadata(cache: true)
-            return false if revoked?
-
-            Fetcher.find(topic, partition, cache: cache)
-          end
-
-          alias_method :committed_offset_metadata, :offset_metadata
-        end
-      end
-    end
-  end
-end