karafka 2.4.11 → 2.4.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 346743e75bc80a6a3e04361aa6b7b5caa540b31697bc25bb138baa490edd0a93
-  data.tar.gz: 6a87b0f7af16210b93732f55e98072b0c2eaf4541b700bb1929864b515a91747
+  metadata.gz: f96e616f91d60d5054e276f52cc2ecde9d4a6f6c8e25b3d61c384936b9239e4f
+  data.tar.gz: 8dfad6cc5d0cb4cdcc885b58e129929f67b0fdc4ee7922c21f9728cbdd863f5b
 SHA512:
-  metadata.gz: c20b7bb58d31b7771e593334783285edf13bfaff977350226aeb5a979fe1fd92482bb27acf72385a05a762d69581260196f953bdcea4aafbd1d50d0842fec9a2
-  data.tar.gz: 07b991c9048f20352c1670a5782c028757e177f7f062ce3a1ce113fd01a97184717b8c7d3612d515993cb048e21b3dcf1804b22852c1b6d9bccebb16463ca2aa
+  metadata.gz: bfea8217fb7ba89158b926417a0dd0cab42460a607b3bf25a62a400ab83e510806b966f81bd2ca76144e5aff62c99fb58b35be94ca1ad5c61ca113e760098243
+  data.tar.gz: ee66d2c6a11dc6baac3cc836ea5455d6c88838f1db7c0f9af9a92c3f5d3c7b8dd9d1f8c42dac4c9908042da3b192d8579466deecc687126640fc7bc5e0aeafb2

data/.github/workflows/ci.yml

@@ -117,7 +117,6 @@ jobs:
           - '3.3'
           - '3.2'
           - '3.1'
-          - '3.0'
     steps:
       - uses: actions/checkout@v4
       - name: Install package dependencies
@@ -165,7 +164,7 @@ jobs:
         run: bin/integrations --exclude '/pro'
 
   integrations_pro:
-    timeout-minutes: 40
+    timeout-minutes: 45
     runs-on: ubuntu-latest
     needs: diffend
     strategy:
@@ -176,7 +175,6 @@ jobs:
           - '3.3'
           - '3.2'
           - '3.1'
-          - '3.0'
     steps:
       - uses: actions/checkout@v4
       - name: Install package dependencies

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Karafka Framework Changelog
 
+## 2.4.12 (2024-09-17)
+- **[Feature]** Provide Adaptive Iterator feature as a fast alternative to Long-Running Jobs (Pro).
+- [Enhancement] Provide `Consumer#each` as a delegation to messages batch.
+- [Enhancement] Verify cancellation request envelope topic similar to the schedule one.
+- [Enhancement] Validate presence of `bootstrap.servers` to avoid incomplete partial reconfiguration.
+- [Enhancement] Support `ActiveJob#enqueue_at` via Scheduled Messages feature (Pro).
+- [Enhancement] Introduce `Karafka::App#debug!` that will switch Karafka and the default producer into extensive debug mode. Useful for CLI debugging.
+- [Enhancement] Support full overwrite of the `BaseConsumer#producer`.
+- [Enhancement] Transfer the time of last poll back to the coordinator for more accurate metrics tracking.
+- [Enhancement] Instrument `Consumer#seek` via `consumer.consuming.seek`.
+- [Fix] Fix incorrect time reference reload in scheduled messages.
+
 ## 2.4.11 (2024-09-04)
 - [Enhancement] Validate envelope target topic type for Scheduled Messages.
 - [Enhancement] Support for enqueue_after_transaction_commit in rails active job.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.11)
+    karafka (2.4.12)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
@@ -36,7 +36,7 @@ GEM
     erubi (1.13.0)
     et-orbi (1.2.11)
       tzinfo
-    factory_bot (6.4.6)
+    factory_bot (6.5.0)
       activesupport (>= 5.0.0)
     ffi (1.17.0)
     fugit (1.11.1)
@@ -55,20 +55,20 @@ GEM
     karafka-testing (2.4.6)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
-    karafka-web (0.10.2)
+    karafka-web (0.10.3)
       erubi (~> 1.4)
       karafka (>= 2.4.10, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
-    logger (1.6.0)
+    logger (1.6.1)
     mini_portile2 (2.8.7)
     minitest (5.25.1)
     ostruct (0.6.0)
     raabro (1.4.0)
     rack (3.1.7)
     rake (13.2.1)
-    roda (3.83.0)
+    roda (3.84.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -93,9 +93,9 @@ GEM
     tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.4)
+    waterdrop (2.8.0)
       karafka-core (>= 2.4.3, < 3.0.0)
-      karafka-rdkafka (>= 0.15.1)
+      karafka-rdkafka (>= 0.17.5)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.18)
 

data/config/locales/errors.yml

@@ -154,6 +154,7 @@ en:
       partitioner_format: 'needs to respond to #call'
       partition_key_type_format: 'needs to be either :key or :partition_key'
       producer_format: 'needs to respond to #call'
+      scheduled_messages_topic_format: 'needs to be a string with a Kafka accepted format'
 
     test:
       missing: needs to be present

data/config/locales/pro_errors.yml

@@ -76,6 +76,14 @@ en:
       direct_assignments_swarm_overbooked: 'cannot allocate partitions in swarm that were not assigned'
       direct_assignments_patterns_active: 'patterns cannot be used with direct assignments'
 
+      adaptive_iterator.active_missing: needs to be present
+      adaptive_iterator.active_format: 'needs to be boolean'
+      adaptive_iterator.marking_method_format: 'needs to be either #mark_as_consumed or #mark_as_consumed!'
+      adaptive_iterator.clean_after_yielding_format: 'needs to be boolean'
+      adaptive_iterator.safety_margin_format: 'needs to be between 1 and 99'
+      adaptive_iterator_with_virtual_partitions: 'cannot be used with virtual partitions'
+      adaptive_iterator_with_long_running_job: 'cannot be used with long running jobs'
+
     consumer_group:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.7.0
+    image: confluentinc/cp-kafka:7.7.1
 
     ports:
       - 9092:9092

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -7,27 +7,32 @@ module ActiveJob
     # Karafka adapter for enqueuing jobs
     # This is here for ease of integration with ActiveJob.
     class KarafkaAdapter
+      include Karafka::Helpers::ConfigImporter.new(
+        dispatcher: %i[internal active_job dispatcher]
+      )
+
       # Enqueues the job using the configured dispatcher
       #
       # @param job [Object] job that should be enqueued
       def enqueue(job)
-        ::Karafka::App.config.internal.active_job.dispatcher.dispatch(job)
+        dispatcher.dispatch(job)
       end
 
       # Enqueues multiple jobs in one go
       # @param jobs [Array<Object>] jobs that we want to enqueue
       # @return [Integer] number of jobs enqueued (required by Rails)
       def enqueue_all(jobs)
-        ::Karafka::App.config.internal.active_job.dispatcher.dispatch_many(jobs)
+        dispatcher.dispatch_many(jobs)
         jobs.size
       end
 
-      # Raises info, that Karafka backend does not support scheduling jobs
+      # Delegates time sensitive dispatch to the dispatcher. OSS will raise error, Pro will handle
+      # this as it supports scheduled messages.
       #
-      # @param _job [Object] job we cannot enqueue
-      # @param _timestamp [Time] time when job should run
-      def enqueue_at(_job, _timestamp)
-        raise NotImplementedError, 'This queueing backend does not support scheduling jobs.'
+      # @param job [Object] job we want to enqueue
+      # @param timestamp [Time] time when job should run
+      def enqueue_at(job, timestamp)
+        dispatcher.dispatch_at(job, timestamp)
       end
 
       # @return [true] should we by default enqueue after the transaction and not during.

data/lib/karafka/active_job/dispatcher.rb

@@ -46,6 +46,19 @@ module Karafka
         end
       end
 
+      # Raises info, that Karafka backend does not support scheduling jobs
+      #
+      # @param _job [Object] job we cannot enqueue
+      # @param _timestamp [Time] time when job should run
+      #
+      # @note Karafka Pro supports this feature
+      def dispatch_at(_job, _timestamp)
+        raise NotImplementedError, <<~ERROR_MESSAGE
+          This queueing backend does not support scheduling jobs.
+          Consider using Karafka Pro, which supports this via the Scheduled Messages feature.
+        ERROR_MESSAGE
+      end
+
       private
 
       # @param job [ActiveJob::Base] job

data/lib/karafka/app.rb

@@ -95,6 +95,23 @@ module Karafka
           end
         RUBY
       end
+
+      # Forces the debug setup onto Karafka and default WaterDrop producer.
+      # This needs to run prior to any operations that would cache state, like consuming or
+      # producing messages.
+      #
+      # @param contexts [String] librdkafka low level debug contexts for granular debugging
+      def debug!(contexts = 'all')
+        logger.level = ::Logger::DEBUG
+        producer.config.logger.level = ::Logger::DEBUG
+
+        config.kafka[:debug] = contexts
+        producer.config.kafka[:debug] = contexts
+
+        consumer_groups.map(&:topics).flat_map(&:to_a).each do |topic|
+          topic.kafka[:debug] = contexts
+        end
+      end
     end
   end
 end

data/lib/karafka/base_consumer.rb

@@ -14,6 +14,8 @@ module Karafka
     def_delegators :producer, :produce_async, :produce_sync, :produce_many_async,
                    :produce_many_sync
 
+    def_delegators :messages, :each
+
     # @return [String] id of the current consumer
     attr_reader :id
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
@@ -291,13 +293,23 @@ module Karafka
       coordinator.manual_seek if manual_seek
       coordinator.seek_offset = nil if reset_offset
 
-      client.seek(
-        Karafka::Messages::Seek.new(
-          topic.name,
-          partition,
-          offset
-        )
+      message = Karafka::Messages::Seek.new(
+        topic.name,
+        partition,
+        offset
       )
+
+      Karafka.monitor.instrument(
+        'consumer.consuming.seek',
+        caller: self,
+        topic: topic.name,
+        partition: partition,
+        message: message,
+        manual_seek: manual_seek,
+        reset_offset: reset_offset
+      ) do
+        client.seek(message)
+      end
     end
 
     # @return [Boolean] true if partition was revoked from the current consumer

data/lib/karafka/connection/client.rb

@@ -112,6 +112,10 @@ module Karafka
           # Fetch message within our time boundaries
           response = poll(time_poll.remaining)
 
+          # We track when last polling happened so we can provide means to detect upcoming
+          # `max.poll.interval.ms` limit
+          @buffer.polled
+
           case response
           when :tick_time
             nil

data/lib/karafka/connection/listener.rb

@@ -336,7 +336,7 @@ module Karafka
         idle_jobs = []
         eofed_jobs = []
 
-        @messages_buffer.each do |topic, partition, messages, eof|
+        @messages_buffer.each do |topic, partition, messages, eof, last_polled_at|
           # In case we did not receive any new messages without eof we skip.
           # We may yield empty array here in case we have reached eof without new messages but in
           # such cases, we can run an eof job
@@ -344,6 +344,7 @@ module Karafka
 
           coordinator = @coordinators.find_or_create(topic, partition)
           coordinator.eofed = eof
+          coordinator.last_polled_at = last_polled_at
 
           # If we did not receive any messages and we did receive eof signal, we run the eofed
           # jobs so user can take actions on reaching eof

data/lib/karafka/connection/messages_buffer.rb

@@ -42,6 +42,7 @@ module Karafka
         # Since it happens "right after" we've received the messages, it is close enough it time
         # to be used as the moment we received messages.
         received_at = Time.now
+        last_polled_at = raw_messages_buffer.last_polled_at
 
         raw_messages_buffer.each do |topic, partition, messages, eof|
           @size += messages.count
@@ -58,7 +59,8 @@ module Karafka
 
           @groups[topic][partition] = {
             eof: eof,
-            messages: built_messages
+            messages: built_messages,
+            last_polled_at: last_polled_at
           }
         end
       end
@@ -69,10 +71,11 @@ module Karafka
       # @yieldparam [Integer] partition number
       # @yieldparam [Array<Karafka::Messages::Message>] messages from a given topic partition
       # @yieldparam [Boolean] true if eof, false otherwise
+      # @yieldparam [Float] last polled at monotonic clock time
       def each
         @groups.each do |topic, partitions|
           partitions.each do |partition, details|
-            yield(topic, partition, details[:messages], details[:eof])
+            yield(topic, partition, details[:messages], details[:eof], details[:last_polled_at])
           end
         end
       end

data/lib/karafka/connection/raw_messages_buffer.rb

@@ -12,11 +12,17 @@ module Karafka
     # @note We store data here in groups per topic partition to handle the revocation case, where
     #    we may need to remove messages from a single topic partition.
     class RawMessagesBuffer
+      include Karafka::Core::Helpers::Time
+
       attr_reader :size
 
+      # @return [Float] last polling time in milliseconds (monotonic)
+      attr_reader :last_polled_at
+
       # @return [Karafka::Connection::MessagesBuffer] buffer instance
       def initialize
         @size = 0
+        @last_polled_at = monotonic_now
 
         @groups = Hash.new do |topic_groups, topic|
           topic_groups[topic] = Hash.new do |partition_groups, partition|
@@ -46,6 +52,11 @@ module Karafka
         @groups[topic][partition][:eof] = true
       end
 
+      # Marks the last polling time that can be accessed via `#last_polled_at`
+      def polled
+        @last_polled_at = monotonic_now
+      end
+
       # Allows to iterate over all the topics and partitions messages
       #
       # @yieldparam [String] topic name

data/lib/karafka/contracts/topic.rb

@@ -51,6 +51,20 @@ module Karafka
         end
       end
 
+      # When users redefine kafka scope settings per topic, they often forget to define the
+      # basic stuff as they assume it is auto-inherited. It is not (unless inherit flag used),
+      # leaving them with things like bootstrap.servers undefined. This checks that bootstrap
+      # servers are defined so we can catch those issues before they cause more problems.
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        kafka = data.fetch(:kafka)
+
+        next if kafka.key?(:'bootstrap.servers')
+
+        [[%w[kafka bootstrap.servers], :missing]]
+      end
+
       virtual do |data, errors|
         next unless errors.empty?
         next unless ::Karafka::App.config.strict_topics_namespacing

data/lib/karafka/helpers/config_importer.rb

@@ -25,6 +25,19 @@ module Karafka
           RUBY
         end
       end
+
+      # @param model [Object] object to which we want to add the config fetcher on a class level
+      def extended(model)
+        super
+
+        @attributes.each do |name, path|
+          model.class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            def self.#{name}
+              @#{name} ||= ::Karafka::App.config.#{path.join('.')}
+            end
+          RUBY
+        end
+      end
     end
   end
 end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -127,6 +127,21 @@ module Karafka
         MSG
       end
 
+      # Prints info about seeking to a particular location
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_consumer_consuming_seek(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        seek_offset = event[:message].offset
+        consumer = event[:caller]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Seeking from #{consumer.class}
+          on topic #{topic}/#{partition} to offset #{seek_offset}
+        MSG
+      end
+
       # Logs info about system signals that Karafka received and prints backtrace for threads in
       # case of ttin
       #

data/lib/karafka/instrumentation/notifications.rb

@@ -56,6 +56,7 @@ module Karafka
         consumer.consumed
         consumer.consuming.pause
         consumer.consuming.retry
+        consumer.consuming.seek
 
         consumer.before_schedule_idle
         consumer.idle

data/lib/karafka/messages/message.rb

@@ -8,15 +8,9 @@ module Karafka
     # heavy-deserialization data without slowing down the whole application.
     class Message
       extend Forwardable
-
-      class << self
-        # @return [Object] general parser
-        # @note We cache it here for performance reasons. It is 2.5x times faster than getting it
-        #   via the config chain.
-        def parser
-          @parser ||= App.config.internal.messages.parser
-        end
-      end
+      extend Helpers::ConfigImporter.new(
+        parser: %i[internal messages parser]
+      )
 
       attr_reader :metadata
       # raw payload needs to be mutable as we want to have option to change it in the parser

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -29,6 +29,9 @@ module Karafka
           partitioner: nil,
           # Allows for usage of `:key` or `:partition_key`
           partition_key_type: :key,
+          # Topic to where this message should go when using scheduled messages. When defined,
+          # it will be used with `enqueue_at`. If not defined it will raise an error.
+          scheduled_messages_topic: nil,
           # Allows for setting a callable producer since at the moment of defining the class,
           # variants may not be available
           #
@@ -83,6 +86,31 @@ module Karafka
           end
         end
 
+        # Will enqueue a job to run in the future
+        #
+        # @param job [Object] job we want to enqueue
+        # @param timestamp [Time] time when job should run
+        def dispatch_at(job, timestamp)
+          target_message = dispatch_details(job).merge!(
+            topic: job.queue_name,
+            payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+          )
+
+          proxy_message = Pro::ScheduledMessages.schedule(
+            message: target_message,
+            epoch: timestamp.to_i,
+            envelope: {
+              # Select the scheduled messages proxy topic
+              topic: fetch_option(job, :scheduled_messages_topic, DEFAULTS)
+            }
+          )
+
+          producer(job).public_send(
+            fetch_option(job, :dispatch_method, DEFAULTS),
+            proxy_message
+          )
+        end
+
         private
 
         # Selects the producer based on options. If callable `:producer` is defined, it will use

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -28,12 +28,21 @@ module Karafka
         optional(:producer) { |val| val.nil? || val.respond_to?(:call) }
         optional(:partitioner) { |val| val.respond_to?(:call) }
         optional(:partition_key_type) { |val| %i[key partition_key partition].include?(val) }
+
+        # Whether this is a legit scheduled messages topic will be validated during the first
+        # dispatch, so we do not repeat validations here
+        optional(:scheduled_messages_topic) do |val|
+          (val.is_a?(String) || val.is_a?(Symbol)) &&
+            ::Karafka::Contracts::TOPIC_REGEXP.match?(val.to_s)
+        end
+
         optional(:dispatch_method) do |val|
           %i[
             produce_async
             produce_sync
           ].include?(val)
         end
+
         optional(:dispatch_many_method) do |val|
           %i[
             produce_many_async

data/lib/karafka/pro/loader.rb

@@ -74,8 +74,8 @@ module Karafka
         #   that need to have some special configuration stuff injected into config, etc
         def features
           [
-            Encryption,
             Cleaner,
+            Encryption,
             RecurringTasks,
             ScheduledMessages
           ]

data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace for adaptive iterator consumer components
+      module AdaptiveIterator
+        # Consumer enhancements needed to wrap the batch iterator for adaptive iterating
+        # It automatically marks as consumed, ensures that we do not reach `max.poll.interval.ms`
+        # and does other stuff to simplify user per-message processing
+        module Consumer
+          # @param args [Array] anything accepted by `Karafka::Messages::Messages#each`
+          def each(*args)
+            adi_config = topic.adaptive_iterator
+
+            tracker = Tracker.new(
+              adi_config.safety_margin,
+              coordinator.last_polled_at,
+              topic.subscription_group.kafka.fetch(:'max.poll.interval.ms')
+            )
+
+            messages.each(*args) do |message|
+              # Always stop if we've lost the assignment
+              return if revoked?
+              # No automatic marking risk when mom is enabled so we can fast stop
+              return if Karafka::App.done? && topic.manual_offset_management?
+
+              # Seek request on done will allow us to stop without marking the offset when user had
+              # the automatic offset marking. This should not be a big network traffic issue for
+              # the end user as we're stopping anyhow but should improve shutdown time
+              if tracker.enough? || Karafka::App.done?
+                # Enough means we no longer have time to process more data without polling as we
+                # risk reaching max poll interval. Instead we seek and we will poll again soon.
+                seek(message.offset, reset_offset: true)
+
+                return
+              end
+
+              tracker.track { yield(message) }
+
+              # Clean if this is what user configured
+              message.clean! if adi_config.clean_after_yielding?
+
+              public_send(adi_config.marking_method, message)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module AdaptiveIterator
+        # Tracker is responsible for monitoring the processing of messages within the poll
+        # interval limitation.
+        # It ensures that the consumer does not exceed the maximum poll interval by tracking the
+        # processing cost and determining when to halt further processing (if needed).
+        class Tracker
+          include Karafka::Core::Helpers::Time
+
+          # Initializes a new Tracker instance.
+          #
+          # @param safety_margin [Float] The safety margin percentage (0-100) to leave as a buffer.
+          # @param last_polled_at [Float] The timestamp of the last polling in milliseconds.
+          # @param max_poll_interval_ms [Integer] The maximum poll interval time in milliseconds.
+          def initialize(
+            safety_margin,
+            last_polled_at,
+            max_poll_interval_ms
+          )
+            @safety_margin = safety_margin / 100.0 # Convert percentage to decimal
+            @last_polled_at = last_polled_at
+            @max_processing_cost = 0
+            @max_poll_interval_ms = max_poll_interval_ms
+          end
+
+          # Tracks the processing time of a block and updates the maximum processing cost.
+          #
+          # @yield Executes the block, measuring the time taken for processing.
+          def track
+            before = monotonic_now
+
+            yield
+
+            time_taken = monotonic_now - before
+
+            return unless time_taken > @max_processing_cost
+
+            @max_processing_cost = time_taken
+          end
+
+          # Determines if there is enough time left to process more messages without exceeding the
+          # maximum poll interval, considering both the safety margin and adaptive margin.
+          #
+          # @return [Boolean] Returns true if it is time to stop processing. False otherwise.
+          def enough?
+            elapsed_time_ms = monotonic_now - @last_polled_at
+            remaining_time_ms = @max_poll_interval_ms - elapsed_time_ms
+
+            safety_margin_ms = @max_poll_interval_ms * @safety_margin
+
+            return true if remaining_time_ms <= safety_margin_ms
+            return true if remaining_time_ms - @max_processing_cost <= safety_margin_ms
+
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/expansions_selector.rb

@@ -25,6 +25,7 @@ module Karafka
           expansions = super
           expansions << Pro::Processing::Piping::Consumer
           expansions << Pro::Processing::OffsetMetadata::Consumer if topic.offset_metadata?
+          expansions << Pro::Processing::AdaptiveIterator::Consumer if topic.adaptive_iterator?
           expansions << Pro::Processing::PeriodicJob::Consumer if topic.periodic_job?
           expansions
         end

data/lib/karafka/pro/routing/features/adaptive_iterator/config.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class AdaptiveIterator < Base
+          # Adaptive Iterator configuration
+          Config = Struct.new(
+            :active,
+            :safety_margin,
+            :marking_method,
+            :clean_after_yielding,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :clean_after_yielding?, :clean_after_yielding
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class AdaptiveIterator < Base
+          # Namespace for adaptive iterator contracts
+          module Contracts
+            # Contract to validate configuration of the adaptive iterator feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:adaptive_iterator) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:safety_margin) { |val| val.is_a?(Integer) && val.positive? && val < 100 }
+                required(:clean_after_yielding) { |val| [true, false].include?(val) }
+
+                required(:marking_method) do |val|
+                  %i[mark_as_consumed mark_as_consumed!].include?(val)
+                end
+              end
+
+              # Since adaptive iterator uses `#seek` and can break processing in the middle, we
+              # cannot use it with virtual partitions that can process data in a distributed
+              # manner
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                adaptive_iterator = data[:adaptive_iterator]
+                virtual_partitions = data[:virtual_partitions]
+
+                next unless adaptive_iterator[:active]
+                next unless virtual_partitions[:active]
+
+                [[%i[adaptive_iterator], :with_virtual_partitions]]
+              end
+
+              # There is no point of using the adaptive iterator with LRJ because of how LRJ works
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                adaptive_iterator = data[:adaptive_iterator]
+                long_running_jobs = data[:long_running_job]
+
+                next unless adaptive_iterator[:active]
+                next unless long_running_jobs[:active]
+
+                [[%i[adaptive_iterator], :with_long_running_job]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class AdaptiveIterator < Base
+          # Topic extension allowing us to enable and configure adaptive iterator
+          module Topic
+            # @param active [Boolean] should we use the automatic adaptive iterator
+            # @param safety_margin [Integer]
+            #   How big of a margin we leave ourselves so we can safely communicate back with
+            #   Kafka, etc. We stop and seek back when we've burned 85% of the time by default.
+            #   We leave 15% of time for post-processing operations so we have space before we
+            #   hit max.poll.interval.ms.
+            # @param marking_method [Symbol] If we should, how should we mark
+            # @param clean_after_yielding [Boolean]  Should we clean post-yielding via the
+            #   cleaner API
+            def adaptive_iterator(
+              active: false,
+              safety_margin: 10,
+              marking_method: :mark_as_consumed,
+              clean_after_yielding: true
+            )
+              @adaptive_iterator ||= Config.new(
+                active: active,
+                safety_margin: safety_margin,
+                marking_method: marking_method,
+                clean_after_yielding: clean_after_yielding
+              )
+            end
+
+            # @return [Boolean] Is adaptive iterator active. It is always `true`, since we use it
+            #   via explicit messages batch wrapper
+            def adaptive_iterator?
+              adaptive_iterator.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus poll guarding
+            #   setup configuration.
+            def to_h
+              super.merge(
+                adaptive_iterator: adaptive_iterator.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/adaptive_iterator.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature that pro-actively monitors remaining time until max poll interval ms and
+        # cost of processing of each message in a batch. When there is no more time to process
+        # more messages from the batch, it will seek back so we do not reach max poll interval.
+        # It can be useful when we reach this once in a while. For a constant long-running jobs,
+        # please use the Long-Running Jobs feature instead.
+        #
+        # It also provides some wrapping over typical operations users do, like stopping if
+        # revoked, auto-marking, etc
+        class AdaptiveIterator < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/scheduled_messages/contracts/message.rb

@@ -28,17 +28,33 @@ module Karafka
             ).fetch('en').fetch('validations').fetch('scheduled_messages_message')
           end
 
-          # Headers we expect in each message of type "message" that goes to our scheduled messages
-          # topic
-          EXPECTED_HEADERS = %w[
+          # Headers we expect in each message of type "schedule" that goes to our scheduled
+          # messages topic
+          EXPECTED_SCHEDULE_HEADERS = %w[
             schedule_schema_version
             schedule_target_epoch
             schedule_source_type
             schedule_target_topic
           ].freeze
 
+          # Headers we expect in each message of type "cancel"
+          EXPECTED_CANCEL_HEADERS = %w[
+            schedule_schema_version
+            schedule_source_type
+          ].freeze
+
           required(:key) { |val| val.is_a?(String) && val.size.positive? }
-          required(:headers) { |val| val.is_a?(Hash) && (val.keys & EXPECTED_HEADERS).size == 4 }
+
+          # Ensure that schedule has all correct keys and that others have other related data
+          required(:headers) do |val|
+            next false unless val.is_a?(Hash)
+
+            if val['schedule_source_type'] == 'message'
+              (val.keys & EXPECTED_SCHEDULE_HEADERS).size >= 4
+            else
+              (val.keys & EXPECTED_CANCEL_HEADERS).size >= 2
+            end
+          end
 
           # Make sure, that schedule_target_epoch is not older than grace period behind us.
           # While this is not ideal verification of scheduling stuff in past, at leats it will
@@ -46,6 +62,9 @@ module Karafka
           virtual do |data, errors|
             next unless errors.empty?
 
+            # Validate epoch only for schedules
+            next unless data[:headers]['schedule_source_type'] == 'schedule'
+
             epoch_time = data[:headers].fetch('schedule_target_epoch').to_i
 
             # We allow for small lag as those will be dispatched but we should prevent dispatching

data/lib/karafka/pro/scheduled_messages/day.rb

@@ -29,7 +29,7 @@ module Karafka
         def initialize
           @created_at = Time.now.to_i
 
-          time = Time.at(@created_at)
+          time = Time.at(@created_at).utc
 
           @starts_at = Time.utc(time.year, time.month, time.day).to_i
           @ends_at = @starts_at + 86_399

data/lib/karafka/pro/scheduled_messages/proxy.rb

@@ -87,12 +87,7 @@ module Karafka
             }.merge(envelope)
 
             enrich(proxy_message, message)
-
-            # Final validation to make sure all user provided extra data and what we have built
-            # complies with our requirements
-            POST_CONTRACT.validate!(proxy_message)
-            # After proxy specific validations we also ensure, that the final form is correct
-            MSG_CONTRACT.validate!(proxy_message, WaterDrop::Errors::MessageInvalidError)
+            validate!(proxy_message)
 
             proxy_message
           end
@@ -106,7 +101,7 @@ module Karafka
           # @note Technically it is a tombstone but we differentiate just for the sake of ability
           #   to debug stuff if needed
           def cancel(key:, envelope: {})
-            {
+            proxy_message = {
               key: key,
               payload: nil,
               headers: {
@@ -114,6 +109,11 @@ module Karafka
                 'schedule_source_type' => 'cancel'
               }
             }.merge(envelope)
+
+            # Ensure user provided envelope is with all expected details
+            validate!(proxy_message)
+
+            proxy_message
           end
 
           # Builds tombstone with the dispatched message details. Those details can be used
@@ -169,6 +169,15 @@ module Karafka
               proxy_message[:partition_key] = message.fetch(attribute).to_s
             end
           end
+
+          # Final validations to make sure all user provided extra data and what we have built
+          # complies with our requirements
+          # @param proxy_message [Hash] our message envelope
+          def validate!(proxy_message)
+            POST_CONTRACT.validate!(proxy_message)
+            # After proxy specific validations we also ensure, that the final form is correct
+            MSG_CONTRACT.validate!(proxy_message, WaterDrop::Errors::MessageInvalidError)
+          end
         end
       end
     end

data/lib/karafka/processing/coordinator.rb

@@ -19,6 +19,9 @@ module Karafka
       # any messages
       attr_accessor :eofed
 
+      # Last polled at time set based on the incoming last poll time
+      attr_accessor :last_polled_at
+
       def_delegators :@pause_tracker, :attempt, :paused?
 
       # @param topic [Karafka::Routing::Topic]
@@ -38,6 +41,7 @@ module Karafka
         @failure = false
         @eofed = false
         @changed_at = monotonic_now
+        @last_polled_at = @changed_at
       end
 
       # Starts the coordinator for given consumption jobs

data/lib/karafka/processing/executor.rb

@@ -174,8 +174,11 @@ module Karafka
           expansions.each { |expansion| consumer.singleton_class.include(expansion) }
 
           consumer.client = @client
-          consumer.producer = ::Karafka::App.producer
           consumer.coordinator = @coordinator
+          # We assign producer only when not available already. It may already be available if
+          # user redefined the `#producer` method for example. This can be useful for example when
+          # having a multi-cluster setup and using a totally custom producer
+          consumer.producer ||= ::Karafka::App.producer
           # Since we have some message-less flows (idle, etc), we initialize consumer with empty
           # messages set. In production we have persistent consumers, so this is not a performance
           # overhead as this will happen only once per consumer lifetime

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.11'
+  VERSION = '2.4.12'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.11
+  version: 2.4.12
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-09-04 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -290,6 +290,8 @@ files:
 - lib/karafka/pro/iterator/expander.rb
 - lib/karafka/pro/iterator/tpl_builder.rb
 - lib/karafka/pro/loader.rb
+- lib/karafka/pro/processing/adaptive_iterator/consumer.rb
+- lib/karafka/pro/processing/adaptive_iterator/tracker.rb
 - lib/karafka/pro/processing/collapser.rb
 - lib/karafka/pro/processing/coordinator.rb
 - lib/karafka/pro/processing/coordinators/errors_tracker.rb
@@ -386,6 +388,10 @@ files:
 - lib/karafka/pro/recurring_tasks/task.rb
 - lib/karafka/pro/routing/features/active_job.rb
 - lib/karafka/pro/routing/features/active_job/builder.rb
+- lib/karafka/pro/routing/features/adaptive_iterator.rb
+- lib/karafka/pro/routing/features/adaptive_iterator/config.rb
+- lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb
+- lib/karafka/pro/routing/features/adaptive_iterator/topic.rb
 - lib/karafka/pro/routing/features/base.rb
 - lib/karafka/pro/routing/features/dead_letter_queue.rb
 - lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb