karafka 2.6.0.beta1 → 2.6.0.rc2

data/lib/karafka/pro/admin/recovery.rb

@@ -159,14 +159,16 @@ module Karafka
         #   )
         #
         # @example Read offsets from a specific point in time
-        #   Karafka::Admin::Recovery.read_committed_offsets('sync', last_committed_at: Time.new(2025, 3, 1))
+        #   Karafka::Admin::Recovery.read_committed_offsets(
+        #     'sync', last_committed_at: Time.new(2025, 3, 1)
+        #   )
         #
         # @example Migrate a stuck consumer group to a new name (two-step workflow)
         #   # Step 1: Read committed offsets from the broken group (bypasses coordinator)
         #   offsets = Karafka::Admin::Recovery.read_committed_offsets('sync')
         #   #=> { 'events' => { 0 => 1400, 1 => 1402 }, 'orders' => { 0 => 890 } }
         #
-        #   # Step 2: Inspect the recovered offsets — verify all expected topics and partitions
+        #   # Step 2: Inspect the recovered offsets - verify all expected topics and partitions
         #   # are present and the offset values look reasonable before committing them
         #
         #   # Step 3: Write the offsets to the target group using standard Admin APIs
@@ -193,11 +195,11 @@ module Karafka
             next unless parsed[:group] == group_id
 
             if parsed[:offset].nil?
-              # Tombstone — offset was deleted, remove from results
+              # Tombstone - offset was deleted, remove from results
               committed[parsed[:topic]].delete(parsed[:partition])
               committed.delete(parsed[:topic]) if committed[parsed[:topic]].empty?
             else
-              # Last write wins — scanning forward means we naturally end up with the most
+              # Last write wins - scanning forward means we naturally end up with the most
               # recent commit per partition
               committed[parsed[:topic]][parsed[:partition]] = parsed[:offset]
             end

data/lib/karafka/pro/base_consumer.rb

@@ -37,8 +37,8 @@ module Karafka
     #
     # Methods here are suppose to be always available or are expected to be redefined
     module BaseConsumer
-      # @return [Karafka::Pro::Processing::ConsumerGroups::Coordinators::ErrorsTracker] tracker for errors that
-      #   occurred during processing until another successful processing
+      # @return [Karafka::Pro::Processing::ConsumerGroups::Coordinators::ErrorsTracker] tracker for
+      #   errors that occurred during processing until another successful processing
       #
       # @note This will always contain **only** details of errors that occurred during `#consume`
       #   because only those are retryable.
@@ -53,9 +53,9 @@ module Karafka
         coordinator.errors_tracker
       end
 
-      # @return [Karafka::Pro::Processing::ConsumerGroups::SubscriptionGroupsCoordinator] Coordinator allowing to
-      #   pause and resume polling of the given subscription group jobs queue for postponing
-      #   further work.
+      # @return [Karafka::Pro::Processing::ConsumerGroups::SubscriptionGroupsCoordinator]
+      #   Coordinator allowing to pause and resume polling of the given subscription group jobs
+      #   queue for postponing further work.
       #
       # @note Since this stops polling, it can cause reaching `max.poll.interval.ms` limitations.
       #

data/lib/karafka/pro/cleaner/messages/metadata.rb

@@ -31,7 +31,6 @@
 module Karafka
   module Pro
     module Cleaner
-      # Cleaner messages components related enhancements
       module Messages
         # Extensions to the message metadata that allow for granular memory control on a per
         # message basis

data/lib/karafka/pro/cli/parallel_segments/base.rb

@@ -52,7 +52,7 @@ module Karafka
           # @return [Hash{String => Array<Karafka::Routing::ConsumerGroup>}] hash with all parallel
           #   consumer groups as values and names of segments origin consumer group as the key.
           def applicable_groups
-            requested_groups = options[:groups].dup || []
+            requested_groups = options[:groups] || []
 
             workable_groups = Karafka::App
               .routes
@@ -68,7 +68,6 @@ module Karafka
               workable_group = workable_groups[requested_group]
 
               if workable_group
-                requested_groups.delete(requested_group)
                 applicable_groups[requested_group] = workable_group
               else
                 raise(

data/lib/karafka/pro/connection/manager.rb

@@ -47,7 +47,6 @@ module Karafka
         # Creates new manager instance
         def initialize
           super
-          @mutex = Mutex.new
           @changes = Hash.new do |h, k|
             h[k] = {
               state: "",
@@ -253,7 +252,11 @@ module Karafka
         #   are also stable. This is a strong indicator that no rebalances or other operations are
         #   happening at a given moment.
         def stable?(sg_listeners)
-          @scale_delay ||= sg_listeners.first.subscription_group.multiplexing.scale_delay
+          # Must stay a local, not a memoized `@scale_delay`: this manager is a single process-wide
+          # instance shared by every subscription-group family, so memoizing here would freeze the
+          # first family's `scale_delay` and apply it to all the others, making them ignore their
+          # own configured value.
+          scale_delay = sg_listeners.first.subscription_group.multiplexing.scale_delay
 
           sg_listeners.all? do |sg_listener|
             # If a listener is not active, we do not take it into consideration when looking at
@@ -262,8 +265,8 @@ module Karafka
 
             state = @changes[sg_listener.subscription_group.id]
 
-            state[:state_age] >= @scale_delay &&
-              (monotonic_now - state[:changed_at]) >= @scale_delay &&
+            state[:state_age] >= scale_delay &&
+              (monotonic_now - state[:changed_at]) >= scale_delay &&
               state[:state] == "up" &&
               state[:join_state] == "steady"
           end

data/lib/karafka/pro/encryption/messages/middleware.rb

@@ -48,6 +48,11 @@ module Karafka
           def call(message)
             payload = message[:payload]
 
+            # Tombstones (records with a nil payload) carry no data to encrypt. We pass them
+            # through untouched so they remain valid tombstones (for example for log compaction)
+            # instead of crashing on `cipher.encrypt(nil)`.
+            return message if payload.nil?
+
             message[:headers] ||= {}
             message[:headers]["encryption"] = version
             message[:payload] = cipher.encrypt(payload)

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -52,6 +52,10 @@ module Karafka
 
             return super unless active && encryption
 
+            # Tombstones (nil payload) have nothing to decrypt; defer to the default parser so they
+            # remain valid tombstones instead of crashing on `cipher.decrypt(nil)`.
+            return super if message.raw_payload.nil?
+
             # Decrypt raw payload so it can be handled by the default parser logic
             decrypted_payload = cipher.decrypt(
               encryption,

data/lib/karafka/pro/instrumentation/callbacks/consumer_groups/decorator.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Instrumentation
+      # Namespace for Pro instrumentation callbacks related components
+      module Callbacks
+        # Namespace for Pro consumer groups callbacks related components
+        module ConsumerGroups
+          # Pro statistics decorator for consumer groups. On top of the standard decoration
+          # (deltas, freeze durations) it invokes Pro statistics enrichments, each encapsulated
+          # in its own component with a `#call` entrypoint. Currently: the lag compensation of
+          # long-paused partitions.
+          class Decorator < Karafka::Instrumentation::Callbacks::ConsumerGroups::Decorator
+            def initialize
+              super
+              @lag_compensator = Pro::Instrumentation::ConsumerGroups::LagCompensation::Compensator.new
+            end
+
+            # @param statistics [Hash] raw librdkafka statistics
+            # @return [Hash] decorated statistics with the Pro enrichments applied
+            #
+            # @note When the lag compensation is disabled, its refresher is not subscribed and
+            #   the registry stays empty, so the compensation is a pass-through by itself
+            def call(statistics)
+              @lag_compensator.call(statistics)
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/compensator.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Instrumentation
+      module ConsumerGroups
+        module LagCompensation
+          # Compensates stale offsets and lags of long-paused partitions in the raw librdkafka
+          # statistics, using the actively refreshed values from the registry. When there is no
+          # refreshed data for a given client, topic or partition, it does nothing, so values
+          # from librdkafka remain untouched.
+          #
+          # It is pure and instant: all the broker queries happen in the refresher on the
+          # listener threads, never on the librdkafka callbacks path.
+          class Compensator
+            # Compensates the statistics in place, so the standard decoration (deltas, freeze
+            # durations) that runs afterwards operates on the compensated values as well
+            #
+            # @param statistics [Hash] raw librdkafka statistics
+            def call(statistics)
+              data = Registry.instance.fetch(statistics["name"])
+
+              return unless data
+
+              topics = statistics["topics"]
+
+              return unless topics
+
+              data.each do |topic, partitions|
+                t_stats = topics[topic]
+
+                next unless t_stats
+
+                partitions.each do |partition, end_offset|
+                  p_stats = t_stats["partitions"][partition.to_s]
+
+                  next unless p_stats
+
+                  compensate(p_stats, end_offset)
+                end
+              end
+            end
+
+            private
+
+            # Compensates a single partition statistics with the refreshed end offset. The
+            # committed and stored offsets are taken from the statistics themselves: they are
+            # maintained by the client commits (not by fetches), so their values stay accurate
+            # while a partition is paused and lags can be derived from them directly
+            #
+            # @param p_stats [Hash] single partition raw statistics
+            # @param end_offset [Integer] refreshed partition end offset
+            def compensate(p_stats, end_offset)
+              # The end offset only grows, so a refreshed value older or equal to what
+              # librdkafka reports means there is nothing to compensate: equal means no new
+              # data in the topic since the last fetch and lower means the partition resumed
+              # and its fetches are fresher than our refreshed snapshot. We compare against the
+              # last stable offset since that is what the statistics report for a read_committed
+              # consumer. Note the refreshed end offset itself is a high watermark (see
+              # `Fetcher`): on a topic with an in-flight transaction it can exceed the fetch-based
+              # last stable offset and briefly overstate the lag until the transaction resolves.
+              return unless end_offset > (p_stats["ls_offset"] || p_stats["hi_offset"] || -1)
+
+              p_stats["ls_offset"] = end_offset
+              # The real high watermark is always at least the last stable offset, so we bump
+              # it when needed to keep the statistics internally consistent, never overstating
+              hi_offset = p_stats["hi_offset"]
+              p_stats["hi_offset"] = end_offset if hi_offset.nil? || end_offset > hi_offset
+
+              committed = p_stats["committed_offset"]
+
+              # Negative means nothing was committed yet for this partition, in which case lag
+              # cannot be derived from the committed offset and we leave it untouched
+              if committed && committed >= 0
+                # Lags derive from the same reference the native librdkafka consumer lag does
+                lag = end_offset - committed
+                p_stats["consumer_lag"] = lag if lag >= 0
+              end
+
+              stored = p_stats["stored_offset"]
+
+              return unless stored && stored >= 0
+
+              lag_stored = end_offset - stored
+              p_stats["consumer_lag_stored"] = lag_stored if lag_stored >= 0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/fetcher.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Instrumentation
+      # Instrumentation components for consumer groups based operation
+      module ConsumerGroups
+        module LagCompensation
+          # Fetches the end offsets of given partitions via the client own connection: no
+          # dedicated instances and no extra Kafka connections are ever created.
+          #
+          # All partitions are resolved with one batched request that librdkafka fans out to the
+          # involved partition leaders internally, so the cost does not grow with the number of
+          # paused partitions.
+          #
+          # End offsets are the only thing the compensation needs from the broker: committed
+          # and stored offsets of paused partitions are maintained by the client commits (not
+          # by fetches), so their statistics values stay accurate while paused and lags can be
+          # derived from them at compensation time.
+          #
+          # The consumer isolation level is forwarded to the query, so on topics without in-flight
+          # transactions the end offset matches the reference the native consumer lag derives from.
+          #
+          # Known edge case (transactional topics): the batched `ListOffsets` this uses resolves
+          # `:latest` to the high watermark regardless of the forwarded isolation level, unlike the
+          # per-partition `query_watermark_offsets` it replaced, which returned the last stable
+          # offset for a read_committed consumer. So while a transaction is IN FLIGHT on a paused
+          # partition, the compensated end offset (and the lag derived from it) reflects the high
+          # watermark and can overstate the read_committed lag by the number of uncommitted
+          # messages. It self-corrects once the transaction commits or aborts and the last stable
+          # offset advances. Non-transactional topics are unaffected (LSO == HWM there).
+          class Fetcher
+            # @param client [Karafka::Connection::Client]
+            # @param paused [Hash{String => Array<Integer>}] paused topics with partitions
+            # @return [Hash{String => Hash{Integer => Integer}}] end offsets of the requested
+            #   partitions
+            def call(client, paused)
+              request = paused.transform_values do |partitions|
+                partitions.map { |partition| { partition: partition, offset: :latest } }
+              end
+
+              return {} if request.empty?
+
+              # Auto-initializes the per-topic partitions hash on first access, same idiom as
+              # the refresher state. Only ever written to and iterated here, never read by a
+              # missing key, so the default block cannot insert a spurious entry.
+              data = Hash.new { |topics, topic| topics[topic] = {} }
+
+              client.read_partition_offsets(request).each do |result|
+                data[result[:topic]][result[:partition]] = result[:offset]
+              end
+
+              data
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/instrumentation/consumer_groups/lag_compensation/refresher.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Instrumentation
+      module ConsumerGroups
+        module LagCompensation
+          # Refreshes offsets and lags data of partitions that stayed paused for at least the
+          # configured pause age, at most once per interval.
+          #
+          # Runs on the listener threads via the `client.events_poll` event, so the fetching
+          # happens on the thread that owns the client connection within its group identity.
+          # Pause state is tracked via the `client.pause` and `client.resume` instrumentation
+          # events, which fire on the same thread.
+          #
+          # On resume or revocation the partition data is dropped immediately, so stale values
+          # never overlay live statistics and revoked partitions are never queried again.
+          # Errors never propagate and are only instrumented: the next attempt happens on the
+          # next interval anyway.
+          class Refresher
+            include Karafka::Core::Helpers::Time
+            include Helpers::ConfigImporter.new(
+              monitor: %i[monitor],
+              interval: %i[internal statistics consumer_groups lag_compensation interval],
+              pause_age: %i[internal statistics consumer_groups lag_compensation pause_age]
+            )
+
+            def initialize
+              @fetcher = Fetcher.new
+              @mutex = Mutex.new
+              @states = Hash.new do |states, subscription_group_id|
+                states[subscription_group_id] = {
+                  # Gates the refreshing to run at most once per interval
+                  runner: Helpers::IntervalRunner.new(interval: interval) do |client, state|
+                    refresh_tick(client, state)
+                  end,
+                  paused: Hash.new { |paused, topic| paused[topic] = {} },
+                  client_name: nil
+                }
+              end
+            end
+
+            # Tracks when a given partition got paused
+            #
+            # @param event [Karafka::Core::Monitoring::Event] pause event
+            def on_client_pause(event)
+              state = state_for(event[:subscription_group].id)
+
+              state[:paused][event[:topic]][event[:partition]] = monotonic_now
+              state[:client_name] = event[:caller].name
+            end
+
+            # Stops refreshing a resumed partition but keeps its last refreshed values in the
+            # registry. The compensator keeps overlaying them (its guard applies them only while
+            # still fresher than the live statistics), so the emitted statistics do not snap back
+            # to the frozen pre-resume values during the gap before librdkafka post-resume
+            # fetches update the offsets. Once the live values catch up the compensator stops
+            # using them, and the kept entries are dropped in bulk on the next rebalance.
+            #
+            # @param event [Karafka::Core::Monitoring::Event] resume event
+            def on_client_resume(event)
+              state = state_for(event[:subscription_group].id)
+              # Fetch with a fallback not to auto-build tracking of an untracked topic
+              partitions = state[:paused].fetch(event[:topic], nil)
+
+              return unless partitions
+              return unless partitions.delete(event[:partition])
+
+              state[:paused].delete(event[:topic]) if partitions.empty?
+            end
+
+            # Once per interval refreshes the data of all the partitions that stayed paused
+            # for at least the configured pause age
+            #
+            # @param event [Karafka::Core::Monitoring::Event] event with the client
+            def on_client_events_poll(event)
+              # Never run during shutdown or quieting: blocking broker queries at that time
+              # would eat into the shutdown time budget for no benefit
+              return if Karafka::App.done?
+
+              state = state_for(event[:subscription_group].id)
+
+              state[:runner].call(event[:caller], state)
+            rescue => e
+              # Refreshing is a best-effort operation: values stay stale until the next
+              # successful run on one of the following intervals. Errors are still
+              # instrumented, so persistent failures (missing ACLs, removed topics, etc)
+              # remain observable
+              monitor.instrument(
+                "error.occurred",
+                caller: self,
+                error: e,
+                type: "lag_compensation.refresher.error"
+              )
+            end
+
+            # Expires all the refreshed data and pause tracking of a client on a rebalance. Its
+            # paused partitions may no longer belong to it and tracking will refill only via
+            # real pause events of partitions this client still owns.
+            #
+            # @param event [Karafka::Core::Monitoring::Event] revocation event
+            def on_rebalance_partitions_revoked(event)
+              state = state_for(event[:subscription_group].id)
+              # Clear (instead of a reassignment) preserves the auto-building behavior
+              state[:paused].clear
+
+              client_name = state[:client_name]
+
+              Registry.instance.evict(client_name) if client_name
+            end
+
+            private
+
+            # @param subscription_group_id [String]
+            # @return [Hash] mutable state of a given subscription group. The mutex guards the
+            #   cross-thread auto-building; aside from creation, the state is accessed only
+            #   from the subscription group own listener thread.
+            def state_for(subscription_group_id)
+              @mutex.synchronize do
+                @states[subscription_group_id]
+              end
+            end
+
+            # Refreshes and stores the data of all the long-paused partitions (if any)
+            #
+            # @param client [Karafka::Connection::Client]
+            # @param state [Hash] subscription group state
+            def refresh_tick(client, state)
+              state[:client_name] = client.name
+
+              long_paused = long_paused_for(state)
+
+              return if long_paused.empty?
+
+              Registry.instance.update(client.name, @fetcher.call(client, long_paused))
+            end
+
+            # @param state [Hash] subscription group state with pause timestamps
+            # @return [Hash{String => Array<Integer>}] partitions that stayed paused for at
+            #   least the configured pause age
+            def long_paused_for(state)
+              threshold = monotonic_now - pause_age
+              result = {}
+
+              state[:paused].each do |topic, partitions|
+                long_paused = partitions.select { |_, at| at <= threshold }.keys
+                result[topic] = long_paused unless long_paused.empty?
+              end
+
+              result
+            end
+          end
+        end
+      end
+    end
+  end
+end