karafka-core 2.5.10 → 2.5.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe7a4974267a2a3b2b200e0763a13042d98695c0586931d00d0016087bc5dfe2
-  data.tar.gz: a5bdd9235a999028ad120d79c2986e05986ee1b9df8a5de6b813fff634bc5619
+  metadata.gz: 6ada2e9b6d4c3d9adfe4aa595f9208198b60eee63bed805501a48d9095da1d90
+  data.tar.gz: 77fb9a1be85a9f120057e784385cfb667ec5e65e0461a2fb059fc4e76e386a0a
 SHA512:
-  metadata.gz: be10ba6481782906bbb076ea69ab9aa05c80f4c295ce82c2faacc53ee27a8ebd01e4c7a200088205058fda7e7efef494d71ac96492d3cf074f579ea331597148
-  data.tar.gz: a51a53327518e282ab42461fccc3aea78407de1418d97376a64b25dda687b616a54f0c6639b542d0c104534876b1c34d9f7f1434486300c40708f129f03ad267
+  metadata.gz: 4f22a703bb815c99259ab70f53adb090b8100dcf7bfb91785706fbd2a91a8bba02a12ef5412d93e4459cfc4ac51e9bbce4f4f1ead392317255f588b823eeb356
+  data.tar.gz: 33cd8fa76ca78714edbcc9ddb052d345ddeb0e63bc569e718ff47feef7db273c52a5288801c35d88c4248b91904a442537b827ea749c72a09fb7724b85bf2bf7

data/.github/workflows/ci.yml

@@ -14,7 +14,7 @@ permissions:
   contents: read
 
 jobs:
-  specs:
+  tests:
     timeout-minutes: 15
     runs-on: ubuntu-latest
     strategy:
@@ -37,7 +37,7 @@ jobs:
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
+        uses: ruby/setup-ruby@eab2afb99481ca09a4e91171a8e0aee0e89bfedd # v1.296.0
         with:
           ruby-version: ${{matrix.ruby}}
           bundler: 'latest'
@@ -69,9 +69,9 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Ruby
-        uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
+        uses: ruby/setup-ruby@eab2afb99481ca09a4e91171a8e0aee0e89bfedd # v1.296.0
         with:
-          ruby-version: '4.0.1'
+          ruby-version: '4.0.2'
           bundler-cache: true
       - name: Run rubocop
         run: bundle exec rubocop
@@ -86,9 +86,9 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Ruby
-        uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
+        uses: ruby/setup-ruby@eab2afb99481ca09a4e91171a8e0aee0e89bfedd # v1.296.0
         with:
-          ruby-version: '4.0.1'
+          ruby-version: '4.0.2'
           bundler-cache: true
       - name: Run yard-lint
         run: bundle exec yard-lint lib/
@@ -101,7 +101,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Node.js
-        uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
         with:
           node-version: '20'
           cache: 'npm'
@@ -116,7 +116,7 @@ jobs:
     if: always()
     needs:
       - rubocop
-      - specs
+      - tests
       - yard-lint
       - lostconf
     steps:

data/.github/workflows/push.yml

@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
+        uses: ruby/setup-ruby@eab2afb99481ca09a4e91171a8e0aee0e89bfedd # v1.296.0
         with:
           bundler-cache: false
 
@@ -32,4 +32,4 @@ jobs:
         run: |
           bundle install --jobs 4 --retry 3
 
-      - uses: rubygems/release-gem@1c162a739e8b4cb21a676e97b087e8268d8fc40b # v1.1.2
+      - uses: rubygems/release-gem@e9a6361a0b14562539327c2a02373edc56dd3169 # v1.1.4

data/.ruby-version

@@ -1 +1 @@
-4.0.1
+4.0.2

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Karafka Core Changelog
 
+## 2.5.11 (2026-04-02)
+- [Enhancement] Specialize `Contract#dig` for common 1-key and 2-key paths to avoid iterator overhead, yielding ~1.5x faster single-key lookups and ~1.45x faster two-key nested lookups.
+- [Enhancement] Replace `Node#build_accessors` `@local_defs` Array with Hash for O(1) membership checks instead of O(n) `Array#include?`, yielding up to ~5x faster accessor lookups at 50 settings.
+- [Enhancement] Use frozen `EMPTY_ARRAY` constant for `Contract#call` and `#validate!` default `scope` parameter to avoid allocating a new Array on every invocation, yielding ~1.36x faster call dispatch and saving 1 Array allocation per call.
+- [Enhancement] Pre-resolve `@events_methods_map` method name before the listener notification loop in `Notifications#instrument` to avoid repeated Hash lookup per listener, yielding ~1.12x faster event dispatch with multiple listeners.
+- [Enhancement] Cache a frozen success `Result` singleton via `Result.success` to eliminate 1 object allocation per successful `Contract#call` on the happy path.
+- [Enhancement] Skip nestings block re-evaluation in `Node#deep_dup` to avoid recreating children that are immediately overwritten, yielding ~14x faster deep_dup and reducing allocations from ~620 to ~66 objects for large configs.
+- [Enhancement] Cache `CallbacksManager#call` values snapshot and invalidate on `add`/`delete` to avoid allocating a new Array on every invocation while preserving thread-safety snapshot semantics, saving 1 Array allocation per call.
+- [Enhancement] Store execution time separately in `Event` and build the merged payload hash lazily on `#payload` access, eliminating 1 Hash allocation per `Notifications#instrument` call when listeners use `#[]` access (the common pattern), yielding ~1.7x faster event dispatch.
+- [Enhancement] Replace `StatisticsDecorator#diff` pending-writes buffer with `keys.each` direct-write iteration, eliminating the buffer and write-back loop for ~13% faster decoration at scale (10 brokers, 20 topics, 2000 partitions).
+- [Enhancement] Reorder `StatisticsDecorator#diff` type checks to test `Numeric` before `Hash`, matching the ~80% numeric value distribution in librdkafka statistics.
+- [Enhancement] Support `only_keys` option in `StatisticsDecorator` to decorate only specified numeric keys (e.g. `consumer_lag`, `committed_offset`). When combined with `excluded_keys`, reduces decoration cost from ~80ms to ~8.5ms per call on large clusters (10 brokers, 20 topics, 2000 partitions) by using structure-aware navigation of the librdkafka statistics tree and direct key access instead of full-hash iteration.
+
 ## 2.5.10 (2026-03-02)
 - [Enhancement] Introduce `MinitestLocator` helper for minitest/spec subject class auto-discovery from test file paths.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-core (2.5.10)
+    karafka-core (2.5.11)
       karafka-rdkafka (>= 0.20.0)
       logger (>= 1.6.0)
 

data/lib/karafka/core/configurable/node.rb

@@ -17,14 +17,16 @@ module Karafka
 
         # @param node_name [Symbol] node name
         # @param nestings [Proc] block for nested settings
-        def initialize(node_name, nestings = ->(_) {})
+        # @param evaluate [Boolean] when false, skip evaluating the nestings block. Used by
+        #   deep_dup to avoid re-creating children that will be overwritten immediately.
+        def initialize(node_name, nestings = ->(_) {}, evaluate: true)
           @node_name = node_name
           @children = []
           @nestings = nestings
           @compiled = false
           @configs_refs = {}
-          @local_defs = []
-          instance_eval(&nestings)
+          @local_defs = {}
+          instance_eval(&nestings) if evaluate
         end
 
         # Allows for a single leaf or nested node definition
@@ -83,7 +85,7 @@ module Karafka
         # and non-side-effect usage on an instance/inherited.
         # @return [Node] duplicated node
         def deep_dup
-          dupped = Node.new(node_name, nestings)
+          dupped = Node.new(node_name, nestings, evaluate: false)
 
           children.each do |value|
             dupped.children << if value.is_a?(Leaf)
@@ -189,8 +191,8 @@ module Karafka
           # object and then we would not redefine it for nodes. This ensures that we only do not
           # redefine our own definitions but we do redefine any user "accidentally" inherited
           # methods
-          if reader_respond ? !@local_defs.include?(reader_name) : true
-            @local_defs << reader_name
+          if reader_respond ? !@local_defs.key?(reader_name) : true
+            @local_defs[reader_name] = true
 
             define_singleton_method(reader_name) do
               @configs_refs[reader_name]

data/lib/karafka/core/contractable/contract.rb

@@ -14,7 +14,12 @@ module Karafka
         # prevent additional array allocation
         DIG_MISS = Object.new
 
-        private_constant :DIG_MISS
+        # Empty array for scope default to avoid allocating a new Array on each
+        # `#call` / `#validate!` invocation. Safe because scope is never mutated – it is only
+        # used in `scope + rule.path` which creates a new Array.
+        EMPTY_ARRAY = [].freeze
+
+        private_constant :DIG_MISS, :EMPTY_ARRAY
 
         # Yaml based error messages data
         setting(:error_messages)
@@ -80,7 +85,7 @@ module Karafka
         # @param scope [Array<String>] scope of this contract (if any) or empty array if no parent
         #   scope is needed if contract starts from root
         # @return [Result] validaton result
-        def call(data, scope: [])
+        def call(data, scope: EMPTY_ARRAY)
           errors = []
 
           self.class.rules.each do |rule|
@@ -94,6 +99,8 @@ module Karafka
             end
           end
 
+          return Result.success if errors.empty?
+
           Result.new(errors, self)
         end
 
@@ -104,7 +111,7 @@ module Karafka
         # @return [Boolean] true
         # @raise [StandardError] any error provided in the error_class that inherits from the
         #   standard error
-        def validate!(data, error_class, scope: [])
+        def validate!(data, error_class, scope: EMPTY_ARRAY)
           result = call(data, scope: scope)
 
           return true if result.success?
@@ -183,15 +190,34 @@ module Karafka
         # @param keys [Array<Symbol>]
         # @return [DIG_MISS, Object] found element or DIGG_MISS indicating that not found
         def dig(data, keys)
-          current = data
+          case keys.length
+          when 1
+            key = keys[0]
 
-          keys.each do |nesting|
-            return DIG_MISS unless current.key?(nesting)
+            return DIG_MISS unless data.key?(key)
 
-            current = current[nesting]
-          end
+            data[key]
+          when 2
+            key1 = keys[0]
+
+            return DIG_MISS unless data.key?(key1)
+
+            mid = data[key1]
+
+            return DIG_MISS unless mid.is_a?(Hash) && mid.key?(keys[1])
 
-          current
+            mid[keys[1]]
+          else
+            current = data
+
+            keys.each do |nesting|
+              return DIG_MISS unless current.key?(nesting)
+
+              current = current[nesting]
+            end
+
+            current
+          end
         end
       end
     end

data/lib/karafka/core/contractable/result.rb

@@ -12,6 +12,15 @@ module Karafka
 
         private_constant :EMPTY_HASH
 
+        class << self
+          # @return [Result] cached frozen success result to avoid allocating a new Result on
+          #   every successful contract validation. Safe because successful results are
+          #   identical regardless of contract (they all have @errors = EMPTY_HASH).
+          def success
+            @success ||= new([], nil).freeze
+          end
+        end
+
         # Builds a result object and remaps (if needed) error keys to proper error messages
         #
         # @param errors [Array<Array>] array with sub-arrays with paths and error keys

data/lib/karafka/core/instrumentation/callbacks_manager.rb

@@ -10,6 +10,7 @@ module Karafka
         # @return [::Karafka::Core::Instrumentation::CallbacksManager]
         def initialize
           @callbacks = {}
+          @values_cache = nil
         end
 
         # Invokes all the callbacks registered one after another
@@ -19,8 +20,10 @@ module Karafka
         #   callbacks and add new at the same time. Since we don't know when and in what thread
         #   things are going to be added to the manager, we need to extract values into an array and
         #   run it. That way we can add new things the same time.
+        #   The values snapshot is cached and invalidated on add/delete to avoid allocating a new
+        #   Array on every call while preserving the thread-safety snapshot semantics.
         def call(*args)
-          @callbacks.values.each { |callback| callback.call(*args) }
+          (@values_cache ||= @callbacks.values).each { |callback| callback.call(*args) }
         end
 
         # Adds a callback to the manager
@@ -29,12 +32,14 @@ module Karafka
         # @param callable [#call] object that responds to a `#call` method
         def add(id, callable)
           @callbacks[id] = callable
+          @values_cache = nil
         end
 
         # Removes the callback from the manager
         # @param id [String] id of the callback we want to remove
         def delete(id)
           @callbacks.delete(id)
+          @values_cache = nil
         end
       end
     end

data/lib/karafka/core/monitoring/event.rb

@@ -5,20 +5,37 @@ module Karafka
     module Monitoring
       # Single notification event wrapping payload with id
       class Event
-        attr_reader :id, :payload
+        attr_reader :id
 
         # @param id [String, Symbol] id of the event
         # @param payload [Hash] event payload
-        def initialize(id, payload)
+        # @param time [Float, nil] execution time, stored separately to avoid eager hash
+        #   allocation. Merged into the payload lazily only when `#payload` is accessed.
+        def initialize(id, payload, time = nil)
           @id = id
-          @payload = payload
+          @raw_payload = payload
+          @time = time
+          @payload = nil
+        end
+
+        # @return [Hash] full payload including time (if set). The merged hash is built lazily
+        #   on first access to avoid allocating a new Hash when listeners only use `#[]`.
+        def payload
+          @payload ||= if @time
+            @raw_payload.empty? ? { time: @time } : @raw_payload.merge(time: @time)
+          else
+            @raw_payload
+          end
         end
 
         # Hash access to the payload data (if present)
+        # Provides direct access to time without triggering payload hash construction.
         #
         # @param name [String, Symbol]
         def [](name)
-          @payload.fetch(name)
+          return @time if name == :time && @time
+
+          @raw_payload.fetch(name)
         end
       end
     end

data/lib/karafka/core/monitoring/notifications.rb

@@ -139,36 +139,25 @@ module Karafka
             return
           end
 
-          final_payload = build_payload(payload, time)
-          event = Event.new(event_id, final_payload)
+          event = Event.new(event_id, payload, time)
 
-          notify_listeners(event_id, event, assigned_listeners)
+          notify_listeners(@events_methods_map[event_id], event, assigned_listeners)
 
           result
         end
 
         private
 
-        # Builds the final payload with time information if available
-        # @param payload [Hash] original payload
-        # @param time [Float, nil] execution time if block was given
-        # @return [Hash] final payload
-        def build_payload(payload, time)
-          return payload unless time
-
-          payload.empty? ? { time: time } : payload.merge(time: time)
-        end
-
         # Notifies all assigned listeners about the event
-        # @param event_id [String]
+        # @param method_name [Symbol] pre-resolved method name for object listeners
         # @param event [Event] event with payload to broadcast to listeners
         # @param assigned_listeners [Array] list of listeners to notify
-        def notify_listeners(event_id, event, assigned_listeners)
+        def notify_listeners(method_name, event, assigned_listeners)
           assigned_listeners.each do |listener|
             if listener.is_a?(Proc)
               listener.call(event)
             else
-              listener.send(@events_methods_map[event_id], event)
+              listener.send(method_name, event)
             end
           end
         end

data/lib/karafka/core/monitoring/statistics_decorator.rb

@@ -6,7 +6,7 @@ module Karafka
       # Many of the librdkafka statistics are absolute values instead of a gauge.
       # This means, that for example number of messages sent is an absolute growing value
       # instead of being a value of messages sent from the last statistics report.
-      # This decorator calculates the diff against previously emited stats, so we get also
+      # This decorator calculates the diff against previously emitted stats, so we get also
       # the diff together with the original values
       #
       # It adds two extra values to numerics:
@@ -28,7 +28,12 @@ module Karafka
         #   duration suffixes. This is useful for skipping large subtrees of the librdkafka
         #   statistics that are not consumed by the application (e.g. broker toppars, window
         #   stats like int_latency, outbuf_latency, throttle, batchsize, batchcnt, req).
-        def initialize(excluded_keys: [])
+        # @param only_keys [Array<String>] when set, only these numeric keys will be decorated
+        #   with delta/freeze duration suffixes. Hash children are still recursed into for
+        #   structural navigation, but only the listed keys receive _d and _fd computation.
+        #   This drastically reduces work at the partition level where most cost occurs.
+        #   When empty (default), all numeric keys are decorated.
+        def initialize(excluded_keys: [], only_keys: [])
           @previous = EMPTY_HASH
           # Operate on ms precision only
           @previous_at = monotonic_now.round
@@ -39,63 +44,222 @@ module Karafka
           @excluded_keys = unless excluded_keys.empty?
             excluded_keys.each_with_object({}) { |k, h| h[k] = true }.freeze
           end
+          # Frozen array for direct-access decoration, nil when empty to use full decoration
+          @only_keys = unless only_keys.empty?
+            only_keys.freeze
+          end
         end
 
-        # @param emited_stats [Hash] original emited statistics
-        # @return [Hash] emited statistics extended with the diff data
-        # @note We modify the emited statistics, instead of creating new. Since we don't expose
+        # @param emitted_stats [Hash] original emitted statistics
+        # @return [Hash] emitted statistics extended with the diff data
+        # @note We modify the emitted statistics, instead of creating new. Since we don't expose
         #   any API to get raw data, users can just assume that the result of this decoration is
         #   the proper raw stats that they can use
-        def call(emited_stats)
+        def call(emitted_stats)
           current_at = monotonic_now.round
           change_d = current_at - @previous_at
 
           diff(
             @previous,
-            emited_stats,
-            [],
-            0,
+            emitted_stats,
             change_d
           )
 
-          @previous = emited_stats
+          @previous = emitted_stats
           @previous_at = current_at
 
-          emited_stats.freeze
+          emitted_stats.freeze
         end
 
         private
 
-        # Calculates the diff of the provided values, appends delta and freeze duration keys,
-        # and modifies in place the emited statistics.
+        # Calculates the diff of the provided values and modifies the emitted statistics
+        # in place to add delta and freeze duration keys.
         #
-        # Uses `each_pair` with a per-call pending-writes buffer instead of `current.keys.each`
-        # to avoid allocating a new Array for every Hash node in the statistics tree. At scale
-        # (thousands of partitions), this reduces allocations from tens of thousands to one per call.
+        # When @only_keys is set, uses a two-phase approach: first recurse into hash
+        # children for structural navigation, then decorate only the specified keys via
+        # direct hash access (no iteration over non-target keys). This drastically reduces
+        # work at the partition level where most of the cost occurs.
         #
-        # The append and suffix_keys_for logic is inlined to reduce method call overhead
-        # (from ~915k method calls to ~39k at 6400 partitions).
+        # When @only_keys is nil, uses full decoration: iterates all keys, decorating every
+        # numeric value found.
         #
         # @param previous [Object] previous value from the given scope in which we are
         # @param current [Object] current scope from emitted statistics
-        # @param pw [Array] pending writes buffer shared across recursive calls
-        # @param pw_start [Integer] starting offset in the buffer for this hash level
         # @param change_d [Integer] time delta in ms since last stats emission
-        def diff(previous, current, pw, pw_start, change_d)
+        def diff(previous, current, change_d)
           return unless current.is_a?(Hash)
 
+          if @only_keys
+            diff_only_keys(previous, current, change_d)
+          else
+            diff_all(previous, current, change_d)
+          end
+        end
+
+        # Full decoration path: iterates all keys, decorating every numeric value.
+        #
+        # Uses `keys.each` to snapshot the current hash's key list, allowing direct writes
+        # to the hash during iteration without a pending-writes buffer.
+        #
+        # Checks Numeric before Hash because ~80% of statistics values are numeric.
+        #
+        # @param previous [Object] previous value from the given scope
+        # @param current [Hash] current scope from emitted statistics
+        # @param change_d [Integer] time delta in ms
+        def diff_all(previous, current, change_d)
           filled_previous = previous || EMPTY_HASH
           cache = @suffix_keys_cache
           excluded = @excluded_keys
-          pw_size = pw_start
 
-          current.each_pair do |key, value|
+          current.keys.each do |key|
             next if excluded&.key?(key)
 
-            if value.is_a?(Hash)
-              diff(filled_previous[key], value, pw, pw_size, change_d)
-              next
+            value = current[key]
+
+            if value.is_a?(Numeric)
+              prev_value = filled_previous[key]
+
+              if prev_value.nil?
+                result = 0
+              elsif prev_value.is_a?(Numeric)
+                result = value - prev_value
+              else
+                next
+              end
+
+              pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
+
+              current[pair[0]] = (result == 0) ? (filled_previous[pair[0]] || 0) + change_d : 0
+              current[pair[1]] = result
+            elsif value.is_a?(Hash)
+              diff_all(filled_previous[key], value, change_d)
             end
+          end
+        end
+
+        # Known librdkafka statistics tree structural keys. Used by diff_only_keys to
+        # navigate directly to known hash paths instead of iterating all keys.
+        KNOWN_HASH_KEYS = { "brokers" => true, "topics" => true, "cgrp" => true }.freeze
+
+        # Known structural keys within a topic hash
+        TOPIC_HASH_KEYS = { "partitions" => true }.freeze
+
+        private_constant :KNOWN_HASH_KEYS, :TOPIC_HASH_KEYS
+
+        # Selective decoration path: navigates known librdkafka statistics tree structure
+        # directly instead of iterating all keys to find hash children. Decorates only the
+        # specified keys via direct access at each level.
+        #
+        # This eliminates the generic recursion overhead (72,000+ is_a?(Hash) checks at the
+        # partition level alone on a 2000-partition cluster) by leveraging knowledge of the
+        # librdkafka statistics layout: root → brokers → broker, root → topics → topic →
+        # partitions → partition, root → cgrp.
+        #
+        # For non-librdkafka hash children (e.g. custom or test data), falls back to generic
+        # recursion to maintain correctness with arbitrary nested structures.
+        #
+        # @param previous [Object] previous value from the given scope
+        # @param current [Hash] current stats hash (root level)
+        # @param change_d [Integer] time delta in ms
+        def diff_only_keys(previous, current, change_d)
+          filled_previous = previous || EMPTY_HASH
+          excluded = @excluded_keys
+
+          # Root level decoration
+          decorate_keys(current, filled_previous, change_d)
+
+          # Brokers: each child is a broker hash (leaf-like for only_keys)
+          unless excluded&.key?("brokers")
+            brokers = current["brokers"]
+
+            if brokers.is_a?(Hash)
+              prev_brokers = filled_previous["brokers"] || EMPTY_HASH
+
+              brokers.each_pair do |name, broker|
+                decorate_keys(broker, prev_brokers[name] || EMPTY_HASH, change_d) if broker.is_a?(Hash)
+              end
+            end
+          end
+
+          # Topics → Partitions
+          unless excluded&.key?("topics")
+            topics = current["topics"]
+
+            if topics.is_a?(Hash)
+              prev_topics = filled_previous["topics"] || EMPTY_HASH
+
+              topics.each_pair do |name, topic|
+                next unless topic.is_a?(Hash)
+
+                prev_topic = prev_topics[name] || EMPTY_HASH
+                decorate_keys(topic, prev_topic, change_d)
+
+                unless excluded&.key?("partitions")
+                  partitions = topic["partitions"]
+
+                  if partitions.is_a?(Hash)
+                    prev_partitions = prev_topic["partitions"] || EMPTY_HASH
+
+                    partitions.each_pair do |pid, partition|
+                      # Leaf level: no hash children, just decorate
+                      decorate_keys(partition, prev_partitions[pid] || EMPTY_HASH, change_d)
+                    end
+                  end
+                end
+              end
+            end
+          end
+
+          # Consumer group (leaf-like)
+          cgrp = current["cgrp"]
+          decorate_keys(cgrp, filled_previous["cgrp"] || EMPTY_HASH, change_d) if cgrp.is_a?(Hash)
+
+          # Fallback: handle any non-standard hash children not in the known structure.
+          # This ensures correctness for arbitrary nested data while the known paths above
+          # provide the fast path for real librdkafka statistics.
+          current.each_pair do |key, value|
+            next if KNOWN_HASH_KEYS.key?(key)
+            next if excluded&.key?(key)
+            next unless value.is_a?(Hash)
+
+            diff_only_keys_generic(filled_previous[key], value, change_d)
+          end
+        end
+
+        # Generic recursive fallback for only_keys mode on non-standard hash children.
+        # Used for hash subtrees not covered by the known librdkafka structure.
+        #
+        # @param previous [Object] previous value
+        # @param current [Hash] current hash to process
+        # @param change_d [Integer] time delta in ms
+        def diff_only_keys_generic(previous, current, change_d)
+          return unless current.is_a?(Hash)
+
+          filled_previous = previous || EMPTY_HASH
+          excluded = @excluded_keys
+
+          decorate_keys(current, filled_previous, change_d)
+
+          current.each_pair do |key, value|
+            next if excluded&.key?(key)
+
+            diff_only_keys_generic(filled_previous[key], value, change_d) if value.is_a?(Hash)
+          end
+        end
+
+        # Decorates only the keys listed in @only_keys via direct hash access.
+        # No iteration over the full hash, no type checking on non-target keys.
+        #
+        # @param current [Hash] current stats node
+        # @param filled_previous [Hash] previous stats node
+        # @param change_d [Integer] time delta in ms
+        def decorate_keys(current, filled_previous, change_d)
+          cache = @suffix_keys_cache
+          only = @only_keys
+
+          only.each do |key|
+            value = current[key]
 
             next unless value.is_a?(Numeric)
 
@@ -109,21 +273,10 @@ module Karafka
               next
             end
 
-            # Inlined suffix_keys_for for reduced method call overhead
             pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
 
-            pw[pw_size] = pair[0]
-            pw[pw_size + 1] = (result == 0) ? (filled_previous[pair[0]] || 0) + change_d : 0
-            pw[pw_size + 2] = pair[1]
-            pw[pw_size + 3] = result
-            pw_size += 4
-          end
-
-          # Apply collected writes for this hash level
-          i = pw_start
-          while i < pw_size
-            current[pw[i]] = pw[i + 1]
-            i += 2
+            current[pair[0]] = (result == 0) ? (filled_previous[pair[0]] || 0) + change_d : 0
+            current[pair[1]] = result
           end
         end
       end

data/lib/karafka/core/version.rb

@@ -4,6 +4,6 @@ module Karafka
   module Core
     # Current Karafka::Core version
     # We follow the versioning schema of given Karafka version
-    VERSION = "2.5.10"
+    VERSION = "2.5.11"
   end
 end

data/test/lib/karafka/core/monitoring/event_test.rb

@@ -9,6 +9,43 @@ describe_current do
   it { assert_equal id, event.id }
   it { assert_equal payload, event.payload }
 
+  describe "when time is provided" do
+    subject(:timed_event) { described_class.new(id, payload, execution_time) }
+
+    let(:execution_time) { rand }
+
+    context "when payload is empty" do
+      let(:payload) { {} }
+
+      it "expect to return only time in payload" do
+        assert_equal({ time: execution_time }, timed_event.payload)
+      end
+    end
+
+    context "when payload is non-empty" do
+      let(:payload) { { key: "value" } }
+
+      it "expect to merge time into payload" do
+        assert_equal({ key: "value", time: execution_time }, timed_event.payload)
+      end
+    end
+
+    describe "#[] with time" do
+      let(:payload) { { key: "value" } }
+
+      it "expect to return time directly via #[] without triggering payload construction" do
+        assert_equal execution_time, timed_event[:time]
+        # We intentionally inspect the internal @payload ivar to ensure that #[] returns time
+        # without triggering the lazy payload memoization as a side effect.
+        assert_nil timed_event.instance_variable_get(:@payload)
+      end
+
+      it "expect to still access raw payload keys" do
+        assert_equal "value", timed_event[:key]
+      end
+    end
+  end
+
   describe "#[]" do
     context "when key is present" do
       let(:payload) { { test: 1 } }

data/test/lib/karafka/core/monitoring/statistics_decorator_test.rb

@@ -3,7 +3,7 @@
 describe_current do
   subject(:decorator) { described_class.new }
 
-  let(:emited_stats1) do
+  let(:emitted_stats1) do
     {
       "string" => "value1",
       "float" => 10.4,
@@ -18,7 +18,7 @@ describe_current do
     }
   end
 
-  let(:emited_stats2) do
+  let(:emitted_stats2) do
     {
       "string" => "value2",
       "float" => 10.8,
@@ -33,7 +33,7 @@ describe_current do
     }
   end
 
-  let(:emited_stats3) do
+  let(:emitted_stats3) do
     {
       "string" => "value3",
       "float" => 11.8,
@@ -51,7 +51,7 @@ describe_current do
   let(:broker_scope) { %w[nested brokers localhost:9092/2] }
 
   context "when it is a first stats emit" do
-    subject(:decorated) { decorator.call(emited_stats1) }
+    subject(:decorated) { decorator.call(emitted_stats1) }
 
     it { assert_equal "value1", decorated["string"] }
     it { refute decorated.key?("string_d") }
@@ -63,8 +63,8 @@ describe_current do
 
   context "when it is a second stats emit" do
     subject(:decorated) do
-      decorator.call(emited_stats1)
-      decorator.call(emited_stats2)
+      decorator.call(emitted_stats1)
+      decorator.call(emitted_stats2)
     end
 
     it { assert_equal "value2", decorated["string"] }
@@ -77,9 +77,9 @@ describe_current do
 
   context "when it is a third stats emit" do
     subject(:decorated) do
-      decorator.call(emited_stats1)
-      decorator.call(emited_stats2)
-      decorator.call(emited_stats3)
+      decorator.call(emitted_stats1)
+      decorator.call(emitted_stats2)
+      decorator.call(emitted_stats3)
     end
 
     it { assert_equal "value3", decorated["string"] }
@@ -97,11 +97,11 @@ describe_current do
 
   context "when a broker is no longer present" do
     subject(:decorated) do
-      decorator.call(emited_stats1)
-      decorator.call(emited_stats2)
+      decorator.call(emitted_stats1)
+      decorator.call(emitted_stats2)
     end
 
-    before { emited_stats2["nested"] = {} }
+    before { emitted_stats2["nested"] = {} }
 
     it { assert_equal "value2", decorated["string"] }
     it { refute decorated.key?("string_d") }
@@ -117,11 +117,11 @@ describe_current do
 
   context "when broker was introduced later on" do
     subject(:decorated) do
-      decorator.call(emited_stats1)
-      decorator.call(emited_stats2)
+      decorator.call(emitted_stats1)
+      decorator.call(emitted_stats2)
     end
 
-    before { emited_stats1["nested"] = {} }
+    before { emitted_stats1["nested"] = {} }
 
     it { assert_equal "value2", decorated["string"] }
     it { refute decorated.key?("string_d") }
@@ -146,7 +146,7 @@ describe_current do
       decorator.call(deep_copy.call)
     end
 
-    let(:deep_copy) { -> { Marshal.load(Marshal.dump(emited_stats1)) } }
+    let(:deep_copy) { -> { Marshal.load(Marshal.dump(emitted_stats1)) } }
 
     it { refute decorated.key?("string_d") }
     it { refute decorated.key?("string_fd") }
@@ -171,7 +171,7 @@ describe_current do
       decorator.call(deep_copy.call)
     end
 
-    let(:deep_copy) { -> { Marshal.load(Marshal.dump(emited_stats1)) } }
+    let(:deep_copy) { -> { Marshal.load(Marshal.dump(emitted_stats1)) } }
 
     it { refute decorated.key?("string_d") }
     it { refute decorated.key?("string_fd") }
@@ -186,13 +186,13 @@ describe_current do
 
   context "when a value type changed from non-numeric to numeric between emissions" do
     subject(:decorated) do
-      decorator.call(emited_stats1)
-      decorator.call(emited_stats2)
+      decorator.call(emitted_stats1)
+      decorator.call(emitted_stats2)
     end
 
     before do
       # In the first emission, txbytes is a string (unusual but defensive)
-      emited_stats1["nested"]["brokers"]["localhost:9092/2"]["txbytes"] = "not_a_number"
+      emitted_stats1["nested"]["brokers"]["localhost:9092/2"]["txbytes"] = "not_a_number"
     end
 
     # When previous value was non-numeric but current is numeric, no delta should be computed
@@ -206,7 +206,7 @@ describe_current do
     subject(:decorator) { described_class.new(excluded_keys: %w[nested]) }
 
     context "when it is a first stats emit" do
-      subject(:decorated) { decorator.call(emited_stats1) }
+      subject(:decorated) { decorator.call(emitted_stats1) }
 
       it { assert_equal 0, decorated["float_d"] }
       it { assert_equal 0, decorated["int_d"] }
@@ -222,8 +222,8 @@ describe_current do
 
     context "when it is a second stats emit" do
       subject(:decorated) do
-        decorator.call(emited_stats1)
-        decorator.call(emited_stats2)
+        decorator.call(emitted_stats1)
+        decorator.call(emitted_stats2)
       end
 
       it { assert_in_delta(0.4, decorated["float_d"].round(10)) }
@@ -241,8 +241,8 @@ describe_current do
     let(:decorator) { described_class.new(excluded_keys: %w[int]) }
 
     subject(:decorated) do
-      decorator.call(emited_stats1)
-      decorator.call(emited_stats2)
+      decorator.call(emitted_stats1)
+      decorator.call(emitted_stats2)
     end
 
     it { assert_in_delta(0.4, decorated["float_d"].round(10)) }
@@ -256,23 +256,242 @@ describe_current do
     let(:decorator) { described_class.new }
 
     subject(:decorated) do
-      decorator.call(emited_stats1)
-      decorator.call(emited_stats2)
+      decorator.call(emitted_stats1)
+      decorator.call(emitted_stats2)
     end
 
     it { assert_equal 18, decorated["int_d"] }
     it { assert_equal 30, decorated.dig(*broker_scope)["txbytes_d"] }
   end
 
+  context "when only_keys are configured" do
+    let(:decorator) { described_class.new(only_keys: %w[int]) }
+
+    context "when it is a first stats emit" do
+      subject(:decorated) { decorator.call(emitted_stats1) }
+
+      it { assert_equal 0, decorated["int_d"] }
+      it { assert_predicate decorated, :frozen? }
+
+      it "does not decorate non-listed numeric keys" do
+        refute decorated.key?("float_d")
+        refute decorated.key?("float_fd")
+      end
+
+      it "preserves non-listed numeric values" do
+        assert_in_delta 10.4, decorated["float"]
+      end
+
+      it "preserves string values" do
+        assert_equal "value1", decorated["string"]
+      end
+    end
+
+    context "when it is a second stats emit" do
+      subject(:decorated) do
+        decorator.call(emitted_stats1)
+        decorator.call(emitted_stats2)
+      end
+
+      it { assert_equal 18, decorated["int_d"] }
+      it { assert_predicate decorated, :frozen? }
+
+      it "does not decorate non-listed numeric keys" do
+        refute decorated.key?("float_d")
+        refute decorated.key?("float_fd")
+      end
+    end
+
+    context "when only_keys targets a nested value" do
+      let(:decorator) { described_class.new(only_keys: %w[txbytes]) }
+
+      subject(:decorated) do
+        decorator.call(emitted_stats1)
+        decorator.call(emitted_stats2)
+      end
+
+      it "decorates matching keys in nested hashes" do
+        assert_equal 30, decorated.dig(*broker_scope)["txbytes_d"]
+      end
+
+      it "does not decorate non-listed root keys" do
+        refute decorated.key?("int_d")
+        refute decorated.key?("float_d")
+      end
+    end
+  end
+
+  context "when only_keys and excluded_keys are combined" do
+    let(:decorator) { described_class.new(excluded_keys: %w[nested], only_keys: %w[int]) }
+
+    subject(:decorated) do
+      decorator.call(emitted_stats1)
+      decorator.call(emitted_stats2)
+    end
+
+    it { assert_equal 18, decorated["int_d"] }
+    it { assert_predicate decorated, :frozen? }
+
+    it "does not decorate non-listed keys" do
+      refute decorated.key?("float_d")
+    end
+
+    it "does not recurse into excluded subtrees" do
+      refute decorated.dig(*broker_scope).key?("txbytes_d")
+    end
+  end
+
+  context "when only_keys value remains unchanged over time" do
+    subject(:decorated) do
+      decorator.call(deep_copy.call)
+      decorator.call(deep_copy.call)
+      sleep(0.01)
+      decorator.call(deep_copy.call)
+    end
+
+    let(:decorator) { described_class.new(only_keys: %w[int]) }
+    let(:deep_copy) { -> { Marshal.load(Marshal.dump(emitted_stats1)) } }
+
+    it { assert_equal 0, decorated["int_d"] }
+    it { assert_in_delta 10, decorated["int_fd"], 5 }
+
+    it "does not decorate non-listed keys" do
+      refute decorated.key?("float_d")
+      refute decorated.key?("float_fd")
+    end
+  end
+
+  context "when only_keys is used with librdkafka-like structure" do
+    let(:decorator) { described_class.new(only_keys: %w[rxmsgs txbytes consumer_lag]) }
+
+    let(:rdkafka_stats1) do
+      {
+        "rxmsgs" => 100,
+        "name" => "consumer-1",
+        "brokers" => {
+          "localhost:9092/0" => {
+            "txbytes" => 500,
+            "rxbytes" => 300,
+            "nodeid" => 0
+          }
+        },
+        "topics" => {
+          "events" => {
+            "age" => 1000,
+            "partitions" => {
+              "0" => {
+                "consumer_lag" => 50,
+                "hi_offset" => 1000,
+                "txmsgs" => 200
+              },
+              "1" => {
+                "consumer_lag" => 30,
+                "hi_offset" => 800,
+                "txmsgs" => 150
+              }
+            }
+          }
+        },
+        "cgrp" => {
+          "stateage" => 5000,
+          "rebalance_cnt" => 2
+        }
+      }
+    end
+
+    let(:rdkafka_stats2) do
+      {
+        "rxmsgs" => 150,
+        "name" => "consumer-1",
+        "brokers" => {
+          "localhost:9092/0" => {
+            "txbytes" => 700,
+            "rxbytes" => 500,
+            "nodeid" => 0
+          }
+        },
+        "topics" => {
+          "events" => {
+            "age" => 2000,
+            "partitions" => {
+              "0" => {
+                "consumer_lag" => 40,
+                "hi_offset" => 1100,
+                "txmsgs" => 250
+              },
+              "1" => {
+                "consumer_lag" => 25,
+                "hi_offset" => 900,
+                "txmsgs" => 200
+              }
+            }
+          }
+        },
+        "cgrp" => {
+          "stateage" => 6000,
+          "rebalance_cnt" => 2
+        }
+      }
+    end
+
+    subject(:decorated) do
+      decorator.call(rdkafka_stats1)
+      decorator.call(rdkafka_stats2)
+    end
+
+    it "decorates root only_keys" do
+      assert_equal 50, decorated["rxmsgs_d"]
+    end
+
+    it "does not decorate non-listed root keys" do
+      refute decorated.key?("name_d")
+    end
+
+    it "decorates broker only_keys" do
+      assert_equal 200, decorated["brokers"]["localhost:9092/0"]["txbytes_d"]
+    end
+
+    it "does not decorate non-listed broker keys" do
+      refute decorated["brokers"]["localhost:9092/0"].key?("rxbytes_d")
+    end
+
+    it "decorates partition only_keys" do
+      assert_equal(-10, decorated.dig("topics", "events", "partitions", "0")["consumer_lag_d"])
+      assert_equal(-5, decorated.dig("topics", "events", "partitions", "1")["consumer_lag_d"])
+    end
+
+    it "does not decorate non-listed partition keys" do
+      refute decorated.dig("topics", "events", "partitions", "0").key?("txmsgs_d")
+      refute decorated.dig("topics", "events", "partitions", "0").key?("hi_offset_d")
+    end
+
+    it "does not decorate non-listed topic keys" do
+      refute decorated.dig("topics", "events").key?("age_d")
+    end
+
+    it { assert_predicate decorated, :frozen? }
+  end
+
+  context "when only_keys previous value type changed to non-numeric" do
+    let(:decorator) { described_class.new(only_keys: %w[val]) }
+
+    subject(:decorated) do
+      decorator.call({ "val" => "not_numeric" })
+      decorator.call({ "val" => 10 })
+    end
+
+    it { refute decorated.key?("val_d") }
+  end
+
   context "when a value type changed from numeric to non-numeric between emissions" do
     subject(:decorated) do
-      decorator.call(emited_stats1)
-      decorator.call(emited_stats2)
+      decorator.call(emitted_stats1)
+      decorator.call(emitted_stats2)
     end
 
     before do
       # In the second emission, txbytes changed to a string
-      emited_stats2["nested"]["brokers"]["localhost:9092/2"]["txbytes"] = "not_a_number"
+      emitted_stats2["nested"]["brokers"]["localhost:9092/2"]["txbytes"] = "not_a_number"
     end
 
     # Non-numeric values are never decorated

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-core
 version: !ruby/object:Gem::Version
-  version: 2.5.10
+  version: 2.5.11
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -144,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Karafka ecosystem core modules
 test_files: []