karafka 2.2.6 → 2.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36fe50a9b0a758b9b916ca668c862085629bf09bcf16f7c0e8a340a9b7333abb
-  data.tar.gz: 61d3f10df4e1545a1a4e0d8190eb6ee2a7b751636bb347cf2703a74149fc710c
+  metadata.gz: b6a42d67752052bcf8f29fb683ed8ffb5fce7675c155b73269c4654681913f7b
+  data.tar.gz: b1bfc5bdad87c27111d8aee50302593133460f0eac47161f3c3872dc726d3b68
 SHA512:
-  metadata.gz: f12fb91f2e8e4e7dba766fbf32c79b734f5cd9f6fa63a66f80863a819dfd6d442024a97a4095bea84ef87bb0cb645ff431ba6d2f04f46d72864c377d61fc76a6
-  data.tar.gz: 4235c1944c98f0bd6359b82b7eab68a69812d70665ab0436c9f67cc5011e4c8edfa2c194a6d65744a4d82f97710beca5ddb871b8671febf61e5c9f591a746bee
+  metadata.gz: b1e46475db36dc2fc837aa68b1639ad43b94a59afc72dd7287c60fac5f504f55c1ed68f9a1527a4eb1f6743801d38ba44ac6eef6c9d69a6b2ae4e4fbf3034b98
+  data.tar.gz: 329fe59c9bbede3367c9bb3624c69652111f9387360d28b8d9e83bb0a184ea0e524d87d3f47afc400446f48993a57a41c4c171ae5e16c5413c7a496e88fc0421

checksums.yaml.gz.sig

@@ -1 +1,4 @@
-��ฟ�R�^Ա�<���q��~	�r�N]C4	��j-8�^���K�4�h�K���g�y���`�������	�׿�G�ejOSz����H�Z�w��*�O�����r��rߣ.<:����3Ń���3i1�����8E��)������T���8.�
+mu
+j.B�7�܆�����mT�c�=k�Q����o�ֽ�E"������V�
+Y���n8v��z��b��E��б�f378��X<����
+���]\C����/� �pcyb�O���v8�
�9��IG"�V!v_T.

data/.github/workflows/ci.yml

@@ -71,6 +71,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           # We run it against the oldest and the newest of a given major to make sure, that there
           # are no syntax-sugars that we would use that were introduced down the road
@@ -115,6 +116,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           - '3.1'
           - '3.0'
@@ -166,6 +168,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           - '3.1'
           - '3.0'

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Karafka framework changelog
 
+## 2.2.7 (2023-10-07)
+- **[Feature]** Introduce Inline Insights to both OSS and Pro. Inline Insights allow you to get the Kafka insights/metrics from the consumer instance and use them to alter the processing flow. In Pro, there's an extra filter flow allowing to ensure, that the insights exist during consumption.
+- [Enhancement] Make sure, that subscription groups ids are unique by including their consumer group id in them similar to how topics ids are handled (not a breaking change).
+- [Enhancement] Expose `#attempt` method on a consumer to directly indicate number of attempt of processing given data.
+- [Enhancement] Support Rails 7.1.
+
 ## 2.2.6 (2023-09-26)
 - [Enhancement] Retry `Karafka::Admin#read_watermark_offsets` fetching upon `not_leader_for_partition` that can occur mostly on newly created topics in KRaft and after crashes during leader selection.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.2.6)
+    karafka (2.2.7)
       karafka-core (>= 2.2.2, < 2.3.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)
@@ -37,9 +37,9 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.7.4)
+    karafka-web (0.7.5)
       erubi (~> 1.4)
-      karafka (>= 2.2.3, < 3.0.0)
+      karafka (>= 2.2.6, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -75,7 +75,7 @@ GEM
     waterdrop (2.6.7)
       karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.11)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   x86_64-linux

data/config/locales/errors.yml

@@ -16,9 +16,11 @@ en:
       max_wait_time_format: needs to be an integer bigger than 0
       kafka_format: needs to be a filled hash
       internal.processing.jobs_builder_format: cannot be nil
-      internal.processing.scheduler: cannot be nil
-      internal.processing.coordinator_class: cannot be nil
-      internal.processing.partitioner_class: cannot be nil
+      internal.processing.scheduler_format: cannot be nil
+      internal.processing.coordinator_class_format: cannot be nil
+      internal.processing.partitioner_class_format: cannot be nil
+      internal.processing.strategy_selector_format: cannot be nil
+      internal.processing.expansions_selector_format: cannot be nil
       internal.active_job.dispatcher: cannot be nil
       internal.active_job.job_options_contract: cannot be nil
       internal.active_job.consumer_class: cannot be nil
@@ -59,8 +61,9 @@ en:
       initial_offset_format: needs to be either earliest or latest
       subscription_group_name_format: must be a non-empty string
       manual_offset_management.active_format: needs to be either true or false
-      consumer_active_job_missing: ActiveJob needs to be available
       manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics
+      inline_insights.active_format: needs to be either true or false
+      consumer_active_job_missing: ActiveJob needs to be available
       dead_letter_queue.max_retries_format: needs to be equal or bigger than 0
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false

data/config/locales/pro_errors.yml

@@ -31,6 +31,9 @@ en:
       patterns.active_format: 'needs to be boolean'
       patterns.type_format: 'needs to be :matcher, :discovered or :regular'
 
+      inline_insights.active_format: 'needs to be boolean'
+      inline_insights.required_format: 'needs to be boolean'
+
     consumer_group:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present

data/lib/karafka/base_consumer.rb

@@ -252,7 +252,13 @@ module Karafka
     #   different flow after there is an error, for example for resources cleanup, small manual
     #   backoff or different instrumentation tracking.
     def retrying?
-      coordinator.pause_tracker.attempt > 1
+      attempt > 1
+    end
+
+    # @return [Integer] attempt of processing given batch. 1 if this is the first attempt or higher
+    #  in case it is a retry
+    def attempt
+      coordinator.pause_tracker.attempt
     end
 
     # Pauses the processing from the last offset to retry on given message

data/lib/karafka/connection/listener.rb

@@ -112,7 +112,7 @@ module Karafka
           # distributing consuming jobs as upon revoking, we might get assigned to the same
           # partitions, thus getting their jobs. The revoking jobs need to finish before
           # appropriate consumers are taken down and re-created
-          build_and_schedule_revoke_lost_partitions_jobs
+          build_and_schedule_revoked_jobs_for_revoked_partitions
 
           # We wait only on jobs from our subscription group. Other groups are independent.
           # This will block on revoked jobs until they are finished. Those are not meant to last
@@ -140,7 +140,7 @@ module Karafka
         # that occurred in the cluster.
         wait_pinging(
           wait_until: -> { @jobs_queue.empty?(@subscription_group.id) },
-          after_ping: -> { build_and_schedule_revoke_lost_partitions_jobs }
+          after_ping: -> { build_and_schedule_revoked_jobs_for_revoked_partitions }
         )
 
         # We do not want to schedule the shutdown jobs prior to finishing all the jobs
@@ -197,7 +197,7 @@ module Karafka
       end
 
       # Enqueues revoking jobs for partitions that were taken away from the running process.
-      def build_and_schedule_revoke_lost_partitions_jobs
+      def build_and_schedule_revoked_jobs_for_revoked_partitions
         revoked_partitions = @client.rebalance_manager.revoked_partitions
 
         # Stop early to save on some execution and array allocation

data/lib/karafka/contracts/config.rb

@@ -73,6 +73,8 @@ module Karafka
           required(:scheduler) { |val| !val.nil? }
           required(:coordinator_class) { |val| !val.nil? }
           required(:partitioner_class) { |val| !val.nil? }
+          required(:strategy_selector) { |val| !val.nil? }
+          required(:expansions_selector) { |val| !val.nil? }
         end
 
         nested(:active_job) do

data/lib/karafka/pro/loader.rb

@@ -49,7 +49,7 @@ module Karafka
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
-        def pre_setup(config)
+        def pre_setup_all(config)
           features.each { |feature| feature.pre_setup(config) }
 
           reconfigure(config)
@@ -60,7 +60,7 @@ module Karafka
         # Runs post setup features configuration operations
         #
         # @param config [Karafka::Core::Configurable::Node]
-        def post_setup(config)
+        def post_setup_all(config)
           features.each { |feature| feature.post_setup(config) }
         end
 

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -24,7 +24,7 @@ module Karafka
             @delay = delay
           end
 
-          # Removes too old messages
+          # Removes too young messages
           #
           # @param messages [Array<Karafka::Messages::Message>]
           def apply!(messages)

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Delayer that checks if we have appropriate insights available. If not, pauses for
+        # 5 seconds so the insights can be loaded from the broker.
+        #
+        # In case it would take more than five seconds to load insights, it will just pause again
+        #
+        # This filter ensures, that we always have inline insights that a consumer can use
+        #
+        # It is relevant in most cases only during the process start, when first poll may not
+        # yield statistics yet but will give some data.
+        class InlineInsightsDelayer < Base
+          # Minimum how long should we pause when there are no metrics
+          PAUSE_TIMEOUT = 5_000
+
+          private_constant :PAUSE_TIMEOUT
+
+          # @param topic [Karafka::Routing::Topic]
+          # @param partition [Integer] partition
+          def initialize(topic, partition)
+            super()
+            @topic = topic
+            @partition = partition
+          end
+
+          # Pauses if inline insights would not be available. Does nothing otherwise
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+            @cursor = messages.first
+
+            # Nothing to do if there were no messages
+            # This can happen when we chain filters
+            return unless @cursor
+
+            insights = ::Karafka::Processing::InlineInsights::Tracker.find(
+              @topic,
+              @partition
+            )
+
+            # If insights are available, also nothing to do here and we can just process
+            return unless insights.empty?
+
+            messages.clear
+
+            @applied = true
+          end
+
+          # @return [Integer] ms timeout in case of pause
+          def timeout
+            @cursor && applied? ? PAUSE_TIMEOUT : 0
+          end
+
+          # Pause when we had to back-off or skip if delay is not needed
+          def action
+            applied? ? :pause : :skip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/config.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Config of this feature
+          Config = Struct.new(
+            :active,
+            :required,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :required?, :required
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Inline Insights related contracts namespace
+          module Contracts
+            # Contract for inline insights topic setup
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested :inline_insights do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:required) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/topic.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Routing topic inline insights API
+          module Topic
+            # @param active [Boolean] should inline insights be activated
+            # @param required [Boolean] are the insights required to operate
+            def inline_insights(active = -1, required: -1)
+              # This weird style of checking allows us to activate inline insights in few ways:
+              #   - inline_insights(true)
+              #   - inline_insights(required: true)
+              #   - inline_insights(required: false)
+              #
+              # In each of those cases inline insights will become active
+              @inline_insights ||= begin
+                config = Config.new(
+                  active: active == true || (active == -1 && required != -1),
+                  required: required == true
+                )
+
+                if config.active? && config.required?
+                  factory = lambda do |topic, partition|
+                    Pro::Processing::Filters::InlineInsightsDelayer.new(topic, partition)
+                  end
+
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Enhanced inline insights
+        # Allows you to set up an automatic filter that will ensure, that metrics are always
+        # available when processing starts.
+        class InlineInsights < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -131,11 +131,18 @@ module Karafka
           topic = @coordinator.topic
 
           strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(topic)
+          expansions = ::Karafka::App.config.internal.processing.expansions_selector.find(topic)
 
           consumer = topic.consumer_class.new
           # We use singleton class as the same consumer class may be used to process different
           # topics with different settings
           consumer.singleton_class.include(strategy)
+
+          # Specific features may expand consumer API beyond the injected strategy. The difference
+          # here is that strategy impacts the flow of states while extra APIs just provide some
+          # extra methods with informations, etc but do no deviate the flow behavior
+          expansions.each { |expansion| consumer.singleton_class.include(expansion) }
+
           consumer.client = @client
           consumer.producer = ::Karafka::App.producer
           consumer.coordinator = @coordinator

data/lib/karafka/processing/expansions_selector.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Selector of appropriate topic setup based features enhancements.
+    #
+    # Those expansions to the consumer API are NOT about the flow of processing. For this we have
+    # strategies. Those are suppose to provide certain extra APIs that user can use to get some
+    # extra non-flow related functionalities.
+    class ExpansionsSelector
+      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+      #   expansions
+      # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance the
+      #   consumer
+      def find(topic)
+        expansions = []
+        expansions << Processing::InlineInsights::Consumer if topic.inline_insights?
+        expansions
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/consumer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Namespace of the Inline Insights feature "non routing" related components
+    #
+    # @note We use both `#insights` because it is the feature name but also `#statistics` to make
+    #   it consistent with the fact that we publish and operate on statistics. User can pick
+    #   whichever name they prefer.
+    module InlineInsights
+      # Module that adds extra methods to the consumer that allow us to fetch the insights
+      module Consumer
+        # @return [Hash] empty hash or hash with given partition insights if already present
+        # @note We cache insights on the consumer, as in some scenarios we may no longer have them
+        #   inside the Tracker, for example under involuntary revocation, incoming statistics may
+        #   no longer have lost partition insights. Since we want to be consistent during single
+        #   batch operations, we want to ensure, that if we have insights they are available
+        #   throughout the whole processing.
+        def insights
+          insights = Tracker.find(topic, partition)
+
+          # If we no longer have new insights but we still have them locally, we can use them
+          return @insights if @insights && insights.empty?
+          # If insights are still the same, we can use them
+          return @insights if @insights.equal?(insights)
+
+          # If we've received new insights that are not empty, we can cache them
+          @insights = insights
+        end
+
+        # @return [Boolean] true if there are insights to work with, otherwise false
+        def insights?
+          !insights.empty?
+        end
+
+        alias statistics insights
+        alias statistics? insights?
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/listener.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module InlineInsights
+      # Listener that adds statistics to our inline tracker
+      class Listener
+        # Adds statistics to the tracker
+        # @param event [Karafka::Core::Monitoring::Event] event with statistics
+        def on_statistics_emitted(event)
+          Tracker.add(
+            event[:consumer_group_id],
+            event[:statistics]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/tracker.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module InlineInsights
+      # Object used to track statistics coming from librdkafka in a way that can be accessible by
+      # the consumers
+      #
+      # We use a single tracker because we do not need state management here as our consumer groups
+      # clients identified by statistics name value are unique. On top of that, having a per
+      # process one that is a singleton allows us to use tracker easily also from other places like
+      # filtering API etc.
+      #
+      # @note We include cache of 5 minutes for revoked partitions to compensate for cases where
+      #   when using LRJ a lost partition data would not be present anymore, however we would still
+      #   be in the processing phase. Since those metrics are published with each `poll`, regular
+      #   processing is not a subject of this issue. For LRJ we keep the reference. The only case
+      #   where this could be switched midway is when LRJ is running for an extended period of time
+      #   after the involuntary revocation. Having a time based cache instead of tracking
+      #   simplifies the design as we do not have to deal with state tracking, especially since
+      #   we would have to track also operations running in a revoked state.
+      #
+      # @note This tracker keeps in memory data about all topics and partitions that it encounters
+      #   because in case of routing patterns, we may start getting statistics prior to registering
+      #   given topic via dynamic routing expansions. In such case we would not have insights
+      #   where they were actually available for us to use.
+      #
+      # @note Memory usage is negligible as long as we can evict expired data. Single metrics set
+      #   for a single partition contains around 4KB of data. This means, that in case of an
+      #   assignment of 1000 partitions, we use around 4MB of space for tracking those metrics.
+      class Tracker
+        include Singleton
+        include Karafka::Core::Helpers::Time
+
+        # Empty hash we want to return in any case where we could not locate appropriate topic
+        # partition statistics.
+        EMPTY_HASH = {}.freeze
+
+        # Empty array to save on memory allocations.
+        EMPTY_ARRAY = [].freeze
+
+        # 5 minutes of cache. We cache last result per consumer group topic partition so we are
+        # not affected by involuntary rebalances during LRJ execution.
+        TTL = 5 * 60 * 1_000
+
+        private_constant :EMPTY_HASH, :EMPTY_ARRAY, :TTL
+
+        class << self
+          extend Forwardable
+
+          def_delegators :instance, :find, :add, :exists?, :clear
+        end
+
+        def initialize
+          @accu = {}
+          @mutex = Mutex.new
+        end
+
+        # Adds each partition statistics into internal accumulator. Single statistics set may
+        # contain data from multiple topics and their partitions because a single client can
+        # operate on multiple topics and partitions.
+        #
+        # We iterate over those topics and partitions and store topics partitions data only.
+        #
+        # @param consumer_group_id [String] id of the consumer group for which statistics were
+        #   emitted.
+        # @param statistics [Hash] librdkafka enriched statistics
+        def add(consumer_group_id, statistics)
+          @mutex.synchronize do
+            statistics.fetch('topics', EMPTY_HASH).each do |topic_name, t_details|
+              t_details.fetch('partitions', EMPTY_HASH).each do |partition_id, p_details|
+                next unless track?(partition_id, p_details)
+
+                key = "#{consumer_group_id}_#{topic_name}_#{partition_id}"
+                @accu[key] = [monotonic_now, p_details]
+              end
+            end
+
+            evict
+          end
+        end
+
+        # Finds statistics about requested consumer group topic partition
+        #
+        # @param topic [Karafka::Routing::Topic]
+        # @param partition [Integer]
+        # @return [Hash] hash with given topic partition statistics or empty hash if not present
+        #
+        # @note We do not enclose it with a mutex mainly because the only thing that could happen
+        #   here that would be a race-condition is a miss that anyhow we need to support due to
+        #   how librdkafka ships metrics and a potential removal of data on heavily revoked LRJ.
+        def find(topic, partition)
+          key = "#{topic.consumer_group.id}_#{topic.name}_#{partition}"
+          @accu.fetch(key, EMPTY_ARRAY).last || EMPTY_HASH
+        end
+
+        # Clears the tracker
+        def clear
+          @mutex.synchronize { @accu.clear }
+        end
+
+        private
+
+        # Evicts expired data from the cache
+        def evict
+          @accu.delete_if { |_, details| monotonic_now - details.first > TTL }
+        end
+
+        # Should we track given partition
+        #
+        # We do not track stopped partitions and the once we do not work with actively
+        # @param partition_id [String] partition id as a string
+        # @param p_details [Hash] partition statistics details
+        # @return [Boolean] true if we should track given partition
+        def track?(partition_id, p_details)
+          return false if partition_id == '-1'
+
+          fetch_state = p_details.fetch('fetch_state')
+
+          return false if fetch_state == 'stopped'
+          return false if fetch_state == 'none'
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/railtie.rb

@@ -33,7 +33,7 @@ if Karafka.rails?
       # server + will support code reloading with each fetched loop. We do it only for karafka
       # based commands as Rails processes and console will have it enabled already
       initializer 'karafka.configure_rails_logger' do
-        # Make Karafka use Rails logger
+        # Make Karafka uses Rails logger
         ::Karafka::App.config.logger = Rails.logger
 
         next unless Rails.env.development?
@@ -42,15 +42,22 @@ if Karafka.rails?
         # If added again, would print stuff twice
         next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
 
-        logger = ActiveSupport::Logger.new($stdout)
+        stdout_logger = ActiveSupport::Logger.new($stdout)
         # Inherit the logger level from Rails, otherwise would always run with the debug level
-        logger.level = Rails.logger.level
+        stdout_logger.level = Rails.logger.level
 
-        Rails.logger.extend(
-          ActiveSupport::Logger.broadcast(
-            logger
+        rails71plus = Rails.gem_version >= Gem::Version.new('7.1.0')
+
+        # Rails 7.1 replaced the broadcast module with a broadcast logger
+        if rails71plus
+          Rails.logger.broadcast_to(stdout_logger)
+        else
+          Rails.logger.extend(
+            ActiveSupport::Logger.broadcast(
+              stdout_logger
+            )
           )
-        )
+        end
       end
 
       initializer 'karafka.configure_rails_auto_load_paths' do |app|

data/lib/karafka/routing/features/base.rb

@@ -23,11 +23,46 @@ module Karafka
 
           # Loads all the features and activates them
           def load_all
+            features.each(&:activate)
+          end
+
+          # @param config [Karafka::Core::Configurable::Node] app config that we can alter with
+          #   particular routing feature specific stuff if needed
+          def pre_setup_all(config)
+            features.each { |feature| feature.pre_setup(config) }
+          end
+
+          # Runs post setup routing features configuration operations
+          #
+          # @param config [Karafka::Core::Configurable::Node]
+          def post_setup_all(config)
+            features.each { |feature| feature.post_setup(config) }
+          end
+
+          private
+
+          # @return [Array<Class>] all available routing features
+          def features
             ObjectSpace
               .each_object(Class)
               .select { |klass| klass < self }
               .sort_by(&:to_s)
-              .each(&:activate)
+          end
+
+          protected
+
+          # Runs pre-setup configuration of a particular routing feature
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config node
+          def pre_setup(_config)
+            true
+          end
+
+          # Runs post-setup configuration of a particular routing feature
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config node
+          def post_setup(_config)
+            true
           end
         end
       end

data/lib/karafka/routing/features/inline_insights/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Config of this feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Inline Insights related contracts namespace
+        module Contracts
+          # Contract for inline insights topic setup
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :inline_insights do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights/topic.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Routing topic inline insights API
+        module Topic
+          # @param active [Boolean] should inline insights be activated
+          def inline_insights(active = false)
+            @inline_insights ||= Config.new(
+              active: active
+            )
+          end
+
+          # @return [Boolean] Are inline insights active
+          def inline_insights?
+            inline_insights.active?
+          end
+
+          # @return [Hash] topic setup hash
+          def to_h
+            super.merge(
+              inline_insights: inline_insights.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Feature allowing us to get visibility during the consumption into metrics of particular
+      # partition we operate on. It can be useful when making context-aware consumers that change
+      # their behaviours based on the lag and other parameters.
+      class InlineInsights < Base
+        class << self
+          # If needed installs the needed listener and initializes tracker
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config
+          def post_setup(_config)
+            ::Karafka::App.monitor.subscribe('app.running') do
+              # Do not activate tracking of statistics if none of our active topics uses it
+              # This prevents us from tracking metrics when user just runs a subset of topics
+              # in a given process and none of those actually utilizes this feature
+              next unless ::Karafka::App
+                          .subscription_groups
+                          .values
+                          .flat_map(&:itself)
+                          .flat_map(&:topics)
+                          .flat_map(&:to_a)
+                          .any?(&:inline_insights?)
+
+              # Initialize the tracker prior to becoming multi-threaded
+              ::Karafka::Processing::InlineInsights::Tracker.instance
+
+              # Subscribe to the statistics reports and collect them
+              ::Karafka.monitor.subscribe(
+                ::Karafka::Processing::InlineInsights::Listener.new
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/subscription_group.rb

@@ -34,7 +34,11 @@ module Karafka
       def initialize(position, topics)
         @name = topics.first.subscription_group_name
         @consumer_group = topics.first.consumer_group
-        @id = "#{@name}_#{position}"
+        # We include the consumer group id here because we want to have unique ids of subscription
+        # groups across the system. Otherwise user could set the same name for multiple
+        # subscription groups in many consumer groups effectively having same id for different
+        # entities
+        @id = "#{@consumer_group.id}_#{@name}_#{position}"
         @position = position
         @topics = topics
         @kafka = build_kafka
@@ -87,7 +91,7 @@ module Karafka
         # If we use static group memberships, there can be a case, where same instance id would
         # be set on many subscription groups as the group instance id from Karafka perspective is
         # set per config. Each instance even if they are subscribed to different topics needs to
-        # have if fully unique. To make sure of that, we just add extra postfix at the end that
+        # have it fully unique. To make sure of that, we just add extra postfix at the end that
         # increments.
         group_instance_id = kafka.fetch(:'group.instance.id', false)
 

data/lib/karafka/setup/attributes_map.rb

@@ -221,6 +221,7 @@ module Karafka
         request.timeout.ms
         resolve_cb
         retries
+        retry.backoff.max.ms
         retry.backoff.ms
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd

data/lib/karafka/setup/config.rb

@@ -208,6 +208,8 @@ module Karafka
           setting :partitioner_class, default: Processing::Partitioner
           # option strategy_selector [Object] processing strategy selector to be used
           setting :strategy_selector, default: Processing::StrategySelector.new
+          # option expansions_selector [Object] processing expansions selector to be used
+          setting :expansions_selector, default: Processing::ExpansionsSelector.new
         end
 
         # Things related to operating on messages
@@ -238,10 +240,14 @@ module Karafka
         def setup(&block)
           # Will prepare and verify license if present
           Licenser.prepare_and_verify(config.license)
+
+          # Pre-setup configure all routing features that would need this
+          Routing::Features::Base.pre_setup_all(config)
+
           # Will configure all the pro components
           # This needs to happen before end user configuration as the end user may overwrite some
           # of the pro defaults with custom components
-          Pro::Loader.pre_setup(config) if Karafka.pro?
+          Pro::Loader.pre_setup_all(config) if Karafka.pro?
 
           configure(&block)
           merge_kafka_defaults!(config)
@@ -253,9 +259,12 @@ module Karafka
           # Refreshes the references that are cached that might have been changed by the config
           ::Karafka.refresh!
 
+          # Post-setup configure all routing features that would need this
+          Routing::Features::Base.post_setup_all(config)
+
           # Runs things that need to be executed after config is defined and all the components
           # are also configured
-          Pro::Loader.post_setup(config) if Karafka.pro?
+          Pro::Loader.post_setup_all(config) if Karafka.pro?
 
           Karafka::App.initialized!
         end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.6'
+  VERSION = '2.2.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.2.6
+  version: 2.2.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-26 00:00:00.000000000 Z
+date: 2023-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -244,6 +244,7 @@ files:
 - lib/karafka/pro/processing/filters/base.rb
 - lib/karafka/pro/processing/filters/delayer.rb
 - lib/karafka/pro/processing/filters/expirer.rb
+- lib/karafka/pro/processing/filters/inline_insights_delayer.rb
 - lib/karafka/pro/processing/filters/throttler.rb
 - lib/karafka/pro/processing/filters/virtual_limiter.rb
 - lib/karafka/pro/processing/filters_applier.rb
@@ -321,6 +322,10 @@ files:
 - lib/karafka/pro/routing/features/filtering/config.rb
 - lib/karafka/pro/routing/features/filtering/contracts/topic.rb
 - lib/karafka/pro/routing/features/filtering/topic.rb
+- lib/karafka/pro/routing/features/inline_insights.rb
+- lib/karafka/pro/routing/features/inline_insights/config.rb
+- lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb
+- lib/karafka/pro/routing/features/inline_insights/topic.rb
 - lib/karafka/pro/routing/features/long_running_job.rb
 - lib/karafka/pro/routing/features/long_running_job/config.rb
 - lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb
@@ -353,6 +358,10 @@ files:
 - lib/karafka/processing/coordinators_buffer.rb
 - lib/karafka/processing/executor.rb
 - lib/karafka/processing/executors_buffer.rb
+- lib/karafka/processing/expansions_selector.rb
+- lib/karafka/processing/inline_insights/consumer.rb
+- lib/karafka/processing/inline_insights/listener.rb
+- lib/karafka/processing/inline_insights/tracker.rb
 - lib/karafka/processing/jobs/base.rb
 - lib/karafka/processing/jobs/consume.rb
 - lib/karafka/processing/jobs/idle.rb
@@ -394,6 +403,10 @@ files:
 - lib/karafka/routing/features/declaratives/config.rb
 - lib/karafka/routing/features/declaratives/contracts/topic.rb
 - lib/karafka/routing/features/declaratives/topic.rb
+- lib/karafka/routing/features/inline_insights.rb
+- lib/karafka/routing/features/inline_insights/config.rb
+- lib/karafka/routing/features/inline_insights/contracts/topic.rb
+- lib/karafka/routing/features/inline_insights/topic.rb
 - lib/karafka/routing/features/manual_offset_management.rb
 - lib/karafka/routing/features/manual_offset_management/config.rb
 - lib/karafka/routing/features/manual_offset_management/contracts/topic.rb
@@ -447,7 +460,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.4.10
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.