karafka 2.2.5 → 2.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 729320b972ba55fe500db0729aa651c6308e1975db33d3ebd4db8061561e38bc
-  data.tar.gz: f26fa836af6b7777692c7fd36fad1c89ce22c7ab60ab243662df40993acd7bd4
+  metadata.gz: b6a42d67752052bcf8f29fb683ed8ffb5fce7675c155b73269c4654681913f7b
+  data.tar.gz: b1bfc5bdad87c27111d8aee50302593133460f0eac47161f3c3872dc726d3b68
 SHA512:
-  metadata.gz: 86cecfe51fbdb21943f75a98a1ee8e755496f9777bf5c4445ad44ecff45fa5be34fc79c0ad6358ded3886a59ebde4f0cd47598fdcdda41f75c5152489cf5ca35
-  data.tar.gz: fec15466a63fc6e49089f339d3f0d9bb3d4d308c7855adf17f23e93ef10e3e09daf6e8453296a53bf74a57d1004883fc2b144c4564342daec8b94b9a7288bd82
+  metadata.gz: b1e46475db36dc2fc837aa68b1639ad43b94a59afc72dd7287c60fac5f504f55c1ed68f9a1527a4eb1f6743801d38ba44ac6eef6c9d69a6b2ae4e4fbf3034b98
+  data.tar.gz: 329fe59c9bbede3367c9bb3624c69652111f9387360d28b8d9e83bb0a184ea0e524d87d3f47afc400446f48993a57a41c4c171ae5e16c5413c7a496e88fc0421

data/.github/workflows/ci.yml

@@ -71,6 +71,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           # We run it against the oldest and the newest of a given major to make sure, that there
           # are no syntax-sugars that we would use that were introduced down the road
@@ -115,6 +116,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           - '3.1'
           - '3.0'
@@ -166,6 +168,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           - '3.1'
           - '3.0'

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Karafka framework changelog
 
+## 2.2.7 (2023-10-07)
+- **[Feature]** Introduce Inline Insights to both OSS and Pro. Inline Insights allow you to get the Kafka insights/metrics from the consumer instance and use them to alter the processing flow. In Pro, there's an extra filter flow allowing to ensure, that the insights exist during consumption.
+- [Enhancement] Make sure, that subscription groups ids are unique by including their consumer group id in them similar to how topics ids are handled (not a breaking change).
+- [Enhancement] Expose `#attempt` method on a consumer to directly indicate number of attempt of processing given data.
+- [Enhancement] Support Rails 7.1.
+
+## 2.2.6 (2023-09-26)
+- [Enhancement] Retry `Karafka::Admin#read_watermark_offsets` fetching upon `not_leader_for_partition` that can occur mostly on newly created topics in KRaft and after crashes during leader selection.
+
 ## 2.2.5 (2023-09-25)
 - [Enhancement] Ensure, that when topic related operations end, the result is usable. There were few cases where admin operations on topics would finish successfully but internal Kafka caches would not report changes for a short period of time.
 - [Enhancement] Stabilize cooperative-sticky early shutdown procedure.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.2.5)
+    karafka (2.2.7)
       karafka-core (>= 2.2.2, < 2.3.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)
@@ -37,9 +37,9 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.7.4)
+    karafka-web (0.7.5)
       erubi (~> 1.4)
-      karafka (>= 2.2.3, < 3.0.0)
+      karafka (>= 2.2.6, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -75,7 +75,7 @@ GEM
     waterdrop (2.6.7)
       karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.11)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   x86_64-linux

data/config/locales/errors.yml

@@ -16,9 +16,11 @@ en:
       max_wait_time_format: needs to be an integer bigger than 0
       kafka_format: needs to be a filled hash
       internal.processing.jobs_builder_format: cannot be nil
-      internal.processing.scheduler: cannot be nil
-      internal.processing.coordinator_class: cannot be nil
-      internal.processing.partitioner_class: cannot be nil
+      internal.processing.scheduler_format: cannot be nil
+      internal.processing.coordinator_class_format: cannot be nil
+      internal.processing.partitioner_class_format: cannot be nil
+      internal.processing.strategy_selector_format: cannot be nil
+      internal.processing.expansions_selector_format: cannot be nil
       internal.active_job.dispatcher: cannot be nil
       internal.active_job.job_options_contract: cannot be nil
       internal.active_job.consumer_class: cannot be nil
@@ -59,8 +61,9 @@ en:
       initial_offset_format: needs to be either earliest or latest
       subscription_group_name_format: must be a non-empty string
       manual_offset_management.active_format: needs to be either true or false
-      consumer_active_job_missing: ActiveJob needs to be available
       manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics
+      inline_insights.active_format: needs to be either true or false
+      consumer_active_job_missing: ActiveJob needs to be available
       dead_letter_queue.max_retries_format: needs to be equal or bigger than 0
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false

data/config/locales/pro_errors.yml

@@ -31,6 +31,9 @@ en:
       patterns.active_format: 'needs to be boolean'
       patterns.type_format: 'needs to be :matcher, :discovered or :regular'
 
+      inline_insights.active_format: 'needs to be boolean'
+      inline_insights.required_format: 'needs to be boolean'
+
     consumer_group:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present

data/lib/karafka/admin.rb

@@ -142,7 +142,12 @@ module Karafka
       # @return [Array<Integer, Integer>] low watermark offset and high watermark offset
       def read_watermark_offsets(name, partition)
         with_consumer do |consumer|
-          consumer.query_watermark_offsets(name, partition)
+          # For newly created topics or in cases where we're trying to get them but there is no
+          # leader, this can fail. It happens more often for new topics under KRaft, however we
+          # still want to make sure things operate as expected even then
+          with_rdkafka_retry(codes: %i[not_leader_for_partition]) do
+            consumer.query_watermark_offsets(name, partition)
+          end
         end
       end
 
@@ -228,6 +233,31 @@ module Karafka
         raise
       end
 
+      # Handles retries for rdkafka related errors that we specify in `:codes`.
+      #
+      # Some operations temporarily fail, especially for cases where we changed something fast
+      # like topic creation or repartitioning. In cases like this it is ok to retry operations that
+      # do not change the state as it will usually recover.
+      #
+      # @param codes [Array<Symbol>] librdkafka error codes on which we want to retry
+      # @param max_attempts [Integer] number of attempts (including initial) after which we should
+      #   give up
+      #
+      # @note This code implements a simple backoff that increases with each attempt.
+      def with_rdkafka_retry(codes:, max_attempts: 5)
+        attempt ||= 0
+        attempt += 1
+
+        yield
+      rescue Rdkafka::RdkafkaError => e
+        raise unless codes.include?(e.code)
+        raise if attempt >= max_attempts
+
+        sleep(max_attempts)
+
+        retry
+      end
+
       # @param type [Symbol] type of config we want
       # @param settings [Hash] extra settings for config (if needed)
       # @return [::Rdkafka::Config] rdkafka config

data/lib/karafka/base_consumer.rb

@@ -252,7 +252,13 @@ module Karafka
     #   different flow after there is an error, for example for resources cleanup, small manual
     #   backoff or different instrumentation tracking.
     def retrying?
-      coordinator.pause_tracker.attempt > 1
+      attempt > 1
+    end
+
+    # @return [Integer] attempt of processing given batch. 1 if this is the first attempt or higher
+    #  in case it is a retry
+    def attempt
+      coordinator.pause_tracker.attempt
     end
 
     # Pauses the processing from the last offset to retry on given message

data/lib/karafka/connection/listener.rb

@@ -112,7 +112,7 @@ module Karafka
           # distributing consuming jobs as upon revoking, we might get assigned to the same
           # partitions, thus getting their jobs. The revoking jobs need to finish before
           # appropriate consumers are taken down and re-created
-          build_and_schedule_revoke_lost_partitions_jobs
+          build_and_schedule_revoked_jobs_for_revoked_partitions
 
           # We wait only on jobs from our subscription group. Other groups are independent.
           # This will block on revoked jobs until they are finished. Those are not meant to last
@@ -140,7 +140,7 @@ module Karafka
         # that occurred in the cluster.
         wait_pinging(
           wait_until: -> { @jobs_queue.empty?(@subscription_group.id) },
-          after_ping: -> { build_and_schedule_revoke_lost_partitions_jobs }
+          after_ping: -> { build_and_schedule_revoked_jobs_for_revoked_partitions }
         )
 
         # We do not want to schedule the shutdown jobs prior to finishing all the jobs
@@ -197,7 +197,7 @@ module Karafka
       end
 
       # Enqueues revoking jobs for partitions that were taken away from the running process.
-      def build_and_schedule_revoke_lost_partitions_jobs
+      def build_and_schedule_revoked_jobs_for_revoked_partitions
         revoked_partitions = @client.rebalance_manager.revoked_partitions
 
         # Stop early to save on some execution and array allocation

data/lib/karafka/contracts/config.rb

@@ -73,6 +73,8 @@ module Karafka
           required(:scheduler) { |val| !val.nil? }
           required(:coordinator_class) { |val| !val.nil? }
           required(:partitioner_class) { |val| !val.nil? }
+          required(:strategy_selector) { |val| !val.nil? }
+          required(:expansions_selector) { |val| !val.nil? }
         end
 
         nested(:active_job) do

data/lib/karafka/pro/loader.rb

@@ -49,7 +49,7 @@ module Karafka
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
-        def pre_setup(config)
+        def pre_setup_all(config)
           features.each { |feature| feature.pre_setup(config) }
 
           reconfigure(config)
@@ -60,7 +60,7 @@ module Karafka
         # Runs post setup features configuration operations
         #
         # @param config [Karafka::Core::Configurable::Node]
-        def post_setup(config)
+        def post_setup_all(config)
           features.each { |feature| feature.post_setup(config) }
         end
 

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -24,7 +24,7 @@ module Karafka
             @delay = delay
           end
 
-          # Removes too old messages
+          # Removes too young messages
           #
           # @param messages [Array<Karafka::Messages::Message>]
           def apply!(messages)

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Delayer that checks if we have appropriate insights available. If not, pauses for
+        # 5 seconds so the insights can be loaded from the broker.
+        #
+        # In case it would take more than five seconds to load insights, it will just pause again
+        #
+        # This filter ensures, that we always have inline insights that a consumer can use
+        #
+        # It is relevant in most cases only during the process start, when first poll may not
+        # yield statistics yet but will give some data.
+        class InlineInsightsDelayer < Base
+          # Minimum how long should we pause when there are no metrics
+          PAUSE_TIMEOUT = 5_000
+
+          private_constant :PAUSE_TIMEOUT
+
+          # @param topic [Karafka::Routing::Topic]
+          # @param partition [Integer] partition
+          def initialize(topic, partition)
+            super()
+            @topic = topic
+            @partition = partition
+          end
+
+          # Pauses if inline insights would not be available. Does nothing otherwise
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+            @cursor = messages.first
+
+            # Nothing to do if there were no messages
+            # This can happen when we chain filters
+            return unless @cursor
+
+            insights = ::Karafka::Processing::InlineInsights::Tracker.find(
+              @topic,
+              @partition
+            )
+
+            # If insights are available, also nothing to do here and we can just process
+            return unless insights.empty?
+
+            messages.clear
+
+            @applied = true
+          end
+
+          # @return [Integer] ms timeout in case of pause
+          def timeout
+            @cursor && applied? ? PAUSE_TIMEOUT : 0
+          end
+
+          # Pause when we had to back-off or skip if delay is not needed
+          def action
+            applied? ? :pause : :skip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/config.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Config of this feature
+          Config = Struct.new(
+            :active,
+            :required,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :required?, :required
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Inline Insights related contracts namespace
+          module Contracts
+            # Contract for inline insights topic setup
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested :inline_insights do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:required) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/topic.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Routing topic inline insights API
+          module Topic
+            # @param active [Boolean] should inline insights be activated
+            # @param required [Boolean] are the insights required to operate
+            def inline_insights(active = -1, required: -1)
+              # This weird style of checking allows us to activate inline insights in few ways:
+              #   - inline_insights(true)
+              #   - inline_insights(required: true)
+              #   - inline_insights(required: false)
+              #
+              # In each of those cases inline insights will become active
+              @inline_insights ||= begin
+                config = Config.new(
+                  active: active == true || (active == -1 && required != -1),
+                  required: required == true
+                )
+
+                if config.active? && config.required?
+                  factory = lambda do |topic, partition|
+                    Pro::Processing::Filters::InlineInsightsDelayer.new(topic, partition)
+                  end
+
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Enhanced inline insights
+        # Allows you to set up an automatic filter that will ensure, that metrics are always
+        # available when processing starts.
+        class InlineInsights < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -131,11 +131,18 @@ module Karafka
           topic = @coordinator.topic
 
           strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(topic)
+          expansions = ::Karafka::App.config.internal.processing.expansions_selector.find(topic)
 
           consumer = topic.consumer_class.new
           # We use singleton class as the same consumer class may be used to process different
           # topics with different settings
           consumer.singleton_class.include(strategy)
+
+          # Specific features may expand consumer API beyond the injected strategy. The difference
+          # here is that strategy impacts the flow of states while extra APIs just provide some
+          # extra methods with informations, etc but do no deviate the flow behavior
+          expansions.each { |expansion| consumer.singleton_class.include(expansion) }
+
           consumer.client = @client
           consumer.producer = ::Karafka::App.producer
           consumer.coordinator = @coordinator

data/lib/karafka/processing/expansions_selector.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Selector of appropriate topic setup based features enhancements.
+    #
+    # Those expansions to the consumer API are NOT about the flow of processing. For this we have
+    # strategies. Those are suppose to provide certain extra APIs that user can use to get some
+    # extra non-flow related functionalities.
+    class ExpansionsSelector
+      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+      #   expansions
+      # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance the
+      #   consumer
+      def find(topic)
+        expansions = []
+        expansions << Processing::InlineInsights::Consumer if topic.inline_insights?
+        expansions
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/consumer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Namespace of the Inline Insights feature "non routing" related components
+    #
+    # @note We use both `#insights` because it is the feature name but also `#statistics` to make
+    #   it consistent with the fact that we publish and operate on statistics. User can pick
+    #   whichever name they prefer.
+    module InlineInsights
+      # Module that adds extra methods to the consumer that allow us to fetch the insights
+      module Consumer
+        # @return [Hash] empty hash or hash with given partition insights if already present
+        # @note We cache insights on the consumer, as in some scenarios we may no longer have them
+        #   inside the Tracker, for example under involuntary revocation, incoming statistics may
+        #   no longer have lost partition insights. Since we want to be consistent during single
+        #   batch operations, we want to ensure, that if we have insights they are available
+        #   throughout the whole processing.
+        def insights
+          insights = Tracker.find(topic, partition)
+
+          # If we no longer have new insights but we still have them locally, we can use them
+          return @insights if @insights && insights.empty?
+          # If insights are still the same, we can use them
+          return @insights if @insights.equal?(insights)
+
+          # If we've received new insights that are not empty, we can cache them
+          @insights = insights
+        end
+
+        # @return [Boolean] true if there are insights to work with, otherwise false
+        def insights?
+          !insights.empty?
+        end
+
+        alias statistics insights
+        alias statistics? insights?
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/listener.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module InlineInsights
+      # Listener that adds statistics to our inline tracker
+      class Listener
+        # Adds statistics to the tracker
+        # @param event [Karafka::Core::Monitoring::Event] event with statistics
+        def on_statistics_emitted(event)
+          Tracker.add(
+            event[:consumer_group_id],
+            event[:statistics]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/tracker.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module InlineInsights
+      # Object used to track statistics coming from librdkafka in a way that can be accessible by
+      # the consumers
+      #
+      # We use a single tracker because we do not need state management here as our consumer groups
+      # clients identified by statistics name value are unique. On top of that, having a per
+      # process one that is a singleton allows us to use tracker easily also from other places like
+      # filtering API etc.
+      #
+      # @note We include cache of 5 minutes for revoked partitions to compensate for cases where
+      #   when using LRJ a lost partition data would not be present anymore, however we would still
+      #   be in the processing phase. Since those metrics are published with each `poll`, regular
+      #   processing is not a subject of this issue. For LRJ we keep the reference. The only case
+      #   where this could be switched midway is when LRJ is running for an extended period of time
+      #   after the involuntary revocation. Having a time based cache instead of tracking
+      #   simplifies the design as we do not have to deal with state tracking, especially since
+      #   we would have to track also operations running in a revoked state.
+      #
+      # @note This tracker keeps in memory data about all topics and partitions that it encounters
+      #   because in case of routing patterns, we may start getting statistics prior to registering
+      #   given topic via dynamic routing expansions. In such case we would not have insights
+      #   where they were actually available for us to use.
+      #
+      # @note Memory usage is negligible as long as we can evict expired data. Single metrics set
+      #   for a single partition contains around 4KB of data. This means, that in case of an
+      #   assignment of 1000 partitions, we use around 4MB of space for tracking those metrics.
+      class Tracker
+        include Singleton
+        include Karafka::Core::Helpers::Time
+
+        # Empty hash we want to return in any case where we could not locate appropriate topic
+        # partition statistics.
+        EMPTY_HASH = {}.freeze
+
+        # Empty array to save on memory allocations.
+        EMPTY_ARRAY = [].freeze
+
+        # 5 minutes of cache. We cache last result per consumer group topic partition so we are
+        # not affected by involuntary rebalances during LRJ execution.
+        TTL = 5 * 60 * 1_000
+
+        private_constant :EMPTY_HASH, :EMPTY_ARRAY, :TTL
+
+        class << self
+          extend Forwardable
+
+          def_delegators :instance, :find, :add, :exists?, :clear
+        end
+
+        def initialize
+          @accu = {}
+          @mutex = Mutex.new
+        end
+
+        # Adds each partition statistics into internal accumulator. Single statistics set may
+        # contain data from multiple topics and their partitions because a single client can
+        # operate on multiple topics and partitions.
+        #
+        # We iterate over those topics and partitions and store topics partitions data only.
+        #
+        # @param consumer_group_id [String] id of the consumer group for which statistics were
+        #   emitted.
+        # @param statistics [Hash] librdkafka enriched statistics
+        def add(consumer_group_id, statistics)
+          @mutex.synchronize do
+            statistics.fetch('topics', EMPTY_HASH).each do |topic_name, t_details|
+              t_details.fetch('partitions', EMPTY_HASH).each do |partition_id, p_details|
+                next unless track?(partition_id, p_details)
+
+                key = "#{consumer_group_id}_#{topic_name}_#{partition_id}"
+                @accu[key] = [monotonic_now, p_details]
+              end
+            end
+
+            evict
+          end
+        end
+
+        # Finds statistics about requested consumer group topic partition
+        #
+        # @param topic [Karafka::Routing::Topic]
+        # @param partition [Integer]
+        # @return [Hash] hash with given topic partition statistics or empty hash if not present
+        #
+        # @note We do not enclose it with a mutex mainly because the only thing that could happen
+        #   here that would be a race-condition is a miss that anyhow we need to support due to
+        #   how librdkafka ships metrics and a potential removal of data on heavily revoked LRJ.
+        def find(topic, partition)
+          key = "#{topic.consumer_group.id}_#{topic.name}_#{partition}"
+          @accu.fetch(key, EMPTY_ARRAY).last || EMPTY_HASH
+        end
+
+        # Clears the tracker
+        def clear
+          @mutex.synchronize { @accu.clear }
+        end
+
+        private
+
+        # Evicts expired data from the cache
+        def evict
+          @accu.delete_if { |_, details| monotonic_now - details.first > TTL }
+        end
+
+        # Should we track given partition
+        #
+        # We do not track stopped partitions and the once we do not work with actively
+        # @param partition_id [String] partition id as a string
+        # @param p_details [Hash] partition statistics details
+        # @return [Boolean] true if we should track given partition
+        def track?(partition_id, p_details)
+          return false if partition_id == '-1'
+
+          fetch_state = p_details.fetch('fetch_state')
+
+          return false if fetch_state == 'stopped'
+          return false if fetch_state == 'none'
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/railtie.rb

@@ -33,7 +33,7 @@ if Karafka.rails?
       # server + will support code reloading with each fetched loop. We do it only for karafka
       # based commands as Rails processes and console will have it enabled already
       initializer 'karafka.configure_rails_logger' do
-        # Make Karafka use Rails logger
+        # Make Karafka uses Rails logger
         ::Karafka::App.config.logger = Rails.logger
 
         next unless Rails.env.development?
@@ -42,15 +42,22 @@ if Karafka.rails?
         # If added again, would print stuff twice
         next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
 
-        logger = ActiveSupport::Logger.new($stdout)
+        stdout_logger = ActiveSupport::Logger.new($stdout)
         # Inherit the logger level from Rails, otherwise would always run with the debug level
-        logger.level = Rails.logger.level
+        stdout_logger.level = Rails.logger.level
 
-        Rails.logger.extend(
-          ActiveSupport::Logger.broadcast(
-            logger
+        rails71plus = Rails.gem_version >= Gem::Version.new('7.1.0')
+
+        # Rails 7.1 replaced the broadcast module with a broadcast logger
+        if rails71plus
+          Rails.logger.broadcast_to(stdout_logger)
+        else
+          Rails.logger.extend(
+            ActiveSupport::Logger.broadcast(
+              stdout_logger
+            )
           )
-        )
+        end
       end
 
       initializer 'karafka.configure_rails_auto_load_paths' do |app|

data/lib/karafka/routing/features/base.rb

@@ -23,11 +23,46 @@ module Karafka
 
           # Loads all the features and activates them
           def load_all
+            features.each(&:activate)
+          end
+
+          # @param config [Karafka::Core::Configurable::Node] app config that we can alter with
+          #   particular routing feature specific stuff if needed
+          def pre_setup_all(config)
+            features.each { |feature| feature.pre_setup(config) }
+          end
+
+          # Runs post setup routing features configuration operations
+          #
+          # @param config [Karafka::Core::Configurable::Node]
+          def post_setup_all(config)
+            features.each { |feature| feature.post_setup(config) }
+          end
+
+          private
+
+          # @return [Array<Class>] all available routing features
+          def features
             ObjectSpace
               .each_object(Class)
               .select { |klass| klass < self }
               .sort_by(&:to_s)
-              .each(&:activate)
+          end
+
+          protected
+
+          # Runs pre-setup configuration of a particular routing feature
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config node
+          def pre_setup(_config)
+            true
+          end
+
+          # Runs post-setup configuration of a particular routing feature
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config node
+          def post_setup(_config)
+            true
           end
         end
       end

data/lib/karafka/routing/features/inline_insights/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Config of this feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Inline Insights related contracts namespace
+        module Contracts
+          # Contract for inline insights topic setup
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :inline_insights do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights/topic.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Routing topic inline insights API
+        module Topic
+          # @param active [Boolean] should inline insights be activated
+          def inline_insights(active = false)
+            @inline_insights ||= Config.new(
+              active: active
+            )
+          end
+
+          # @return [Boolean] Are inline insights active
+          def inline_insights?
+            inline_insights.active?
+          end
+
+          # @return [Hash] topic setup hash
+          def to_h
+            super.merge(
+              inline_insights: inline_insights.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Feature allowing us to get visibility during the consumption into metrics of particular
+      # partition we operate on. It can be useful when making context-aware consumers that change
+      # their behaviours based on the lag and other parameters.
+      class InlineInsights < Base
+        class << self
+          # If needed installs the needed listener and initializes tracker
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config
+          def post_setup(_config)
+            ::Karafka::App.monitor.subscribe('app.running') do
+              # Do not activate tracking of statistics if none of our active topics uses it
+              # This prevents us from tracking metrics when user just runs a subset of topics
+              # in a given process and none of those actually utilizes this feature
+              next unless ::Karafka::App
+                          .subscription_groups
+                          .values
+                          .flat_map(&:itself)
+                          .flat_map(&:topics)
+                          .flat_map(&:to_a)
+                          .any?(&:inline_insights?)
+
+              # Initialize the tracker prior to becoming multi-threaded
+              ::Karafka::Processing::InlineInsights::Tracker.instance
+
+              # Subscribe to the statistics reports and collect them
+              ::Karafka.monitor.subscribe(
+                ::Karafka::Processing::InlineInsights::Listener.new
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/subscription_group.rb

@@ -34,7 +34,11 @@ module Karafka
       def initialize(position, topics)
         @name = topics.first.subscription_group_name
         @consumer_group = topics.first.consumer_group
-        @id = "#{@name}_#{position}"
+        # We include the consumer group id here because we want to have unique ids of subscription
+        # groups across the system. Otherwise user could set the same name for multiple
+        # subscription groups in many consumer groups effectively having same id for different
+        # entities
+        @id = "#{@consumer_group.id}_#{@name}_#{position}"
         @position = position
         @topics = topics
         @kafka = build_kafka
@@ -87,7 +91,7 @@ module Karafka
         # If we use static group memberships, there can be a case, where same instance id would
         # be set on many subscription groups as the group instance id from Karafka perspective is
         # set per config. Each instance even if they are subscribed to different topics needs to
-        # have if fully unique. To make sure of that, we just add extra postfix at the end that
+        # have it fully unique. To make sure of that, we just add extra postfix at the end that
         # increments.
         group_instance_id = kafka.fetch(:'group.instance.id', false)
 

data/lib/karafka/setup/attributes_map.rb

@@ -221,6 +221,7 @@ module Karafka
         request.timeout.ms
         resolve_cb
         retries
+        retry.backoff.max.ms
         retry.backoff.ms
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd

data/lib/karafka/setup/config.rb

@@ -208,6 +208,8 @@ module Karafka
           setting :partitioner_class, default: Processing::Partitioner
           # option strategy_selector [Object] processing strategy selector to be used
           setting :strategy_selector, default: Processing::StrategySelector.new
+          # option expansions_selector [Object] processing expansions selector to be used
+          setting :expansions_selector, default: Processing::ExpansionsSelector.new
         end
 
         # Things related to operating on messages
@@ -238,10 +240,14 @@ module Karafka
         def setup(&block)
           # Will prepare and verify license if present
           Licenser.prepare_and_verify(config.license)
+
+          # Pre-setup configure all routing features that would need this
+          Routing::Features::Base.pre_setup_all(config)
+
           # Will configure all the pro components
           # This needs to happen before end user configuration as the end user may overwrite some
           # of the pro defaults with custom components
-          Pro::Loader.pre_setup(config) if Karafka.pro?
+          Pro::Loader.pre_setup_all(config) if Karafka.pro?
 
           configure(&block)
           merge_kafka_defaults!(config)
@@ -253,9 +259,12 @@ module Karafka
           # Refreshes the references that are cached that might have been changed by the config
           ::Karafka.refresh!
 
+          # Post-setup configure all routing features that would need this
+          Routing::Features::Base.post_setup_all(config)
+
           # Runs things that need to be executed after config is defined and all the components
           # are also configured
-          Pro::Loader.post_setup(config) if Karafka.pro?
+          Pro::Loader.post_setup_all(config) if Karafka.pro?
 
           Karafka::App.initialized!
         end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.5'
+  VERSION = '2.2.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.2.5
+  version: 2.2.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-25 00:00:00.000000000 Z
+date: 2023-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -244,6 +244,7 @@ files:
 - lib/karafka/pro/processing/filters/base.rb
 - lib/karafka/pro/processing/filters/delayer.rb
 - lib/karafka/pro/processing/filters/expirer.rb
+- lib/karafka/pro/processing/filters/inline_insights_delayer.rb
 - lib/karafka/pro/processing/filters/throttler.rb
 - lib/karafka/pro/processing/filters/virtual_limiter.rb
 - lib/karafka/pro/processing/filters_applier.rb
@@ -321,6 +322,10 @@ files:
 - lib/karafka/pro/routing/features/filtering/config.rb
 - lib/karafka/pro/routing/features/filtering/contracts/topic.rb
 - lib/karafka/pro/routing/features/filtering/topic.rb
+- lib/karafka/pro/routing/features/inline_insights.rb
+- lib/karafka/pro/routing/features/inline_insights/config.rb
+- lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb
+- lib/karafka/pro/routing/features/inline_insights/topic.rb
 - lib/karafka/pro/routing/features/long_running_job.rb
 - lib/karafka/pro/routing/features/long_running_job/config.rb
 - lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb
@@ -353,6 +358,10 @@ files:
 - lib/karafka/processing/coordinators_buffer.rb
 - lib/karafka/processing/executor.rb
 - lib/karafka/processing/executors_buffer.rb
+- lib/karafka/processing/expansions_selector.rb
+- lib/karafka/processing/inline_insights/consumer.rb
+- lib/karafka/processing/inline_insights/listener.rb
+- lib/karafka/processing/inline_insights/tracker.rb
 - lib/karafka/processing/jobs/base.rb
 - lib/karafka/processing/jobs/consume.rb
 - lib/karafka/processing/jobs/idle.rb
@@ -394,6 +403,10 @@ files:
 - lib/karafka/routing/features/declaratives/config.rb
 - lib/karafka/routing/features/declaratives/contracts/topic.rb
 - lib/karafka/routing/features/declaratives/topic.rb
+- lib/karafka/routing/features/inline_insights.rb
+- lib/karafka/routing/features/inline_insights/config.rb
+- lib/karafka/routing/features/inline_insights/contracts/topic.rb
+- lib/karafka/routing/features/inline_insights/topic.rb
 - lib/karafka/routing/features/manual_offset_management.rb
 - lib/karafka/routing/features/manual_offset_management/config.rb
 - lib/karafka/routing/features/manual_offset_management/contracts/topic.rb
@@ -447,7 +460,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.4.10
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.