karafka 2.6.0.beta1 → 2.6.0.rc1

data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb

@@ -52,6 +52,39 @@ module Karafka
                 tasks_cfg = App.config.recurring_tasks
                 topics_cfg = tasks_cfg.topics
 
+                # Declarative topic definitions live independently of routing. We declare the
+                # broker-side structure here so it is managed by the CLI topics commands without
+                # relying on the (to be retired) routing `config(...)` bridge.
+                #
+                # We guard each declaration with `find_topic` because `Declaratives::Builder#topic`
+                # re-applies its block on an already existing declaration (additive semantics),
+                # unlike the old bridge which used `||=` and kept the first declaration. Without the
+                # guard, our framework defaults would clobber a topic the user may have declared
+                # themselves via `App.declaratives.draw`. Skipping when already present preserves
+                # first-declaration-wins.
+                App.declaratives.draw do
+                  # Keep older data for a day and compact to the last state available
+                  unless find_topic(topics_cfg.schedules.name)
+                    topic(topics_cfg.schedules.name) do
+                      config(
+                        "cleanup.policy": "compact,delete",
+                        "retention.ms": 86_400_000
+                      )
+                    end
+                  end
+
+                  # Keep cron logs of executions for a week and after that remove. Week should be
+                  # enough and should not produce too much data.
+                  unless find_topic(topics_cfg.logs.name)
+                    topic(topics_cfg.logs.name) do
+                      config(
+                        "cleanup.policy": "delete",
+                        "retention.ms": 604_800_000
+                      )
+                    end
+                  end
+                end
+
                 consumer_group tasks_cfg.group_id do
                   # Registers the primary topic that we use to control schedules execution. This is
                   # the one that we use to trigger recurring tasks.
@@ -87,12 +120,6 @@ module Karafka
                     pause_timeout(60 * 1_000)
                     pause_max_timeout(60 * 1_000)
 
-                    # Keep older data for a day and compact to the last state available
-                    config(
-                      "cleanup.policy": "compact,delete",
-                      "retention.ms": 86_400_000
-                    )
-
                     # This is the core of execution. Since we're producers of states, we need a way
                     # to tick without having new data
                     periodic(
@@ -113,13 +140,6 @@ module Karafka
                     active(false)
                     deserializer tasks_cfg.deserializer
                     target.recurring_tasks(true)
-
-                    # Keep cron logs of executions for a week and after that remove. Week should be
-                    # enough and should not produce too much data.
-                    config(
-                      "cleanup.policy": "delete",
-                      "retention.ms": 604_800_000
-                    )
                   end
 
                   yield(schedules_topic, logs_topic) if block && block.arity.positive?

data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb

@@ -38,9 +38,9 @@ module Karafka
             module Builder
               # Enabled scheduled messages operations and adds needed topics and other stuff.
               #
-              # @param topic_name [String, false] name for scheduled messages topic that is also used
-              #   as a group identifier. Users can have multiple schedule topics flows to prevent key
-              #   collisions, prioritize and do other stuff. `false` if not active.
+              # @param topic_name [String, false] name for scheduled messages topic that is also
+              #   used as a group identifier. Users can have multiple schedule topics flows to
+              #   prevent key collisions, prioritize and do other stuff. `false` if not active.
               # @param block [Proc] optional reconfiguration of the topics definitions.
               # @note Namespace for topics should include the divider as it is not automatically
               #   added.
@@ -54,6 +54,54 @@ module Karafka
                 # should have same partition count.
                 default_partitions = 5
                 msg_cfg = App.config.scheduled_messages
+                states_topic_name = "#{topic_name}#{msg_cfg.states_postfix}"
+
+                # Declarative topic definitions live independently of routing. We declare the
+                # broker-side structure here so it is managed by the CLI topics commands without
+                # relying on the (to be retired) routing `config(...)` bridge.
+                #
+                # We guard each declaration with `find_topic` because `Declaratives::Builder#topic`
+                # re-applies its block on an already existing declaration (additive semantics),
+                # unlike the old bridge which used `||=` and kept the first declaration. Without the
+                # guard, our framework defaults would clobber a topic the user may have declared
+                # themselves via `App.declaratives.draw`. Skipping when already present preserves
+                # first-declaration-wins.
+                App.declaratives.draw do
+                  # This is a setup that should allow messages to be compacted fairly fast. Since
+                  # each dispatched message should be removed via tombstone, they do not have to
+                  # be present in the topic for too long.
+                  unless find_topic(topic_name)
+                    topic(topic_name) do
+                      partitions default_partitions
+                      config(
+                        # Will ensure, that after tombstone is present, given scheduled message,
+                        # that has been dispatched is removed by Kafka
+                        "cleanup.policy": "compact",
+                        # When 10% or more dispatches are done, compact data
+                        "min.cleanable.dirty.ratio": 0.1,
+                        # Frequent segment rotation to support intense compaction
+                        "segment.ms": 3_600_000,
+                        "delete.retention.ms": 3_600_000,
+                        "segment.bytes": 52_428_800
+                      )
+                    end
+                  end
+
+                  # Holds states of scheduler per each of the partitions. Same partition count as
+                  # the messages topic since they tick independently per partition.
+                  unless find_topic(states_topic_name)
+                    topic(states_topic_name) do
+                      partitions default_partitions
+                      config(
+                        "cleanup.policy": "compact",
+                        "min.cleanable.dirty.ratio": 0.1,
+                        "segment.ms": 3_600_000,
+                        "delete.retention.ms": 3_600_000,
+                        "segment.bytes": 52_428_800
+                      )
+                    end
+                  end
+                end
 
                 consumer_group msg_cfg.group_id do
                   # Registers the primary topic that we use to control schedules execution. This is
@@ -93,22 +141,6 @@ module Karafka
 
                     max_wait_time(1_000)
 
-                    # This is a setup that should allow messages to be compacted fairly fast. Since
-                    # each dispatched message should be removed via tombstone, they do not have to
-                    # be present in the topic for too long.
-                    config(
-                      partitions: default_partitions,
-                      # Will ensure, that after tombstone is present, given scheduled message, that
-                      # has been dispatched is removed by Kafka
-                      "cleanup.policy": "compact",
-                      # When 10% or more dispatches are done, compact data
-                      "min.cleanable.dirty.ratio": 0.1,
-                      # Frequent segment rotation to support intense compaction
-                      "segment.ms": 3_600_000,
-                      "delete.retention.ms": 3_600_000,
-                      "segment.bytes": 52_428_800
-                    )
-
                     # This is the core of execution. Since we dispatch data in time intervals, we
                     # need to be able to do this even when no new data is coming
                     periodic(
@@ -126,17 +158,9 @@ module Karafka
                   # Holds states of scheduler per each of the partitions since they tick
                   # independently. We only hold future statistics not to have to deal with
                   # any type of state restoration
-                  states_topic = topic("#{topic_name}#{msg_cfg.states_postfix}") do
+                  states_topic = topic(states_topic_name) do
                     active(false)
                     target.scheduled_messages(true)
-                    config(
-                      partitions: default_partitions,
-                      "cleanup.policy": "compact",
-                      "min.cleanable.dirty.ratio": 0.1,
-                      "segment.ms": 3_600_000,
-                      "delete.retention.ms": 3_600_000,
-                      "segment.bytes": 52_428_800
-                    )
                     deserializers(
                       payload: msg_cfg.deserializers.payload
                     )

data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb

@@ -46,7 +46,8 @@ module Karafka
                 super
               end
 
-              # @param active [Boolean] should this topic be considered related to scheduled messages
+              # @param active [Boolean] should this topic be considered related to scheduled
+              #   messages
               def scheduled_messages(active = false)
                 @scheduled_messages ||= Config.new(active: active)
               end

data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb

@@ -45,16 +45,17 @@ module Karafka
                 super
               end
 
-              # @param max_partitions [Integer] max number of virtual partitions that can come out of
-              #   the single distribution flow. When set to more than the Karafka threading, will
+              # @param max_partitions [Integer] max number of virtual partitions that can come out
+              #   of the single distribution flow. When set to more than the Karafka threading, will
               #   create more work than workers. When less, can ensure we have spare resources to
               #   process other things in parallel.
               # @param partitioner [nil, #call] nil or callable partitioner
               # @param offset_metadata_strategy [Symbol] how we should match the metadata for the
-              #   offset. `:exact` will match the offset matching metadata and `:current` will select
-              #   the most recently reported metadata
+              #   offset. `:exact` will match the offset matching metadata and `:current` will
+              #   select the most recently reported metadata
               # @param reducer [nil, #call] reducer for VPs key. It allows for using a custom
-              #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+              #   reducer to achieve enhanced parallelization when the default reducer is not
+              #   enough.
               # @param distribution [Symbol] the strategy to use for virtual partitioning. Can be
               #   either `:consistent` or `:balanced`. The `:balanced` strategy ensures balanced
               #   distribution of work across available workers while maintaining message order

data/lib/karafka/pro/scheduled_messages/consumer.rb

@@ -72,7 +72,13 @@ module Karafka
 
           @states_reporter.call
 
-          recent_timestamp = messages.last.timestamp.to_i
+          # Clamp to the wall clock: a future-dated message timestamp (producer clock skew or an
+          # explicitly set future timestamp) must not be treated as proof of catch-up. Otherwise a
+          # single future-dated message read mid-backlog would flip `loaded!` while older tombstones
+          # at higher offsets are still unread, dispatching schedules whose cancelling tombstones
+          # have not been loaded yet (double dispatch). Mirrors the `[timestamp, now].min` clamp in
+          # the batch metadata builder.
+          recent_timestamp = [messages.last.timestamp.to_i, Time.now.to_i].min
           post_started_timestamp = @tracker.started_at + GRACE_PERIOD
 
           # If we started getting messages that are beyond the current time, it means we have

data/lib/karafka/pro/scheduled_messages/schema_validator.rb

@@ -43,7 +43,12 @@ module Karafka
           def call(message)
             message_version = message.headers["schedule_schema_version"]
 
-            return if message_version <= ScheduledMessages::SCHEMA_VERSION
+            # Versions are semver, so they need to be compared segment-wise and not as strings
+            # ("1.2.0" is older than "1.10.0" but lexicographically greater)
+            message_gem_version = Gem::Version.new(message_version)
+            current_gem_version = Gem::Version.new(ScheduledMessages::SCHEMA_VERSION)
+
+            return if message_gem_version <= current_gem_version
 
             raise Errors::IncompatibleSchemaError, message_version
           end

data/lib/karafka/pro/swarm/liveness_listener.rb

@@ -47,28 +47,50 @@ module Karafka
       #   - 1 - polling ttl exceeded
       #   - 2 - consuming ttl exceeded
       #   - 3 - memory limit exceeded
+      #   - 4 - stability ttl exceeded (stuck in non-steady librdkafka join state)
       #
       # @note This listener should not break anything if subscribed in the supervisor prior to
       #   forking as it relies on server events for operations.
       class LivenessListener < Karafka::Swarm::LivenessListener
+        # Default time in ms after which we consider consumption hanging (5 minutes)
+        DEFAULT_CONSUMING_TTL = 5 * 60 * 1_000
+        # Default max time in ms between polls before the node is considered dead (5 minutes)
+        DEFAULT_POLLING_TTL = 5 * 60 * 1_000
+        # Multiplier applied to max.poll.interval.ms when deriving the default stability_ttl.
+        STABILITY_TTL_MAX_POLL_MULTIPLIER = 2
+
+        private_constant(
+          :DEFAULT_CONSUMING_TTL,
+          :DEFAULT_POLLING_TTL,
+          :STABILITY_TTL_MAX_POLL_MULTIPLIER
+        )
+
         # @param memory_limit [Integer] max memory in MB for this process to be considered healthy
-        # @param consuming_ttl [Integer] time in ms after which we consider consumption hanging.
-        #   It allows us to define max consumption time after which supervisor should consider
-        #   given process as hanging
-        # @param polling_ttl [Integer] max time in ms for polling. If polling (any) does not
-        #   happen that often, process should be considered dead.
-        # @note The default TTL matches the default `max.poll.interval.ms`
+        # @param consuming_ttl [Integer] see Kubernetes::LivenessListener for full documentation.
+        # @param polling_ttl [Integer] see Kubernetes::LivenessListener for full documentation.
+        # @param stability_ttl [Integer, nil] see Kubernetes::LivenessListener for full
+        #   documentation, including the derivation rules (max `max.poll.interval.ms` across
+        #   all active subscription groups, times 2, resolved lazily on first health
+        #   evaluation). Same semantics apply here; the node is reported unhealthy with
+        #   status code 4 instead of returning HTTP 500.
         def initialize(
           memory_limit: Float::INFINITY,
-          consuming_ttl: 5 * 60 * 1_000,
-          polling_ttl: 5 * 60 * 1_000
+          consuming_ttl: DEFAULT_CONSUMING_TTL,
+          polling_ttl: DEFAULT_POLLING_TTL,
+          stability_ttl: nil
         )
           @polling_ttl = polling_ttl
           @consuming_ttl = consuming_ttl
+          # When nil, resolved lazily via #stability_ttl on first health evaluation because
+          # the default derives from the routing subscription groups, which may not be drawn
+          # yet when the listener is constructed in karafka.rb
+          @stability_ttl = stability_ttl
           # We cast it just in case someone would provide '10MB' or something similar
           @memory_limit = memory_limit.is_a?(String) ? memory_limit.to_i : memory_limit
           @pollings = {}
           @consumptions = {}
+          @join_states = {}
+          @instabilities = {}
 
           super()
         end
@@ -117,6 +139,27 @@ module Karafka
           RUBY
         end
 
+        # @see Karafka::Instrumentation::Vendors::Kubernetes::LivenessListener#on_statistics_emitted
+        # @param event [Karafka::Core::Monitoring::Event]
+        def on_statistics_emitted(event)
+          cgrp = event[:statistics]&.dig("cgrp")
+          return unless cgrp
+
+          sg_id = event[:subscription_group_id]
+          join_state = cgrp["join_state"]
+          return unless join_state
+
+          synchronize do
+            if join_state == "steady" || join_state == "init" || join_state == "wait-metadata"
+              @instabilities.delete(sg_id)
+              @join_states.delete(sg_id)
+            elsif @join_states[sg_id] != join_state
+              @join_states[sg_id] = join_state
+              @instabilities[sg_id] = monotonic_now
+            end
+          end
+        end
+
         # @param _event [Karafka::Core::Monitoring::Event]
         def on_error_occurred(_event)
           clear_consumption_tick
@@ -143,6 +186,13 @@ module Karafka
           clear_polling_tick
         end
 
+        # @see Karafka::Instrumentation::Vendors::Kubernetes::LivenessListener
+        #   #on_connection_listener_after_fetch_loop
+        # @param event [Karafka::Core::Monitoring::Event]
+        def on_connection_listener_after_fetch_loop(event)
+          clear_instability_tracking(event[:subscription_group])
+        end
+
         private
 
         # Reports the current status to the supervisor periodically
@@ -158,10 +208,21 @@ module Karafka
           end
         end
 
-        # @return [Integer] object id of the current fiber
-        # @note We use fiber object id instead of thread object id to ensure fiber-safety.
-        #   Multiple fibers can run on the same thread, and using thread id would cause them
-        #   to overwrite each other's timestamps.
+        # @see Karafka::Instrumentation::Vendors::Kubernetes::LivenessListener
+        #   #clear_instability_tracking
+        # @param subscription_group [Karafka::Routing::SubscriptionGroup, nil]
+        def clear_instability_tracking(subscription_group)
+          return unless subscription_group
+
+          sg_id = subscription_group.id
+
+          synchronize do
+            @instabilities.delete(sg_id)
+            @join_states.delete(sg_id)
+          end
+        end
+
+        # @see Karafka::Instrumentation::Vendors::Kubernetes::LivenessListener#fiber_id
         def fiber_id
           Fiber.current.object_id
         end
@@ -195,17 +256,61 @@ module Karafka
         end
 
         # Did we exceed any of the ttls
-        # @return [String] 204 string if ok, 500 otherwise
+        # @return [Integer] 0 if healthy, positive error code otherwise:
+        #   1 - polling TTL exceeded
+        #   2 - consuming TTL exceeded
+        #   3 - memory limit exceeded
+        #   4 - stability TTL exceeded (stuck in non-steady join state)
+        # @note Reads `@pollings`, `@consumptions` and `@instabilities` without acquiring the
+        #   mutex. This is intentional and consistent across all three: `status` is called from
+        #   inside `periodically`, which already holds `@mutex` for scheduling, so re-entering
+        #   `synchronize` here would deadlock. Concurrent writes from `mark_*` / `clear_*` /
+        #   `on_statistics_emitted` are safe to observe partially on MRI (GVL atomises Hash ops).
+        #   This is the asymmetry vs `Kubernetes::LivenessListener`, which can lock its reads
+        #   because it has no equivalent periodic-scheduling lock. Do not "fix" this without
+        #   resolving the periodic-scheduling lock first - JRuby is not a supported runtime.
         def status
           time = monotonic_now
 
           return 1 if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
           return 2 if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
           return 3 if rss_mb > @memory_limit
+          return 4 if @instabilities.values.any? { |start| (time - start) > stability_ttl }
 
           0
         end
 
+        # @return [Integer] effective stability ttl in ms. Resolved lazily (instead of in
+        #   `#initialize`) because the default derives from the routing subscription groups,
+        #   which may not be drawn yet when the listener is constructed in karafka.rb. By the
+        #   time status is evaluated, routing is final.
+        def stability_ttl
+          @stability_ttl ||= default_stability_ttl
+        end
+
+        # @return [Integer] default stability_ttl derived from the configured
+        #   max.poll.interval.ms, multiplied by 2 for headroom above the slowest legitimate
+        #   rebalance phase. Uses the maximum value across all active subscription groups
+        #   (their kafka configs already have librdkafka defaults injected), so per
+        #   subscription group overrides set via the routing DSL are respected. Falls back to
+        #   the root kafka config when no subscription groups are available.
+        def default_stability_ttl
+          max_poll_interval = Karafka::App
+            .subscription_groups
+            .values
+            .flatten
+            .map { |sg| sg.kafka.fetch(:"max.poll.interval.ms") }
+            .max
+
+          max_poll_interval ||= begin
+            kafka_config = Karafka::App.config.kafka.dup
+            Karafka::Setup::DefaultsInjector.consumer(kafka_config)
+            kafka_config.fetch(:"max.poll.interval.ms")
+          end
+
+          max_poll_interval * STABILITY_TTL_MAX_POLL_MULTIPLIER
+        end
+
         # @return [Integer] RSS in MB for the current process
         def rss_mb
           RUBY_PLATFORM.include?("linux") ? rss_mb_linux : rss_mb_macos

data/lib/karafka/pro.rb

@@ -1,11 +1,5 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
-# All of the commercial components are present in the lib/karafka/pro directory of this repository
-# and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
 module Karafka
   # Namespace for pro components, licensed under the commercial license agreement.
   module Pro

data/lib/karafka/processing/consumer_groups/coordinator.rb

@@ -2,16 +2,14 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       # Basic coordinator that allows us to provide coordination objects into consumers.
       #
       # This is a wrapping layer to simplify management of work to be handled around consumption.
       #
       # @note This coordinator needs to be thread safe. Some operations are performed only in the
-      #   listener thread, but we go with thread-safe by default for all not to worry about potential
-      #   future mistakes.
+      #   listener thread, but we go with thread-safe by default for all not to worry about
+      #   potential future mistakes.
       class Coordinator
         extend Forwardable
         include Core::Helpers::Time
@@ -29,7 +27,8 @@ module Karafka
 
         # @param topic [Karafka::Routing::Topic]
         # @param partition [Integer]
-        # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
+        # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic
+        #   partition
         def initialize(topic, partition, pause_tracker)
           @topic = topic
           @partition = partition

data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       # Coordinators builder used to build coordinators per topic partition
       #
@@ -50,6 +48,12 @@ module Karafka
         def revoke(topic_name, partition)
           return unless @coordinators[topic_name].key?(partition)
 
+          # Reset the partition's pause tracker attempt count. The tracker lives in the pauses
+          # manager keyed by topic-partition and would otherwise be reused as-is if we reclaim this
+          # partition, carrying a stale retry attempt count across the rebalance (which, with DLQ,
+          # would send the next failure straight to the dead letter queue, skipping the retries)
+          @pauses_manager.revoke(@topics.find(topic_name), partition)
+
           # The fact that we delete here does not change the fact that the executor still holds the
           # reference to this coordinator. We delete it here, as we will no longer process any
           # new stuff with it and we may need a new coordinator if we regain this partition, but the

data/lib/karafka/processing/consumer_groups/executor.rb

@@ -7,8 +7,8 @@ module Karafka
     # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       # Executors:
-      # - run consumers code (for `#call`) or run given preparation / teardown operations when needed
-      #   from separate threads.
+      # - run consumers code (for `#call`) or run given preparation / teardown operations when
+      #   needed from separate threads.
       # - they re-create consumer instances in case of partitions that were revoked and assigned
       #   back.
       #
@@ -54,8 +54,8 @@ module Karafka
           @coordinator = coordinator
         end
 
-        # Allows us to prepare the consumer in the listener thread prior to the job being send to
-        # be scheduled. It also allows to run some code that is time sensitive and cannot wait in the
+        # Allows us to prepare the consumer in the listener thread prior to the job being send to be
+        # scheduled. It also allows to run some code that is time sensitive and cannot wait in the
         # queue as it could cause starvation.
         #
         # @param messages [Array<Karafka::Messages::Message>]

data/lib/karafka/processing/consumer_groups/executors_buffer.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       # Buffer for executors of a given subscription group. It wraps around the concept of building
       # and caching them, so we can re-use them instead of creating new each time.
@@ -63,8 +61,8 @@ module Karafka
         #
         # @param topic [String] topic name
         # @param partition [Integer] partition number
-        # @return [Array<Executor, Pro::Processing::ConsumerGroups::Executor>] executors in use for this
-        #   topic + partition
+        # @return [Array<Executor, Pro::Processing::ConsumerGroups::Executor>] executors in use for
+        #   this topic + partition
         def find_all(topic, partition)
           @buffer[topic][partition].values
         end

data/lib/karafka/processing/consumer_groups/expansions_selector.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       # Selector of appropriate topic setup based features enhancements.
       #

data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       # Namespace of the Inline Insights feature "non routing" related components
       #

data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module InlineInsights
         # Listener that adds statistics to our inline tracker

data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb

@@ -2,31 +2,29 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module InlineInsights
         # Object used to track statistics coming from librdkafka in a way that can be accessible by
         # the consumers
         #
-        # We use a single tracker because we do not need state management here as our consumer groups
-        # clients identified by statistics name value are unique. On top of that, having a per
-        # process one that is a singleton allows us to use tracker easily also from other places like
-        # filtering API etc.
+        # We use a single tracker because we do not need state management here as our consumer
+        # groups clients identified by statistics name value are unique. On top of that, having a
+        # per process one that is a singleton allows us to use tracker easily also from other places
+        # like filtering API etc.
         #
         # @note We include cache of 5 minutes for revoked partitions to compensate for cases where
-        #   when using LRJ a lost partition data would not be present anymore, however we would still
-        #   be in the processing phase. Since those metrics are published with each `poll`, regular
-        #   processing is not a subject of this issue. For LRJ we keep the reference. The only case
-        #   where this could be switched midway is when LRJ is running for an extended period of time
-        #   after the involuntary revocation. Having a time based cache instead of tracking
-        #   simplifies the design as we do not have to deal with state tracking, especially since
-        #   we would have to track also operations running in a revoked state.
+        #   when using LRJ a lost partition data would not be present anymore, however we would
+        #   still be in the processing phase. Since those metrics are published with each `poll`,
+        #   regular processing is not a subject of this issue. For LRJ we keep the reference. The
+        #   only case where this could be switched midway is when LRJ is running for an extended
+        #   period of time after the involuntary revocation. Having a time based cache instead of
+        #   tracking simplifies the design as we do not have to deal with state tracking, especially
+        #   since we would have to track also operations running in a revoked state.
         #
         # @note This tracker keeps in memory data about all topics and partitions that it encounters
-        #   because in case of routing patterns, we may start getting statistics prior to registering
-        #   given topic via dynamic routing expansions. In such case we would not have insights
-        #   where they were actually available for us to use.
+        #   because in case of routing patterns, we may start getting statistics prior to
+        #   registering given topic via dynamic routing expansions. In such case we would not have
+        #   insights where they were actually available for us to use.
         #
         # @note Memory usage is negligible as long as we can evict expired data. Single metrics set
         #   for a single partition contains around 4KB of data. This means, that in case of an

data/lib/karafka/processing/consumer_groups/jobs/consume.rb

@@ -2,10 +2,7 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
-      # Consumer-group-specific job types
       module Jobs
         # The main job type. It runs the executor that triggers given topic partition messages
         # processing in an underlying consumer instance.
@@ -15,8 +12,8 @@ module Karafka
 
           self.action = :consume
 
-          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is suppose to run a given
-          #   job
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is suppose
+          #   to run a given job
           # @param messages [Karafka::Messages::Messages] karafka messages batch
           # @return [Consume]
           def initialize(executor, messages)