karafka 2.2.6 → 2.2.8.beta1

data/lib/karafka/pro/routing/features/inline_insights/config.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Config of this feature
+          Config = Struct.new(
+            :active,
+            :required,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :required?, :required
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Inline Insights related contracts namespace
+          module Contracts
+            # Contract for inline insights topic setup
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested :inline_insights do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:required) { |val| [true, false].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights/topic.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class InlineInsights < Base
+          # Routing topic inline insights API
+          module Topic
+            # @param active [Boolean] should inline insights be activated
+            # @param required [Boolean] are the insights required to operate
+            def inline_insights(active = -1, required: -1)
+              # This weird style of checking allows us to activate inline insights in few ways:
+              #   - inline_insights(true)
+              #   - inline_insights(required: true)
+              #   - inline_insights(required: false)
+              #
+              # In each of those cases inline insights will become active
+              @inline_insights ||= begin
+                config = Config.new(
+                  active: active == true || (active == -1 && required != -1),
+                  required: required == true
+                )
+
+                if config.active? && config.required?
+                  factory = lambda do |topic, partition|
+                    Pro::Processing::Filters::InlineInsightsDelayer.new(topic, partition)
+                  end
+
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/inline_insights.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Enhanced inline insights
+        # Allows you to set up an automatic filter that will ensure, that metrics are always
+        # available when processing starts.
+        class InlineInsights < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/executor.rb

@@ -131,11 +131,18 @@ module Karafka
           topic = @coordinator.topic
 
           strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(topic)
+          expansions = ::Karafka::App.config.internal.processing.expansions_selector.find(topic)
 
           consumer = topic.consumer_class.new
           # We use singleton class as the same consumer class may be used to process different
           # topics with different settings
           consumer.singleton_class.include(strategy)
+
+          # Specific features may expand consumer API beyond the injected strategy. The difference
+          # here is that strategy impacts the flow of states while extra APIs just provide some
+          # extra methods with informations, etc but do no deviate the flow behavior
+          expansions.each { |expansion| consumer.singleton_class.include(expansion) }
+
           consumer.client = @client
           consumer.producer = ::Karafka::App.producer
           consumer.coordinator = @coordinator

data/lib/karafka/processing/expansions_selector.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Selector of appropriate topic setup based features enhancements.
+    #
+    # Those expansions to the consumer API are NOT about the flow of processing. For this we have
+    # strategies. Those are suppose to provide certain extra APIs that user can use to get some
+    # extra non-flow related functionalities.
+    class ExpansionsSelector
+      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+      #   expansions
+      # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance the
+      #   consumer
+      def find(topic)
+        expansions = []
+        expansions << Processing::InlineInsights::Consumer if topic.inline_insights?
+        expansions
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/consumer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Namespace of the Inline Insights feature "non routing" related components
+    #
+    # @note We use both `#insights` because it is the feature name but also `#statistics` to make
+    #   it consistent with the fact that we publish and operate on statistics. User can pick
+    #   whichever name they prefer.
+    module InlineInsights
+      # Module that adds extra methods to the consumer that allow us to fetch the insights
+      module Consumer
+        # @return [Hash] empty hash or hash with given partition insights if already present
+        # @note We cache insights on the consumer, as in some scenarios we may no longer have them
+        #   inside the Tracker, for example under involuntary revocation, incoming statistics may
+        #   no longer have lost partition insights. Since we want to be consistent during single
+        #   batch operations, we want to ensure, that if we have insights they are available
+        #   throughout the whole processing.
+        def insights
+          insights = Tracker.find(topic, partition)
+
+          # If we no longer have new insights but we still have them locally, we can use them
+          return @insights if @insights && insights.empty?
+          # If insights are still the same, we can use them
+          return @insights if @insights.equal?(insights)
+
+          # If we've received new insights that are not empty, we can cache them
+          @insights = insights
+        end
+
+        # @return [Boolean] true if there are insights to work with, otherwise false
+        def insights?
+          !insights.empty?
+        end
+
+        alias statistics insights
+        alias statistics? insights?
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/listener.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module InlineInsights
+      # Listener that adds statistics to our inline tracker
+      class Listener
+        # Adds statistics to the tracker
+        # @param event [Karafka::Core::Monitoring::Event] event with statistics
+        def on_statistics_emitted(event)
+          Tracker.add(
+            event[:consumer_group_id],
+            event[:statistics]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/inline_insights/tracker.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module InlineInsights
+      # Object used to track statistics coming from librdkafka in a way that can be accessible by
+      # the consumers
+      #
+      # We use a single tracker because we do not need state management here as our consumer groups
+      # clients identified by statistics name value are unique. On top of that, having a per
+      # process one that is a singleton allows us to use tracker easily also from other places like
+      # filtering API etc.
+      #
+      # @note We include cache of 5 minutes for revoked partitions to compensate for cases where
+      #   when using LRJ a lost partition data would not be present anymore, however we would still
+      #   be in the processing phase. Since those metrics are published with each `poll`, regular
+      #   processing is not a subject of this issue. For LRJ we keep the reference. The only case
+      #   where this could be switched midway is when LRJ is running for an extended period of time
+      #   after the involuntary revocation. Having a time based cache instead of tracking
+      #   simplifies the design as we do not have to deal with state tracking, especially since
+      #   we would have to track also operations running in a revoked state.
+      #
+      # @note This tracker keeps in memory data about all topics and partitions that it encounters
+      #   because in case of routing patterns, we may start getting statistics prior to registering
+      #   given topic via dynamic routing expansions. In such case we would not have insights
+      #   where they were actually available for us to use.
+      #
+      # @note Memory usage is negligible as long as we can evict expired data. Single metrics set
+      #   for a single partition contains around 4KB of data. This means, that in case of an
+      #   assignment of 1000 partitions, we use around 4MB of space for tracking those metrics.
+      class Tracker
+        include Singleton
+        include Karafka::Core::Helpers::Time
+
+        # Empty hash we want to return in any case where we could not locate appropriate topic
+        # partition statistics.
+        EMPTY_HASH = {}.freeze
+
+        # Empty array to save on memory allocations.
+        EMPTY_ARRAY = [].freeze
+
+        # 5 minutes of cache. We cache last result per consumer group topic partition so we are
+        # not affected by involuntary rebalances during LRJ execution.
+        TTL = 5 * 60 * 1_000
+
+        private_constant :EMPTY_HASH, :EMPTY_ARRAY, :TTL
+
+        class << self
+          extend Forwardable
+
+          def_delegators :instance, :find, :add, :exists?, :clear
+        end
+
+        def initialize
+          @accu = {}
+          @mutex = Mutex.new
+        end
+
+        # Adds each partition statistics into internal accumulator. Single statistics set may
+        # contain data from multiple topics and their partitions because a single client can
+        # operate on multiple topics and partitions.
+        #
+        # We iterate over those topics and partitions and store topics partitions data only.
+        #
+        # @param consumer_group_id [String] id of the consumer group for which statistics were
+        #   emitted.
+        # @param statistics [Hash] librdkafka enriched statistics
+        def add(consumer_group_id, statistics)
+          @mutex.synchronize do
+            statistics.fetch('topics', EMPTY_HASH).each do |topic_name, t_details|
+              t_details.fetch('partitions', EMPTY_HASH).each do |partition_id, p_details|
+                next unless track?(partition_id, p_details)
+
+                key = "#{consumer_group_id}_#{topic_name}_#{partition_id}"
+                @accu[key] = [monotonic_now, p_details]
+              end
+            end
+
+            evict
+          end
+        end
+
+        # Finds statistics about requested consumer group topic partition
+        #
+        # @param topic [Karafka::Routing::Topic]
+        # @param partition [Integer]
+        # @return [Hash] hash with given topic partition statistics or empty hash if not present
+        #
+        # @note We do not enclose it with a mutex mainly because the only thing that could happen
+        #   here that would be a race-condition is a miss that anyhow we need to support due to
+        #   how librdkafka ships metrics and a potential removal of data on heavily revoked LRJ.
+        def find(topic, partition)
+          key = "#{topic.consumer_group.id}_#{topic.name}_#{partition}"
+          @accu.fetch(key, EMPTY_ARRAY).last || EMPTY_HASH
+        end
+
+        # Clears the tracker
+        def clear
+          @mutex.synchronize { @accu.clear }
+        end
+
+        private
+
+        # Evicts expired data from the cache
+        def evict
+          @accu.delete_if { |_, details| monotonic_now - details.first > TTL }
+        end
+
+        # Should we track given partition
+        #
+        # We do not track stopped partitions and the once we do not work with actively
+        # @param partition_id [String] partition id as a string
+        # @param p_details [Hash] partition statistics details
+        # @return [Boolean] true if we should track given partition
+        def track?(partition_id, p_details)
+          return false if partition_id == '-1'
+
+          fetch_state = p_details.fetch('fetch_state')
+
+          return false if fetch_state == 'stopped'
+          return false if fetch_state == 'none'
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/railtie.rb

@@ -33,7 +33,7 @@ if Karafka.rails?
       # server + will support code reloading with each fetched loop. We do it only for karafka
       # based commands as Rails processes and console will have it enabled already
       initializer 'karafka.configure_rails_logger' do
-        # Make Karafka use Rails logger
+        # Make Karafka uses Rails logger
         ::Karafka::App.config.logger = Rails.logger
 
         next unless Rails.env.development?
@@ -42,15 +42,22 @@ if Karafka.rails?
         # If added again, would print stuff twice
         next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
 
-        logger = ActiveSupport::Logger.new($stdout)
+        stdout_logger = ActiveSupport::Logger.new($stdout)
         # Inherit the logger level from Rails, otherwise would always run with the debug level
-        logger.level = Rails.logger.level
+        stdout_logger.level = Rails.logger.level
 
-        Rails.logger.extend(
-          ActiveSupport::Logger.broadcast(
-            logger
+        rails71plus = Rails.gem_version >= Gem::Version.new('7.1.0')
+
+        # Rails 7.1 replaced the broadcast module with a broadcast logger
+        if rails71plus
+          Rails.logger.broadcast_to(stdout_logger)
+        else
+          Rails.logger.extend(
+            ActiveSupport::Logger.broadcast(
+              stdout_logger
+            )
           )
-        )
+        end
       end
 
       initializer 'karafka.configure_rails_auto_load_paths' do |app|

data/lib/karafka/routing/features/base.rb

@@ -23,11 +23,46 @@ module Karafka
 
           # Loads all the features and activates them
           def load_all
+            features.each(&:activate)
+          end
+
+          # @param config [Karafka::Core::Configurable::Node] app config that we can alter with
+          #   particular routing feature specific stuff if needed
+          def pre_setup_all(config)
+            features.each { |feature| feature.pre_setup(config) }
+          end
+
+          # Runs post setup routing features configuration operations
+          #
+          # @param config [Karafka::Core::Configurable::Node]
+          def post_setup_all(config)
+            features.each { |feature| feature.post_setup(config) }
+          end
+
+          private
+
+          # @return [Array<Class>] all available routing features
+          def features
             ObjectSpace
               .each_object(Class)
               .select { |klass| klass < self }
               .sort_by(&:to_s)
-              .each(&:activate)
+          end
+
+          protected
+
+          # Runs pre-setup configuration of a particular routing feature
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config node
+          def pre_setup(_config)
+            true
+          end
+
+          # Runs post-setup configuration of a particular routing feature
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config node
+          def post_setup(_config)
+            true
           end
         end
       end

data/lib/karafka/routing/features/inline_insights/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Config of this feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Inline Insights related contracts namespace
+        module Contracts
+          # Contract for inline insights topic setup
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :inline_insights do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights/topic.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class InlineInsights < Base
+        # Routing topic inline insights API
+        module Topic
+          # @param active [Boolean] should inline insights be activated
+          def inline_insights(active = false)
+            @inline_insights ||= Config.new(
+              active: active
+            )
+          end
+
+          # @return [Boolean] Are inline insights active
+          def inline_insights?
+            inline_insights.active?
+          end
+
+          # @return [Hash] topic setup hash
+          def to_h
+            super.merge(
+              inline_insights: inline_insights.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/inline_insights.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Feature allowing us to get visibility during the consumption into metrics of particular
+      # partition we operate on. It can be useful when making context-aware consumers that change
+      # their behaviours based on the lag and other parameters.
+      class InlineInsights < Base
+        class << self
+          # If needed installs the needed listener and initializes tracker
+          #
+          # @param _config [Karafka::Core::Configurable::Node] app config
+          def post_setup(_config)
+            ::Karafka::App.monitor.subscribe('app.running') do
+              # Do not activate tracking of statistics if none of our active topics uses it
+              # This prevents us from tracking metrics when user just runs a subset of topics
+              # in a given process and none of those actually utilizes this feature
+              next unless ::Karafka::App
+                          .subscription_groups
+                          .values
+                          .flat_map(&:itself)
+                          .flat_map(&:topics)
+                          .flat_map(&:to_a)
+                          .any?(&:inline_insights?)
+
+              # Initialize the tracker prior to becoming multi-threaded
+              ::Karafka::Processing::InlineInsights::Tracker.instance
+
+              # Subscribe to the statistics reports and collect them
+              ::Karafka.monitor.subscribe(
+                ::Karafka::Processing::InlineInsights::Listener.new
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/subscription_group.rb

@@ -34,7 +34,11 @@ module Karafka
       def initialize(position, topics)
         @name = topics.first.subscription_group_name
         @consumer_group = topics.first.consumer_group
-        @id = "#{@name}_#{position}"
+        # We include the consumer group id here because we want to have unique ids of subscription
+        # groups across the system. Otherwise user could set the same name for multiple
+        # subscription groups in many consumer groups effectively having same id for different
+        # entities
+        @id = "#{@consumer_group.id}_#{@name}_#{position}"
         @position = position
         @topics = topics
         @kafka = build_kafka
@@ -87,7 +91,7 @@ module Karafka
         # If we use static group memberships, there can be a case, where same instance id would
         # be set on many subscription groups as the group instance id from Karafka perspective is
         # set per config. Each instance even if they are subscribed to different topics needs to
-        # have if fully unique. To make sure of that, we just add extra postfix at the end that
+        # have it fully unique. To make sure of that, we just add extra postfix at the end that
         # increments.
         group_instance_id = kafka.fetch(:'group.instance.id', false)
 

data/lib/karafka/setup/attributes_map.rb

@@ -221,6 +221,7 @@ module Karafka
         request.timeout.ms
         resolve_cb
         retries
+        retry.backoff.max.ms
         retry.backoff.ms
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd

data/lib/karafka/setup/config.rb

@@ -208,6 +208,8 @@ module Karafka
           setting :partitioner_class, default: Processing::Partitioner
           # option strategy_selector [Object] processing strategy selector to be used
           setting :strategy_selector, default: Processing::StrategySelector.new
+          # option expansions_selector [Object] processing expansions selector to be used
+          setting :expansions_selector, default: Processing::ExpansionsSelector.new
         end
 
         # Things related to operating on messages
@@ -238,10 +240,14 @@ module Karafka
         def setup(&block)
           # Will prepare and verify license if present
           Licenser.prepare_and_verify(config.license)
+
+          # Pre-setup configure all routing features that would need this
+          Routing::Features::Base.pre_setup_all(config)
+
           # Will configure all the pro components
           # This needs to happen before end user configuration as the end user may overwrite some
           # of the pro defaults with custom components
-          Pro::Loader.pre_setup(config) if Karafka.pro?
+          Pro::Loader.pre_setup_all(config) if Karafka.pro?
 
           configure(&block)
           merge_kafka_defaults!(config)
@@ -253,9 +259,12 @@ module Karafka
           # Refreshes the references that are cached that might have been changed by the config
           ::Karafka.refresh!
 
+          # Post-setup configure all routing features that would need this
+          Routing::Features::Base.post_setup_all(config)
+
           # Runs things that need to be executed after config is defined and all the components
           # are also configured
-          Pro::Loader.post_setup(config) if Karafka.pro?
+          Pro::Loader.post_setup_all(config) if Karafka.pro?
 
           Karafka::App.initialized!
         end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.6'
+  VERSION = '2.2.8.beta1'
 end