karafka 2.2.6 → 2.2.8.beta1

data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb

@@ -0,0 +1,331 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      # Namespace for Appsignal instrumentation
+      module Appsignal
+        # Listener that ships metrics to Appsignal
+        class MetricsListener < Base
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace
+
+          # Value object for storing a single rdkafka metric publishing details
+          RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
+
+          setting :namespace, default: 'karafka'
+
+          setting :client, default: Client.new
+
+          setting :rd_kafka_metrics, default: [
+            # Broker metrics
+            RdKafkaMetric.new(:count, :brokers, 'requests_retries', 'txretries_d'),
+            RdKafkaMetric.new(:count, :brokers, 'transmission_errors', 'txerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'receive_errors', 'rxerrs_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection_connects', 'connects_d'),
+            RdKafkaMetric.new(:count, :brokers, 'connection_disconnects', 'disconnects_d'),
+            RdKafkaMetric.new(:gauge, :brokers, 'network_latency_avg', %w[rtt avg]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network_latency_p95', %w[rtt p95]),
+            RdKafkaMetric.new(:gauge, :brokers, 'network_latency_p99', %w[rtt p99]),
+
+            # Topics partitions metrics
+            RdKafkaMetric.new(:gauge, :topics, 'consumer_lag', 'consumer_lag_stored'),
+            RdKafkaMetric.new(:gauge, :topics, 'consumer_lag_delta', 'consumer_lag_stored_d')
+          ].freeze
+
+          # Metrics that sum values on topics levels and not on partition levels
+          setting :aggregated_rd_kafka_metrics, default: [
+            # Topic aggregated metrics
+            RdKafkaMetric.new(:gauge, :topics, 'consumer_aggregated_lag', 'consumer_lag_stored')
+          ].freeze
+
+          configure
+
+          # Before each consumption process, lets start a transaction associated with it
+          # We also set some basic metadata about the given consumption that can be useful for
+          # debugging
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consume(event)
+            consumer = event.payload[:caller]
+
+            start_transaction(consumer, 'consume')
+
+            client.metadata = {
+              batch_size: consumer.messages.size,
+              first_offset: consumer.messages.metadata.first_offset,
+              last_offset: consumer.messages.metadata.last_offset,
+              consumer_group: consumer.topic.consumer_group.id,
+              topic: consumer.topic.name,
+              partition: consumer.partition,
+              attempt: consumer.coordinator.pause_tracker.attempt
+            }
+          end
+
+          # Once we're done with consumption, we bump counters about that
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_consumed(event)
+            consumer = event.payload[:caller]
+            messages = consumer.messages
+            metadata = messages.metadata
+
+            with_multiple_resolutions(consumer) do |tags|
+              count('consumer_messages', messages.size, tags)
+              count('consumer_batches', 1, tags)
+              gauge('consumer_offsets', metadata.last_offset, tags)
+            end
+
+            stop_transaction
+          end
+
+          # Register minute based probe only on app running. Otherwise if we would always register
+          # minute probe, it would report on processes using Karafka but not running the
+          # consumption process
+          #
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_running(_event)
+            return if @probe_registered
+
+            @probe_registered = true
+
+            # Registers the minutely probe for one-every-minute metrics
+            client.register_probe(:karafka, -> { minute_probe })
+          end
+
+          # Keeps track of revocation user code execution
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoke(event)
+            consumer = event.payload[:caller]
+            start_transaction(consumer, 'revoked')
+          end
+
+          # Finishes the revocation transaction
+          #
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_revoked(_event)
+            stop_transaction
+          end
+
+          # Keeps track of revocation user code execution
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutting_down(event)
+            consumer = event.payload[:caller]
+            start_transaction(consumer, 'shutdown')
+          end
+
+          # Finishes the shutdown transaction
+          #
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_consumer_shutdown(_event)
+            stop_transaction
+          end
+
+          # Counts DLQ dispatches
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_dead_letter_queue_dispatched(event)
+            consumer = event.payload[:caller]
+
+            with_multiple_resolutions(consumer) do |tags|
+              count('consumer_dead', 1, tags)
+            end
+          end
+
+          # Reports on **any** error that occurs. This also includes non-user related errors
+          # originating from the framework.
+          #
+          # @param event [Karafka::Core::Monitoring::Event] error event details
+          def on_error_occurred(event)
+            # If this is a user consumption related error, we bump the counters for metrics
+            if event[:type] == 'consumer.consume.error'
+              consumer = event.payload[:caller]
+
+              with_multiple_resolutions(consumer) do |tags|
+                count('consumer_errors', 1, tags)
+              end
+            end
+
+            stop_transaction
+          end
+
+          # Hooks up to Karafka instrumentation for emitted statistics
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_statistics_emitted(event)
+            statistics = event[:statistics]
+            consumer_group_id = event[:consumer_group_id]
+
+            rd_kafka_metrics.each do |metric|
+              report_metric(metric, statistics, consumer_group_id)
+            end
+
+            report_aggregated_topics_metrics(statistics, consumer_group_id)
+          end
+
+          # Reports a given metric statistics to Appsignal
+          # @param metric [RdKafkaMetric] metric value object
+          # @param statistics [Hash] hash with all the statistics emitted
+          # @param consumer_group_id [String] cg in context which we operate
+          def report_metric(metric, statistics, consumer_group_id)
+            case metric.scope
+            when :root
+              # Do nothing on the root metrics as the same metrics are reported in a granular
+              # way from other places
+              nil
+            when :brokers
+              statistics.fetch('brokers').each_value do |broker_statistics|
+                # Skip bootstrap nodes
+                # Bootstrap nodes have nodeid -1, other nodes have positive
+                # node ids
+                next if broker_statistics['nodeid'] == -1
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  broker_statistics.dig(*metric.key_location),
+                  {
+                    broker: broker_statistics['nodename']
+                  }
+                )
+              end
+            when :topics
+              statistics.fetch('topics').each do |topic_name, topic_values|
+                topic_values['partitions'].each do |partition_name, partition_statistics|
+                  next if partition_name == '-1'
+                  # Skip until lag info is available
+                  next if partition_statistics['consumer_lag'] == -1
+                  next if partition_statistics['consumer_lag_stored'] == -1
+
+                  # Skip if we do not own the fetch assignment
+                  next if partition_statistics['fetch_state'] == 'stopped'
+                  next if partition_statistics['fetch_state'] == 'none'
+
+                  public_send(
+                    metric.type,
+                    metric.name,
+                    partition_statistics.dig(*metric.key_location),
+                    {
+                      consumer_group: consumer_group_id,
+                      topic: topic_name,
+                      partition: partition_name
+                    }
+                  )
+                end
+              end
+            else
+              raise ArgumentError, metric.scope
+            end
+          end
+
+          # Publishes aggregated topic-level metrics that are sum of per partition metrics
+          #
+          # @param statistics [Hash] hash with all the statistics emitted
+          # @param consumer_group_id [String] cg in context which we operate
+          def report_aggregated_topics_metrics(statistics, consumer_group_id)
+            config.aggregated_rd_kafka_metrics.each do |metric|
+              statistics.fetch('topics').each do |topic_name, topic_values|
+                sum = 0
+
+                topic_values['partitions'].each do |partition_name, partition_statistics|
+                  next if partition_name == '-1'
+                  # Skip until lag info is available
+                  next if partition_statistics['consumer_lag'] == -1
+                  next if partition_statistics['consumer_lag_stored'] == -1
+
+                  sum += partition_statistics.dig(*metric.key_location)
+                end
+
+                public_send(
+                  metric.type,
+                  metric.name,
+                  sum,
+                  {
+                    consumer_group: consumer_group_id,
+                    topic: topic_name
+                  }
+                )
+              end
+            end
+          end
+
+          # Increments a counter with a namespace key, value and tags
+          #
+          # @param key [String] key we want to use (without the namespace)
+          # @param value [Integer] count value
+          # @param tags [Hash] additional extra tags
+          def count(key, value, tags)
+            client.count(
+              namespaced_metric(key),
+              value,
+              tags
+            )
+          end
+
+          # Sets the gauge value
+          #
+          # @param key [String] key we want to use (without the namespace)
+          # @param value [Integer] gauge value
+          # @param tags [Hash] additional extra tags
+          def gauge(key, value, tags)
+            client.gauge(
+              namespaced_metric(key),
+              value,
+              tags
+            )
+          end
+
+          private
+
+          # Wraps metric name in listener's namespace
+          # @param metric_name [String] RdKafkaMetric name
+          # @return [String]
+          def namespaced_metric(metric_name)
+            "#{namespace}_#{metric_name}"
+          end
+
+          # Starts the transaction for monitoring user code
+          #
+          # @param consumer [Karafka::BaseConsumer] karafka consumer instance
+          # @param action_name [String] lifecycle user method name
+          def start_transaction(consumer, action_name)
+            client.start_transaction(
+              "#{consumer.class}##{action_name}"
+            )
+          end
+
+          # Stops the transaction wrapping user code
+          def stop_transaction
+            client.stop_transaction
+          end
+
+          # @param consumer [Karafka::BaseConsumer] Karafka consumer instance
+          def with_multiple_resolutions(consumer)
+            topic_name = consumer.topic.name
+            consumer_group_id = consumer.topic.consumer_group.id
+            partition = consumer.partition
+
+            tags = {
+              consumer_group: consumer_group_id,
+              topic: topic_name
+            }
+
+            yield(tags)
+            yield(tags.merge(partition: partition))
+          end
+
+          # Sends minute based probing metrics
+          def minute_probe
+            concurrency = Karafka::App.config.concurrency
+
+            count('processes_count', 1, {})
+            count('threads_count', concurrency, {})
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -67,7 +67,7 @@ module Karafka
             configure(&block)
           end
 
-          # Hooks up to WaterDrop instrumentation for emitted statistics
+          # Hooks up to Karafka instrumentation for emitted statistics
           #
           # @param event [Karafka::Core::Monitoring::Event]
           def on_statistics_emitted(event)
@@ -101,7 +101,7 @@ module Karafka
             time_taken = event[:time]
             messages_count = event[:messages_buffer].size
 
-            consumer_group_id = event[:subscription_group].consumer_group_id
+            consumer_group_id = event[:subscription_group].consumer_group.id
 
             extra_tags = ["consumer_group:#{consumer_group_id}"]
 

data/lib/karafka/patches/rdkafka/bindings.rb

@@ -18,15 +18,23 @@ module Karafka
           # @param client_ptr [FFI::Pointer]
           # @param code [Integer]
           # @param partitions_ptr [FFI::Pointer]
-          def on_cooperative_rebalance(client_ptr, code, partitions_ptr)
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          # @param opaque [Rdkafka::Opaque]
+          def on_cooperative_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
             case code
             when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              opaque&.call_on_partitions_assign(tpl)
               RB.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
+              opaque&.call_on_partitions_assigned(tpl)
             when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              opaque&.call_on_partitions_revoke(tpl)
               RB.rd_kafka_commit(client_ptr, nil, false)
               RB.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
+              opaque&.call_on_partitions_revoked(tpl)
             else
+              opaque&.call_on_partitions_assign(tpl)
               RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+              opaque&.call_on_partitions_assigned(tpl)
             end
           end
 
@@ -35,45 +43,25 @@ module Karafka
           # @param client_ptr [FFI::Pointer]
           # @param code [Integer]
           # @param partitions_ptr [FFI::Pointer]
-          def on_eager_rebalance(client_ptr, code, partitions_ptr)
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          # @param opaque [Rdkafka::Opaque]
+          def on_eager_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
             case code
             when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              opaque&.call_on_partitions_assign(tpl)
               RB.rd_kafka_assign(client_ptr, partitions_ptr)
+              opaque&.call_on_partitions_assigned(tpl)
             when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              opaque&.call_on_partitions_revoke(tpl)
               RB.rd_kafka_commit(client_ptr, nil, false)
               RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+              opaque&.call_on_partitions_revoked(tpl)
             else
+              opaque&.call_on_partitions_assign(tpl)
               RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+              opaque&.call_on_partitions_assigned(tpl)
             end
           end
-
-          # Trigger Karafka callbacks
-          #
-          # @param code [Integer]
-          # @param opaque [Rdkafka::Opaque]
-          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
-          def trigger_callbacks(code, opaque, tpl)
-            Karafka.monitor.instrument(
-              'connection.client.rebalance_callback',
-              caller: self,
-              code: code,
-              tpl: tpl
-            ) do
-              case code
-              when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
-                opaque.call_on_partitions_assigned(tpl)
-              when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
-                opaque.call_on_partitions_revoked(tpl)
-              end
-            end
-          rescue StandardError => e
-            Karafka.monitor.instrument(
-              'error.occurred',
-              caller: self,
-              error: e,
-              type: 'connection.client.rebalance_callback.error'
-            )
-          end
         end
 
         # This patch changes few things:
@@ -89,19 +77,14 @@ module Karafka
         ) do |client_ptr, code, partitions_ptr, opaque_ptr|
           # Patch reference
           pr = ::Karafka::Patches::Rdkafka::Bindings
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
+          opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
 
           if RB.rd_kafka_rebalance_protocol(client_ptr) == 'COOPERATIVE'
-            pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr)
+            pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
           else
-            pr.on_eager_rebalance(client_ptr, code, partitions_ptr)
+            pr.on_eager_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
           end
-
-          opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
-          return unless opaque
-
-          tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
-
-          pr.trigger_callbacks(code, opaque, tpl)
         end
       end
     end

data/lib/karafka/patches/rdkafka/opaque.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Patches
+    module Rdkafka
+      # Patches allowing us to run events on both pre and post rebalance events.
+      # Thanks to that, we can easily connect to the whole flow despite of the moment when things
+      # are happening
+      module Opaque
+        # Handles pre-assign phase of rebalance
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def call_on_partitions_assign(tpl)
+          return unless consumer_rebalance_listener
+          return unless consumer_rebalance_listener.respond_to?(:on_partitions_assign)
+
+          consumer_rebalance_listener.on_partitions_assign(tpl)
+        end
+
+        # Handles pre-revoke phase of rebalance
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def call_on_partitions_revoke(tpl)
+          return unless consumer_rebalance_listener
+          return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoke)
+
+          consumer_rebalance_listener.on_partitions_revoke(tpl)
+        end
+      end
+    end
+  end
+end
+
+::Rdkafka::Opaque.include(
+  Karafka::Patches::Rdkafka::Opaque
+)

data/lib/karafka/pro/loader.rb

@@ -49,7 +49,7 @@ module Karafka
         # Loads all the pro components and configures them wherever it is expected
         # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
-        def pre_setup(config)
+        def pre_setup_all(config)
           features.each { |feature| feature.pre_setup(config) }
 
           reconfigure(config)
@@ -60,7 +60,7 @@ module Karafka
         # Runs post setup features configuration operations
         #
         # @param config [Karafka::Core::Configurable::Node]
-        def post_setup(config)
+        def post_setup_all(config)
           features.each { |feature| feature.post_setup(config) }
         end
 

data/lib/karafka/pro/processing/coordinator.rb

@@ -17,6 +17,10 @@ module Karafka
       # Pro coordinator that provides extra orchestration methods useful for parallel processing
       # within the same partition
       class Coordinator < ::Karafka::Processing::Coordinator
+        extend Forwardable
+
+        def_delegators :@collapser, :collapsed?, :collapse_until!
+
         attr_reader :filter, :virtual_offset_manager
 
         # @param args [Object] anything the base coordinator accepts
@@ -57,7 +61,7 @@ module Karafka
 
           # We keep the old processed offsets until the collapsing is done and regular processing
           # with virtualization is restored
-          @virtual_offset_manager.clear if topic.virtual_partitions? && !@collapser.collapsed?
+          @virtual_offset_manager.clear if topic.virtual_partitions? && !collapsed?
 
           @last_message = messages.last
         end
@@ -68,12 +72,7 @@ module Karafka
         # @param error [StandardError] error from the failure
         def failure!(consumer, error)
           super
-          @collapser.collapse_until!(@last_message.offset + 1)
-        end
-
-        # @return [Boolean] are we in a collapsed state at the moment
-        def collapsed?
-          @collapser.collapsed?
+          collapse_until!(@last_message.offset + 1)
         end
 
         # @return [Boolean] did any of the filters apply any logic that would cause use to run

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -24,7 +24,7 @@ module Karafka
             @delay = delay
           end
 
-          # Removes too old messages
+          # Removes too young messages
           #
           # @param messages [Array<Karafka::Messages::Message>]
           def apply!(messages)

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Filters
+        # Delayer that checks if we have appropriate insights available. If not, pauses for
+        # 5 seconds so the insights can be loaded from the broker.
+        #
+        # In case it would take more than five seconds to load insights, it will just pause again
+        #
+        # This filter ensures, that we always have inline insights that a consumer can use
+        #
+        # It is relevant in most cases only during the process start, when first poll may not
+        # yield statistics yet but will give some data.
+        class InlineInsightsDelayer < Base
+          # Minimum how long should we pause when there are no metrics
+          PAUSE_TIMEOUT = 5_000
+
+          private_constant :PAUSE_TIMEOUT
+
+          # @param topic [Karafka::Routing::Topic]
+          # @param partition [Integer] partition
+          def initialize(topic, partition)
+            super()
+            @topic = topic
+            @partition = partition
+          end
+
+          # Pauses if inline insights would not be available. Does nothing otherwise
+          #
+          # @param messages [Array<Karafka::Messages::Message>]
+          def apply!(messages)
+            @applied = false
+            @cursor = messages.first
+
+            # Nothing to do if there were no messages
+            # This can happen when we chain filters
+            return unless @cursor
+
+            insights = ::Karafka::Processing::InlineInsights::Tracker.find(
+              @topic,
+              @partition
+            )
+
+            # If insights are available, also nothing to do here and we can just process
+            return unless insights.empty?
+
+            messages.clear
+
+            @applied = true
+          end
+
+          # @return [Integer] ms timeout in case of pause
+          def timeout
+            @cursor && applied? ? PAUSE_TIMEOUT : 0
+          end
+
+          # Pause when we had to back-off or skip if delay is not needed
+          def action
+            applied? ? :pause : :skip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -72,6 +72,15 @@ module Karafka
               coordinator.collapsed?
             end
 
+            # @param offset [Integer] first offset from which we should not operate in a collapsed
+            #   mode.
+            # @note Keep in mind, that if a batch contains this but also messages earlier messages
+            #   that should be collapsed, all will continue to operate in a collapsed mode until
+            #   first full batch with only messages that should not be collapsed.
+            def collapse_until!(offset)
+              coordinator.collapse_until!(offset)
+            end
+
             # @return [Boolean] true if any of virtual partition we're operating in the entangled
             #   mode has already failed and we know we are failing collectively.
             #   Useful for early stop to minimize number of things processed twice.
@@ -84,6 +93,17 @@ module Karafka
               coordinator.failure?
             end
 
+            # Allows for cross-virtual-partition consumers locks
+            #
+            # This is not needed in the non-VP flows because there is always only one consumer
+            # per partition at the same time, so no coordination is needed directly for the
+            # end users
+            #
+            # @param block [Proc] block we want to run in a mutex to prevent race-conditions
+            def synchronize(&block)
+              coordinator.synchronize(&block)
+            end
+
             private
 
             # Prior to adding work to the queue, registers all the messages offsets into the