karafka 2.2.6 → 2.2.8.beta1

data/lib/karafka/connection/listener.rb

@@ -112,7 +112,7 @@ module Karafka
           # distributing consuming jobs as upon revoking, we might get assigned to the same
           # partitions, thus getting their jobs. The revoking jobs need to finish before
           # appropriate consumers are taken down and re-created
-          build_and_schedule_revoke_lost_partitions_jobs
+          build_and_schedule_revoked_jobs_for_revoked_partitions
 
           # We wait only on jobs from our subscription group. Other groups are independent.
           # This will block on revoked jobs until they are finished. Those are not meant to last
@@ -140,7 +140,7 @@ module Karafka
         # that occurred in the cluster.
         wait_pinging(
           wait_until: -> { @jobs_queue.empty?(@subscription_group.id) },
-          after_ping: -> { build_and_schedule_revoke_lost_partitions_jobs }
+          after_ping: -> { build_and_schedule_revoked_jobs_for_revoked_partitions }
         )
 
         # We do not want to schedule the shutdown jobs prior to finishing all the jobs
@@ -197,7 +197,7 @@ module Karafka
       end
 
       # Enqueues revoking jobs for partitions that were taken away from the running process.
-      def build_and_schedule_revoke_lost_partitions_jobs
+      def build_and_schedule_revoked_jobs_for_revoked_partitions
         revoked_partitions = @client.rebalance_manager.revoked_partitions
 
         # Stop early to save on some execution and array allocation

data/lib/karafka/connection/rebalance_manager.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Connection
-    # Manager for tracking changes in the partitions assignment.
+    # Manager for tracking changes in the partitions assignment after the assignment is done.
     #
     # We need tracking of those to clean up consumers that will no longer process given partitions
     # as they were taken away.
@@ -17,6 +17,10 @@ module Karafka
     #
     # @note For cooperative-sticky `#assigned_partitions` holds only the recently assigned
     #   partitions, not all the partitions that are owned
+    #
+    # @note We have to have the `subscription_group` reference because we have a global pipeline
+    #   for notifications and we need to make sure we track changes only for things that are of
+    #   relevance to our subscription group
     class RebalanceManager
       # Empty array for internal usage not to create new objects
       EMPTY_ARRAY = [].freeze
@@ -25,12 +29,17 @@ module Karafka
 
       private_constant :EMPTY_ARRAY
 
+      # @param subscription_group_id [String] subscription group id
       # @return [RebalanceManager]
-      def initialize
+      def initialize(subscription_group_id)
         @assigned_partitions = {}
         @revoked_partitions = {}
         @changed = false
         @active = false
+        @subscription_group_id = subscription_group_id
+
+        # Connects itself to the instrumentation pipeline so rebalances can be tracked
+        ::Karafka.monitor.subscribe(self)
       end
 
       # Resets the rebalance manager state
@@ -55,36 +64,42 @@ module Karafka
         @active
       end
 
-      # Callback that kicks in inside of rdkafka, when new partitions are assigned.
+      # We consider as lost only partitions that were taken away and not re-assigned back to us
+      def lost_partitions
+        lost_partitions = {}
+
+        revoked_partitions.each do |topic, partitions|
+          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
+        end
+
+        lost_partitions
+      end
+
+      # Callback that kicks in inside of rdkafka, when new partitions were assigned.
       #
       # @private
-      # @param partitions [Rdkafka::Consumer::TopicPartitionList]
-      def on_partitions_assigned(partitions)
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_rebalance_partitions_assigned(event)
+        # Apply changes only for our subscription group
+        return unless event[:subscription_group_id] == @subscription_group_id
+
         @active = true
-        @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @assigned_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end
 
-      # Callback that kicks in inside of rdkafka, when partitions are revoked.
+      # Callback that kicks in inside of rdkafka, when partitions were revoked.
       #
       # @private
-      # @param partitions [Rdkafka::Consumer::TopicPartitionList]
-      def on_partitions_revoked(partitions)
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_rebalance_partitions_revoked(event)
+        # Apply changes only for our subscription group
+        return unless event[:subscription_group_id] == @subscription_group_id
+
         @active = true
-        @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end
-
-      # We consider as lost only partitions that were taken away and not re-assigned back to us
-      def lost_partitions
-        lost_partitions = {}
-
-        revoked_partitions.each do |topic, partitions|
-          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
-        end
-
-        lost_partitions
-      end
     end
   end
 end

data/lib/karafka/contracts/config.rb

@@ -73,6 +73,8 @@ module Karafka
           required(:scheduler) { |val| !val.nil? }
           required(:coordinator_class) { |val| !val.nil? }
           required(:partitioner_class) { |val| !val.nil? }
+          required(:strategy_selector) { |val| !val.nil? }
+          required(:expansions_selector) { |val| !val.nil? }
         end
 
         nested(:active_job) do

data/lib/karafka/errors.rb

@@ -60,5 +60,8 @@ module Karafka
     # Raised when we run operations that require certain result but despite successfully finishing
     # it is not yet available due to some synchronization mechanisms and caches
     ResultNotVisibleError = Class.new(BaseError)
+
+    # Raised when there is an attempt to run an unrecognized CLI command
+    UnrecognizedCommandError = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/callbacks/rebalance.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    module Callbacks
+      # Callback that connects to the librdkafka rebalance callback and converts those events into
+      # our internal events
+      class Rebalance
+        # @param subscription_group_id [String] id of the current subscription group instance
+        # @param consumer_group_id [String] id of the current consumer group
+        def initialize(subscription_group_id, consumer_group_id)
+          @subscription_group_id = subscription_group_id
+          @consumer_group_id = consumer_group_id
+        end
+
+        # Publishes an event that partitions are going to be revoked.
+        # At this stage we can still commit offsets, etc.
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def on_partitions_revoke(tpl)
+          instrument('partitions_revoke', tpl)
+        end
+
+        # Publishes an event that partitions are going to be assigned
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def on_partitions_assign(tpl)
+          instrument('partitions_assign', tpl)
+        end
+
+        # Publishes an event that partitions were revoked. This is after we've lost them, so no
+        # option to commit offsets.
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def on_partitions_revoked(tpl)
+          instrument('partitions_revoked', tpl)
+        end
+
+        # Publishes an event that partitions were assigned.
+        #
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def on_partitions_assigned(tpl)
+          instrument('partitions_assigned', tpl)
+        end
+
+        private
+
+        # Publishes info that a rebalance event of a given type has happened
+        #
+        # @param name [String] name of the event
+        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+        def instrument(name, tpl)
+          ::Karafka.monitor.instrument(
+            "rebalance.#{name}",
+            caller: self,
+            subscription_group_id: @subscription_group_id,
+            consumer_group_id: @consumer_group_id,
+            tpl: tpl
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/notifications.rb

@@ -35,10 +35,14 @@ module Karafka
         connection.listener.fetch_loop
         connection.listener.fetch_loop.received
 
-        connection.client.rebalance_callback
         connection.client.poll.error
         connection.client.unsubscribe.error
 
+        rebalance.partitions_assign
+        rebalance.partitions_assigned
+        rebalance.partitions_revoke
+        rebalance.partitions_revoked
+
         consumer.consume
         consumer.consumed
         consumer.consuming.pause

data/lib/karafka/instrumentation/vendors/appsignal/base.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'client'
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      # Namespace for Appsignal instrumentation
+      module Appsignal
+        # Base for all the instrumentation listeners
+        class Base
+          include ::Karafka::Core::Configurable
+          extend Forwardable
+
+          # @param block [Proc] configuration block
+          def initialize(&block)
+            configure
+            setup(&block) if block
+          end
+
+          # @param block [Proc] configuration block
+          # @note We define this alias to be consistent with `Karafka#setup`
+          def setup(&block)
+            configure(&block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/appsignal/client.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      module Appsignal
+        # Appsignal client wrapper
+        # We wrap the native client so we can inject our own stub in specs when needed
+        #
+        # It also abstracts away the notion of transactions and their management
+        #
+        # @note This client is abstract, it has no notion of Karafka whatsoever
+        class Client
+          # Starts an appsignal transaction with a given action name
+          #
+          # @param action_name [String] action name. For processing this should be equal to
+          #   consumer class + method name
+          def start_transaction(action_name)
+            transaction = ::Appsignal::Transaction.create(
+              SecureRandom.uuid,
+              ::Appsignal::Transaction::BACKGROUND_JOB,
+              ::Appsignal::Transaction::GenericRequest.new({})
+            )
+
+            transaction.set_action_if_nil(action_name)
+          end
+
+          # Stops the current transaction (if any)
+          def stop_transaction
+            return unless transaction?
+
+            ::Appsignal::Transaction.complete_current!
+          end
+
+          # Sets metadata on a current transaction (if any)
+          #
+          # @param metadata_hash [Hash] hash with metadata we want to set
+          def metadata=(metadata_hash)
+            return unless transaction?
+
+            transaction = ::Appsignal::Transaction.current
+
+            stringify_hash(metadata_hash).each do |key, value|
+              transaction.set_metadata(key, value)
+            end
+          end
+
+          # Increments counter with the given value and tags
+          #
+          # @param key [String] key we want to use
+          # @param value [Integer] increment value
+          # @param tags [Hash] additional extra tags
+          def count(key, value, tags)
+            ::Appsignal.increment_counter(
+              key,
+              value,
+              stringify_hash(tags)
+            )
+          end
+
+          # Sets gauge with the given value and tags
+          #
+          # @param key [String] key we want to use
+          # @param value [Integer] gauge value
+          # @param tags [Hash] additional extra tags
+          def gauge(key, value, tags)
+            ::Appsignal.set_gauge(
+              key,
+              value,
+              stringify_hash(tags)
+            )
+          end
+
+          # Sends the error that occurred to Appsignal
+          #
+          # @param error [Object] error we want to ship to Appsignal
+          def send_error(error)
+            # If we have an active transaction we should use it instead of creating a generic one
+            # That way proper namespace and other data may be transferred
+            #
+            # In case there is no transaction, a new generic background job one will be used
+            if transaction?
+              transaction.set_error(error)
+            else
+              ::Appsignal.send_error(error) do |transaction|
+                transaction.set_namespace(::Appsignal::Transaction::BACKGROUND_JOB)
+              end
+            end
+          end
+
+          # Registers the probe under a given name
+          # @param name [Symbol] probe name
+          # @param probe [Proc] code to run every minute
+          def register_probe(name, probe)
+            ::Appsignal::Minutely.probes.register(name, probe)
+          end
+
+          private
+
+          # @return [Boolean] do we have a transaction
+          def transaction?
+            ::Appsignal::Transaction.current?
+          end
+
+          # @return [::Appsignal::Transaction, nil] transaction or nil if not started
+          def transaction
+            ::Appsignal::Transaction.current
+          end
+
+          # Converts both keys and values of a hash into strings
+          # @param hash [Hash]
+          # @return [Hash]
+          def stringify_hash(hash)
+            hash
+              .transform_values(&:to_s)
+              .transform_keys!(&:to_s)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/vendors/appsignal/dashboard.json

@@ -0,0 +1,222 @@
+{
+  "title": "Karafka",
+  "description": "This dashboard gives an overview of the overall Karafka status.\nCheckout topics lag, throughput/performance per consumer,\nprocess/worker counts, and many other things.\n",
+  "visuals": [
+    {
+      "title": "Consumed messages",
+      "description": "",
+      "line_label": "%topic%[%consumer_group%]",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_consumer_messages",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Consumed batches",
+      "line_label": "%topic%[%consumer_group%]",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_consumer_batches",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Lags",
+      "line_label": "%topic%[%consumer_group%]",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_consumer_aggregated_lag",
+          "fields": [
+            {
+              "field": "GAUGE"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Errors and DLQ",
+      "description": "",
+      "line_label": "%name% %topic%[%consumer_group%]",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_consumer_errors",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        },
+        {
+          "name": "karafka_consumer_dead",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": [
+            {
+              "key": "consumer_group",
+              "value": "*"
+            },
+            {
+              "key": "topic",
+              "value": "*"
+            }
+          ]
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Connection stability",
+      "description": "",
+      "line_label": "%name%",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_connection_disconnects",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_connection_connects",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_requests_retries",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_transmission_errors",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_receive_errors",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "title": "Concurrency",
+      "line_label": "%name%",
+      "display": "LINE",
+      "format": "number",
+      "draw_null_as_zero": true,
+      "metrics": [
+        {
+          "name": "karafka_processes_count",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        },
+        {
+          "name": "karafka_threads_count",
+          "fields": [
+            {
+              "field": "COUNTER"
+            }
+          ],
+          "tags": []
+        }
+      ],
+      "type": "timeseries"
+    }
+  ]
+}

data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Karafka
+  module Instrumentation
+    module Vendors
+      # Namespace for Appsignal instrumentation
+      module Appsignal
+        # Listener for reporting errors from both consumers and producers
+        # Since we have the same API for WaterDrop and Karafka, we can use one listener with
+        # independent instances
+        class ErrorsListener < Base
+          def_delegators :config, :client
+
+          setting :client, default: Client.new
+
+          configure
+
+          # Sends error details to Appsignal
+          #
+          # @param event [Karafka::Core::Monitoring::Event]
+          def on_error_occurred(event)
+            client.send_error(event[:error])
+          end
+        end
+      end
+    end
+  end
+end