karafka 2.2.13 → 2.3.0.alpha1

data/lib/karafka/errors.rb

@@ -63,5 +63,18 @@ module Karafka
 
     # Raised when there is an attempt to run an unrecognized CLI command
     UnrecognizedCommandError = Class.new(BaseError)
+
+    # Raised when we attempt to perform operation that is only allowed inside of a transaction and
+    # there is no transaction around us
+    TransactionRequiredError = Class.new(BaseError)
+
+    # Raised in case user would want to perform nested transactions.
+    TransactionAlreadyInitializedError = Class.new(BaseError)
+
+    # Raised in case a listener that was paused is being resumed
+    InvalidListenerResumeError = Class.new(BaseError)
+
+    # Raised when we want to un-pause listener that was not paused
+    InvalidListenerPauseError = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/assignments_tracker.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Keeps track of active assignments and materializes them by returning the routing topics
+    # with appropriate partitions that are assigned at a given moment
+    #
+    # It is auto-subscribed as part of Karafka itself.
+    #
+    # It is not heavy from the computational point of view, as it only operates during rebalances.
+    #
+    # We keep assignments as flat topics structure because we can go from topics to both
+    # subscription and consumer groups if needed.
+    class AssignmentsTracker
+      include Singleton
+
+      def initialize
+        @mutex = Mutex.new
+        @assignments = Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      # Returns all the active/current assignments of this given process
+      #
+      # @return [Hash<Karafka::Routing::Topic, Array<Integer>>]
+      #
+      # @note Keep in mind, that those assignments can change any time, especially when working
+      #   with multiple consumer groups or subscription groups.
+      #
+      # @note We return a copy because we modify internals and we do not want user to tamper with
+      #   the data accidentally
+      def current
+        assignments = {}
+
+        @assignments.each do |topic, partitions|
+          assignments[topic] = partitions.dup.freeze
+        end
+
+        assignments.freeze
+      end
+
+      # Clears all the assignments
+      def clear
+        @mutex.synchronize do
+          @assignments.clear
+        end
+      end
+
+      # When client is under reset due to critical issues, remove all of its assignments as we will
+      #   get a new set of assignments
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_client_reset(event)
+        sg = event[:subscription_group]
+
+        @mutex.synchronize do
+          @assignments.delete_if do |topic, _partitions|
+            topic.subscription_group.id == sg.id
+          end
+        end
+      end
+
+      # Removes partitions from the current assignments hash
+      #
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_rebalance_partitions_revoked(event)
+        sg = event[:subscription_group]
+
+        @mutex.synchronize do
+          event[:tpl].to_h.each do |topic, partitions|
+            topic = sg.topics.find(topic)
+
+            @assignments[topic] -= partitions.map(&:partition)
+            @assignments[topic].sort!
+            # Remove completely topics for which we do not have any assignments left
+            @assignments.delete_if { |_topic, cur_partitions| cur_partitions.empty? }
+          end
+        end
+      end
+
+      # # Adds partitions to the current assignments hash
+      #
+      # @param event [Karafka::Core::Monitoring::Event]
+      def on_rebalance_partitions_assigned(event)
+        sg = event[:subscription_group]
+
+        @mutex.synchronize do
+          event[:tpl].to_h.each do |topic, partitions|
+            topic = sg.topics.find(topic)
+
+            @assignments[topic] += partitions.map(&:partition)
+            @assignments[topic].sort!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/callbacks/rebalance.rb

@@ -6,11 +6,10 @@ module Karafka
       # Callback that connects to the librdkafka rebalance callback and converts those events into
       # our internal events
       class Rebalance
-        # @param subscription_group_id [String] id of the current subscription group instance
-        # @param consumer_group_id [String] id of the current consumer group
-        def initialize(subscription_group_id, consumer_group_id)
-          @subscription_group_id = subscription_group_id
-          @consumer_group_id = consumer_group_id
+        # @param subscription_group [Karafka::Routes::SubscriptionGroup] subscription group for
+        #   which we want to manage rebalances
+        def initialize(subscription_group)
+          @subscription_group = subscription_group
         end
 
         # Publishes an event that partitions are going to be revoked.
@@ -53,8 +52,12 @@ module Karafka
           ::Karafka.monitor.instrument(
             "rebalance.#{name}",
             caller: self,
-            subscription_group_id: @subscription_group_id,
-            consumer_group_id: @consumer_group_id,
+            # We keep the id references here for backwards compatibility as some of the monitors
+            # may use the id references
+            subscription_group_id: @subscription_group.id,
+            subscription_group: @subscription_group,
+            consumer_group_id: @subscription_group.consumer_group.id,
+            consumer_group: @subscription_group.consumer_group,
             tpl: tpl
           )
         end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -241,21 +241,15 @@ module Karafka
         when 'consumer.revoked.error'
           error "Consumer on revoked failed due to an error: #{error}"
           error details
-        when 'consumer.before_enqueue.error'
-          error "Consumer before enqueue failed due to an error: #{error}"
-          error details
-        when 'consumer.before_consume.error'
-          error "Consumer before consume failed due to an error: #{error}"
-          error details
-        when 'consumer.after_consume.error'
-          error "Consumer after consume failed due to an error: #{error}"
-          error details
         when 'consumer.idle.error'
           error "Consumer idle failed due to an error: #{error}"
           error details
         when 'consumer.shutdown.error'
           error "Consumer on shutdown failed due to an error: #{error}"
           error details
+        when 'consumer.tick.error'
+          error "Consumer tick failed due to an error: #{error}"
+          error details
         when 'worker.process.error'
           fatal "Worker processing failed due to an error: #{error}"
           fatal details

data/lib/karafka/instrumentation/notifications.rb

@@ -20,6 +20,7 @@ module Karafka
         active_job.consume
         active_job.consumed
 
+        app.initializing
         app.initialized
         app.running
         app.quieting
@@ -30,27 +31,31 @@ module Karafka
 
         client.pause
         client.resume
+        client.reset
 
         connection.listener.before_fetch_loop
         connection.listener.fetch_loop
         connection.listener.fetch_loop.received
+        connection.listener.after_fetch_loop
 
-        connection.client.poll.error
-        connection.client.unsubscribe.error
-
-        rebalance.partitions_assign
-        rebalance.partitions_assigned
-        rebalance.partitions_revoke
-        rebalance.partitions_revoked
-
-        consumer.before_enqueue
+        consumer.before_schedule_consume
         consumer.consume
         consumer.consumed
         consumer.consuming.pause
         consumer.consuming.retry
+
+        consumer.before_schedule_idle
         consumer.idle
+
+        consumer.before_schedule_revoked
         consumer.revoke
         consumer.revoked
+
+        consumer.before_schedule_tick
+        consumer.tick
+        consumer.ticked
+
+        consumer.before_schedule_shutdown
         consumer.shutting_down
         consumer.shutdown
 
@@ -61,6 +66,11 @@ module Karafka
 
         process.notice_signal
 
+        rebalance.partitions_assign
+        rebalance.partitions_assigned
+        rebalance.partitions_revoke
+        rebalance.partitions_revoked
+
         statistics.emitted
 
         worker.process

data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb

@@ -42,6 +42,16 @@ module Karafka
 
           configure
 
+          # Types of errors originating from user code in the consumer flow
+          USER_CONSUMER_ERROR_TYPES = %w[
+            consumer.consume.error
+            consumer.revoked.error
+            consumer.shutdown.error
+            consumer.tick.error
+          ].freeze
+
+          private_constant :USER_CONSUMER_ERROR_TYPES
+
           # Before each consumption process, lets start a transaction associated with it
           # We also set some basic metadata about the given consumption that can be useful for
           # debugging
@@ -94,34 +104,27 @@ module Karafka
             client.register_probe(:karafka, -> { minute_probe })
           end
 
-          # Keeps track of revocation user code execution
-          #
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoke(event)
-            consumer = event.payload[:caller]
-            start_transaction(consumer, 'revoked')
-          end
-
-          # Finishes the revocation transaction
-          #
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoked(_event)
-            stop_transaction
-          end
-
-          # Keeps track of revocation user code execution
-          #
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutting_down(event)
-            consumer = event.payload[:caller]
-            start_transaction(consumer, 'shutdown')
-          end
+          [
+            %i[revoke revoked revoked],
+            %i[shutting_down shutdown shutdown],
+            %i[tick ticked tick]
+          ].each do |before, after, name|
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # Keeps track of user code execution
+              #
+              # @param event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{before}(event)
+                consumer = event.payload[:caller]
+                start_transaction(consumer, '#{name}')
+              end
 
-          # Finishes the shutdown transaction
-          #
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutdown(_event)
-            stop_transaction
+              # Finishes the transaction
+              #
+              # @param _event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{after}(_event)
+                stop_transaction
+              end
+            RUBY
           end
 
           # Counts DLQ dispatches
@@ -141,7 +144,7 @@ module Karafka
           # @param event [Karafka::Core::Monitoring::Event] error event details
           def on_error_occurred(event)
             # If this is a user consumption related error, we bump the counters for metrics
-            if event[:type] == 'consumer.consume.error'
+            if USER_CONSUMER_ERROR_TYPES.include?(event[:type])
               consumer = event.payload[:caller]
 
               with_multiple_resolutions(consumer) do |tags|

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -55,7 +55,24 @@ module Karafka
             consumer = job.executor.topic.consumer
             topic = job.executor.topic.name
 
-            current_span.resource = "#{consumer}#consume"
+            action = case job_type
+                     when 'Periodic'
+                       'tick'
+                     when 'PeriodicNonBlocking'
+                       'tick'
+                     when 'Shutdown'
+                       'shutdown'
+                     when 'Revoked'
+                       'revoked'
+                     when 'RevokedNonBlocking'
+                       'revoked'
+                     when 'Idle'
+                       'idle'
+                     else
+                       'consume'
+                     end
+
+            current_span.resource = "#{consumer}##{action}"
             info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
 
             pop_tags
@@ -94,14 +111,16 @@ module Karafka
               error "Consumer consuming error: #{error}"
             when 'consumer.revoked.error'
               error "Consumer on revoked failed due to an error: #{error}"
-            when 'consumer.before_enqueue.error'
-              error "Consumer before enqueue failed due to an error: #{error}"
+            when 'consumer.before_schedule.error'
+              error "Consumer before schedule failed due to an error: #{error}"
             when 'consumer.before_consume.error'
               error "Consumer before consume failed due to an error: #{error}"
             when 'consumer.after_consume.error'
               error "Consumer after consume failed due to an error: #{error}"
             when 'consumer.shutdown.error'
               error "Consumer on shutdown failed due to an error: #{error}"
+            when 'consumer.tick.error'
+              error "Consumer tick failed due to an error: #{error}"
             when 'worker.process.error'
               fatal "Worker processing failed due to an error: #{error}"
             when 'connection.listener.fetch_loop.error'

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -128,18 +128,21 @@ module Karafka
             histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
           end
 
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoked(event)
-            tags = default_tags + consumer_tags(event.payload[:caller])
-
-            count('consumer.revoked', 1, tags: tags)
-          end
-
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutdown(event)
-            tags = default_tags + consumer_tags(event.payload[:caller])
-
-            count('consumer.shutdown', 1, tags: tags)
+          {
+            revoked: :revoked,
+            shutdown: :shutdown,
+            ticked: :tick
+          }.each do |after, name|
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # Keeps track of user code execution
+              #
+              # @param event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{after}(event)
+                tags = default_tags + consumer_tags(event.payload[:caller])
+
+                count('consumer.#{name}', 1, tags: tags)
+              end
+            RUBY
           end
 
           # Worker related metrics

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -15,6 +15,14 @@ module Karafka
         # data would be processed, but process itself would still be active. This listener allows
         # for defining of a ttl that gets bumped on each poll loop and before and after processing
         # of a given messages batch.
+        #
+        # @note This listener will bind itself only when Karafka will actually attempt to start
+        #   and moves from initializing to running. Before that, the TCP server will NOT be active.
+        #   This is done on purpose to mitigate a case where users would subscribe this listener
+        #   in `karafka.rb` without checking the recommendations of conditional assignment.
+        #
+        # @note In case of usage within an embedding with Puma, you need to select different port
+        #   then the one used by Puma itself.
         class LivenessListener
           include ::Karafka::Core::Helpers::Time
 
@@ -40,12 +48,18 @@ module Karafka
             consuming_ttl: 5 * 60 * 1_000,
             polling_ttl: 5 * 60 * 1_000
           )
-            @server = TCPServer.new(*[hostname, port].compact)
+            @hostname = hostname
+            @port = port
             @polling_ttl = polling_ttl
             @consuming_ttl = consuming_ttl
             @mutex = Mutex.new
             @pollings = {}
             @consumptions = {}
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_running(_event)
+            @server = TCPServer.new(*[@hostname, @port].compact)
 
             Thread.new do
               loop do
@@ -54,42 +68,37 @@ module Karafka
             end
           end
 
-          # Tick on each fetch
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_connection_listener_fetch_loop(_event)
-            mark_polling_tick
-          end
-
-          # Tick on starting work
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_consume(_event)
-            mark_consumption_tick
-          end
-
-          # Tick on finished work
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_consumed(_event)
-            clear_consumption_tick
-          end
-
+          # Stop the http server when we stop the process
           # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoke(_event)
-            mark_consumption_tick
+          def on_app_stopped(_event)
+            @server.close
           end
 
+          # Tick on each fetch
           # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoked(_event)
-            clear_consumption_tick
+          def on_connection_listener_fetch_loop(_event)
+            mark_polling_tick
           end
 
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutting_down(_event)
-            mark_consumption_tick
-          end
+          {
+            consume: :consumed,
+            revoke: :revoked,
+            shutting_down: :shutdown,
+            tick: :ticked
+          }.each do |before, after|
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # Tick on starting work
+              # @param _event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{before}(_event)
+                mark_consumption_tick
+              end
 
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutdown(_event)
-            clear_consumption_tick
+              # Tick on finished work
+              # @param _event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{after}(_event)
+                clear_consumption_tick
+              end
+            RUBY
           end
 
           # @param _event [Karafka::Core::Monitoring::Event]
@@ -98,12 +107,6 @@ module Karafka
             clear_polling_tick
           end
 
-          # Stop the http server when we stop the process
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_app_stopped(_event)
-            @server.close
-          end
-
           private
 
           # Wraps the logic with a mutex

data/lib/karafka/pro/base_consumer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Extra methods always used in the base consumer in the pro mode
+    #
+    # We do not define those methods as part of the strategies flows, because they are injected
+    # (strategies) on singletons and often used only in one of the strategy variants
+    #
+    # Methods here are suppose to be always available or are expected to be redefined
+    module BaseConsumer
+      # Runs the on-schedule tick periodic operations
+      # This method is an alias but is part of the naming convention used for other flows, this
+      # is why we do not reference the `handle_before_schedule_tick` directly
+      def on_before_schedule_tick
+        handle_before_schedule_tick
+      end
+
+      # Used by the executor to trigger consumer tick
+      # @private
+      def on_tick
+        handle_tick
+      rescue StandardError => e
+        Karafka.monitor.instrument(
+          'error.occurred',
+          error: e,
+          caller: self,
+          type: 'consumer.tick.error'
+        )
+      end
+
+      # By default we do nothing when ticking
+      def tick; end
+    end
+  end
+end