karafka 2.2.13 → 2.3.0.alpha1

data/lib/karafka/app.rb

@@ -36,6 +36,13 @@ module Karafka
       # Just a nicer name for the consumer groups
       alias routes consumer_groups
 
+      # Returns current assignments of this process. Both topics and partitions
+      #
+      # @return [Hash<Karafka::Routing::Topic, Array<Integer>>]
+      def assignments
+        Instrumentation::AssignmentsTracker.instance.current
+      end
+
       # Allow for easier status management via `Karafka::App` by aliasing status methods here
       Status::STATES.each do |state, transition|
         class_eval <<~RUBY, __FILE__, __LINE__ + 1
@@ -68,9 +75,11 @@ module Karafka
         monitor
         pro?
       ].each do |delegated|
-        define_method(delegated) do
-          Karafka.send(delegated)
-        end
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          def #{delegated}
+            Karafka.#{delegated}
+          end
+        RUBY
       end
     end
   end

data/lib/karafka/base_consumer.rb

@@ -11,6 +11,9 @@ module Karafka
 
     def_delegators :@coordinator, :topic, :partition
 
+    def_delegators :producer, :produce_async, :produce_sync, :produce_many_async,
+                   :produce_many_sync
+
     # @return [String] id of the current consumer
     attr_reader :id
     # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
@@ -34,16 +37,9 @@ module Karafka
     # @note This should not be used by the end users as it is part of the lifecycle of things and
     #   not as a part of the public api. This should not perform any extensive operations as it is
     #   blocking and running in the listener thread.
-    def on_before_enqueue
+    def on_before_schedule_consume
       @used = true
-      handle_before_enqueue
-    rescue StandardError => e
-      Karafka.monitor.instrument(
-        'error.occurred',
-        error: e,
-        caller: self,
-        type: 'consumer.before_enqueue.error'
-      )
+      handle_before_schedule_consume
     end
 
     # Can be used to run preparation code in the worker
@@ -59,13 +55,6 @@ module Karafka
       # We run this after the full metadata setup, so we can use all the messages information
       # if needed
       handle_before_consume
-    rescue StandardError => e
-      Karafka.monitor.instrument(
-        'error.occurred',
-        error: e,
-        caller: self,
-        type: 'consumer.before_consume.error'
-      )
     end
 
     # Executes the default consumer flow.
@@ -94,13 +83,13 @@ module Karafka
     #   not as part of the public api.
     def on_after_consume
       handle_after_consume
-    rescue StandardError => e
-      Karafka.monitor.instrument(
-        'error.occurred',
-        error: e,
-        caller: self,
-        type: 'consumer.after_consume.error'
-      )
+    end
+
+    # Can be used to run code prior to scheduling of idle execution
+    #
+    # @private
+    def on_before_schedule_idle
+      handle_before_schedule_idle
     end
 
     # Trigger method for running on idle runs without messages
@@ -108,13 +97,13 @@ module Karafka
     # @private
     def on_idle
       handle_idle
-    rescue StandardError => e
-      Karafka.monitor.instrument(
-        'error.occurred',
-        error: e,
-        caller: self,
-        type: 'consumer.idle.error'
-      )
+    end
+
+    # Can be used to run code prior to scheduling of revoked execution
+    #
+    # @private
+    def on_before_schedule_revoked
+      handle_before_schedule_revoked
     end
 
     # Trigger method for running on partition revocation.
@@ -131,6 +120,13 @@ module Karafka
       )
     end
 
+    # Can be used to run code prior to scheduling of revoked execution
+    #
+    # @private
+    def on_before_schedule_shutdown
+      handle_before_schedule_shutdown
+    end
+
     # Trigger method for running on shutdown.
     #
     # @private
@@ -226,9 +222,14 @@ module Karafka
     # @param manual_seek [Boolean] Flag to differentiate between user seek and system/strategy
     #   based seek. User seek operations should take precedence over system actions, hence we need
     #   to know who invoked it.
+    # @param reset_offset [Boolean] should we reset offset when seeking backwards. It is false by
+    #   default to prevent marking in the offset that was earlier than the highest marked offset
+    #   for given consumer group. It can be set to true if we want to reprocess data once again and
+    #   want to make sure that the marking starts from where we moved to.
     # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
-    def seek(offset, manual_seek = true)
+    def seek(offset, manual_seek = true, reset_offset: false)
       coordinator.manual_seek if manual_seek
+      coordinator.seek_offset = nil if reset_offset
 
       client.seek(
         Karafka::Messages::Seek.new(

data/lib/karafka/cli/base.rb

@@ -101,7 +101,7 @@ module Karafka
 
         # @return [Array<String>] names and aliases for command matching
         def names
-          ((@aliases || []) << name).flatten.map(&:to_s)
+          ((@aliases || []) << name).flatten.map(&:to_s).uniq
         end
       end
     end

data/lib/karafka/connection/client.rb

@@ -10,6 +10,10 @@ module Karafka
     class Client
       attr_reader :rebalance_manager
 
+      # @return [Karafka::Routing::SubscriptionGroup] subscription group to which this client
+      #   belongs to
+      attr_reader :subscription_group
+
       # @return [String] underlying consumer name
       # @note Consumer name may change in case we regenerate it
       attr_reader :name
@@ -20,16 +24,7 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
-      # 1 minute of max wait for the first rebalance before a forceful attempt
-      # This applies only to a case when a short-lived Karafka instance with a client would be
-      # closed before first rebalance. Mitigates a librdkafka bug.
-      COOPERATIVE_STICKY_MAX_WAIT = 60_000
-
-      # We want to make sure we never close several clients in the same moment to prevent
-      # potential race conditions and other issues
-      SHUTDOWN_MUTEX = Mutex.new
-
-      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :COOPERATIVE_STICKY_MAX_WAIT
+      private_constant :MAX_POLL_RETRIES
 
       # Creates a new consumer instance.
       #
@@ -45,12 +40,8 @@ module Karafka
         @buffer = RawMessagesBuffer.new
         @tick_interval = ::Karafka::App.config.internal.tick_interval
         @rebalance_manager = RebalanceManager.new(@subscription_group.id)
-        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(
-          @subscription_group.id,
-          @subscription_group.consumer_group.id
-        )
+        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group)
         @events_poller = Helpers::IntervalRunner.new { events_poll }
-        @kafka = build_consumer
         # There are few operations that can happen in parallel from the listener threads as well
         # as from the workers. They are not fully thread-safe because they may be composed out of
         # few calls to Kafka or out of few internal state changes. That is why we mutex them.
@@ -125,13 +116,19 @@ module Karafka
       # Stores offset for a given partition of a given topic based on the provided message.
       #
       # @param message [Karafka::Messages::Message]
-      def store_offset(message)
-        internal_store_offset(message)
+      # @param offset_metadata [String, nil] offset storage metadata or nil if none
+      def store_offset(message, offset_metadata = nil)
+        internal_store_offset(message, offset_metadata)
       end
 
       # @return [Boolean] true if our current assignment has been lost involuntarily.
       def assignment_lost?
-        @kafka.assignment_lost?
+        kafka.assignment_lost?
+      end
+
+      # @return [Rdkafka::Consumer::TopicPartitionList] current active assignment
+      def assignment
+        kafka.assignment
       end
 
       # Commits the offset on a current consumer in a non-blocking or blocking way.
@@ -202,7 +199,7 @@ module Karafka
 
           @paused_tpls[topic][partition] = tpl
 
-          @kafka.pause(tpl)
+          kafka.pause(tpl)
 
           # If offset is not provided, will pause where it finished.
           # This makes librdkafka not purge buffers and can provide significant network savings
@@ -243,43 +240,23 @@ module Karafka
             partition: partition
           )
 
-          @kafka.resume(tpl)
+          kafka.resume(tpl)
         end
       end
 
       # Gracefully stops topic consumption.
-      #
-      # @note Stopping running consumers without a really important reason is not recommended
-      #   as until all the consumers are stopped, the server will keep running serving only
-      #   part of the messages
       def stop
-        # This ensures, that we do not stop the underlying client until it passes the first
-        # rebalance for cooperative-sticky. Otherwise librdkafka may crash
-        #
-        # We set a timeout just in case the rebalance would never happen or would last for an
-        # extensive time period.
-        #
-        # @see https://github.com/confluentinc/librdkafka/issues/4312
+        # In case of cooperative-sticky, there is a bug in librdkafka that may hang it.
+        # To mitigate it we first need to unsubscribe so we will not receive any assignments and
+        # only then we should be good to go.
+        # @see https://github.com/confluentinc/librdkafka/issues/4527
         if @subscription_group.kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
-          active_wait = false
-
-          (COOPERATIVE_STICKY_MAX_WAIT / 100).times do
-            # If we're past the first rebalance, no need to wait
-            if @rebalance_manager.active?
-              # We give it a a bit of time because librdkafka has a tendency to do some-post
-              # callback work that from its perspective is still under rebalance
-              sleep(5) if active_wait
-
-              break
-            end
-
-            active_wait = true
-
-            # poll to trigger potential rebalances that could occur during stopping and to trigger
-            # potential callbacks
-            poll(100)
+          unsubscribe
 
+          until assignment.empty?
             sleep(0.1)
+
+            ping
           end
         end
 
@@ -288,33 +265,40 @@ module Karafka
 
       # Marks given message as consumed.
       #
-      # @param [Karafka::Messages::Message] message that we want to mark as processed
+      # @param message [Karafka::Messages::Message] message that we want to mark as processed
+      # @param metadata [String, nil] offset storage metadata or nil if none
       # @return [Boolean] true if successful. False if we no longer own given partition
       # @note This method won't trigger automatic offsets commits, rather relying on the offset
       #   check-pointing trigger that happens with each batch processed. It will however check the
       #   `librdkafka` assignment ownership to increase accuracy for involuntary revocations.
-      def mark_as_consumed(message)
-        store_offset(message) && !assignment_lost?
+      def mark_as_consumed(message, metadata = nil)
+        store_offset(message, metadata) && !assignment_lost?
       end
 
       # Marks a given message as consumed and commits the offsets in a blocking way.
       #
-      # @param [Karafka::Messages::Message] message that we want to mark as processed
+      # @param message [Karafka::Messages::Message] message that we want to mark as processed
+      # @param metadata [String, nil] offset storage metadata or nil if none
       # @return [Boolean] true if successful. False if we no longer own given partition
-      def mark_as_consumed!(message)
-        return false unless mark_as_consumed(message)
+      def mark_as_consumed!(message, metadata = nil)
+        return false unless mark_as_consumed(message, metadata)
 
         commit_offsets!
       end
 
       # Closes and resets the client completely.
       def reset
-        close
+        Karafka.monitor.instrument(
+          'client.reset',
+          caller: self,
+          subscription_group: @subscription_group
+        ) do
+          close
 
-        @events_poller.reset
-        @closed = false
-        @paused_tpls.clear
-        @kafka = build_consumer
+          @events_poller.reset
+          @closed = false
+          @paused_tpls.clear
+        end
       end
 
       # Runs a single poll on the main queue and consumer queue ignoring all the potential errors
@@ -340,7 +324,27 @@ module Karafka
       # @note It is non-blocking when timeout 0 and will not wait if queue empty. It costs up to
       #   2ms when no callbacks are triggered.
       def events_poll(timeout = 0)
-        @kafka.events_poll(timeout)
+        kafka.events_poll(timeout)
+      end
+
+      # Returns pointer to the consumer group metadata. It is used only in the context of
+      # exactly-once-semantics in transactions, this is why it is never remapped to Ruby
+      # @return [FFI::Pointer]
+      def consumer_group_metadata_pointer
+        kafka.consumer_group_metadata_pointer
+      end
+
+      # Return the current committed offset per partition for this consumer group.
+      # The offset field of each requested partition will either be set to stored offset or to
+      # -1001 in case there was no stored offset for that partition.
+      #
+      # @param tpl [Rdkafka::Consumer::TopicPartitionList] for which we want to get committed
+      # @return [Rdkafka::Consumer::TopicPartitionList]
+      # @raise [Rdkafka::RdkafkaError] When getting the committed positions fails.
+      # @note It is recommended to use this only on rebalances to get positions with metadata
+      #   when working with metadata as this is synchronous
+      def committed(tpl = nil)
+        Proxy.new(kafka).committed(tpl)
       end
 
       private
@@ -349,9 +353,10 @@ module Karafka
       #
       # Non thread-safe offset storing method
       # @param message [Karafka::Messages::Message]
+      # @param metadata [String, nil] offset storage metadata or nil if none
       # @return [Boolean] true if we could store the offset (if we still own the partition)
-      def internal_store_offset(message)
-        @kafka.store_offset(message)
+      def internal_store_offset(message, metadata)
+        kafka.store_offset(message, metadata)
         true
       rescue Rdkafka::RdkafkaError => e
         return false if e.code == :assignment_lost
@@ -367,7 +372,7 @@ module Karafka
       #   even when no stored, because with sync commit, it refreshes the ownership state of the
       #   consumer in a sync way.
       def internal_commit_offsets(async: true)
-        @kafka.commit(nil, async)
+        kafka.commit(nil, async)
 
         true
       rescue Rdkafka::RdkafkaError => e
@@ -404,7 +409,7 @@ module Karafka
             message.partition => message.offset
           )
 
-          proxy = Proxy.new(@kafka)
+          proxy = Proxy.new(kafka)
 
           # Now we can overwrite the seek message offset with our resolved offset and we can
           # then seek to the appropriate message
@@ -426,29 +431,29 @@ module Karafka
         # seeking and pausing
         return if message.offset == topic_partition_position(message.topic, message.partition)
 
-        @kafka.seek(message)
+        kafka.seek(message)
       end
 
       # Commits the stored offsets in a sync way and closes the consumer.
       def close
-        # Allow only one client to be closed at the same time
-        SHUTDOWN_MUTEX.synchronize do
-          # Once client is closed, we should not close it again
-          # This could only happen in case of a race-condition when forceful shutdown happens
-          # and triggers this from a different thread
-          return if @closed
+        # Once client is closed, we should not close it again
+        # This could only happen in case of a race-condition when forceful shutdown happens
+        # and triggers this from a different thread
+        return if @closed
 
-          @closed = true
+        @closed = true
 
-          # Remove callbacks runners that were registered
-          ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
-          ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
+        return unless @kafka
 
-          @kafka.close
-          @buffer.clear
-          # @note We do not clear rebalance manager here as we may still have revocation info
-          # here that we want to consider valid prior to running another reconnection
-        end
+        # Remove callbacks runners that were registered
+        ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
+        ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
+
+        kafka.close
+        @kafka = nil
+        @buffer.clear
+        # @note We do not clear rebalance manager here as we may still have revocation info
+        # here that we want to consider valid prior to running another reconnection
       end
 
       # Unsubscribes from all the subscriptions
@@ -456,7 +461,7 @@ module Karafka
       # @note We do not re-raise since this is supposed to be only used on close and can be safely
       #   ignored. We do however want to instrument on it
       def unsubscribe
-        @kafka.unsubscribe
+        kafka.unsubscribe
       rescue ::Rdkafka::RdkafkaError => e
         Karafka.monitor.instrument(
           'error.occurred',
@@ -470,7 +475,7 @@ module Karafka
       # @param partition [Integer]
       # @return [Rdkafka::Consumer::TopicPartitionList]
       def topic_partition_list(topic, partition)
-        rdkafka_partition = @kafka
+        rdkafka_partition = kafka
                             .assignment
                             .to_h[topic]
                             &.detect { |part| part.partition == partition }
@@ -489,7 +494,7 @@ module Karafka
         rd_partition = ::Rdkafka::Consumer::Partition.new(partition, nil, 0)
         tpl = ::Rdkafka::Consumer::TopicPartitionList.new(topic => [rd_partition])
 
-        @kafka.position(tpl).to_h.fetch(topic).first.offset || -1
+        kafka.position(tpl).to_h.fetch(topic).first.offset || -1
       end
 
       # Performs a single poll operation and handles retries and errors
@@ -517,7 +522,7 @@ module Karafka
         # blocking events from being handled.
         poll_tick = timeout > @tick_interval ? @tick_interval : timeout
 
-        result = @kafka.poll(poll_tick)
+        result = kafka.poll(poll_tick)
 
         # If we've got a message, we can return it
         return result if result
@@ -644,6 +649,11 @@ module Karafka
 
         @buffer.uniq!
       end
+
+      # @return [Rdkafka::Consumer] librdkafka consumer instance
+      def kafka
+        @kafka ||= build_consumer
+      end
     end
   end
 end

data/lib/karafka/connection/conductor.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Conductor is responsible for time orchestration of listeners manager.
+    # It blocks when manager is not needed as there were no state changes that could cause any
+    # listeners config changes and unblocks when things change or when certain time passed.
+    # The time based unblocking allows for building of complex managers that could be state aware
+    class Conductor
+      # @param max_interval [Integer] after how many milliseconds of doing nothing should we wake
+      #   up the manager despite no state changes
+      def initialize(max_interval = 30_000)
+        @lock = RUBY_VERSION < '3.2' ? Processing::TimedQueue.new : Queue.new
+        @timeout = max_interval / 1_000.0
+      end
+
+      # Waits in a blocking way until it is time to manage listeners
+      def wait
+        @lock.pop(timeout: @timeout)
+      end
+
+      # Releases wait lock on state change
+      def signal
+        @lock << true
+      end
+    end
+  end
+end