karafka-rdkafka 0.28.0-aarch64-linux-gnu → 0.28.2-aarch64-linux-gnu

data/lib/rdkafka/callbacks/delete_records_handler.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DELETERECORDS_RESULT` events
+    # @private
+    class DeleteRecordsHandler < BaseHandler
+      class << self
+        # Resolves the delete-records handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DeleteRecords_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DeleteRecordsHandle.remove(handle_ptr.address))
+
+          # An operation-level error (e.g. timeout or a closed client) is delivered on the event
+          # itself, with no per-partition result to parse.
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DeleteRecordsReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/list_consumer_groups_handler.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT` events
+    # @private
+    class ListConsumerGroupsHandler < BaseHandler
+      class << self
+        # Resolves the list-consumer-groups handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_ListConsumerGroups_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::ListConsumerGroupsHandle.remove(handle_ptr.address))
+
+          # An operation-level error (e.g. timeout or a closed client) is delivered on the event
+          # itself; per the source feature's all-or-nothing contract we surface it as the result
+          # rather than returning a partial listing.
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::ListConsumerGroupsReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks.rb

@@ -219,12 +219,14 @@ module Rdkafka
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT then DeleteTopicHandler
         when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT then CreatePartitionsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT then DeleteGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETERECORDS_RESULT then DeleteRecordsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT then CreateAclHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT then DeleteAclHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT then DescribeAclHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT then DescribeConfigsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT then IncrementalAlterConfigsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT then ListOffsetsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT then ListConsumerGroupsHandler
         end
 
         handler&.call(event_ptr)
@@ -286,9 +288,9 @@ module Rdkafka
       end
     end
 
-    # @private
+    # @!visibility private
     @@mutex = Mutex.new
-    # @private
+    # @!visibility private
     @@current_pid = nil
 
     class << self

data/lib/rdkafka/clients.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Tracks live public clients so they can be closed before Ruby begins forced finalization.
+  # @private
+  module Clients
+    # Ruby rewrote `ObjectSpace::WeakMap` in 3.3. On 3.2 and earlier its `[]` can hand back a
+    # foreign or already-freed object for a key, which raises `NoMethodError` or segfaults the VM
+    # the moment the value is touched. Track weakly only where that is safe and fall back to a
+    # strong registry - pruned as clients close - on older Rubies. The strong registry holds an
+    # open, un-closed client until shutdown, which is exactly when this cleanup needs it anyway.
+    WEAK_TRACKING = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3")
+
+    @pid = Process.pid
+    @mutex = Mutex.new
+
+    if WEAK_TRACKING
+      @clients = ObjectSpace::WeakMap.new
+      @tokens = [].freeze
+    else
+      @clients = {}.freeze
+    end
+
+    # Registers a successfully constructed client so it can be closed during process shutdown.
+    # @param client [Consumer, Producer, Admin] client to close during process shutdown
+    # @return [Consumer, Producer, Admin] the registered client
+    def self.register(client)
+      reset_after_fork
+
+      @mutex.synchronize do
+        if WEAK_TRACKING
+          live_clients
+
+          # The strongly-held token is the key and the client is the weak value, so registering a
+          # client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
+          # look the value up by token instead.
+          token = Object.new
+          @clients[token] = client
+          @tokens = (@tokens + [token]).freeze
+        else
+          # Strong references, so drop clients that have already been closed to bound growth.
+          live = @clients.reject { |_id, existing| existing.closed? }
+          live[client.object_id] = client
+          @clients = live.freeze
+        end
+      end
+
+      client
+    end
+
+    # Closes each live client registered in this process.
+    # @return [nil]
+    # @raise [StandardError] the first close error, after attempting every client
+    def self.close_all
+      return unless @pid == Process.pid
+
+      clients = @mutex.synchronize { current_clients }
+      first_error = nil
+
+      clients.each do |client|
+        client.close unless client.closed?
+      rescue => error
+        first_error ||= error
+      end
+
+      raise first_error if first_error
+    end
+
+    # Returns the currently registered live clients.
+    # @return [Array<Consumer, Producer, Admin>] live clients
+    # @private
+    def self.current_clients
+      WEAK_TRACKING ? live_clients : @clients.values
+    end
+    private_class_method :current_clients
+
+    # Returns the live clients and releases tokens whose clients have been collected.
+    # Weak-tracking Rubies only.
+    # @return [Array<Consumer, Producer, Admin>] live clients
+    # @private
+    def self.live_clients
+      clients = []
+      tokens = []
+
+      @tokens.each do |token|
+        client = @clients[token]
+        clients << client if client
+        tokens << token if client
+      end
+
+      @tokens = tokens.freeze
+      clients
+    end
+    private_class_method :live_clients
+
+    # Replaces inherited state before registering a client in a forked child.
+    # @return [nil]
+    # @private
+    def self.reset_after_fork
+      return if @pid == Process.pid
+
+      if WEAK_TRACKING
+        @clients = ObjectSpace::WeakMap.new
+        @tokens = [].freeze
+      else
+        @clients = {}.freeze
+      end
+
+      @pid = Process.pid
+      @mutex = Mutex.new
+    end
+    private_class_method :reset_after_fork
+  end
+end

data/lib/rdkafka/config.rb

@@ -3,24 +3,24 @@
 module Rdkafka
   # Configuration for a Kafka consumer or producer. You can create an instance and use
   # the consumer and producer methods to create a client. Documentation of the available
-  # configuration options is available on https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
+  # configuration options is available on
+  # https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md.
   class Config
-    # @private
+    # @!visibility private
     @@logger = Logger.new($stdout)
-    # @private
+    # @!visibility private
     @@statistics_callback = nil
-    # @private
+    # @!visibility private
     @@error_callback = nil
-    # @private
+    # @!visibility private
     @@opaques = ObjectSpace::WeakMap.new
-    # @private
+    # @!visibility private
     @@log_queue = Queue.new
-    # We memoize thread on the first log flush
-    # This allows us also to restart logger thread on forks
+    # We memoize thread on first log flush. This allows us also to restart logger thread on forks
     @@log_thread = nil
-    # @private
+    # @!visibility private
     @@log_mutex = Mutex.new
-    # @private
+    # @!visibility private
     @@oauthbearer_token_refresh_callback = nil
 
     # Returns the current logger, by default this is a logger to stdout.
@@ -69,7 +69,8 @@ module Rdkafka
 
     # Set a callback that will be called every time the underlying client emits statistics.
     # You can configure if and how often this happens using `statistics.interval.ms`.
-    # The callback is called with a hash that's documented here: https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
+    # The callback is called with a hash that's documented here:
+    # https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
     #
     # @param callback [Proc, #call, nil] callable object or nil to clear
     # @return [nil]
@@ -86,7 +87,8 @@ module Rdkafka
     end
 
     # Set a callback that will be called every time the underlying client emits an error.
-    # If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
+    # If this callback is not set, global errors such as brokers becoming unavailable will only be
+    # sent to the logger, as defined by librdkafka.
     # The callback is called with an instance of RdKafka::Error.
     #
     # @param callback [Proc, #call, nil] callable object to handle errors or nil to clear
@@ -207,21 +209,16 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
       end
 
-      # Create native client
-      kafka = native_kafka(config, :rd_kafka_consumer)
-
-      # Redirect the main queue to the consumer queue
-      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
-
-      # Return consumer with Kafka client
-      Rdkafka::Consumer.new(
-        Rdkafka::NativeKafka.new(
-          kafka,
-          run_polling_thread: false,
-          opaque: opaque,
-          auto_start: native_kafka_auto_start
-        )
-      )
+      build_native_client(
+        config,
+        :rd_kafka_consumer,
+        opaque: opaque,
+        run_polling_thread: false,
+        auto_start: native_kafka_auto_start
+      ) do |kafka, native_kafka|
+        Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka) if @consumer_poll_set
+        Rdkafka::Consumer.new(native_kafka)
+      end
     end
 
     # Create a producer with this configuration.
@@ -245,19 +242,17 @@ module Rdkafka
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
 
-      kafka = native_kafka(config, :rd_kafka_producer)
-
-      Rdkafka::Producer.new(
-        Rdkafka::NativeKafka.new(
-          kafka,
-          run_polling_thread: run_polling_thread,
-          opaque: opaque,
-          auto_start: native_kafka_auto_start,
-          timeout_ms: native_kafka_poll_timeout_ms
-        ),
-        partitioner_name
-      ).tap do |producer|
-        opaque.producer = producer
+      build_native_client(
+        config,
+        :rd_kafka_producer,
+        opaque: opaque,
+        run_polling_thread: run_polling_thread,
+        auto_start: native_kafka_auto_start,
+        timeout_ms: native_kafka_poll_timeout_ms
+      ) do |_kafka, native_kafka|
+        Rdkafka::Producer.new(native_kafka, partitioner_name).tap do |producer|
+          opaque.producer = producer
+        end
       end
     end
 
@@ -277,17 +272,16 @@ module Rdkafka
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
 
-      kafka = native_kafka(config, :rd_kafka_producer)
-
-      Rdkafka::Admin.new(
-        Rdkafka::NativeKafka.new(
-          kafka,
-          run_polling_thread: run_polling_thread,
-          opaque: opaque,
-          auto_start: native_kafka_auto_start,
-          timeout_ms: native_kafka_poll_timeout_ms
-        )
-      )
+      build_native_client(
+        config,
+        :rd_kafka_producer,
+        opaque: opaque,
+        run_polling_thread: run_polling_thread,
+        auto_start: native_kafka_auto_start,
+        timeout_ms: native_kafka_poll_timeout_ms
+      ) do |_kafka, native_kafka|
+        Rdkafka::Admin.new(native_kafka)
+      end
     end
 
     # Returns all configuration properties and their current values for this config.
@@ -295,6 +289,9 @@ module Rdkafka
     # Uses `rd_kafka_conf_dump` to retrieve every property (including defaults and
     # internal properties like `client.software.name`) as a flat Hash.
     #
+    # @return [Hash{Symbol => String}] property names mapped to their current values
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
     # @note The librdkafka C API does not distinguish between producer-only, consumer-only,
     #   and global properties at the configuration level. All properties are returned
     #   regardless of the intended client type.
@@ -302,10 +299,6 @@ module Rdkafka
     # @note The returned Hash may include sensitive values such as authentication
     #   credentials and key passwords. Do not log or serialize the returned data
     #   unless you have explicitly redacted secret entries.
-    #
-    # @return [Hash{Symbol => String}] property names mapped to their current values
-    #
-    # @raise [ConfigError] When the configuration contains invalid options
     def describe_properties
       config = nil
       dump_ptr = nil
@@ -330,7 +323,8 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_destroy(config) if config
     end
 
-    # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
+    # Error that is returned by the underlying rdkafka error if an invalid configuration option is
+    # present.
     class ConfigError < RuntimeError; end
 
     # Error that is returned by the underlying rdkafka library if the client cannot be created.
@@ -341,6 +335,56 @@ module Rdkafka
 
     private
 
+    # Builds a public client around a native Kafka handle, destroying the current owner if setup
+    # fails before ownership is transferred to the caller.
+    # @param config [FFI::Pointer] native rdkafka configuration
+    # @param type [Symbol] native client type
+    # @param opaque [Object] callback context retained for the native client
+    # @param run_polling_thread [Boolean] whether to run a Ruby polling thread
+    # @param auto_start [Boolean] whether to start the polling thread after wrapping the client
+    # @param timeout_ms [Integer] polling timeout in milliseconds
+    # @yieldparam kafka [FFI::Pointer] native Kafka handle
+    # @yieldparam native_kafka [NativeKafka] Ruby owner of the native handle
+    # @return [Consumer, Producer, Admin] the constructed public client
+    # @private
+    def build_native_client(
+      config,
+      type,
+      opaque:,
+      run_polling_thread:,
+      auto_start:,
+      timeout_ms: Defaults::NATIVE_KAFKA_POLL_TIMEOUT_MS
+    )
+      kafka = native_kafka(config, type)
+      native_kafka = nil
+      client = nil
+
+      begin
+        native_kafka = Rdkafka::NativeKafka.new(
+          kafka,
+          run_polling_thread: run_polling_thread,
+          opaque: opaque,
+          auto_start: false,
+          timeout_ms: timeout_ms
+        )
+        client = yield(kafka, native_kafka)
+        native_kafka.start if auto_start
+        Rdkafka::Clients.register(client)
+      rescue Exception
+        owner = client || native_kafka
+
+        begin
+          owner ? owner.close : Rdkafka::Bindings.rd_kafka_destroy(kafka)
+        rescue Exception => err
+          # Never let a teardown failure replace the error that caused it: that error is what the
+          # caller needs to see, and it is re-raised below regardless.
+          Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
+        end
+
+        raise
+      end
+    end
+
     # This method is only intended to be used to create a client,
     # using it in another way will leak memory.
     #
@@ -427,6 +471,9 @@ module Rdkafka
 
       # Return handle which should be closed using rd_kafka_destroy after usage.
       handle
+    rescue Exception
+      Rdkafka::Bindings.rd_kafka_destroy(handle) if handle && !handle.null?
+      raise
     end
   end
 

data/lib/rdkafka/consumer/allocator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Consumer
+    # Creates FFI pointers from the process-global allocator so that memory handed to librdkafka
+    # can be released by it with the matching `free`.
+    module Allocator
+      extend FFI::Library
+
+      ffi_lib FFI::CURRENT_PROCESS # use process-global malloc not FFI::Library::LIBC
+      attach_function :malloc, [:size_t], :pointer
+
+      class << self
+        # Return an FFI pointer to a string allocated from the global allocator
+        # so that it can be released by another library.
+        #
+        # @private
+        #
+        # @param string [String] Ruby string that will be copied into the new
+        #   string pointer.
+        #
+        # @return [FFI::Pointer]
+        def string_pointer(string)
+          bytes = string.to_s
+          pointer = malloc(bytes.bytesize + 1)
+          Kernel.raise(NoMemoryError, "malloc failed") if pointer.null?
+
+          pointer.put_bytes(0, bytes)
+          pointer.put_char(bytes.bytesize, 0) # NUL terminator
+
+          pointer.autorelease = false
+
+          pointer
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -6,7 +6,8 @@ module Rdkafka
     class TopicPartitionList
       # Create a topic partition list.
       #
-      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
+      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an
+      #   empty list
       #
       # @return [TopicPartitionList]
       def initialize(data = nil)
@@ -34,10 +35,12 @@ module Rdkafka
       end
 
       # Add a topic with optionally partitions to the list.
-      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      # Calling this method multiple times for the same topic will overwrite the previous
+      # configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
+      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or
+      #   partition count
       #
       # @return [nil]
       #
@@ -61,11 +64,12 @@ module Rdkafka
       end
 
       # Add a topic with partitions and offsets set to the list
-      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      # Calling this method multiple times for the same topic will overwrite the previous
+      # configuraton.
       #
       # @param topic [String] The topic's name
-      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The topic's
-      #   partitions and offsets (Hash) or partitions with offsets and metadata (Array)
+      # @param partitions_with_offsets [Hash{Integer => Integer}, Array<Consumer::Partition>] The
+      #   topic's partitions and offsets (Hash) or partitions with offsets and metadata (Array)
       # @return [nil]
       def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
         @data[topic.to_s] = partitions_with_offsets.map do |p, o|
@@ -73,7 +77,8 @@ module Rdkafka
         end
       end
 
-      # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
+      # Return a `Hash` with the topics as keys and and an array of partition information as the
+      # value if present.
       #
       # @return [Hash{String => Array<Partition>,nil}]
       def to_h
@@ -97,7 +102,8 @@ module Rdkafka
       #
       # @private
       #
-      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied.
+      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will
+      #   be copied.
       #
       # @return [TopicPartitionList]
       def self.from_native_tpl(pointer)
@@ -118,7 +124,6 @@ module Rdkafka
             else
               elem[:offset]
             end
-
             partition = Partition.new(
               elem[:partition],
               offset,
@@ -157,10 +162,9 @@ module Rdkafka
                 # Remove the respond to check after karafka 2.3.0 is released
                 if p.respond_to?(:metadata) && p.metadata
                   part = Rdkafka::Bindings::TopicPartition.new(ref)
-                  str_ptr = FFI::MemoryPointer.from_string(p.metadata)
-                  # released here:
+                  # The metadata string is owned by librdkafka once handed over and released here:
                   # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
-                  str_ptr.autorelease = false
+                  str_ptr = Allocator.string_pointer(p.metadata)
                   part[:metadata] = str_ptr
                   part[:metadata_size] = p.metadata.bytesize
                 end