karafka-rdkafka 0.30.2-aarch64-linux-gnu → 0.31.0-aarch64-linux-gnu

data/lib/rdkafka/callbacks/describe_cluster_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT` events
+    # @private
+    class DescribeClusterHandler < BaseHandler
+      class << self
+        # Resolves the describe cluster handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeCluster_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DescribeClusterHandle.remove(handle_ptr.address))
+
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DescribeClusterReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/describe_consumer_groups_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DESCRIBECONSUMERGROUPS_RESULT` events
+    # @private
+    class DescribeConsumerGroupsHandler < BaseHandler
+      class << self
+        # Resolves the describe-consumer-groups handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeConsumerGroups_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DescribeConsumerGroupsHandle.remove(handle_ptr.address))
+
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DescribeConsumerGroupsReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks/describe_topics_handler.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Callbacks
+    # Handles `RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT` events
+    # @private
+    class DescribeTopicsHandler < BaseHandler
+      class << self
+        # Resolves the describe topics handle from its result event
+        # @param event_ptr [FFI::Pointer] pointer to the event
+        # @return [void]
+        def call(event_ptr)
+          result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeTopics_result(event_ptr)
+          handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+          return unless (handle = Rdkafka::Admin::DescribeTopicsHandle.remove(handle_ptr.address))
+
+          return if resolve_operation_error(event_ptr, handle)
+
+          handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+          # Parsing must copy everything out of event-owned memory before the event is destroyed.
+          # An exception here is captured and re-raised on the waiting thread, since it cannot
+          # unwind through librdkafka native frames.
+          handle.result = begin
+            Rdkafka::Admin::DescribeTopicsReport.new(result_ptr)
+          rescue => e
+            e
+          end
+
+          handle.unlock
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/callbacks.rb

@@ -225,6 +225,9 @@ module Rdkafka
         when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT then IncrementalAlterConfigsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT then ListOffsetsHandler
         when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT then ListConsumerGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONSUMERGROUPS_RESULT then DescribeConsumerGroupsHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT then DescribeClusterHandler
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT then DescribeTopicsHandler
         end
 
         handler&.call(event_ptr)
@@ -253,6 +256,9 @@ module Rdkafka
           delivery_handle[:response] = message[:err]
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
+          delivery_handle[:status] = Rdkafka::Bindings.rd_kafka_message_status(message_ptr)
+          delivery_handle[:latency] = Rdkafka::Bindings.rd_kafka_message_latency(message_ptr)
+          delivery_handle[:broker_id] = Rdkafka::Bindings.rd_kafka_message_broker_id(message_ptr)
           # The topic is not stored on the handle here: it is already set as the handle's
           # `topic` Ruby attribute during produce, which spares a native string copy that
           # would otherwise be allocated and retained for every message.
@@ -266,7 +272,10 @@ module Rdkafka
                   message[:offset],
                   topic_name,
                   message[:err],
-                  delivery_handle.label
+                  delivery_handle.label,
+                  status: delivery_handle[:status],
+                  latency: delivery_handle[:latency],
+                  broker_id: delivery_handle[:broker_id]
                 ),
                 delivery_handle
               )

data/lib/rdkafka/clients.rb

@@ -4,22 +4,10 @@ module Rdkafka
   # Tracks live public clients so they can be closed before Ruby begins forced finalization.
   # @private
   module Clients
-    # Ruby rewrote `ObjectSpace::WeakMap` in 3.3. On 3.2 and earlier its `[]` can hand back a
-    # foreign or already-freed object for a key, which raises `NoMethodError` or segfaults the VM
-    # the moment the value is touched. Track weakly only where that is safe and fall back to a
-    # strong registry - pruned as clients close - on older Rubies. The strong registry holds an
-    # open, un-closed client until shutdown, which is exactly when this cleanup needs it anyway.
-    WEAK_TRACKING = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3")
-
     @pid = Process.pid
     @mutex = Mutex.new
-
-    if WEAK_TRACKING
-      @clients = ObjectSpace::WeakMap.new
-      @tokens = [].freeze
-    else
-      @clients = {}.freeze
-    end
+    @clients = ObjectSpace::WeakMap.new
+    @tokens = [].freeze
 
     # Registers a successfully constructed client so it can be closed during process shutdown.
     # @param client [Consumer, Producer, Admin] client to close during process shutdown
@@ -28,21 +16,14 @@ module Rdkafka
       reset_after_fork
 
       @mutex.synchronize do
-        if WEAK_TRACKING
-          live_clients
-
-          # The strongly-held token is the key and the client is the weak value, so registering a
-          # client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
-          # look the value up by token instead.
-          token = Object.new
-          @clients[token] = client
-          @tokens = (@tokens + [token]).freeze
-        else
-          # Strong references, so drop clients that have already been closed to bound growth.
-          live = @clients.reject { |_id, existing| existing.closed? }
-          live[client.object_id] = client
-          @clients = live.freeze
-        end
+        live_clients
+
+        # The strongly-held token is the key and the client is the weak value, so registering a
+        # client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
+        # look the value up by token instead.
+        token = Object.new
+        @clients[token] = client
+        @tokens = (@tokens + [token]).freeze
       end
 
       client
@@ -54,7 +35,7 @@ module Rdkafka
     def self.close_all
       return unless @pid == Process.pid
 
-      clients = @mutex.synchronize { current_clients }
+      clients = @mutex.synchronize { live_clients }
       first_error = nil
 
       clients.each do |client|
@@ -66,16 +47,7 @@ module Rdkafka
       raise first_error if first_error
     end
 
-    # Returns the currently registered live clients.
-    # @return [Array<Consumer, Producer, Admin>] live clients
-    # @private
-    def self.current_clients
-      WEAK_TRACKING ? live_clients : @clients.values
-    end
-    private_class_method :current_clients
-
     # Returns the live clients and releases tokens whose clients have been collected.
-    # Weak-tracking Rubies only.
     # @return [Array<Consumer, Producer, Admin>] live clients
     # @private
     def self.live_clients
@@ -99,13 +71,8 @@ module Rdkafka
     def self.reset_after_fork
       return if @pid == Process.pid
 
-      if WEAK_TRACKING
-        @clients = ObjectSpace::WeakMap.new
-        @tokens = [].freeze
-      else
-        @clients = {}.freeze
-      end
-
+      @clients = ObjectSpace::WeakMap.new
+      @tokens = [].freeze
       @pid = Process.pid
       @mutex = Mutex.new
     end

data/lib/rdkafka/consumer/message.rb

@@ -35,19 +35,16 @@ module Rdkafka
       # @param native_message [Rdkafka::Bindings::Message] native message struct from librdkafka
       def initialize(native_message)
         # Set topic
-        unless native_message[:rkt].null?
-          @topic = Rdkafka::Bindings.rd_kafka_topic_name(native_message[:rkt])
-        end
+        rkt = native_message[:rkt]
+        @topic = Rdkafka::Bindings.rd_kafka_topic_name(rkt) unless rkt.null?
         # Set partition
         @partition = native_message[:partition]
         # Set payload
-        unless native_message[:payload].null?
-          @payload = native_message[:payload].read_string(native_message[:len])
-        end
+        payload = native_message[:payload]
+        @payload = payload.read_string(native_message[:len]) unless payload.null?
         # Set key
-        unless native_message[:key].null?
-          @key = native_message[:key].read_string(native_message[:key_len])
-        end
+        key = native_message[:key]
+        @key = key.read_string(native_message[:key_len]) unless key.null?
         # Set offset
         @offset = native_message[:offset]
         # Set timestamp

data/lib/rdkafka/producer/delivery_handle.rb

@@ -7,7 +7,10 @@ module Rdkafka
       layout :pending, :bool,
         :response, :int,
         :partition, :int,
-        :offset, :int64
+        :offset, :int64,
+        :status, :int,
+        :latency, :int64,
+        :broker_id, :int32
 
       # @return [Object, nil] label set during message production or nil by default
       attr_accessor :label
@@ -24,23 +27,22 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        if self[:response] == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
-          DeliveryReport.new(
-            self[:partition],
-            self[:offset],
-            topic,
-            nil,
-            label
-          )
+        error = if self[:response] == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          nil
         else
-          DeliveryReport.new(
-            self[:partition],
-            self[:offset],
-            topic,
-            Rdkafka::RdkafkaError.build(self[:response]),
-            label
-          )
+          Rdkafka::RdkafkaError.build(self[:response])
         end
+
+        DeliveryReport.new(
+          self[:partition],
+          self[:offset],
+          topic,
+          error,
+          label,
+          status: self[:status],
+          latency: self[:latency],
+          broker_id: self[:broker_id]
+        )
       end
     end
   end

data/lib/rdkafka/producer/delivery_report.rb

@@ -25,6 +25,26 @@ module Rdkafka
       # @return [Object, nil] label set during message production or nil by default
       attr_reader :label
 
+      # Persistence status of the message, one of `Bindings::RD_KAFKA_MSG_STATUS_*`:
+      #   - `RD_KAFKA_MSG_STATUS_NOT_PERSISTED` - the message was not written to the broker. A retry
+      #     risks reordering, but not a duplicate.
+      #   - `RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED` - the message may have been written (e.g. the
+      #     request timed out before an ack). A retry risks reordering and a duplicate.
+      #   - `RD_KAFKA_MSG_STATUS_PERSISTED` - the broker acknowledged the message. Trust this only
+      #     with `acks=all`.
+      # `nil` when the report was not built from a delivery callback.
+      # @return [Integer, nil]
+      attr_reader :status
+
+      # Time from the `produce` call until the delivery report, in microseconds, or `nil` when
+      #   not available
+      # @return [Integer, nil]
+      attr_reader :latency
+
+      # Id of the broker the message was produced to, or `nil` when not known
+      # @return [Integer, nil]
+      attr_reader :broker_id
+
       # We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
       # where the topic name is under `#topic` method. That way we have a consistent name that
       # is present in both places
@@ -38,12 +58,42 @@ module Rdkafka
       # @param topic_name [String, nil] topic name
       # @param error [Integer, nil] error code if any
       # @param label [Object, nil] user-defined label
-      def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
+      # @param status [Integer, nil] `Bindings::RD_KAFKA_MSG_STATUS_*` persistence status
+      # @param latency [Integer, nil] produce latency in microseconds, negative when not available
+      # @param broker_id [Integer, nil] broker id, negative when not known
+      def initialize(
+        partition,
+        offset,
+        topic_name = nil,
+        error = nil,
+        label = nil,
+        status: nil,
+        latency: nil,
+        broker_id: nil
+      )
         @partition = partition
         @offset = offset
         @topic_name = topic_name
         @error = error
         @label = label
+        @status = status
+        @latency = (latency.nil? || latency.negative?) ? nil : latency
+        @broker_id = (broker_id.nil? || broker_id.negative?) ? nil : broker_id
+      end
+
+      # @return [Boolean] true when the broker acknowledged the message
+      def persisted?
+        status == Bindings::RD_KAFKA_MSG_STATUS_PERSISTED
+      end
+
+      # @return [Boolean] true when the message may have been written, so a retry may duplicate it
+      def possibly_persisted?
+        status == Bindings::RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED
+      end
+
+      # @return [Boolean] true when the message was not written, so a retry does not duplicate it
+      def not_persisted?
+        status == Bindings::RD_KAFKA_MSG_STATUS_NOT_PERSISTED
       end
     end
   end

data/lib/rdkafka/share_consumer.rb

@@ -167,6 +167,12 @@ module Rdkafka
 
     # Unsubscribes from all subscribed topics and leaves the share group assignment.
     #
+    # Closes the share sessions like {#close} does: acknowledgements made so far (every delivered
+    # record in implicit mode) are committed, and records that were delivered but not
+    # acknowledged, or acquired by a fetch still in flight, are released to the other members
+    # right away. Blocks until the sessions are closed, which takes up to `fetch.wait.max.ms`
+    # when a fetch is in flight.
+    #
     # @return [nil]
     # @raise [RdkafkaError] When unsubscribing fails
     def unsubscribe

data/lib/rdkafka/version.rb

@@ -2,9 +2,9 @@
 
 module Rdkafka
   # Current rdkafka-ruby gem version
-  VERSION = "0.30.2"
+  VERSION = "0.31.0"
   # Target librdkafka version to be used
-  LIBRDKAFKA_VERSION = "2.15.1"
+  LIBRDKAFKA_VERSION = "2.16.0"
   # SHA256 hash of the librdkafka source tarball for verification
-  LIBRDKAFKA_SOURCE_SHA256 = "23c8575c7d1ced07246cb9cf200c11325b72201fd4134a02414ca869fbdd8ed3"
+  LIBRDKAFKA_SOURCE_SHA256 = "e6b61de61d3282879a88e4ee3d9f634a8b05bf78a9198dfc25c4820c0c9ed231"
 end

data/lib/rdkafka.rb

@@ -41,6 +41,13 @@ require "rdkafka/admin/list_offsets_handle"
 require "rdkafka/admin/list_offsets_report"
 require "rdkafka/admin/list_consumer_groups_handle"
 require "rdkafka/admin/list_consumer_groups_report"
+require "rdkafka/admin/describe_result_parsing"
+require "rdkafka/admin/describe_cluster_handle"
+require "rdkafka/admin/describe_cluster_report"
+require "rdkafka/admin/describe_topics_handle"
+require "rdkafka/admin/describe_topics_report"
+require "rdkafka/admin/describe_consumer_groups_handle"
+require "rdkafka/admin/describe_consumer_groups_report"
 require "rdkafka/admin/acl_binding_result"
 require "rdkafka/admin/config_binding_result"
 require "rdkafka/admin/config_resource_binding_result"
@@ -61,6 +68,9 @@ require "rdkafka/callbacks/describe_configs_handler"
 require "rdkafka/callbacks/incremental_alter_configs_handler"
 require "rdkafka/callbacks/list_offsets_handler"
 require "rdkafka/callbacks/list_consumer_groups_handler"
+require "rdkafka/callbacks/describe_cluster_handler"
+require "rdkafka/callbacks/describe_topics_handler"
+require "rdkafka/callbacks/describe_consumer_groups_handler"
 require "rdkafka/config"
 require "rdkafka/consumer"
 require "rdkafka/consumer/headers"

data/package-lock.json

@@ -103,9 +103,9 @@
       }
     },
     "node_modules/fastq": {
-      "version": "1.20.2",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.2.tgz",
-      "integrity": "sha512-UpGiiODyCGprM8EPP6JodP6jC9Rws6TCuiDOD+nn0CJhR8guI3g/ozo4ugL0vJ+Yz1UtJuuRPqvQuybVOF1VQA==",
+      "version": "1.20.3",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
+      "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==",
       "dev": true,
       "license": "ISC",
       "dependencies": {
@@ -286,9 +286,9 @@
       }
     },
     "node_modules/smol-toml": {
-      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz",
-      "integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==",
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.9.0.tgz",
+      "integrity": "sha512-hpd+HLON7HdZXqYchMM/+LaTTbdK0AU3NngIJ4KVyWbY9bfQqdL9cD+4yf6dUoU2Ap4VsU0JkQi6FxAI1B2mXQ==",
       "dev": true,
       "license": "BSD-3-Clause",
       "engines": {
@@ -312,9 +312,9 @@
       }
     },
     "node_modules/yaml": {
-      "version": "2.9.0",
-      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz",
-      "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==",
+      "version": "2.9.1",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz",
+      "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==",
       "dev": true,
       "license": "ISC",
       "bin": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.30.2
+  version: 0.31.0
 platform: aarch64-linux-gnu
 authors:
 - Thijs Cadier
@@ -122,8 +122,15 @@ files:
 - lib/rdkafka/admin/delete_topic_report.rb
 - lib/rdkafka/admin/describe_acl_handle.rb
 - lib/rdkafka/admin/describe_acl_report.rb
+- lib/rdkafka/admin/describe_cluster_handle.rb
+- lib/rdkafka/admin/describe_cluster_report.rb
 - lib/rdkafka/admin/describe_configs_handle.rb
 - lib/rdkafka/admin/describe_configs_report.rb
+- lib/rdkafka/admin/describe_consumer_groups_handle.rb
+- lib/rdkafka/admin/describe_consumer_groups_report.rb
+- lib/rdkafka/admin/describe_result_parsing.rb
+- lib/rdkafka/admin/describe_topics_handle.rb
+- lib/rdkafka/admin/describe_topics_report.rb
 - lib/rdkafka/admin/incremental_alter_configs_handle.rb
 - lib/rdkafka/admin/incremental_alter_configs_report.rb
 - lib/rdkafka/admin/list_consumer_groups_handle.rb
@@ -131,6 +138,21 @@ files:
 - lib/rdkafka/admin/list_offsets_handle.rb
 - lib/rdkafka/admin/list_offsets_report.rb
 - lib/rdkafka/bindings.rb
+- lib/rdkafka/bindings/admin.rb
+- lib/rdkafka/bindings/admin/acls.rb
+- lib/rdkafka/bindings/admin/configs.rb
+- lib/rdkafka/bindings/admin/list_offsets.rb
+- lib/rdkafka/bindings/callbacks.rb
+- lib/rdkafka/bindings/configuration.rb
+- lib/rdkafka/bindings/consumer.rb
+- lib/rdkafka/bindings/errors.rb
+- lib/rdkafka/bindings/handle.rb
+- lib/rdkafka/bindings/messages.rb
+- lib/rdkafka/bindings/metadata.rb
+- lib/rdkafka/bindings/polling.rb
+- lib/rdkafka/bindings/producer.rb
+- lib/rdkafka/bindings/share_consumer.rb
+- lib/rdkafka/bindings/structs.rb
 - lib/rdkafka/callbacks.rb
 - lib/rdkafka/callbacks/alter_consumer_group_offsets_handler.rb
 - lib/rdkafka/callbacks/base_handler.rb
@@ -143,7 +165,10 @@ files:
 - lib/rdkafka/callbacks/delete_records_handler.rb
 - lib/rdkafka/callbacks/delete_topic_handler.rb
 - lib/rdkafka/callbacks/describe_acl_handler.rb
+- lib/rdkafka/callbacks/describe_cluster_handler.rb
 - lib/rdkafka/callbacks/describe_configs_handler.rb
+- lib/rdkafka/callbacks/describe_consumer_groups_handler.rb
+- lib/rdkafka/callbacks/describe_topics_handler.rb
 - lib/rdkafka/callbacks/incremental_alter_configs_handler.rb
 - lib/rdkafka/callbacks/list_consumer_groups_handler.rb
 - lib/rdkafka/callbacks/list_offsets_handler.rb
@@ -191,7 +216,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="