rdkafka 0.16.0.beta1 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3135d4f2663517517330d165948e9761ffc0ecd20942f911a6ee9541c437ee7e
-  data.tar.gz: 9a489c2400c4054e9cec0d6c8d24f75bce407d370f8c393b7b31dcd0dbf7c361
+  metadata.gz: 07f1bd822870a6f28cd441b95870683442b3d5857f12d85e04057fa29a29121a
+  data.tar.gz: f734cec92e64dfdfbb6a7e3e1875ad402982aeea6d9e8e14befeb50144f2c125
 SHA512:
-  metadata.gz: be8eba2aec012af189d893ebf6ddcd4e4dec117aecd7f36afa76bc6ee4c2a3fa93f4ac4160efa16e8c91efd7011ee41cda00a5e2146e18044a166035850cd490
-  data.tar.gz: d1761a6ab9c7d9ee539d79679dc730cc392fc2d7369d3f78b9c8d8f1c800855ab15ae7c472259b43e95139aa30d4777026c0529671fd85f98f3e5fceeaf53e62
+  metadata.gz: 644f1435d988fd962cae3031d5d7c7a31de2d618e80def8f48913bda6309e08b81ff62741933b9ada5c40f59f79502e0a9dfee09efcc57fe7845269f178d1738
+  data.tar.gz: 3f83530ade65c38064b1c3a9922c173f973f57b49544fce3fd645a1c6a47ee18506450b33299e6417510f052123114fced56689a6ce2c59abd7d4be11b752c48

data/.github/workflows/ci.yml

@@ -22,11 +22,11 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.4.0-preview1'
           - '3.3'
           - '3.2'
           - '3.1'
           - '3.0'
-          - '2.7'
         include:
           - ruby: '3.3'
             coverage: 'true'

data/.ruby-version

@@ -1 +1 @@
-3.3.1
+3.3.3

data/CHANGELOG.md

@@ -1,10 +1,18 @@
 # Rdkafka Changelog
 
-## 0.16.0 (Unreleased)
+## 0.16.0 (2024-06-13)
+- **[Breaking]** Retire support for Ruby 2.7.
+- **[Breaking]** Messages without headers returned by `#poll` contain frozen empty hash.
+- **[Breaking]** `HashWithSymbolKeysTreatedLikeStrings` has been removed so headers are regular hashes with string keys.
+- **[Feature]** Support incremental config describe + alter API.
 - **[Feature]** Oauthbearer token refresh callback (bruce-szalwinski-he)
+- **[Feature]** Provide ability to use topic config on a producer for custom behaviors per dispatch.
+- [Enhancement] Use topic config reference cache for messages production to prevent topic objects allocation with each message.
+- [Enhancement] Provide `Rrdkafka::Admin#describe_errors` to get errors descriptions (mensfeld)
 - [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
 - [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
 - [Enhancement] name polling Thread as `rdkafka.native_kafka#<name>` (nijikon)
+- [Enhancement] Save two objects on message produced and lower CPU usage on message produced with small improvements.
 - [Change] Allow for native kafka thread operations deferring and manual start for consumer, producer and admin.
 - [Change] The `wait_timeout` argument in `AbstractHandle.wait` method is deprecated and will be removed in future versions without replacement. We don't rely on it's value anymore (nijikon)
 - [Fix] Background logger stops working after forking causing memory leaks (mensfeld)

data/README.md

@@ -163,6 +163,7 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka |
 |-|-|
+| 0.16.0 (2024-06-13) | 2.3.0 (2023-10-25) |
 | 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) |
 | 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) |
 | 0.13.0 (2023-07-24) | 2.0.2 (2023-01-20) |

data/lib/rdkafka/admin/config_binding_result.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # A single config binding result that represents its values extracted from C
+    class ConfigBindingResult
+      attr_reader :name, :value, :read_only, :default, :sensitive, :synonym, :synonyms
+
+      # @param config_ptr [FFI::Pointer] config pointer
+      def initialize(config_ptr)
+        @name = Bindings.rd_kafka_ConfigEntry_name(config_ptr)
+        @value = Bindings.rd_kafka_ConfigEntry_value(config_ptr)
+        @read_only = Bindings.rd_kafka_ConfigEntry_is_read_only(config_ptr)
+        @default = Bindings.rd_kafka_ConfigEntry_is_default(config_ptr)
+        @sensitive = Bindings.rd_kafka_ConfigEntry_is_sensitive(config_ptr)
+        @synonym = Bindings.rd_kafka_ConfigEntry_is_synonym(config_ptr)
+        @synonyms = []
+
+        # The code below builds up the config synonyms using same config binding
+        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        synonym_ptr = Bindings.rd_kafka_ConfigEntry_synonyms(config_ptr, pointer_to_size_t)
+        synonyms_ptr = synonym_ptr.read_array_of_pointer(pointer_to_size_t.read_int)
+
+        (1..pointer_to_size_t.read_int).map do |ar|
+          self.class.new synonyms_ptr[ar - 1]
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/config_resource_binding_result.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # A simple binding that represents the requested config resource
+    class ConfigResourceBindingResult
+      attr_reader :name, :type, :configs, :configs_count
+
+      def initialize(config_resource_ptr)
+        ffi_binding = Bindings::ConfigResource.new(config_resource_ptr)
+
+        @name = ffi_binding[:name]
+        @type = ffi_binding[:type]
+        @configs = []
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/describe_acl_report.rb

@@ -10,6 +10,7 @@ module Rdkafka
 
       def initialize(acls:, acls_count:)
         @acls=[]
+
         if acls != FFI::Pointer::NULL
           acl_binding_result_pointers  = acls.read_array_of_pointer(acls_count)
           (1..acls_count).map do |acl_index|

data/lib/rdkafka/admin/describe_configs_handle.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class DescribeConfigsHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :response_string, :pointer,
+             :config_entries, :pointer,
+             :entry_count, :int
+
+      # @return [String] the name of the operation.
+      def operation_name
+        "describe configs"
+      end
+
+      # @return [DescribeAclReport] instance with an array of acls that matches the request filters.
+      def create_result
+        DescribeConfigsReport.new(
+          config_entries: self[:config_entries],
+          entry_count: self[:entry_count]
+        )
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+          self[:response],
+          broker_message: self[:response_string].read_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/describe_configs_report.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class DescribeConfigsReport
+      attr_reader :resources
+
+      def initialize(config_entries:, entry_count:)
+        @resources=[]
+
+        return if config_entries == FFI::Pointer::NULL
+
+        config_entries
+          .read_array_of_pointer(entry_count)
+          .each { |config_resource_result_ptr| validate!(config_resource_result_ptr) }
+          .each do |config_resource_result_ptr|
+            config_resource_result = ConfigResourceBindingResult.new(config_resource_result_ptr)
+
+            pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+            configs_ptr = Bindings.rd_kafka_ConfigResource_configs(
+              config_resource_result_ptr,
+              pointer_to_size_t
+            )
+
+            configs_ptr
+              .read_array_of_pointer(pointer_to_size_t.read_int)
+              .map { |config_ptr| ConfigBindingResult.new(config_ptr) }
+              .each { |config_binding| config_resource_result.configs << config_binding }
+
+            @resources << config_resource_result
+          end
+      ensure
+        return if config_entries == FFI::Pointer::NULL
+
+        Bindings.rd_kafka_ConfigResource_destroy_array(config_entries, entry_count)
+      end
+
+      private
+
+      def validate!(config_resource_result_ptr)
+        code = Bindings.rd_kafka_ConfigResource_error(config_resource_result_ptr)
+
+        return if code.zero?
+
+        raise(
+          RdkafkaError.new(
+            code,
+            Bindings.rd_kafka_ConfigResource_error_string(config_resource_result_ptr)
+          )
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/incremental_alter_configs_handle.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class IncrementalAlterConfigsHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :response_string, :pointer,
+             :config_entries, :pointer,
+             :entry_count, :int
+
+      # @return [String] the name of the operation.
+      def operation_name
+        "incremental alter configs"
+      end
+
+      # @return [DescribeAclReport] instance with an array of acls that matches the request filters.
+      def create_result
+        IncrementalAlterConfigsReport.new(
+          config_entries: self[:config_entries],
+          entry_count: self[:entry_count]
+        )
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+          self[:response],
+          broker_message: self[:response_string].read_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/incremental_alter_configs_report.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class IncrementalAlterConfigsReport
+      attr_reader :resources
+
+      def initialize(config_entries:, entry_count:)
+        @resources=[]
+
+        return if config_entries == FFI::Pointer::NULL
+
+        config_entries
+          .read_array_of_pointer(entry_count)
+          .each { |config_resource_result_ptr| validate!(config_resource_result_ptr) }
+          .each do |config_resource_result_ptr|
+            config_resource_result = ConfigResourceBindingResult.new(config_resource_result_ptr)
+
+            pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+            configs_ptr = Bindings.rd_kafka_ConfigResource_configs(
+              config_resource_result_ptr,
+              pointer_to_size_t
+            )
+
+            configs_ptr
+              .read_array_of_pointer(pointer_to_size_t.read_int)
+              .map { |config_ptr| ConfigBindingResult.new(config_ptr) }
+              .each { |config_binding| config_resource_result.configs << config_binding }
+
+            @resources << config_resource_result
+          end
+      ensure
+        return if config_entries == FFI::Pointer::NULL
+
+        Bindings.rd_kafka_ConfigResource_destroy_array(config_entries, entry_count)
+      end
+
+      private
+
+      def validate!(config_resource_result_ptr)
+        code = Bindings.rd_kafka_ConfigResource_error(config_resource_result_ptr)
+
+        return if code.zero?
+
+        raise(
+          RdkafkaError.new(
+            code,
+            Bindings.rd_kafka_ConfigResource_error_string(config_resource_result_ptr)
+          )
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin.rb

@@ -4,6 +4,50 @@ module Rdkafka
   class Admin
     include Helpers::OAuth
 
+    class << self
+      # Allows us to retrieve librdkafka errors with descriptions
+      # Useful for debugging and building UIs, etc.
+      #
+      # @return [Hash<Integer, Hash>] hash with errors mapped by code
+      def describe_errors
+        # Memory pointers for the array of structures and count
+        p_error_descs = FFI::MemoryPointer.new(:pointer)
+        p_count = FFI::MemoryPointer.new(:size_t)
+
+        # Call the attached function
+        Bindings.rd_kafka_get_err_descs(p_error_descs, p_count)
+
+        # Retrieve the number of items in the array
+        count = p_count.read_uint
+
+        # Get the pointer to the array of error descriptions
+        array_of_errors = FFI::Pointer.new(Bindings::NativeErrorDesc, p_error_descs.read_pointer)
+
+        errors = {}
+
+        count.times do |i|
+          # Get the pointer to each struct
+          error_ptr = array_of_errors[i]
+
+          # Create a new instance of NativeErrorDesc for each item
+          error_desc = Bindings::NativeErrorDesc.new(error_ptr)
+
+          # Read values from the struct
+          code = error_desc[:code]
+
+          name = ''
+          desc = ''
+
+          name = error_desc[:name].read_string unless error_desc[:name].null?
+          desc = error_desc[:desc].read_string unless error_desc[:desc].null?
+
+          errors[code] = { code: code, name: name, description: desc }
+        end
+
+        errors
+      end
+    end
+
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
@@ -620,6 +664,166 @@ module Rdkafka
       describe_acl_handle
     end
 
+
+    # Describe configs
+    #
+    # @param resources [Array<Hash>]  Array where elements are hashes with two keys:
+    #   - `:resource_type` - numerical resource type based on Kafka API
+    #   - `:resource_name` - string with resource name
+    # @return [DescribeConfigsHandle] Describe config handle that can be used to wait for the
+    #   result of fetching resources with their appropriate configs
+    #
+    # @raise [RdkafkaError]
+    #
+    # @note Several resources can be requested at one go, but only one broker at a time
+    def describe_configs(resources)
+      closed_admin_check(__method__)
+
+      handle = DescribeConfigsHandle.new
+      handle[:pending] = true
+      handle[:response] = -1
+
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+
+      if queue_ptr.null?
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+          inner,
+          Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS
+        )
+      end
+
+      DescribeConfigsHandle.register(handle)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+
+      pointer_array = resources.map do |resource_details|
+        Rdkafka::Bindings.rd_kafka_ConfigResource_new(
+          resource_details.fetch(:resource_type),
+          FFI::MemoryPointer.from_string(
+            resource_details.fetch(:resource_name)
+          )
+        )
+      end
+
+      configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
+      configs_array_ptr.write_array_of_pointer(pointer_array)
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_DescribeConfigs(
+            inner,
+            configs_array_ptr,
+            pointer_array.size,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        DescribeConfigsHandle.remove(handle.to_ptr.address)
+
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
+          configs_array_ptr,
+          pointer_array.size
+        ) if configs_array_ptr
+      end
+
+      handle
+    end
+
+    # Alters in an incremental way all the configs provided for given resources
+    #
+    # @param resources_with_configs [Array<Hash>] resources with the configs key that contains
+    # name, value and the proper op_type to perform on this value.
+    #
+    # @return [IncrementalAlterConfigsHandle] Incremental alter configs handle that can be used to
+    #   wait for the result of altering resources with their appropriate configs
+    #
+    # @raise [RdkafkaError]
+    #
+    # @note Several resources can be requested at one go, but only one broker at a time
+    # @note The results won't contain altered values but only the altered resources
+    def incremental_alter_configs(resources_with_configs)
+      closed_admin_check(__method__)
+
+      handle = IncrementalAlterConfigsHandle.new
+      handle[:pending] = true
+      handle[:response] = -1
+
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
+
+      if queue_ptr.null?
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(
+          inner,
+          Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS
+        )
+      end
+
+      IncrementalAlterConfigsHandle.register(handle)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
+
+      # Tu poprawnie tworzyc
+      pointer_array = resources_with_configs.map do |resource_details|
+        # First build the appropriate resource representation
+        resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
+          resource_details.fetch(:resource_type),
+          FFI::MemoryPointer.from_string(
+            resource_details.fetch(:resource_name)
+          )
+        )
+
+        resource_details.fetch(:configs).each do |config|
+          Bindings.rd_kafka_ConfigResource_add_incremental_config(
+            resource_ptr,
+            config.fetch(:name),
+            config.fetch(:op_type),
+            config.fetch(:value)
+          )
+        end
+
+        resource_ptr
+      end
+
+      configs_array_ptr = FFI::MemoryPointer.new(:pointer, pointer_array.size)
+      configs_array_ptr.write_array_of_pointer(pointer_array)
+
+
+      begin
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
+            inner,
+            configs_array_ptr,
+            pointer_array.size,
+            admin_options_ptr,
+            queue_ptr
+          )
+        end
+      rescue Exception
+        IncrementalAlterConfigsHandle.remove(handle.to_ptr.address)
+
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_ConfigResource_destroy_array(
+          configs_array_ptr,
+          pointer_array.size
+        ) if configs_array_ptr
+      end
+
+      handle
+    end
+
     private
 
     def closed_admin_check(method)

data/lib/rdkafka/bindings.rb

@@ -89,10 +89,58 @@ module Rdkafka
     attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
     attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
 
+    # Configs management
+    #
+    # Structs for management of configurations
+    # Each configuration is attached to a resource and one resource can have many configuration
+    # details. Each resource will also have separate errors results if obtaining configuration
+    # was not possible for any reason
+    class ConfigResource < FFI::Struct
+      layout :type, :int,
+             :name, :string
+    end
+
+    attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
+    attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
+    attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_configs, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigEntry_name, [:pointer], :string
+    attach_function :rd_kafka_ConfigEntry_value, [:pointer], :string
+    attach_function :rd_kafka_ConfigEntry_is_read_only, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_is_default, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_is_sensitive, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_is_synonym, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_synonyms, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_error, [:pointer], :int
+    attach_function :rd_kafka_ConfigResource_error_string, [:pointer], :string
+    attach_function :rd_kafka_IncrementalAlterConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_IncrementalAlterConfigs_result_resources, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_add_incremental_config, [:pointer, :string, :int32, :string], :pointer
+    attach_function :rd_kafka_event_IncrementalAlterConfigs_result, [:pointer], :pointer
+
+    RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS = 5
+    RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT = 104
+
+    RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
+    RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
+
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET      = 0
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE   = 1
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND   = 2
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
+
     # Errors
+    class NativeErrorDesc < FFI::Struct
+      layout :code, :int,
+             :name, :pointer,
+             :desc, :pointer
+    end
 
     attach_function :rd_kafka_err2name, [:int], :string
     attach_function :rd_kafka_err2str, [:int], :string
+    attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
 
     # Configuration
 
@@ -119,6 +167,9 @@ module Rdkafka
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
     attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
+    # Per topic configs
+    attach_function :rd_kafka_topic_conf_new, [], :pointer
+    attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
 
     LogCallback = FFI::Function.new(
       :void, [:pointer, :int, :string, :string]