rdkafka 0.15.0 → 0.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d906b2e71dae5b5f45459e915c48dc8cb88e0d51ebb90ded80cef3c8e5531b77
-  data.tar.gz: 8f0df2688bbc3b264de22b5943b18462ad41898781cc12e6e534804409133ce0
+  metadata.gz: 8636c80e1798cf24b34cf25a20ca24f35e2951fb179843a3a85a94fe0274ca76
+  data.tar.gz: f115aa7fff4961d42280a7ad6fd78fed40568b936139d588ae2362a2f0f45c25
 SHA512:
-  metadata.gz: c35d392b326f4d47077f419bced92b929436be548651afc9364f5ada2eda51883ad75feeeb30183369aa6b15db3ac4630410f408eae449d1b0cc5a007cf011fc
-  data.tar.gz: 1487bb54713e6330ce55fd95f656dffd2edc34bcd8bc151d94faf2d6f043b8183276e3407d5f45c6152e99463c8323ccb83c15fff7007ebb96c4b369533002d5
+  metadata.gz: e5b5368a732e42b1c57aff93a7172c95b0bad93ac646dcba495c0509c5b6c29cf8753601d1f04f028c98579846e5db7e7119ae9a6a4cca2f41441316b60b5c9c
+  data.tar.gz: 8596d6944d5151df3ad875d93dbd7cf2aee00c1ded85e053e96880b8c5420ca6ba18a72f57cc867a7bfedf38060be3c9ea4334d79d7a21b2503a078bce1d266a

data/.github/workflows/ci.yml

@@ -22,16 +22,15 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.3.0-preview2'
+          - '3.3'
           - '3.2'
           - '3.1'
           - '3.1.0'
           - '3.0'
           - '3.0.0'
           - '2.7'
-          - '2.7.0'
         include:
-          - ruby: '3.2'
+          - ruby: '3.3'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v4

data/.ruby-version

@@ -1 +1 @@
-3.2.2
+3.3.0

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Rdkafka Changelog
 
+## 0.15.1 (2024-01-30)
+- [Enhancement] Provide support for Nix OS (alexandriainfantino)
+- [Enhancement] Replace `rd_kafka_offset_store` with `rd_kafka_offsets_store` (mensfeld)
+- [Enhancement] Alias `topic_name` as `topic` in the delivery report (mensfeld)
+- [Enhancement] Provide `label` producer handler and report reference for improved traceability (mensfeld)
+- [Enhancement] Include the error when invoking `create_result` on producer handle (mensfeld)
+- [Enhancement] Skip intermediate array creation on delivery report callback execution (one per message) (mensfeld).
+- [Enhancement] Report `-1` instead of `nil` in case `partition_count` failure (mensfeld).
+- [Fix] Fix return type on `#rd_kafka_poll` (mensfeld)
+- [Fix] `uint8_t` does not exist on Apple Silicon (mensfeld)
+- [Fix] Missing ACL `RD_KAFKA_RESOURCE_BROKER` constant reference (mensfeld)
+- [Fix] Partition cache caches invalid nil result for `PARTITIONS_COUNT_TTL` (mensfeld)
+- [Change] Rename `matching_acl_pattern_type` to `matching_acl_resource_pattern_type` to align the whole API (mensfeld)
+
 ## 0.15.0 (2023-12-03)
 - **[Feature]** Add `Admin#metadata` (mensfeld)
 - **[Feature]** Add `Admin#create_partitions` (mensfeld)

data/README.md

@@ -32,6 +32,7 @@ The most important pieces of a Kafka client are implemented, and we aim to provi
   * [Message Publishing Libraries](#message-publishing-libraries)
 - [Development](#development)
 - [Example](#example)
+- [Versions](#versions)
 
 ## Project Scope
 
@@ -147,3 +148,14 @@ To see everything working, run these in separate tabs:
 bundle exec rake consume_messages
 bundle exec rake produce_messages
 ```
+
+## Versions
+
+| rdkafka-ruby | librdkafka |
+|-|-|
+| 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) |
+| 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) |
+| 0.13.0 (2023-07-24) | 2.0.2 (2023-01-20) |
+| 0.12.0 (2022-06-17) | 1.9.0 (2022-06-16) |
+| 0.11.0 (2021-11-17) | 1.8.2 (2021-10-18) |
+| 0.10.0 (2021-09-07) | 1.5.0 (2020-07-20) |

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.2
+    image: confluentinc/cp-kafka:7.5.3
 
     ports:
       - 9092:9092

data/ext/Rakefile

@@ -1,40 +1,57 @@
 # frozen_string_literal: true
 
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
-require "mini_portile2"
 require "fileutils"
 require "open-uri"
 
 task :default => :clean do
-  # Download and compile librdkafka
-  recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
+  # For nix users, nix can't locate the file paths because the packages it's requiring aren't managed by the system but are
+  # managed by nix itself, so using the normal file paths doesn't work for nix users.
+  #
+  # Mini_portile causes an issue because it's dependencies are downloaded on the fly and therefore don't exist/aren't
+  # accessible in the nix environment
+  if ENV.fetch('RDKAFKA_EXT_PATH', '').empty?
+    # Download and compile librdkafka if RDKAFKA_EXT_PATH is not set
+    require "mini_portile2"
+    recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
-  # Use default homebrew openssl if we're on mac and the directory exists
-  # and each of flags is not empty
-  if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
-    ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
-    ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
-  end
+    # Use default homebrew openssl if we're on mac and the directory exists
+    # and each of flags is not empty
+    if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
+      ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
+      ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
+    end
 
-  recipe.files << {
-    :url => "https://codeload.github.com/confluentinc/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
-    :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
-  }
-  recipe.configure_options = ["--host=#{recipe.host}"]
-  recipe.cook
-  # Move dynamic library we're interested in
-  if recipe.host.include?('darwin')
-    from_extension = '1.dylib'
-    to_extension   = 'dylib'
+    recipe.files << {
+      :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
+      :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
+    }
+    recipe.configure_options = ["--host=#{recipe.host}"]
+    recipe.cook
+    # Move dynamic library we're interested in
+    if recipe.host.include?('darwin')
+      from_extension = '1.dylib'
+      to_extension   = 'dylib'
+    else
+      from_extension = 'so.1'
+      to_extension = 'so'
+    end
+    lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
+    FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
+    # Cleanup files created by miniportile we don't need in the gem
+    FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
+    FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
   else
-    from_extension = 'so.1'
-    to_extension = 'so'
+    # Otherwise, copy existing libraries to ./ext
+    if ENV['RDKAFKA_EXT_PATH'].nil? || ENV['RDKAFKA_EXT_PATH'].empty?
+      raise "RDKAFKA_EXT_PATH must be set in your nix config when running under nix"
+    end
+    files = [
+      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.dylib'),
+      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.so')
+    ]
+    files.each { |ext| FileUtils.cp(ext, File.dirname(__FILE__)) if File.exist?(ext) }
   end
-  lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
-  FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
-  # Cleanup files created by miniportile we don't need in the gem
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
 end
 
 task :clean do

data/lib/rdkafka/admin/acl_binding_result.rb

@@ -2,36 +2,50 @@
 
 module Rdkafka
   class Admin
-
     # Extracts attributes of rd_kafka_AclBinding_t
     #
     class AclBindingResult
-      attr_reader :result_error, :error_string, :matching_acl_resource_type, :matching_acl_resource_name, :matching_acl_pattern_type, :matching_acl_principal, :matching_acl_host, :matching_acl_operation, :matching_acl_permission_type
+      attr_reader :result_error, :error_string, :matching_acl_resource_type,
+                  :matching_acl_resource_name, :matching_acl_resource_pattern_type,
+                  :matching_acl_principal, :matching_acl_host, :matching_acl_operation,
+                  :matching_acl_permission_type
+
+      # This attribute was initially released under the name that is now an alias
+      # We keep it for backwards compatibility but it was changed for the consistency
+      alias matching_acl_pattern_type matching_acl_resource_pattern_type
 
       def initialize(matching_acl)
-          rd_kafka_error_pointer        = Rdkafka::Bindings.rd_kafka_AclBinding_error(matching_acl)
-          @result_error                 = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
-          error_string                 = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
-          if error_string != FFI::Pointer::NULL
-            @error_string = error_string.read_string
-          end
-          @matching_acl_resource_type   = Rdkafka::Bindings.rd_kafka_AclBinding_restype(matching_acl)
-          matching_acl_resource_name    = Rdkafka::Bindings.rd_kafka_AclBinding_name(matching_acl)
-          if matching_acl_resource_name != FFI::Pointer::NULL
-            @matching_acl_resource_name = matching_acl_resource_name.read_string
-          end
-          @matching_acl_pattern_type    = Rdkafka::Bindings.rd_kafka_AclBinding_resource_pattern_type(matching_acl)
-          matching_acl_principal        = Rdkafka::Bindings.rd_kafka_AclBinding_principal(matching_acl)
-          if matching_acl_principal != FFI::Pointer::NULL
-            @matching_acl_principal = matching_acl_principal.read_string
-          end
-          matching_acl_host             = Rdkafka::Bindings.rd_kafka_AclBinding_host(matching_acl)
-          if matching_acl_host != FFI::Pointer::NULL
-            @matching_acl_host = matching_acl_host.read_string
-          end
-          @matching_acl_operation       = Rdkafka::Bindings.rd_kafka_AclBinding_operation(matching_acl)
-          @matching_acl_permission_type = Rdkafka::Bindings.rd_kafka_AclBinding_permission_type(matching_acl)
+        rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_AclBinding_error(matching_acl)
+        @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
+        error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
+
+        if error_string != FFI::Pointer::NULL
+          @error_string = error_string.read_string
         end
+
+        @matching_acl_resource_type = Rdkafka::Bindings.rd_kafka_AclBinding_restype(matching_acl)
+        matching_acl_resource_name = Rdkafka::Bindings.rd_kafka_AclBinding_name(matching_acl)
+
+        if matching_acl_resource_name != FFI::Pointer::NULL
+          @matching_acl_resource_name = matching_acl_resource_name.read_string
+        end
+
+        @matching_acl_resource_pattern_type = Rdkafka::Bindings.rd_kafka_AclBinding_resource_pattern_type(matching_acl)
+        matching_acl_principal = Rdkafka::Bindings.rd_kafka_AclBinding_principal(matching_acl)
+
+        if matching_acl_principal != FFI::Pointer::NULL
+          @matching_acl_principal = matching_acl_principal.read_string
+        end
+
+        matching_acl_host = Rdkafka::Bindings.rd_kafka_AclBinding_host(matching_acl)
+
+        if matching_acl_host != FFI::Pointer::NULL
+          @matching_acl_host = matching_acl_host.read_string
+        end
+
+        @matching_acl_operation = Rdkafka::Bindings.rd_kafka_AclBinding_operation(matching_acl)
+        @matching_acl_permission_type = Rdkafka::Bindings.rd_kafka_AclBinding_permission_type(matching_acl)
       end
+    end
   end
 end

data/lib/rdkafka/bindings.rb

@@ -32,7 +32,7 @@ module Rdkafka
     # Polling
 
     attach_function :rd_kafka_flush, [:pointer, :int], :int, blocking: true
-    attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
+    attach_function :rd_kafka_poll, [:pointer, :int], :int, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
     # Metadata
@@ -185,12 +185,15 @@ module Rdkafka
     attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
-    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int, blocking: true
+    attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
     attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
     attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
+    # those two are used for eos support
+    attach_function :rd_kafka_consumer_group_metadata, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_consumer_group_metadata_destroy, [:pointer], :void, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -394,6 +397,7 @@ module Rdkafka
     RD_KAFKA_RESOURCE_ANY   = 1
     RD_KAFKA_RESOURCE_TOPIC = 2
     RD_KAFKA_RESOURCE_GROUP = 3
+    RD_KAFKA_RESOURCE_BROKER = 4
 
     # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
 
@@ -436,9 +440,9 @@ module Rdkafka
     class NativeError < FFI::Struct # rd_kafka_error_t
       layout :code, :int32,
              :errstr, :pointer,
-             :fatal, :uint8_t,
-             :retriable, :uint8_t,
-             :txn_requires_abort, :uint8_t
+             :fatal, :u_int8_t,
+             :retriable, :u_int8_t,
+             :txn_requires_abort, :u_int8_t
     end
 
     attach_function :rd_kafka_group_result_error, [:pointer], NativeError.by_ref # rd_kafka_group_result_t* => rd_kafka_error_t*

data/lib/rdkafka/callbacks.rb

@@ -261,7 +261,6 @@ module Rdkafka
           end
         end
       end
-
     end
 
     # FFI Function used for Message Delivery callbacks
@@ -289,7 +288,16 @@ module Rdkafka
 
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], topic_name, message[:err]), delivery_handle)
+            opaque.call_delivery_callback(
+              Rdkafka::Producer::DeliveryReport.new(
+                message[:partition],
+                message[:offset],
+                topic_name,
+                message[:err],
+                delivery_handle.label
+              ),
+              delivery_handle
+            )
           end
         end
       end

data/lib/rdkafka/consumer.rb

@@ -19,10 +19,6 @@ module Rdkafka
       @native_kafka = native_kafka
     end
 
-    def finalizer
-      ->(_) { close }
-    end
-
     # @return [String] consumer name
     def name
       @name ||= @native_kafka.with_inner do |inner|
@@ -30,6 +26,10 @@ module Rdkafka
       end
     end
 
+    def finalizer
+      ->(_) { close }
+    end
+
     # Close this consumer
     # @return [nil]
     def close
@@ -239,7 +239,7 @@ module Rdkafka
     # @param timeout_ms [Integer] The timeout for fetching this information.
     # @return [TopicPartitionList]
     # @raise [RdkafkaError] When getting the committed positions fails.
-    def committed(list=nil, timeout_ms=1200)
+    def committed(list=nil, timeout_ms=2000)
       closed_consumer_check(__method__)
 
       if list.nil?
@@ -387,27 +387,26 @@ module Rdkafka
     def store_offset(message)
       closed_consumer_check(__method__)
 
-      # rd_kafka_offset_store is one of the few calls that does not support
-      # a string as the topic, so create a native topic for it.
-      native_topic = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_topic_new(
+      list = TopicPartitionList.new
+      list.add_topic_and_partitions_with_offsets(
+        message.topic,
+        message.partition => message.offset + 1
+      )
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_offsets_store(
           inner,
-          message.topic,
-          nil
+          tpl
         )
       end
-      response = Rdkafka::Bindings.rd_kafka_offset_store(
-        native_topic,
-        message.partition,
-        message.offset
-      )
+
       if response != 0
         raise Rdkafka::RdkafkaError.new(response)
       end
     ensure
-      if native_topic && !native_topic.null?
-        Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
-      end
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Seek to a particular message. The next poll on the topic/partition will return the
@@ -673,6 +672,22 @@ module Rdkafka
       end
     end
 
+    # Returns pointer to the consumer group metadata. It is used only in the context of
+    # exactly-once-semantics in transactions, this is why it is never remapped to Ruby
+    #
+    # This API is **not** usable by itself from Ruby
+    #
+    # @note This pointer **needs** to be removed with `#rd_kafka_consumer_group_metadata_destroy`
+    #
+    # @private
+    def consumer_group_metadata_pointer
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Bindings.rd_kafka_consumer_group_metadata(inner)
+      end
+    end
+
     private
 
     def closed_consumer_check(method)

data/lib/rdkafka/producer/delivery_handle.rb

@@ -11,6 +11,9 @@ module Rdkafka
              :offset, :int64,
              :topic_name, :pointer
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_accessor :label
+
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
         "delivery"
@@ -18,7 +21,15 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        DeliveryReport.new(self[:partition], self[:offset], self[:topic_name].read_string)
+        DeliveryReport.new(
+          self[:partition],
+          self[:offset],
+          # For part of errors, we will not get a topic name reference and in cases like this
+          # we should not return it
+          self[:topic_name].null? ? nil : self[:topic_name].read_string,
+          self[:response] != 0 ? RdkafkaError.new(self[:response]) : nil,
+          label
+        )
       end
     end
   end

data/lib/rdkafka/producer/delivery_report.rb

@@ -12,21 +12,34 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # The name of the topic this message was produced to.
-      # @return [String]
+      # The name of the topic this message was produced to or nil in case of reports with errors
+      # where topic was not reached.
+      #
+      # @return [String, nil]
       attr_reader :topic_name
 
       # Error in case happen during produce.
       # @return [Integer]
       attr_reader :error
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_reader :label
+
+      # We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
+      # where the topic name is under `#topic` method. That way we have a consistent name that
+      # is present in both places
+      #
+      # We do not remove the original `#topic_name` because of backwards compatibility
+      alias topic topic_name
+
       private
 
-      def initialize(partition, offset, topic_name = nil, error = nil)
+      def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
         @partition = partition
         @offset = offset
         @topic_name = topic_name
         @error = error
+        @label = label
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -23,6 +23,9 @@ module Rdkafka
     attr_reader :delivery_callback_arity
 
     # @private
+    # @param native_kafka [NativeKafka]
+    # @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
+    #   the "consistent_random" default
     def initialize(native_kafka, partitioner_name)
       @native_kafka = native_kafka
       @partitioner_name = partitioner_name || "consistent_random"
@@ -37,10 +40,16 @@ module Rdkafka
           topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
         end
 
-        cache[topic] = [
-          monotonic_now,
-          topic_metadata ? topic_metadata[:partition_count] : nil
-        ]
+        partition_count = topic_metadata ? topic_metadata[:partition_count] : -1
+
+        # This approach caches the failure to fetch only for 1 second. This will make sure, that
+        # we do not cache the failure for too long but also "buys" us a bit of time in case there
+        # would be issues in the cluster so we won't overaload it with consecutive requests
+        cache[topic] = if partition_count.positive?
+                         [monotonic_now, partition_count]
+                       else
+                         [monotonic_now - PARTITIONS_COUNT_TTL + 5, partition_count]
+                       end
       end
     end
 
@@ -134,14 +143,15 @@ module Rdkafka
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.
-    # @return [Integer] partition count for a given topic
+    # @return [Integer] partition count for a given topic or `-1` if it could not be obtained.
     #
     # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
     #   auto-created after returning nil.
     #
     # @note We cache the partition count for a given topic for given time.
     #   This prevents us in case someone uses `partition_key` from querying for the count with
-    #   each message. Instead we query once every 30 seconds at most
+    #   each message. Instead we query once every 30 seconds at most if we have a valid partition
+    #   count or every 5 seconds in case we were not able to obtain number of partitions
     def partition_count(topic)
       closed_producer_check(__method__)
 
@@ -164,11 +174,12 @@ module Rdkafka
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
+    # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
-    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
+    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil, label: nil)
       closed_producer_check(__method__)
 
       # Start by checking and converting the input
@@ -190,7 +201,7 @@ module Rdkafka
       if partition_key
         partition_count = partition_count(topic)
         # If the topic is not present, set to -1
-        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
       end
 
       # If partition is nil, use -1 to let librdafka set the partition randomly or
@@ -210,6 +221,7 @@ module Rdkafka
                       end
 
       delivery_handle = DeliveryHandle.new
+      delivery_handle.label = label
       delivery_handle[:pending] = true
       delivery_handle[:response] = -1
       delivery_handle[:partition] = -1
@@ -256,13 +268,27 @@ module Rdkafka
       delivery_handle
     end
 
+    # Calls (if registered) the delivery callback
+    #
+    # @param delivery_report [Producer::DeliveryReport]
+    # @param delivery_handle [Producer::DeliveryHandle]
     def call_delivery_callback(delivery_report, delivery_handle)
       return unless @delivery_callback
 
-      args = [delivery_report, delivery_handle].take(@delivery_callback_arity)
-      @delivery_callback.call(*args)
+      case @delivery_callback_arity
+      when 0
+        @delivery_callback.call
+      when 1
+        @delivery_callback.call(delivery_report)
+      else
+        @delivery_callback.call(delivery_report, delivery_handle)
+      end
     end
 
+    # Figures out the arity of a given block/method
+    #
+    # @param callback [#call, Proc]
+    # @return [Integer] arity of the provided block/method
     def arity(callback)
       return callback.arity if callback.respond_to?(:arity)
 
@@ -271,6 +297,10 @@ module Rdkafka
 
     private
 
+    # Ensures, no operations can happen on a closed producer
+    #
+    # @param method [Symbol] name of the method that invoked producer
+    # @raise [Rdkafka::ClosedProducerError]
     def closed_producer_check(method)
       raise Rdkafka::ClosedProducerError.new(method) if closed?
     end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.15.0"
+  VERSION = "0.15.1"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/rdkafka.gemspec

@@ -3,10 +3,10 @@
 require File.expand_path('lib/rdkafka/version', __dir__)
 
 Gem::Specification.new do |gem|
-  gem.authors = ['Thijs Cadier']
+  gem.authors = ['Thijs Cadier', 'Maciej Mensfeld']
   gem.email = ["contact@karafka.io"]
   gem.description = "Modern Kafka client library for Ruby based on librdkafka"
-  gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
+  gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.7+."
   gem.license = 'MIT'
 
   gem.files = `git ls-files`.split($\)

data/spec/rdkafka/admin/delete_acl_report_spec.rb

@@ -50,6 +50,7 @@ describe Rdkafka::Admin::DeleteAclReport do
   end
 
   it "should get deleted acl resource pattern type as Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL" do
+    expect(subject.deleted_acls[0].matching_acl_resource_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
     expect(subject.deleted_acls[0].matching_acl_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
   end
 

data/spec/rdkafka/admin/describe_acl_report_spec.rb

@@ -51,6 +51,7 @@ describe Rdkafka::Admin::DescribeAclReport do
   end
 
   it "should get matching acl resource pattern type as Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL" do
+    expect(subject.acls[0].matching_acl_resource_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
     expect(subject.acls[0].matching_acl_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
   end
 

data/spec/rdkafka/consumer_spec.rb

@@ -480,6 +480,8 @@ describe Rdkafka::Consumer do
       end
 
       describe "#store_offset" do
+        let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': false).consumer }
+
         before do
           config = {}
           config[:'enable.auto.offset.store'] = false
@@ -542,6 +544,14 @@ describe Rdkafka::Consumer do
             }.to raise_error(Rdkafka::RdkafkaError)
           end
         end
+
+        context "when trying to use with enable.auto.offset.store set to true" do
+          let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': true).consumer }
+
+          it "expect to raise invalid configuration error" do
+            expect { consumer.store_offset(message) }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
+          end
+        end
       end
     end
   end
@@ -1123,6 +1133,16 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe '#consumer_group_metadata_pointer' do
+    let(:pointer) { consumer.consumer_group_metadata_pointer }
+
+    after { Rdkafka::Bindings.rd_kafka_consumer_group_metadata_destroy(pointer) }
+
+    it 'expect to return a pointer' do
+      expect(pointer).to be_a(FFI::Pointer)
+    end
+  end
+
   describe "a rebalance listener" do
     let(:consumer) do
       config = rdkafka_consumer_config

data/spec/rdkafka/producer/delivery_report_spec.rb

@@ -15,6 +15,10 @@ describe Rdkafka::Producer::DeliveryReport do
     expect(subject.topic_name).to eq "topic"
   end
 
+  it "should get the same topic name under topic alias" do
+    expect(subject.topic).to eq "topic"
+  end
+
   it "should get the error" do
     expect(subject.error).to eq -1
   end

data/spec/rdkafka/producer_spec.rb

@@ -34,6 +34,7 @@ describe Rdkafka::Producer do
 
         producer.delivery_callback = lambda do |report|
           expect(report).not_to be_nil
+          expect(report.label).to eq "label"
           expect(report.partition).to eq 1
           expect(report.offset).to be >= 0
           expect(report.topic_name).to eq "produce_test_topic"
@@ -44,9 +45,12 @@ describe Rdkafka::Producer do
         handle = producer.produce(
           topic:   "produce_test_topic",
           payload: "payload",
-          key:     "key"
+          key:     "key",
+          label:   "label"
         )
 
+        expect(handle.label).to eq "label"
+
         # Wait for it to be delivered
         handle.wait(max_wait_timeout: 15)
 
@@ -175,11 +179,13 @@ describe Rdkafka::Producer do
     handle = producer.produce(
       topic:   "produce_test_topic",
       payload: "payload",
-      key:     "key"
+      key:     "key",
+      label:   "label"
     )
 
     # Should be pending at first
     expect(handle.pending?).to be true
+    expect(handle.label).to eq "label"
 
     # Check delivery handle and report
     report = handle.wait(max_wait_timeout: 5)
@@ -187,6 +193,7 @@ describe Rdkafka::Producer do
     expect(report).not_to be_nil
     expect(report.partition).to eq 1
     expect(report.offset).to be >= 0
+    expect(report.label).to eq "label"
 
     # Flush and close producer
     producer.flush
@@ -558,6 +565,23 @@ describe Rdkafka::Producer do
     end
   end
 
+  context "when not being able to deliver the message" do
+    let(:producer) do
+      rdkafka_producer_config(
+        "bootstrap.servers": "localhost:9093",
+        "message.timeout.ms": 100
+      ).producer
+    end
+
+    it "should contain the error in the response when not deliverable" do
+      handler = producer.produce(topic: 'produce_test_topic', payload: nil, label: 'na')
+      # Wait for the async callbacks and delivery registry to update
+      sleep(2)
+      expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
+      expect(handler.create_result.label).to eq('na')
+    end
+  end
+
   describe '#partition_count' do
     it { expect(producer.partition_count('consume_test_topic')).to eq(3) }
 

metadata

@@ -1,10 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.15.1
 platform: ruby
 authors:
 - Thijs Cadier
+- Maciej Mensfeld
 autorequire: 
 bindir: bin
 cert_chain:
@@ -35,7 +36,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-12-03 00:00:00.000000000 Z
+date: 2024-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -277,12 +278,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
-  and Ruby 2.4+.
+  and Ruby 2.7+.
 test_files:
 - spec/rdkafka/abstract_handle_spec.rb
 - spec/rdkafka/admin/create_acl_handle_spec.rb